A fork of Gitea (see branch `mj`) adding Majority Judgment Polls 𐄷 over Issues and Merge Requests. https://git.mieuxvoter.fr
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

426 lines
11 KiB

9 years ago
9 years ago
Move macaron to chi (#14293) Use [chi](https://github.com/go-chi/chi) instead of the forked [macaron](https://gitea.com/macaron/macaron). Since macaron and chi have conflicts with session share, this big PR becomes a have-to thing. According my previous idea, we can replace macaron step by step but I'm wrong. :( Below is a list of big changes on this PR. - [x] Define `context.ResponseWriter` interface with an implementation `context.Response`. - [x] Use chi instead of macaron, and also a customize `Route` to wrap chi so that the router usage is similar as before. - [x] Create different routers for `web`, `api`, `internal` and `install` so that the codes will be more clear and no magic . - [x] Use https://github.com/unrolled/render instead of macaron's internal render - [x] Use https://github.com/NYTimes/gziphandler instead of https://gitea.com/macaron/gzip - [x] Use https://gitea.com/go-chi/session which is a modified version of https://gitea.com/macaron/session and removed `nodb` support since it will not be maintained. **BREAK** - [x] Use https://gitea.com/go-chi/captcha which is a modified version of https://gitea.com/macaron/captcha - [x] Use https://gitea.com/go-chi/cache which is a modified version of https://gitea.com/macaron/cache - [x] Use https://gitea.com/go-chi/binding which is a modified version of https://gitea.com/macaron/binding - [x] Use https://github.com/go-chi/cors instead of https://gitea.com/macaron/cors - [x] Dropped https://gitea.com/macaron/i18n and make a new one in `code.gitea.io/gitea/modules/translation` - [x] Move validation form structs from `code.gitea.io/gitea/modules/auth` to `code.gitea.io/gitea/modules/forms` to avoid dependency cycle. - [x] Removed macaron log service because it's not need any more. **BREAK** - [x] All form structs have to be get by `web.GetForm(ctx)` in the route function but not as a function parameter on routes definition. - [x] Move Git HTTP protocol implementation to use routers directly. - [x] Fix the problem that chi routes don't support trailing slash but macaron did. - [x] `/api/v1/swagger` now will be redirect to `/api/swagger` but not render directly so that `APIContext` will not create a html render. Notices: - Chi router don't support request with trailing slash - Integration test `TestUserHeatmap` maybe mysql version related. It's failed on my macOS(mysql 5.7.29 installed via brew) but succeed on CI. Co-authored-by: 6543 <6543@obermui.de>
2 years ago
9 years ago
7 years ago
9 years ago
  1. // Copyright 2014 The Gogs Authors. All rights reserved.
  2. // Copyright 2016 The Gitea Authors. All rights reserved.
  3. // Use of this source code is governed by a MIT-style
  4. // license that can be found in the LICENSE file.
  5. package cmd
  6. import (
  7. "fmt"
  8. "io/ioutil"
  9. "os"
  10. "path"
  11. "path/filepath"
  12. "strings"
  13. "time"
  14. "code.gitea.io/gitea/models"
  15. "code.gitea.io/gitea/modules/log"
  16. "code.gitea.io/gitea/modules/setting"
  17. "code.gitea.io/gitea/modules/storage"
  18. "code.gitea.io/gitea/modules/util"
  19. "gitea.com/go-chi/session"
  20. jsoniter "github.com/json-iterator/go"
  21. archiver "github.com/mholt/archiver/v3"
  22. "github.com/urfave/cli"
  23. )
  24. func addFile(w archiver.Writer, filePath string, absPath string, verbose bool) error {
  25. if verbose {
  26. log.Info("Adding file %s\n", filePath)
  27. }
  28. file, err := os.Open(absPath)
  29. if err != nil {
  30. return err
  31. }
  32. defer file.Close()
  33. fileInfo, err := file.Stat()
  34. if err != nil {
  35. return err
  36. }
  37. return w.Write(archiver.File{
  38. FileInfo: archiver.FileInfo{
  39. FileInfo: fileInfo,
  40. CustomName: filePath,
  41. },
  42. ReadCloser: file,
  43. })
  44. }
  45. func isSubdir(upper string, lower string) (bool, error) {
  46. if relPath, err := filepath.Rel(upper, lower); err != nil {
  47. return false, err
  48. } else if relPath == "." || !strings.HasPrefix(relPath, ".") {
  49. return true, nil
  50. }
  51. return false, nil
  52. }
  53. type outputType struct {
  54. Enum []string
  55. Default string
  56. selected string
  57. }
  58. func (o outputType) Join() string {
  59. return strings.Join(o.Enum, ", ")
  60. }
  61. func (o *outputType) Set(value string) error {
  62. for _, enum := range o.Enum {
  63. if enum == value {
  64. o.selected = value
  65. return nil
  66. }
  67. }
  68. return fmt.Errorf("allowed values are %s", o.Join())
  69. }
  70. func (o outputType) String() string {
  71. if o.selected == "" {
  72. return o.Default
  73. }
  74. return o.selected
  75. }
  76. var outputTypeEnum = &outputType{
  77. Enum: []string{"zip", "tar", "tar.gz", "tar.xz", "tar.bz2"},
  78. Default: "zip",
  79. }
  80. // CmdDump represents the available dump sub-command.
  81. var CmdDump = cli.Command{
  82. Name: "dump",
  83. Usage: "Dump Gitea files and database",
  84. Description: `Dump compresses all related files and database into zip file.
  85. It can be used for backup and capture Gitea server image to send to maintainer`,
  86. Action: runDump,
  87. Flags: []cli.Flag{
  88. cli.StringFlag{
  89. Name: "file, f",
  90. Value: fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()),
  91. Usage: "Name of the dump file which will be created. Supply '-' for stdout. See type for available types.",
  92. },
  93. cli.BoolFlag{
  94. Name: "verbose, V",
  95. Usage: "Show process details",
  96. },
  97. cli.StringFlag{
  98. Name: "tempdir, t",
  99. Value: os.TempDir(),
  100. Usage: "Temporary dir path",
  101. },
  102. cli.StringFlag{
  103. Name: "database, d",
  104. Usage: "Specify the database SQL syntax",
  105. },
  106. cli.BoolFlag{
  107. Name: "skip-repository, R",
  108. Usage: "Skip the repository dumping",
  109. },
  110. cli.BoolFlag{
  111. Name: "skip-log, L",
  112. Usage: "Skip the log dumping",
  113. },
  114. cli.BoolFlag{
  115. Name: "skip-custom-dir",
  116. Usage: "Skip custom directory",
  117. },
  118. cli.BoolFlag{
  119. Name: "skip-lfs-data",
  120. Usage: "Skip LFS data",
  121. },
  122. cli.BoolFlag{
  123. Name: "skip-attachment-data",
  124. Usage: "Skip attachment data",
  125. },
  126. cli.GenericFlag{
  127. Name: "type",
  128. Value: outputTypeEnum,
  129. Usage: fmt.Sprintf("Dump output format: %s", outputTypeEnum.Join()),
  130. },
  131. },
  132. }
  133. func fatal(format string, args ...interface{}) {
  134. fmt.Fprintf(os.Stderr, format+"\n", args...)
  135. log.Fatal(format, args...)
  136. }
  137. func runDump(ctx *cli.Context) error {
  138. var file *os.File
  139. fileName := ctx.String("file")
  140. if fileName == "-" {
  141. file = os.Stdout
  142. err := log.DelLogger("console")
  143. if err != nil {
  144. fatal("Deleting default logger failed. Can not write to stdout: %v", err)
  145. }
  146. }
  147. setting.NewContext()
  148. // make sure we are logging to the console no matter what the configuration tells us do to
  149. if _, err := setting.Cfg.Section("log").NewKey("MODE", "console"); err != nil {
  150. fatal("Setting logging mode to console failed: %v", err)
  151. }
  152. if _, err := setting.Cfg.Section("log.console").NewKey("STDERR", "true"); err != nil {
  153. fatal("Setting console logger to stderr failed: %v", err)
  154. }
  155. if !setting.InstallLock {
  156. log.Error("Is '%s' really the right config path?\n", setting.CustomConf)
  157. return fmt.Errorf("gitea is not initialized")
  158. }
  159. setting.NewServices() // cannot access session settings otherwise
  160. err := models.SetEngine()
  161. if err != nil {
  162. return err
  163. }
  164. if err := storage.Init(); err != nil {
  165. return err
  166. }
  167. if file == nil {
  168. file, err = os.Create(fileName)
  169. if err != nil {
  170. fatal("Unable to open %s: %v", fileName, err)
  171. }
  172. }
  173. defer file.Close()
  174. absFileName, err := filepath.Abs(fileName)
  175. if err != nil {
  176. return err
  177. }
  178. verbose := ctx.Bool("verbose")
  179. outType := ctx.String("type")
  180. var iface interface{}
  181. if fileName == "-" {
  182. iface, err = archiver.ByExtension(fmt.Sprintf(".%s", outType))
  183. } else {
  184. iface, err = archiver.ByExtension(fileName)
  185. }
  186. if err != nil {
  187. fatal("Unable to get archiver for extension: %v", err)
  188. }
  189. w, _ := iface.(archiver.Writer)
  190. if err := w.Create(file); err != nil {
  191. fatal("Creating archiver.Writer failed: %v", err)
  192. }
  193. defer w.Close()
  194. if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") {
  195. log.Info("Skip dumping local repositories")
  196. } else {
  197. log.Info("Dumping local repositories... %s", setting.RepoRootPath)
  198. if err := addRecursiveExclude(w, "repos", setting.RepoRootPath, []string{absFileName}, verbose); err != nil {
  199. fatal("Failed to include repositories: %v", err)
  200. }
  201. if ctx.IsSet("skip-lfs-data") && ctx.Bool("skip-lfs-data") {
  202. log.Info("Skip dumping LFS data")
  203. } else if err := storage.LFS.IterateObjects(func(objPath string, object storage.Object) error {
  204. info, err := object.Stat()
  205. if err != nil {
  206. return err
  207. }
  208. return w.Write(archiver.File{
  209. FileInfo: archiver.FileInfo{
  210. FileInfo: info,
  211. CustomName: path.Join("data", "lfs", objPath),
  212. },
  213. ReadCloser: object,
  214. })
  215. }); err != nil {
  216. fatal("Failed to dump LFS objects: %v", err)
  217. }
  218. }
  219. tmpDir := ctx.String("tempdir")
  220. if _, err := os.Stat(tmpDir); os.IsNotExist(err) {
  221. fatal("Path does not exist: %s", tmpDir)
  222. }
  223. dbDump, err := ioutil.TempFile(tmpDir, "gitea-db.sql")
  224. if err != nil {
  225. fatal("Failed to create tmp file: %v", err)
  226. }
  227. defer func() {
  228. if err := util.Remove(dbDump.Name()); err != nil {
  229. log.Warn("Unable to remove temporary file: %s: Error: %v", dbDump.Name(), err)
  230. }
  231. }()
  232. targetDBType := ctx.String("database")
  233. if len(targetDBType) > 0 && targetDBType != setting.Database.Type {
  234. log.Info("Dumping database %s => %s...", setting.Database.Type, targetDBType)
  235. } else {
  236. log.Info("Dumping database...")
  237. }
  238. if err := models.DumpDatabase(dbDump.Name(), targetDBType); err != nil {
  239. fatal("Failed to dump database: %v", err)
  240. }
  241. if err := addFile(w, "gitea-db.sql", dbDump.Name(), verbose); err != nil {
  242. fatal("Failed to include gitea-db.sql: %v", err)
  243. }
  244. if len(setting.CustomConf) > 0 {
  245. log.Info("Adding custom configuration file from %s", setting.CustomConf)
  246. if err := addFile(w, "app.ini", setting.CustomConf, verbose); err != nil {
  247. fatal("Failed to include specified app.ini: %v", err)
  248. }
  249. }
  250. if ctx.IsSet("skip-custom-dir") && ctx.Bool("skip-custom-dir") {
  251. log.Info("Skiping custom directory")
  252. } else {
  253. customDir, err := os.Stat(setting.CustomPath)
  254. if err == nil && customDir.IsDir() {
  255. if is, _ := isSubdir(setting.AppDataPath, setting.CustomPath); !is {
  256. if err := addRecursiveExclude(w, "custom", setting.CustomPath, []string{absFileName}, verbose); err != nil {
  257. fatal("Failed to include custom: %v", err)
  258. }
  259. } else {
  260. log.Info("Custom dir %s is inside data dir %s, skipped", setting.CustomPath, setting.AppDataPath)
  261. }
  262. } else {
  263. log.Info("Custom dir %s doesn't exist, skipped", setting.CustomPath)
  264. }
  265. }
  266. isExist, err := util.IsExist(setting.AppDataPath)
  267. if err != nil {
  268. log.Error("Unable to check if %s exists. Error: %v", setting.AppDataPath, err)
  269. }
  270. if isExist {
  271. log.Info("Packing data directory...%s", setting.AppDataPath)
  272. var excludes []string
  273. if setting.Cfg.Section("session").Key("PROVIDER").Value() == "file" {
  274. var opts session.Options
  275. json := jsoniter.ConfigCompatibleWithStandardLibrary
  276. if err = json.Unmarshal([]byte(setting.SessionConfig.ProviderConfig), &opts); err != nil {
  277. return err
  278. }
  279. excludes = append(excludes, opts.ProviderConfig)
  280. }
  281. excludes = append(excludes, setting.RepoRootPath)
  282. excludes = append(excludes, setting.LFS.Path)
  283. excludes = append(excludes, setting.Attachment.Path)
  284. excludes = append(excludes, setting.LogRootPath)
  285. excludes = append(excludes, absFileName)
  286. if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil {
  287. fatal("Failed to include data directory: %v", err)
  288. }
  289. }
  290. if ctx.IsSet("skip-attachment-data") && ctx.Bool("skip-attachment-data") {
  291. log.Info("Skip dumping attachment data")
  292. } else if err := storage.Attachments.IterateObjects(func(objPath string, object storage.Object) error {
  293. info, err := object.Stat()
  294. if err != nil {
  295. return err
  296. }
  297. return w.Write(archiver.File{
  298. FileInfo: archiver.FileInfo{
  299. FileInfo: info,
  300. CustomName: path.Join("data", "attachments", objPath),
  301. },
  302. ReadCloser: object,
  303. })
  304. }); err != nil {
  305. fatal("Failed to dump attachments: %v", err)
  306. }
  307. // Doesn't check if LogRootPath exists before processing --skip-log intentionally,
  308. // ensuring that it's clear the dump is skipped whether the directory's initialized
  309. // yet or not.
  310. if ctx.IsSet("skip-log") && ctx.Bool("skip-log") {
  311. log.Info("Skip dumping log files")
  312. } else {
  313. isExist, err := util.IsExist(setting.LogRootPath)
  314. if err != nil {
  315. log.Error("Unable to check if %s exists. Error: %v", setting.LogRootPath, err)
  316. }
  317. if isExist {
  318. if err := addRecursiveExclude(w, "log", setting.LogRootPath, []string{absFileName}, verbose); err != nil {
  319. fatal("Failed to include log: %v", err)
  320. }
  321. }
  322. }
  323. if fileName != "-" {
  324. if err = w.Close(); err != nil {
  325. _ = util.Remove(fileName)
  326. fatal("Failed to save %s: %v", fileName, err)
  327. }
  328. if err := os.Chmod(fileName, 0600); err != nil {
  329. log.Info("Can't change file access permissions mask to 0600: %v", err)
  330. }
  331. }
  332. if fileName != "-" {
  333. log.Info("Finish dumping in file %s", fileName)
  334. } else {
  335. log.Info("Finish dumping to stdout")
  336. }
  337. return nil
  338. }
  339. func contains(slice []string, s string) bool {
  340. for _, v := range slice {
  341. if v == s {
  342. return true
  343. }
  344. }
  345. return false
  346. }
  347. // addRecursiveExclude zips absPath to specified insidePath inside writer excluding excludeAbsPath
  348. func addRecursiveExclude(w archiver.Writer, insidePath, absPath string, excludeAbsPath []string, verbose bool) error {
  349. absPath, err := filepath.Abs(absPath)
  350. if err != nil {
  351. return err
  352. }
  353. dir, err := os.Open(absPath)
  354. if err != nil {
  355. return err
  356. }
  357. defer dir.Close()
  358. files, err := dir.Readdir(0)
  359. if err != nil {
  360. return err
  361. }
  362. for _, file := range files {
  363. currentAbsPath := path.Join(absPath, file.Name())
  364. currentInsidePath := path.Join(insidePath, file.Name())
  365. if file.IsDir() {
  366. if !contains(excludeAbsPath, currentAbsPath) {
  367. if err := addFile(w, currentInsidePath, currentAbsPath, false); err != nil {
  368. return err
  369. }
  370. if err = addRecursiveExclude(w, currentInsidePath, currentAbsPath, excludeAbsPath, verbose); err != nil {
  371. return err
  372. }
  373. }
  374. } else {
  375. if err = addFile(w, currentInsidePath, currentAbsPath, verbose); err != nil {
  376. return err
  377. }
  378. }
  379. }
  380. return nil
  381. }