// Copyright 2014 The Gogs Authors. All rights reserved. // Copyright 2016 The Gitea Authors. All rights reserved. // Use of this source code is governed by a MIT-style // license that can be found in the LICENSE file. package cmd import ( "encoding/json" "fmt" "io/ioutil" "os" "path" "path/filepath" "strings" "time" "code.gitea.io/gitea/models" "code.gitea.io/gitea/modules/log" "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/storage" "code.gitea.io/gitea/modules/util" "gitea.com/macaron/session" archiver "github.com/mholt/archiver/v3" "github.com/urfave/cli" ) func addFile(w archiver.Writer, filePath string, absPath string, verbose bool) error { if verbose { log.Info("Adding file %s\n", filePath) } file, err := os.Open(absPath) if err != nil { return err } defer file.Close() fileInfo, err := file.Stat() if err != nil { return err } return w.Write(archiver.File{ FileInfo: archiver.FileInfo{ FileInfo: fileInfo, CustomName: filePath, }, ReadCloser: file, }) } func addRecursive(w archiver.Writer, dirPath string, absPath string, verbose bool) error { if verbose { log.Info("Adding dir %s\n", dirPath) } dir, err := os.Open(absPath) if err != nil { return fmt.Errorf("Could not open directory %s: %s", absPath, err) } defer dir.Close() files, err := dir.Readdir(0) if err != nil { return fmt.Errorf("Unable to list files in %s: %s", absPath, err) } if err := addFile(w, dirPath, absPath, false); err != nil { return err } for _, fileInfo := range files { if fileInfo.IsDir() { err = addRecursive(w, filepath.Join(dirPath, fileInfo.Name()), filepath.Join(absPath, fileInfo.Name()), verbose) } else { err = addFile(w, filepath.Join(dirPath, fileInfo.Name()), filepath.Join(absPath, fileInfo.Name()), verbose) } if err != nil { return err } } return nil } func isSubdir(upper string, lower string) (bool, error) { if relPath, err := filepath.Rel(upper, lower); err != nil { return false, err } else if relPath == "." || !strings.HasPrefix(relPath, ".") { return true, nil } return false, nil } type outputType struct { Enum []string Default string selected string } func (o outputType) Join() string { return strings.Join(o.Enum, ", ") } func (o *outputType) Set(value string) error { for _, enum := range o.Enum { if enum == value { o.selected = value return nil } } return fmt.Errorf("allowed values are %s", o.Join()) } func (o outputType) String() string { if o.selected == "" { return o.Default } return o.selected } var outputTypeEnum = &outputType{ Enum: []string{"zip", "tar", "tar.gz", "tar.xz", "tar.bz2"}, Default: "zip", } // CmdDump represents the available dump sub-command. var CmdDump = cli.Command{ Name: "dump", Usage: "Dump Gitea files and database", Description: `Dump compresses all related files and database into zip file. It can be used for backup and capture Gitea server image to send to maintainer`, Action: runDump, Flags: []cli.Flag{ cli.StringFlag{ Name: "file, f", Value: fmt.Sprintf("gitea-dump-%d.zip", time.Now().Unix()), Usage: "Name of the dump file which will be created. Supply '-' for stdout. See type for available types.", }, cli.BoolFlag{ Name: "verbose, V", Usage: "Show process details", }, cli.StringFlag{ Name: "tempdir, t", Value: os.TempDir(), Usage: "Temporary dir path", }, cli.StringFlag{ Name: "database, d", Usage: "Specify the database SQL syntax", }, cli.BoolFlag{ Name: "skip-repository, R", Usage: "Skip the repository dumping", }, cli.BoolFlag{ Name: "skip-log, L", Usage: "Skip the log dumping", }, cli.GenericFlag{ Name: "type", Value: outputTypeEnum, Usage: fmt.Sprintf("Dump output format: %s", outputTypeEnum.Join()), }, }, } func fatal(format string, args ...interface{}) { fmt.Fprintf(os.Stderr, format+"\n", args...) log.Fatal(format, args...) } func runDump(ctx *cli.Context) error { var file *os.File fileName := ctx.String("file") if fileName == "-" { file = os.Stdout err := log.DelLogger("console") if err != nil { fatal("Deleting default logger failed. Can not write to stdout: %v", err) } } setting.NewContext() // make sure we are logging to the console no matter what the configuration tells us do to if _, err := setting.Cfg.Section("log").NewKey("MODE", "console"); err != nil { fatal("Setting logging mode to console failed: %v", err) } if _, err := setting.Cfg.Section("log.console").NewKey("STDERR", "true"); err != nil { fatal("Setting console logger to stderr failed: %v", err) } if !setting.InstallLock { log.Error("Is '%s' really the right config path?\n", setting.CustomConf) return fmt.Errorf("gitea is not initialized") } setting.NewServices() // cannot access session settings otherwise err := models.SetEngine() if err != nil { return err } if err := storage.Init(); err != nil { return err } if file == nil { file, err = os.Create(fileName) if err != nil { fatal("Unable to open %s: %v", fileName, err) } } defer file.Close() verbose := ctx.Bool("verbose") outType := ctx.String("type") var iface interface{} if fileName == "-" { iface, err = archiver.ByExtension(fmt.Sprintf(".%s", outType)) } else { iface, err = archiver.ByExtension(fileName) } if err != nil { fatal("Unable to get archiver for extension: %v", err) } w, _ := iface.(archiver.Writer) if err := w.Create(file); err != nil { fatal("Creating archiver.Writer failed: %v", err) } defer w.Close() if ctx.IsSet("skip-repository") && ctx.Bool("skip-repository") { log.Info("Skip dumping local repositories") } else { log.Info("Dumping local repositories... %s", setting.RepoRootPath) if err := addRecursive(w, "repos", setting.RepoRootPath, verbose); err != nil { fatal("Failed to include repositories: %v", err) } if err := storage.LFS.IterateObjects(func(objPath string, object storage.Object) error { info, err := object.Stat() if err != nil { return err } return w.Write(archiver.File{ FileInfo: archiver.FileInfo{ FileInfo: info, CustomName: path.Join("data", "lfs", objPath), }, ReadCloser: object, }) }); err != nil { fatal("Failed to dump LFS objects: %v", err) } } tmpDir := ctx.String("tempdir") if _, err := os.Stat(tmpDir); os.IsNotExist(err) { fatal("Path does not exist: %s", tmpDir) } dbDump, err := ioutil.TempFile(tmpDir, "gitea-db.sql") if err != nil { fatal("Failed to create tmp file: %v", err) } defer func() { if err := util.Remove(dbDump.Name()); err != nil { log.Warn("Unable to remove temporary file: %s: Error: %v", dbDump.Name(), err) } }() targetDBType := ctx.String("database") if len(targetDBType) > 0 && targetDBType != setting.Database.Type { log.Info("Dumping database %s => %s...", setting.Database.Type, targetDBType) } else { log.Info("Dumping database...") } if err := models.DumpDatabase(dbDump.Name(), targetDBType); err != nil { fatal("Failed to dump database: %v", err) } if err := addFile(w, "gitea-db.sql", dbDump.Name(), verbose); err != nil { fatal("Failed to include gitea-db.sql: %v", err) } if len(setting.CustomConf) > 0 { log.Info("Adding custom configuration file from %s", setting.CustomConf) if err := addFile(w, "app.ini", setting.CustomConf, verbose); err != nil { fatal("Failed to include specified app.ini: %v", err) } } customDir, err := os.Stat(setting.CustomPath) if err == nil && customDir.IsDir() { if is, _ := isSubdir(setting.AppDataPath, setting.CustomPath); !is { if err := addRecursive(w, "custom", setting.CustomPath, verbose); err != nil { fatal("Failed to include custom: %v", err) } } else { log.Info("Custom dir %s is inside data dir %s, skipped", setting.CustomPath, setting.AppDataPath) } } else { log.Info("Custom dir %s doesn't exist, skipped", setting.CustomPath) } isExist, err := util.IsExist(setting.AppDataPath) if err != nil { log.Error("Unable to check if %s exists. Error: %v", setting.AppDataPath, err) } if isExist { log.Info("Packing data directory...%s", setting.AppDataPath) var excludes []string if setting.Cfg.Section("session").Key("PROVIDER").Value() == "file" { var opts session.Options if err = json.Unmarshal([]byte(setting.SessionConfig.ProviderConfig), &opts); err != nil { return err } excludes = append(excludes, opts.ProviderConfig) } excludes = append(excludes, setting.RepoRootPath) excludes = append(excludes, setting.LFS.Path) excludes = append(excludes, setting.Attachment.Path) excludes = append(excludes, setting.LogRootPath) if err := addRecursiveExclude(w, "data", setting.AppDataPath, excludes, verbose); err != nil { fatal("Failed to include data directory: %v", err) } } if err := storage.Attachments.IterateObjects(func(objPath string, object storage.Object) error { info, err := object.Stat() if err != nil { return err } return w.Write(archiver.File{ FileInfo: archiver.FileInfo{ FileInfo: info, CustomName: path.Join("data", "attachments", objPath), }, ReadCloser: object, }) }); err != nil { fatal("Failed to dump attachments: %v", err) } // Doesn't check if LogRootPath exists before processing --skip-log intentionally, // ensuring that it's clear the dump is skipped whether the directory's initialized // yet or not. if ctx.IsSet("skip-log") && ctx.Bool("skip-log") { log.Info("Skip dumping log files") } else { isExist, err := util.IsExist(setting.LogRootPath) if err != nil { log.Error("Unable to check if %s exists. Error: %v", setting.LogRootPath, err) } if isExist { if err := addRecursive(w, "log", setting.LogRootPath, verbose); err != nil { fatal("Failed to include log: %v", err) } } } if fileName != "-" { if err = w.Close(); err != nil { _ = util.Remove(fileName) fatal("Failed to save %s: %v", fileName, err) } if err := os.Chmod(fileName, 0600); err != nil { log.Info("Can't change file access permissions mask to 0600: %v", err) } } if fileName != "-" { log.Info("Finish dumping in file %s", fileName) } else { log.Info("Finish dumping to stdout") } return nil } func contains(slice []string, s string) bool { for _, v := range slice { if v == s { return true } } return false } // addRecursiveExclude zips absPath to specified insidePath inside writer excluding excludeAbsPath func addRecursiveExclude(w archiver.Writer, insidePath, absPath string, excludeAbsPath []string, verbose bool) error { absPath, err := filepath.Abs(absPath) if err != nil { return err } dir, err := os.Open(absPath) if err != nil { return err } defer dir.Close() files, err := dir.Readdir(0) if err != nil { return err } for _, file := range files { currentAbsPath := path.Join(absPath, file.Name()) currentInsidePath := path.Join(insidePath, file.Name()) if file.IsDir() { if !contains(excludeAbsPath, currentAbsPath) { if err := addFile(w, currentInsidePath, currentAbsPath, false); err != nil { return err } if err = addRecursiveExclude(w, currentInsidePath, currentAbsPath, excludeAbsPath, verbose); err != nil { return err } } } else { if err = addFile(w, currentInsidePath, currentAbsPath, verbose); err != nil { return err } } } return nil }