diff --git a/cmd/doctor.go b/cmd/doctor.go index 2a93db27d..2ca2bb5e7 100644 --- a/cmd/doctor.go +++ b/cmd/doctor.go @@ -26,6 +26,7 @@ import ( "code.gitea.io/gitea/modules/setting" "code.gitea.io/gitea/modules/util" "xorm.io/builder" + "xorm.io/xorm" "github.com/urfave/cli" ) @@ -62,6 +63,27 @@ var CmdDoctor = cli.Command{ Usage: `Name of the log file (default: "doctor.log"). Set to "-" to output to stdout, set to "" to disable`, }, }, + Subcommands: []cli.Command{ + cmdRecreateTable, + }, +} + +var cmdRecreateTable = cli.Command{ + Name: "recreate-table", + Usage: "Recreate tables from XORM definitions and copy the data.", + ArgsUsage: "[TABLE]... : (TABLEs to recreate - leave blank for all)", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "debug", + Usage: "Print SQL commands sent", + }, + }, + Description: `The database definitions Gitea uses change across versions, sometimes changing default values and leaving old unused columns. + +This command will cause Xorm to recreate tables, copying over the data and deleting the old table. + +You should back-up your database before doing this and ensure that your database is up-to-date first.`, + Action: runRecreateTable, } type check struct { @@ -136,6 +158,47 @@ var checklist = []check{ // more checks please append here } +func runRecreateTable(ctx *cli.Context) error { + // Redirect the default golog to here + golog.SetFlags(0) + golog.SetPrefix("") + golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT))) + + setting.NewContext() + setting.InitDBConfig() + + setting.EnableXORMLog = ctx.Bool("debug") + setting.Database.LogSQL = ctx.Bool("debug") + setting.Cfg.Section("log").Key("XORM").SetValue(",") + + setting.NewXORMLogService(!ctx.Bool("debug")) + if err := models.SetEngine(); err != nil { + fmt.Println(err) + fmt.Println("Check if you are using the right config file. You can use a --config directive to specify one.") + return nil + } + + args := ctx.Args() + names := make([]string, 0, ctx.NArg()) + for i := 0; i < ctx.NArg(); i++ { + names = append(names, args.Get(i)) + } + + beans, err := models.NamesToBean(names...) + if err != nil { + return err + } + recreateTables := migrations.RecreateTables(beans...) + + return models.NewEngine(context.Background(), func(x *xorm.Engine) error { + if err := migrations.EnsureUpToDate(x); err != nil { + return err + } + return recreateTables(x) + }) + +} + func runDoctor(ctx *cli.Context) error { // Silence the default loggers diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md index e458b11ba..49df30edb 100644 --- a/docs/content/doc/usage/command-line.en-us.md +++ b/docs/content/doc/usage/command-line.en-us.md @@ -319,6 +319,36 @@ var checklist = []check{ This function will receive a command line context and return a list of details about the problems or error. +##### doctor recreate-table + +Sometimes when there are migrations the old columns and default values may be left +unchanged in the database schema. This may lead to warning such as: + +``` +2020/08/02 11:32:29 ...rm/session_schema.go:360:Sync2() [W] Table user Column keep_activity_private db default is , struct default is 0 +``` + +You can cause Gitea to recreate these tables and copy the old data into the new table +with the defaults set appropriately by using: + +``` +gitea doctor recreate-table user +``` + +You can ask gitea to recreate multiple tables using: + +``` +gitea doctor recreate-table table1 table2 ... +``` + +And if you would like Gitea to recreate all tables simply call: + +``` +gitea doctor recreate-table +``` + +It is highly recommended to back-up your database before running these commands. + #### manager Manage running server operations: diff --git a/integrations/migration-test/migration_test.go b/integrations/migration-test/migration_test.go index 976a59a57..940e4738a 100644 --- a/integrations/migration-test/migration_test.go +++ b/integrations/migration-test/migration_test.go @@ -258,6 +258,30 @@ func doMigrationTest(t *testing.T, version string) { err = models.NewEngine(context.Background(), wrappedMigrate) assert.NoError(t, err) currentEngine.Close() + + err = models.SetEngine() + assert.NoError(t, err) + + beans, _ := models.NamesToBean() + + err = models.NewEngine(context.Background(), func(x *xorm.Engine) error { + currentEngine = x + return migrations.RecreateTables(beans...)(x) + }) + assert.NoError(t, err) + currentEngine.Close() + + // We do this a second time to ensure that there is not a problem with retained indices + err = models.SetEngine() + assert.NoError(t, err) + + err = models.NewEngine(context.Background(), func(x *xorm.Engine) error { + currentEngine = x + return migrations.RecreateTables(beans...)(x) + }) + assert.NoError(t, err) + + currentEngine.Close() } func TestMigrations(t *testing.T) { diff --git a/models/migrations/migrations.go b/models/migrations/migrations.go index aca3891f3..9b94c49cc 100644 --- a/models/migrations/migrations.go +++ b/models/migrations/migrations.go @@ -7,6 +7,7 @@ package migrations import ( "fmt" + "reflect" "regexp" "strings" @@ -327,6 +328,221 @@ Please try upgrading to a lower version first (suggested v1.6.4), then upgrade t return nil } +// RecreateTables will recreate the tables for the provided beans using the newly provided bean definition and move all data to that new table +// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION +func RecreateTables(beans ...interface{}) func(*xorm.Engine) error { + return func(x *xorm.Engine) error { + sess := x.NewSession() + defer sess.Close() + if err := sess.Begin(); err != nil { + return err + } + sess = sess.StoreEngine("InnoDB") + for _, bean := range beans { + log.Info("Recreating Table: %s for Bean: %s", x.TableName(bean), reflect.Indirect(reflect.ValueOf(bean)).Type().Name()) + if err := recreateTable(sess, bean); err != nil { + return err + } + } + return sess.Commit() + } +} + +// recreateTable will recreate the table using the newly provided bean definition and move all data to that new table +// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION +// WARNING: YOU MUST COMMIT THE SESSION AT THE END +func recreateTable(sess *xorm.Session, bean interface{}) error { + // TODO: This will not work if there are foreign keys + + tableName := sess.Engine().TableName(bean) + tempTableName := fmt.Sprintf("tmp_recreate__%s", tableName) + + // We need to move the old table away and create a new one with the correct columns + // We will need to do this in stages to prevent data loss + // + // First create the temporary table + if err := sess.Table(tempTableName).CreateTable(bean); err != nil { + log.Error("Unable to create table %s. Error: %v", tempTableName, err) + return err + } + + if err := sess.Table(tempTableName).CreateUniques(bean); err != nil { + log.Error("Unable to create uniques for table %s. Error: %v", tempTableName, err) + return err + } + + if err := sess.Table(tempTableName).CreateIndexes(bean); err != nil { + log.Error("Unable to create indexes for table %s. Error: %v", tempTableName, err) + return err + } + + // Work out the column names from the bean - these are the columns to select from the old table and install into the new table + table, err := sess.Engine().TableInfo(bean) + if err != nil { + log.Error("Unable to get table info. Error: %v", err) + + return err + } + newTableColumns := table.Columns() + if len(newTableColumns) == 0 { + return fmt.Errorf("no columns in new table") + } + hasID := false + for _, column := range newTableColumns { + hasID = hasID || (column.IsPrimaryKey && column.IsAutoIncrement) + } + + if hasID && setting.Database.UseMSSQL { + if _, err := sess.Exec(fmt.Sprintf("SET IDENTITY_INSERT `%s` ON", tempTableName)); err != nil { + log.Error("Unable to set identity insert for table %s. Error: %v", tempTableName, err) + return err + } + } + + sqlStringBuilder := &strings.Builder{} + _, _ = sqlStringBuilder.WriteString("INSERT INTO `") + _, _ = sqlStringBuilder.WriteString(tempTableName) + _, _ = sqlStringBuilder.WriteString("` (`") + _, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name) + _, _ = sqlStringBuilder.WriteString("`") + for _, column := range newTableColumns[1:] { + _, _ = sqlStringBuilder.WriteString(", `") + _, _ = sqlStringBuilder.WriteString(column.Name) + _, _ = sqlStringBuilder.WriteString("`") + } + _, _ = sqlStringBuilder.WriteString(")") + _, _ = sqlStringBuilder.WriteString(" SELECT ") + if newTableColumns[0].Default != "" { + _, _ = sqlStringBuilder.WriteString("COALESCE(`") + _, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name) + _, _ = sqlStringBuilder.WriteString("`, ") + _, _ = sqlStringBuilder.WriteString(newTableColumns[0].Default) + _, _ = sqlStringBuilder.WriteString(")") + } else { + _, _ = sqlStringBuilder.WriteString("`") + _, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name) + _, _ = sqlStringBuilder.WriteString("`") + } + + for _, column := range newTableColumns[1:] { + if column.Default != "" { + _, _ = sqlStringBuilder.WriteString(", COALESCE(`") + _, _ = sqlStringBuilder.WriteString(column.Name) + _, _ = sqlStringBuilder.WriteString("`, ") + _, _ = sqlStringBuilder.WriteString(column.Default) + _, _ = sqlStringBuilder.WriteString(")") + } else { + _, _ = sqlStringBuilder.WriteString(", `") + _, _ = sqlStringBuilder.WriteString(column.Name) + _, _ = sqlStringBuilder.WriteString("`") + } + } + _, _ = sqlStringBuilder.WriteString(" FROM `") + _, _ = sqlStringBuilder.WriteString(tableName) + _, _ = sqlStringBuilder.WriteString("`") + + if _, err := sess.Exec(sqlStringBuilder.String()); err != nil { + log.Error("Unable to set copy data in to temp table %s. Error: %v", tempTableName, err) + return err + } + + if hasID && setting.Database.UseMSSQL { + if _, err := sess.Exec(fmt.Sprintf("SET IDENTITY_INSERT `%s` OFF", tempTableName)); err != nil { + log.Error("Unable to switch off identity insert for table %s. Error: %v", tempTableName, err) + return err + } + } + + switch { + case setting.Database.UseSQLite3: + // SQLite will drop all the constraints on the old table + if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil { + log.Error("Unable to drop old table %s. Error: %v", tableName, err) + return err + } + + if err := sess.Table(tempTableName).DropIndexes(bean); err != nil { + log.Error("Unable to drop indexes on temporary table %s. Error: %v", tempTableName, err) + return err + } + + if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + + if err := sess.Table(tableName).CreateIndexes(bean); err != nil { + log.Error("Unable to recreate indexes on table %s. Error: %v", tableName, err) + return err + } + + if err := sess.Table(tableName).CreateUniques(bean); err != nil { + log.Error("Unable to recreate uniques on table %s. Error: %v", tableName, err) + return err + } + + case setting.Database.UseMySQL: + // MySQL will drop all the constraints on the old table + if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil { + log.Error("Unable to drop old table %s. Error: %v", tableName, err) + return err + } + + // SQLite and MySQL will move all the constraints from the temporary table to the new table + if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + case setting.Database.UsePostgreSQL: + // CASCADE causes postgres to drop all the constraints on the old table + if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil { + log.Error("Unable to drop old table %s. Error: %v", tableName, err) + return err + } + + // CASCADE causes postgres to move all the constraints from the temporary table to the new table + if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + + var indices []string + schema := sess.Engine().Dialect().URI().Schema + sess.Engine().SetSchema("") + if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + sess.Engine().SetSchema(schema) + + for _, index := range indices { + newIndexName := strings.Replace(index, "tmp_recreate__", "", 1) + if _, err := sess.Exec(fmt.Sprintf("ALTER INDEX `%s` RENAME TO `%s`", index, newIndexName)); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", index, newIndexName, err) + return err + } + } + + case setting.Database.UseMSSQL: + // MSSQL will drop all the constraints on the old table + if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil { + log.Error("Unable to drop old table %s. Error: %v", tableName, err) + return err + } + + // MSSQL sp_rename will move all the constraints from the temporary table to the new table + if _, err := sess.Exec(fmt.Sprintf("sp_rename `%s`,`%s`", tempTableName, tableName)); err != nil { + log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err) + return err + } + + default: + log.Fatal("Unrecognized DB") + } + return nil +} + +// WARNING: YOU MUST COMMIT THE SESSION AT THE END func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...string) (err error) { if tableName == "" || len(columnNames) == 0 { return nil diff --git a/models/models.go b/models/models.go index e0dd3ed2a..27a0c660b 100644 --- a/models/models.go +++ b/models/models.go @@ -10,6 +10,8 @@ import ( "database/sql" "errors" "fmt" + "reflect" + "strings" "code.gitea.io/gitea/modules/setting" @@ -214,6 +216,36 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e return nil } +// NamesToBean return a list of beans or an error +func NamesToBean(names ...string) ([]interface{}, error) { + beans := []interface{}{} + if len(names) == 0 { + beans = append(beans, tables...) + return beans, nil + } + // Need to map provided names to beans... + beanMap := make(map[string]interface{}) + for _, bean := range tables { + + beanMap[strings.ToLower(reflect.Indirect(reflect.ValueOf(bean)).Type().Name())] = bean + beanMap[strings.ToLower(x.TableName(bean))] = bean + beanMap[strings.ToLower(x.TableName(bean, true))] = bean + } + + gotBean := make(map[interface{}]bool) + for _, name := range names { + bean, ok := beanMap[strings.ToLower(strings.TrimSpace(name))] + if !ok { + return nil, fmt.Errorf("No table found that matches: %s", name) + } + if !gotBean[bean] { + beans = append(beans, bean) + gotBean[bean] = true + } + } + return beans, nil +} + // Statistic contains the database statistics type Statistic struct { Counter struct {