Add command to recreate tables

Provides new command: `gitea doctor recreate-table` which will recreate
db tables and copy the old data in to the new table.

This function can be used to remove the old warning of struct defaults being
out of date.

Fix go-gitea#8868
Fix go-gitea#3265
Fix go-gitea#8894

Signed-off-by: Andrew Thornton <[email protected]>

Author: zeripath

cmd/doctor.go

@@ -61,6 +61,21 @@ var CmdDoctor = cli.Command{
 			Usage: `Name of the log file (default: "doctor.log"). Set to "-" to output to stdout, set to "" to disable`,
 		},
 	},
+	Subcommands: []cli.Command{
+		cmdRecreateTable,
+	},
+}
+
+var cmdRecreateTable = cli.Command{
+	Name:      "recreate-table",
+	Usage:     "Recreate tables from XORM definitions and copy the data.",
+	ArgsUsage: "[TABLE]... : (TABLEs to recreate - leave blank for all)",
+	Description: `The database definitions Gitea uses change across versions, sometimes changing default values and leaving old unused columns.
+
+This command will cause Xorm to recreate tables, copying over the data and deleting the old table.
+
+You should back-up your database before doing this and ensure that your database is up-to-date first.`,
+	Action: runRecreateTable,
 }
 
 type check struct {
@@ -129,6 +144,37 @@ var checklist = []check{
 	// more checks please append here
 }
 
+func runRecreateTable(ctx *cli.Context) error {
+	// Redirect the default golog to here
+	golog.SetFlags(0)
+	golog.SetPrefix("")
+	golog.SetOutput(log.NewLoggerAsWriter("INFO", log.GetLogger(log.DEFAULT)))
+
+	setting.EnableXORMLog = false
+	if err := initDBDisableConsole(true); err != nil {
+		fmt.Println(err)
+		fmt.Println("Check if you are using the right config file. You can use a --config directive to specify one.")
+		return nil
+	}
+
+	if err := models.NewEngine(context.Background(), migrations.EnsureUpToDate); err != nil {
+		return err
+	}
+
+	args := ctx.Args()
+	names := make([]string, 0, ctx.NArg())
+	for i := 0; i < ctx.NArg(); i++ {
+		names = append(names, args.Get(i))
+	}
+
+	beans, err := models.NamesToBean(names...)
+	if err != nil {
+		return err
+	}
+
+	return models.NewEngine(context.Background(), migrations.RecreateTables(beans...))
+}
+
 func runDoctor(ctx *cli.Context) error {
 
 	// Silence the default loggers

docs/content/doc/usage/command-line.en-us.md

@@ -319,6 +319,36 @@ var checklist = []check{
 
 This function will receive a command line context and return a list of details about the problems or error.
 
+##### doctor recreate-table
+
+Sometimes when there are migrations the old columns and default values may be left
+unchanged in the database schema. This may lead to warning such as:
+
+```
+2020/08/02 11:32:29 ...rm/session_schema.go:360:Sync2() [W] Table user Column keep_activity_private db default is , struct default is 0
+```
+
+You can cause Gitea to recreate these tables and copy the old data into the new table
+with the defaults set appropriately by using:
+
+```
+gitea doctor recreate-table user
+```
+
+You can ask gitea to recreate multiple tables using:
+
+```
+gitea doctor recreate-table table1 table2 ...
+```
+
+And if you would like Gitea to recreate all tables simply call:
+
+```
+gitea doctor recreate-table
+```
+
+It is highly recommended to back-up your database before running these commands.
+
 #### manager
 
 Manage running server operations:

integrations/migration-test/migration_test.go

@@ -256,6 +256,12 @@ func doMigrationTest(t *testing.T, version string) {
 
 	err = models.NewEngine(context.Background(), wrappedMigrate)
 	assert.NoError(t, err)
+
+	beans, _ := models.NamesToBean()
+
+	err = models.NewEngine(context.Background(), migrations.RecreateTables(beans...))
+	assert.NoError(t, err)
+
 	currentEngine.Close()
 }
 

models/migrations/migrations.go

@@ -7,6 +7,7 @@ package migrations
 
 import (
 	"fmt"
+	"reflect"
 	"regexp"
 	"strings"
 
@@ -317,6 +318,153 @@ Please try upgrading to a lower version first (suggested v1.6.4), then upgrade t
 	return nil
 }
 
+// RecreateTables will recreate the tables for the provided beans using the newly provided bean definition and move all data to that new table
+// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION
+func RecreateTables(beans ...interface{}) func(*xorm.Engine) error {
+	return func(x *xorm.Engine) error {
+		sess := x.NewSession()
+		defer sess.Close()
+		if err := sess.Begin(); err != nil {
+			return err
+		}
+		for _, bean := range beans {
+			log.Info("Recreating Table: %s for Bean: %s", x.TableName(bean), reflect.Indirect(reflect.ValueOf(bean)).Type().Name())
+			if err := recreateTable(sess, bean); err != nil {
+				return err
+			}
+		}
+		return sess.Commit()
+	}
+}
+
+// recreateTable will recreate the table using the newly provided bean definition and move all data to that new table
+// WARNING: YOU MUST PROVIDE THE FULL BEAN DEFINITION
+// WARNING: YOU MUST COMMIT THE SESSION AT THE END
+func recreateTable(sess *xorm.Session, bean interface{}) error {
+	// TODO: This will not work if there are foreign keys
+
+	tableName := sess.Engine().TableName(bean)
+	tempTableName := fmt.Sprintf("temporary__%s__temporary", tableName)
+
+	// We need to move the old table away and create a new one with the correct columns
+	// We will need to do this in stages to prevent data loss
+	//
+	// First let's update the old table to ensure it has all the necessary columns
+	if err := sess.CreateTable(bean); err != nil {
+		return err
+	}
+
+	// Now we create the temporary table
+	if err := sess.Table(tempTableName).CreateTable(bean); err != nil {
+		return err
+	}
+
+	// Work out the column names from the bean - these are the columns to select from the old table and install into the new table
+	table, err := sess.Engine().TableInfo(bean)
+	if err != nil {
+		return err
+	}
+	newTableColumns := table.Columns()
+	if len(newTableColumns) == 0 {
+		return fmt.Errorf("no columns in new table")
+	}
+
+	if setting.Database.UseMSSQL {
+		sess.Exec(fmt.Sprintf("SET IDENTITY_INSERT %s ON", tempTableName))
+	}
+
+	sqlStringBuilder := &strings.Builder{}
+	_, _ = sqlStringBuilder.WriteString("INSERT INTO ")
+	_, _ = sqlStringBuilder.WriteString(tempTableName)
+	if setting.Database.UseMSSQL {
+		_, _ = sqlStringBuilder.WriteString(" (`")
+		_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name)
+		_, _ = sqlStringBuilder.WriteString("`")
+		for _, column := range newTableColumns[1:] {
+			_, _ = sqlStringBuilder.WriteString(", `")
+			_, _ = sqlStringBuilder.WriteString(column.Name)
+			_, _ = sqlStringBuilder.WriteString("`")
+		}
+		_, _ = sqlStringBuilder.WriteString(")")
+	}
+	_, _ = sqlStringBuilder.WriteString(" SELECT ")
+	if newTableColumns[0].Default != "" {
+		_, _ = sqlStringBuilder.WriteString("COALESCE(`")
+		_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name)
+		_, _ = sqlStringBuilder.WriteString("`, ")
+		_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Default)
+		_, _ = sqlStringBuilder.WriteString(")")
+	} else {
+		_, _ = sqlStringBuilder.WriteString("`")
+		_, _ = sqlStringBuilder.WriteString(newTableColumns[0].Name)
+		_, _ = sqlStringBuilder.WriteString("`")
+	}
+
+	for _, column := range newTableColumns[1:] {
+		if column.Default != "" {
+			_, _ = sqlStringBuilder.WriteString(", COALESCE(`")
+			_, _ = sqlStringBuilder.WriteString(column.Name)
+			_, _ = sqlStringBuilder.WriteString("`, ")
+			_, _ = sqlStringBuilder.WriteString(column.Default)
+			_, _ = sqlStringBuilder.WriteString(")")
+		} else {
+			_, _ = sqlStringBuilder.WriteString(", `")
+			_, _ = sqlStringBuilder.WriteString(column.Name)
+			_, _ = sqlStringBuilder.WriteString("`")
+		}
+	}
+	_, _ = sqlStringBuilder.WriteString(" FROM ")
+	_, _ = sqlStringBuilder.WriteString(tableName)
+
+	if _, err := sess.Exec(sqlStringBuilder.String()); err != nil {
+		return err
+	}
+
+	if setting.Database.UseMSSQL {
+		sess.Exec(fmt.Sprintf("SET IDENTITY_INSERT %s OFF", tempTableName))
+	}
+
+	switch {
+	case setting.Database.UseSQLite3:
+		fallthrough
+	case setting.Database.UseMySQL:
+		// SQLite and MySQL will drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
+			return err
+		}
+
+		// SQLite and MySQL will move all the constraints from the temporary table to the new table
+		if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil {
+			return err
+		}
+	case setting.Database.UsePostgreSQL:
+		// CASCADE causes postgres to drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
+			return err
+		}
+
+		// CASCADE causes postgres to move all the constraints from the temporary table to the new table
+		if _, err := sess.Exec(fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", tempTableName, tableName)); err != nil {
+			return err
+		}
+	case setting.Database.UseMSSQL:
+		// MSSQL will drop all the constraints on the old table
+		if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {
+			return err
+		}
+
+		// MSSQL sp_rename will move all the constraints from the temporary table to the new table
+		if _, err := sess.Exec(fmt.Sprintf("sp_rename '[%s]','[%s]'", tempTableName, tableName)); err != nil {
+			return err
+		}
+
+	default:
+		log.Fatal("Unrecognized DB")
+	}
+	return nil
+}
+
+// WARNING: YOU MUST COMMIT THE SESSION AT THE END
 func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...string) (err error) {
 	if tableName == "" || len(columnNames) == 0 {
 		return nil
@@ -465,7 +613,6 @@ func dropTableColumns(sess *xorm.Session, tableName string, columnNames ...strin
 			return fmt.Errorf("Drop table `%s` columns %v: %v", tableName, columnNames, err)
 		}
 
-		return sess.Commit()
 	default:
 		log.Fatal("Unrecognized DB")
 	}

models/migrations/v102.go

@@ -11,5 +11,8 @@ import (
 func dropColumnHeadUserNameOnPullRequest(x *xorm.Engine) error {
 	sess := x.NewSession()
 	defer sess.Close()
-	return dropTableColumns(sess, "pull_request", "head_user_name")
+	if err := dropTableColumns(sess, "pull_request", "head_user_name"); err != nil {
+		return err
+	}
+	return sess.Commit()
 }

models/models.go

@@ -10,6 +10,8 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"reflect"
+	"strings"
 
 	"code.gitea.io/gitea/modules/setting"
 
@@ -210,6 +212,38 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e
 	return nil
 }
 
+// NamesToBean return a list of beans or an error
+func NamesToBean(names ...string) ([]interface{}, error) {
+	beans := []interface{}{}
+	if len(names) == 0 {
+		for _, bean := range tables {
+			beans = append(beans, bean)
+		}
+		return beans, nil
+	}
+	// Need to map provided names to beans...
+	beanMap := make(map[string]interface{})
+	for _, bean := range tables {
+
+		beanMap[strings.ToLower(reflect.Indirect(reflect.ValueOf(bean)).Type().Name())] = bean
+		beanMap[strings.ToLower(x.TableName(bean))] = bean
+		beanMap[strings.ToLower(x.TableName(bean, true))] = bean
+	}
+
+	gotBean := make(map[interface{}]bool)
+	for _, name := range names {
+		bean, ok := beanMap[strings.ToLower(strings.TrimSpace(name))]
+		if !ok {
+			return nil, fmt.Errorf("No table found that matches: %s", name)
+		}
+		if !gotBean[bean] {
+			beans = append(beans, bean)
+			gotBean[bean] = true
+		}
+	}
+	return beans, nil
+}
+
 // Statistic contains the database statistics
 type Statistic struct {
 	Counter struct {

models/repo_language_stats.go

@@ -19,7 +19,7 @@ type LanguageStat struct {
 	RepoID      int64 `xorm:"UNIQUE(s) INDEX NOT NULL"`
 	CommitID    string
 	IsPrimary   bool
-	Language    string             `xorm:"VARCHAR(30) UNIQUE(s) INDEX NOT NULL"`
+	Language    string             `xorm:"VARCHAR(50) UNIQUE(s) INDEX NOT NULL"`
 	Percentage  float32            `xorm:"-"`
 	Size        int64              `xorm:"NOT NULL DEFAULT 0"`
 	Color       string             `xorm:"-"`