Add repounit fixer for go-gitea#16961 (go-gitea#17136)

Backport go-gitea#17136

Unfortunately a number of people appear to have been bitten by the bug in the dump
command.

This PR adds a doctor command to attempt to fix the broken repo_units

Fix go-gitea#16961

Signed-off-by: Andrew Thornton <[email protected]>

Author: zeripath

models/login_source.go

@@ -71,9 +71,9 @@ var (
 	_ convert.Conversion = &SSPIConfig{}
 )
 
-// jsonUnmarshalHandleDoubleEncode - due to a bug in xorm (see https://gitea.com/xorm/xorm/pulls/1957) - it's
+// JSONUnmarshalHandleDoubleEncode - due to a bug in xorm (see https://gitea.com/xorm/xorm/pulls/1957) - it's
 // possible that a Blob may be double encoded or gain an unwanted prefix of 0xff 0xfe.
-func jsonUnmarshalHandleDoubleEncode(bs []byte, v interface{}) error {
+func JSONUnmarshalHandleDoubleEncode(bs []byte, v interface{}) error {
 	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	err := json.Unmarshal(bs, v)
 	if err != nil {
@@ -108,7 +108,7 @@ type LDAPConfig struct {
 
 // FromDB fills up a LDAPConfig from serialized format.
 func (cfg *LDAPConfig) FromDB(bs []byte) error {
-	err := jsonUnmarshalHandleDoubleEncode(bs, &cfg)
+	err := JSONUnmarshalHandleDoubleEncode(bs, &cfg)
 	if err != nil {
 		return err
 	}
@@ -149,7 +149,7 @@ type SMTPConfig struct {
 
 // FromDB fills up an SMTPConfig from serialized format.
 func (cfg *SMTPConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalHandleDoubleEncode(bs, cfg)
+	return JSONUnmarshalHandleDoubleEncode(bs, cfg)
 }
 
 // ToDB exports an SMTPConfig to a serialized format.
@@ -166,7 +166,7 @@ type PAMConfig struct {
 
 // FromDB fills up a PAMConfig from serialized format.
 func (cfg *PAMConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalHandleDoubleEncode(bs, cfg)
+	return JSONUnmarshalHandleDoubleEncode(bs, cfg)
 }
 
 // ToDB exports a PAMConfig to a serialized format.
@@ -187,7 +187,7 @@ type OAuth2Config struct {
 
 // FromDB fills up an OAuth2Config from serialized format.
 func (cfg *OAuth2Config) FromDB(bs []byte) error {
-	return jsonUnmarshalHandleDoubleEncode(bs, cfg)
+	return JSONUnmarshalHandleDoubleEncode(bs, cfg)
 }
 
 // ToDB exports an SMTPConfig to a serialized format.
@@ -207,7 +207,7 @@ type SSPIConfig struct {
 
 // FromDB fills up an SSPIConfig from serialized format.
 func (cfg *SSPIConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalHandleDoubleEncode(bs, cfg)
+	return JSONUnmarshalHandleDoubleEncode(bs, cfg)
 }
 
 // ToDB exports an SSPIConfig to a serialized format.

models/repo_unit.go

@@ -28,7 +28,7 @@ type UnitConfig struct{}
 
 // FromDB fills up a UnitConfig from serialized format.
 func (cfg *UnitConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalHandleDoubleEncode(bs, &cfg)
+	return JSONUnmarshalHandleDoubleEncode(bs, &cfg)
 }
 
 // ToDB exports a UnitConfig to a serialized format.
@@ -44,7 +44,7 @@ type ExternalWikiConfig struct {
 
 // FromDB fills up a ExternalWikiConfig from serialized format.
 func (cfg *ExternalWikiConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalHandleDoubleEncode(bs, &cfg)
+	return JSONUnmarshalHandleDoubleEncode(bs, &cfg)
 }
 
 // ToDB exports a ExternalWikiConfig to a serialized format.
@@ -62,7 +62,7 @@ type ExternalTrackerConfig struct {
 
 // FromDB fills up a ExternalTrackerConfig from serialized format.
 func (cfg *ExternalTrackerConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalHandleDoubleEncode(bs, &cfg)
+	return JSONUnmarshalHandleDoubleEncode(bs, &cfg)
 }
 
 // ToDB exports a ExternalTrackerConfig to a serialized format.
@@ -80,7 +80,7 @@ type IssuesConfig struct {
 
 // FromDB fills up a IssuesConfig from serialized format.
 func (cfg *IssuesConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalHandleDoubleEncode(bs, &cfg)
+	return JSONUnmarshalHandleDoubleEncode(bs, &cfg)
 }
 
 // ToDB exports a IssuesConfig to a serialized format.
@@ -104,7 +104,7 @@ type PullRequestsConfig struct {
 
 // FromDB fills up a PullRequestsConfig from serialized format.
 func (cfg *PullRequestsConfig) FromDB(bs []byte) error {
-	return jsonUnmarshalHandleDoubleEncode(bs, &cfg)
+	return JSONUnmarshalHandleDoubleEncode(bs, &cfg)
 }
 
 // ToDB exports a PullRequestsConfig to a serialized format.
@@ -219,3 +219,8 @@ func getUnitsByRepoID(e Engine, repoID int64) (units []*RepoUnit, err error) {
 
 	return units, nil
 }
+
+func UpdateRepoUnit(unit *RepoUnit) error {
+	_, err := x.ID(unit.ID).Update(unit)
+	return err
+}

modules/doctor/fix16961.go

@@ -0,0 +1,259 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package doctor
+
+import (
+	"bytes"
+	"fmt"
+	"strconv"
+
+	"code.gitea.io/gitea/models"
+	"code.gitea.io/gitea/modules/log"
+	"code.gitea.io/gitea/modules/timeutil"
+	"xorm.io/builder"
+)
+
+// #16831 revealed that the dump command that was broken in 1.14.3-1.14.6 and 1.15.0 (#15885).
+// This led to repo_unit and login_source cfg not being converted to JSON in the dump
+// Unfortunately although it was hoped that there were only a few users affected it
+// appears that many users are affected.
+
+// We therefore need to provide a doctor command to fix this repeated issue #16961
+
+func fixBrokenRepoUnits(logger log.Logger, autofix bool) error {
+	// RepoUnit describes all units of a repository
+	type RepoUnit struct {
+		ID          int64
+		RepoID      int64
+		Type        models.UnitType
+		Config      []byte
+		CreatedUnix timeutil.TimeStamp `xorm:"INDEX CREATED"`
+	}
+
+	count := 0
+
+	err := models.Iterate(
+		models.DefaultDBContext(),
+		new(RepoUnit),
+		builder.Eq{"1": "1"},
+		func(idx int, bean interface{}) error {
+			unit := bean.(*RepoUnit)
+
+			bs := unit.Config
+			repoUnit := &models.RepoUnit{
+				ID:          unit.ID,
+				RepoID:      unit.RepoID,
+				Type:        unit.Type,
+				CreatedUnix: unit.CreatedUnix,
+			}
+
+			switch models.UnitType(unit.Type) {
+			case models.UnitTypeCode, models.UnitTypeReleases, models.UnitTypeWiki, models.UnitTypeProjects:
+				cfg := &models.UnitConfig{}
+				repoUnit.Config = cfg
+
+				err := models.JSONUnmarshalHandleDoubleEncode(bs, &cfg)
+				if err == nil {
+					return nil
+				}
+
+				// Handle #16961
+				if string(bs) != "&{}" {
+					return err
+				}
+			case models.UnitTypeExternalWiki:
+				cfg := &models.ExternalWikiConfig{}
+				repoUnit.Config = cfg
+				err := models.JSONUnmarshalHandleDoubleEncode(bs, &cfg)
+				if err == nil {
+					return nil
+				}
+
+				if len(bs) < 3 {
+					return err
+				}
+				if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
+					return err
+				}
+				cfg.ExternalWikiURL = string(bs[2 : len(bs)-1])
+			case models.UnitTypeExternalTracker:
+				cfg := &models.ExternalTrackerConfig{}
+				repoUnit.Config = cfg
+				err := models.JSONUnmarshalHandleDoubleEncode(bs, &cfg)
+				if err == nil {
+					return nil
+				}
+				// Handle #16961
+				if len(bs) < 3 {
+					return err
+				}
+
+				if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
+					return err
+				}
+
+				parts := bytes.Split(bs[2:len(bs)-1], []byte{' '})
+				if len(parts) != 3 {
+					return err
+				}
+
+				cfg.ExternalTrackerURL = string(bytes.Join(parts[:len(parts)-2], []byte{' '}))
+				cfg.ExternalTrackerFormat = string(parts[len(parts)-2])
+				cfg.ExternalTrackerStyle = string(parts[len(parts)-1])
+			case models.UnitTypePullRequests:
+				cfg := &models.PullRequestsConfig{}
+				repoUnit.Config = cfg
+				err := models.JSONUnmarshalHandleDoubleEncode(bs, &cfg)
+				if err == nil {
+					return nil
+				}
+
+				// Handle #16961
+				if len(bs) < 3 {
+					return err
+				}
+
+				if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
+					return err
+				}
+
+				// PullRequestsConfig was the following in 1.14
+				// type PullRequestsConfig struct {
+				// 	IgnoreWhitespaceConflicts bool
+				// 	AllowMerge                bool
+				// 	AllowRebase               bool
+				// 	AllowRebaseMerge          bool
+				// 	AllowSquash               bool
+				// 	AllowManualMerge          bool
+				// 	AutodetectManualMerge     bool
+				// }
+				//
+				// 1.15 added in addition:
+				// DefaultDeleteBranchAfterMerge bool
+				// DefaultMergeStyle             MergeStyle
+				parts := bytes.Split(bs[2:len(bs)-1], []byte{' '})
+				if len(parts) < 7 {
+					return err
+				}
+
+				var parseErr error
+				cfg.IgnoreWhitespaceConflicts, parseErr = strconv.ParseBool(string(parts[0]))
+				if parseErr != nil {
+					return err
+				}
+				cfg.AllowMerge, parseErr = strconv.ParseBool(string(parts[1]))
+				if parseErr != nil {
+					return err
+				}
+				cfg.AllowRebase, parseErr = strconv.ParseBool(string(parts[2]))
+				if parseErr != nil {
+					return err
+				}
+				cfg.AllowRebaseMerge, parseErr = strconv.ParseBool(string(parts[3]))
+				if parseErr != nil {
+					return err
+				}
+				cfg.AllowSquash, parseErr = strconv.ParseBool(string(parts[4]))
+				if parseErr != nil {
+					return err
+				}
+				cfg.AllowManualMerge, parseErr = strconv.ParseBool(string(parts[5]))
+				if parseErr != nil {
+					return err
+				}
+				cfg.AutodetectManualMerge, parseErr = strconv.ParseBool(string(parts[6]))
+				if parseErr != nil {
+					return err
+				}
+
+				// 1.14 unit
+				if len(parts) == 7 {
+					count++
+					if !autofix {
+						return nil
+					}
+					return models.UpdateRepoUnit(repoUnit)
+				}
+
+				if len(parts) < 9 {
+					return err
+				}
+
+				cfg.DefaultDeleteBranchAfterMerge, parseErr = strconv.ParseBool(string(parts[7]))
+				if parseErr != nil {
+					return err
+				}
+
+				cfg.DefaultMergeStyle = models.MergeStyle(string(bytes.Join(parts[8:], []byte{' '})))
+			case models.UnitTypeIssues:
+				cfg := &models.IssuesConfig{}
+				repoUnit.Config = cfg
+				err := models.JSONUnmarshalHandleDoubleEncode(bs, &cfg)
+				if err == nil {
+					return nil
+				}
+
+				// Handle #16961
+				if len(bs) < 3 {
+					return err
+				}
+
+				if bs[0] != '&' || bs[1] != '{' || bs[len(bs)-1] != '}' {
+					return err
+				}
+
+				parts := bytes.Split(bs[2:len(bs)-1], []byte{' '})
+				if len(parts) != 3 {
+					return err
+				}
+				var parseErr error
+				cfg.EnableTimetracker, parseErr = strconv.ParseBool(string(parts[0]))
+				if parseErr != nil {
+					return err
+				}
+				cfg.AllowOnlyContributorsToTrackTime, parseErr = strconv.ParseBool(string(parts[1]))
+				if parseErr != nil {
+					return err
+				}
+				cfg.EnableDependencies, parseErr = strconv.ParseBool(string(parts[2]))
+				if parseErr != nil {
+					return err
+				}
+			default:
+				panic(fmt.Sprintf("unrecognized repo unit type: %v", unit.Type))
+			}
+
+			count++
+			if !autofix {
+				return nil
+			}
+
+			return models.UpdateRepoUnit(repoUnit)
+		},
+	)
+
+	if err != nil {
+		logger.Critical("Unable to iterate acrosss repounits to fix the broken units: Error %v", err)
+		return err
+	}
+
+	if !autofix {
+		logger.Warn("Found %d broken repo_units", count)
+		return nil
+	}
+	logger.Info("Fixed %d broken repo_units", count)
+
+	return nil
+}
+
+func init() {
+	Register(&Check{
+		Title:     "Check for incorrectly dumped repo_units (See #16961)",
+		Name:      "fix-broken-repo-units",
+		IsDefault: false,
+		Run:       fixBrokenRepoUnits,
+		Priority:  7,
+	})
+}