Remove legacy unknwon/com package (#19298)

Follows: #19284
* The `CopyDir` is only used inside test code
* Rewrite `ToSnakeCase` with more test cases
* The `RedisCacher` only put strings into cache, here we use internal `toStr` to replace the legacy `ToStr`
* The `UniqueQueue` can use string as ID directly, no need to call `ToStr`

Author: wxiaoguang

cmd/manager_logging.go

@@ -11,6 +11,7 @@ import (
 
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/private"
+
 	"github.com/urfave/cli"
 )
 

contrib/pr/checkout.go

@@ -112,7 +112,7 @@ func runPR() {
 	unittest.LoadFixtures()
 	util.RemoveAll(setting.RepoRootPath)
 	util.RemoveAll(models.LocalCopyPath())
-	util.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath)
+	unittest.CopyDir(path.Join(curDir, "integrations/gitea-repositories-meta"), setting.RepoRootPath)
 
 	log.Printf("[PR] Setting up router\n")
 	// routers.GlobalInit()

go.mod

@@ -78,7 +78,6 @@ require (
 	github.com/stretchr/testify v1.7.0
 	github.com/syndtr/goleveldb v1.0.0
 	github.com/tstranex/u2f v1.0.0
-	github.com/unknwon/com v1.0.1
 	github.com/unknwon/i18n v0.0.0-20210904045753-ff3a8617e361
 	github.com/unknwon/paginater v0.0.0-20200328080006-042474bd0eae
 	github.com/unrolled/render v1.4.1
@@ -251,6 +250,7 @@ require (
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 // indirect
 	github.com/toqueteos/webbrowser v1.2.0 // indirect
 	github.com/ulikunitz/xz v0.5.10 // indirect
+	github.com/unknwon/com v1.0.1 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	github.com/xanzy/ssh-agent v0.3.1 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect

integrations/integration_test.go

@@ -254,7 +254,7 @@ func prepareTestEnv(t testing.TB, skip ...int) func() {
 	assert.NoError(t, unittest.LoadFixtures())
 	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
 
-	assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
+	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
 	if err != nil {
 		assert.NoError(t, err, "unable to read the new repo root: %v\n", err)
@@ -550,7 +550,7 @@ func resetFixtures(t *testing.T) {
 	assert.NoError(t, queue.GetManager().FlushAll(context.Background(), -1))
 	assert.NoError(t, unittest.LoadFixtures())
 	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
-	assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
+	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
 	if err != nil {
 		assert.NoError(t, err, "unable to read the new repo root: %v\n", err)

integrations/migration-test/migration_test.go

@@ -21,6 +21,7 @@ import (
 	"code.gitea.io/gitea/integrations"
 	"code.gitea.io/gitea/models/db"
 	"code.gitea.io/gitea/models/migrations"
+	"code.gitea.io/gitea/models/unittest"
 	"code.gitea.io/gitea/modules/base"
 	"code.gitea.io/gitea/modules/charset"
 	"code.gitea.io/gitea/modules/git"
@@ -60,7 +61,7 @@ func initMigrationTest(t *testing.T) func() {
 
 	assert.True(t, len(setting.RepoRootPath) != 0)
 	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
-	assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
+	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"), setting.RepoRootPath))
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
 	if err != nil {
 		assert.NoError(t, err, "unable to read the new repo root: %v\n", err)

models/migrations/migrations_test.go

@@ -203,7 +203,7 @@ func prepareTestEnv(t *testing.T, skip int, syncModels ...interface{}) (*xorm.En
 	deferFn := PrintCurrentTest(t, ourSkip)
 	assert.NoError(t, os.RemoveAll(setting.RepoRootPath))
 
-	assert.NoError(t, util.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"),
+	assert.NoError(t, unittest.CopyDir(path.Join(filepath.Dir(setting.AppPath), "integrations/gitea-repositories-meta"),
 		setting.RepoRootPath))
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
 	if err != nil {

models/unittest/fscopy.go

@@ -0,0 +1,103 @@
+// Copyright 2022 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package unittest
+
+import (
+	"errors"
+	"io"
+	"os"
+	"path"
+	"strings"
+
+	"code.gitea.io/gitea/modules/util"
+)
+
+// Copy copies file from source to target path.
+func Copy(src, dest string) error {
+	// Gather file information to set back later.
+	si, err := os.Lstat(src)
+	if err != nil {
+		return err
+	}
+
+	// Handle symbolic link.
+	if si.Mode()&os.ModeSymlink != 0 {
+		target, err := os.Readlink(src)
+		if err != nil {
+			return err
+		}
+		// NOTE: os.Chmod and os.Chtimes don't recognize symbolic link,
+		// which will lead "no such file or directory" error.
+		return os.Symlink(target, dest)
+	}
+
+	sr, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+	defer sr.Close()
+
+	dw, err := os.Create(dest)
+	if err != nil {
+		return err
+	}
+	defer dw.Close()
+
+	if _, err = io.Copy(dw, sr); err != nil {
+		return err
+	}
+
+	// Set back file information.
+	if err = os.Chtimes(dest, si.ModTime(), si.ModTime()); err != nil {
+		return err
+	}
+	return os.Chmod(dest, si.Mode())
+}
+
+// CopyDir copy files recursively from source to target directory.
+//
+// The filter accepts a function that process the path info.
+// and should return true for need to filter.
+//
+// It returns error when error occurs in underlying functions.
+func CopyDir(srcPath, destPath string, filters ...func(filePath string) bool) error {
+	// Check if target directory exists.
+	if _, err := os.Stat(destPath); !errors.Is(err, os.ErrNotExist) {
+		return errors.New("file or directory already exists: " + destPath)
+	}
+
+	err := os.MkdirAll(destPath, os.ModePerm)
+	if err != nil {
+		return err
+	}
+
+	// Gather directory info.
+	infos, err := util.StatDir(srcPath, true)
+	if err != nil {
+		return err
+	}
+
+	var filter func(filePath string) bool
+	if len(filters) > 0 {
+		filter = filters[0]
+	}
+
+	for _, info := range infos {
+		if filter != nil && filter(info) {
+			continue
+		}
+
+		curPath := path.Join(destPath, info)
+		if strings.HasSuffix(info, "/") {
+			err = os.MkdirAll(curPath, os.ModePerm)
+		} else {
+			err = Copy(path.Join(srcPath, info), curPath)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

models/unittest/testdb.go

@@ -104,7 +104,7 @@ func MainTest(m *testing.M, pathToGiteaRoot string, fixtureFiles ...string) {
 	if err = util.RemoveAll(repoRootPath); err != nil {
 		fatalTestError("util.RemoveAll: %v\n", err)
 	}
-	if err = util.CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
+	if err = CopyDir(filepath.Join(pathToGiteaRoot, "integrations", "gitea-repositories-meta"), setting.RepoRootPath); err != nil {
 		fatalTestError("util.CopyDir: %v\n", err)
 	}
 
@@ -175,7 +175,7 @@ func PrepareTestEnv(t testing.TB) {
 	assert.NoError(t, PrepareTestDatabase())
 	assert.NoError(t, util.RemoveAll(setting.RepoRootPath))
 	metaPath := filepath.Join(giteaRoot, "integrations", "gitea-repositories-meta")
-	assert.NoError(t, util.CopyDir(metaPath, setting.RepoRootPath))
+	assert.NoError(t, CopyDir(metaPath, setting.RepoRootPath))
 
 	ownerDirs, err := os.ReadDir(setting.RepoRootPath)
 	assert.NoError(t, err)

modules/cache/cache_redis.go

@@ -6,11 +6,11 @@ package cache
 
 import (
 	"fmt"
+	"strconv"
 	"time"
 
 	"code.gitea.io/gitea/modules/graceful"
 	"code.gitea.io/gitea/modules/nosql"
-	"code.gitea.io/gitea/modules/util"
 
 	"gitea.com/go-chi/cache"
 	"github.com/go-redis/redis/v8"
@@ -24,20 +24,37 @@ type RedisCacher struct {
 	occupyMode bool
 }
 
-// Put puts value into cache with key and expire time.
+// toStr convert string/int/int64 interface to string. it's only used by the RedisCacher.Put internally
+func toStr(v interface{}) string {
+	if v == nil {
+		return ""
+	}
+	switch v := v.(type) {
+	case string:
+		return v
+	case []byte:
+		return string(v)
+	case int:
+		return strconv.FormatInt(int64(v), 10)
+	case int64:
+		return strconv.FormatInt(v, 10)
+	default:
+		return fmt.Sprint(v) // as what the old com.ToStr does in most cases
+	}
+}
+
+// Put puts value (string type) into cache with key and expire time.
 // If expired is 0, it lives forever.
 func (c *RedisCacher) Put(key string, val interface{}, expire int64) error {
+	// this function is not well-designed, it only puts string values into cache
 	key = c.prefix + key
 	if expire == 0 {
-		if err := c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), 0).Err(); err != nil {
+		if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), 0).Err(); err != nil {
 			return err
 		}
 	} else {
-		dur, err := time.ParseDuration(util.ToStr(expire) + "s")
-		if err != nil {
-			return err
-		}
-		if err = c.c.Set(graceful.GetManager().HammerContext(), key, util.ToStr(val), dur).Err(); err != nil {
+		dur := time.Duration(expire) * time.Second
+		if err := c.c.Set(graceful.GetManager().HammerContext(), key, toStr(val), dur).Err(); err != nil {
 			return err
 		}
 	}

modules/context/csrf.go

@@ -22,8 +22,10 @@ import (
 	"encoding/base32"
 	"fmt"
 	"net/http"
+	"strconv"
 	"time"
 
+	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/util"
 	"code.gitea.io/gitea/modules/web/middleware"
@@ -215,9 +217,16 @@ func Csrfer(opt CsrfOptions, ctx *Context) CSRF {
 	}
 
 	x.ID = "0"
-	uid := ctx.Session.Get(opt.SessionKey)
-	if uid != nil {
-		x.ID = util.ToStr(uid)
+	uidAny := ctx.Session.Get(opt.SessionKey)
+	if uidAny != nil {
+		switch uidVal := uidAny.(type) {
+		case string:
+			x.ID = uidVal
+		case int64:
+			x.ID = strconv.FormatInt(uidVal, 10)
+		default:
+			log.Error("invalid uid type in session: %T", uidAny)
+		}
 	}
 
 	needsNew := false

modules/nosql/manager.go

@@ -11,6 +11,7 @@ import (
 	"time"
 
 	"code.gitea.io/gitea/modules/process"
+
 	"github.com/go-redis/redis/v8"
 	"github.com/syndtr/goleveldb/leveldb"
 )

modules/sync/unique_queue.go

@@ -5,8 +5,6 @@
 
 package sync
 
-import "code.gitea.io/gitea/modules/util"
-
 // UniqueQueue is a queue which guarantees only one instance of same
 // identity is in the line. Instances with same identity will be
 // discarded if there is already one in the line.
@@ -53,10 +51,10 @@ func (q *UniqueQueue) IsClosed() <-chan struct{} {
 }
 
 // IDs returns the current ids in the pool
-func (q *UniqueQueue) IDs() []interface{} {
+func (q *UniqueQueue) IDs() []string {
 	q.table.lock.Lock()
 	defer q.table.lock.Unlock()
-	ids := make([]interface{}, 0, len(q.table.pool))
+	ids := make([]string, 0, len(q.table.pool))
 	for id := range q.table.pool {
 		ids = append(ids, id)
 	}
@@ -70,38 +68,37 @@ func (q *UniqueQueue) Queue() <-chan string {
 
 // Exist returns true if there is an instance with given identity
 // exists in the queue.
-func (q *UniqueQueue) Exist(id interface{}) bool {
-	return q.table.IsRunning(util.ToStr(id))
+func (q *UniqueQueue) Exist(id string) bool {
+	return q.table.IsRunning(id)
 }
 
 // AddFunc adds new instance to the queue with a custom runnable function,
 // the queue is blocked until the function exits.
-func (q *UniqueQueue) AddFunc(id interface{}, fn func()) {
-	idStr := util.ToStr(id)
+func (q *UniqueQueue) AddFunc(id string, fn func()) {
 	q.table.lock.Lock()
-	if _, ok := q.table.pool[idStr]; ok {
+	if _, ok := q.table.pool[id]; ok {
 		q.table.lock.Unlock()
 		return
 	}
-	q.table.pool[idStr] = struct{}{}
+	q.table.pool[id] = struct{}{}
 	if fn != nil {
 		fn()
 	}
 	q.table.lock.Unlock()
 	select {
 	case <-q.closed:
 		return
-	case q.queue <- idStr:
+	case q.queue <- id:
 		return
 	}
 }
 
 // Add adds new instance to the queue.
-func (q *UniqueQueue) Add(id interface{}) {
+func (q *UniqueQueue) Add(id string) {
 	q.AddFunc(id, nil)
 }
 
 // Remove removes instance from the queue.
-func (q *UniqueQueue) Remove(id interface{}) {
-	q.table.Stop(util.ToStr(id))
+func (q *UniqueQueue) Remove(id string) {
+	q.table.Stop(id)
 }