Merge branch 'main' into feature-jwt-asymmetric

Author: techknowlogick

custom/conf/app.example.ini

@@ -1155,20 +1155,20 @@ PATH =
 ;STARTUP_TIMEOUT = 30s
 ;;
 ;; Issue indexer queue, currently support: channel, levelqueue or redis, default is levelqueue (deprecated - use [queue.issue_indexer])
-;ISSUE_INDEXER_QUEUE_TYPE = levelqueue
+;ISSUE_INDEXER_QUEUE_TYPE = levelqueue; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 ;;
 ;; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the path where the queue will be saved.
 ;; This can be overridden by `ISSUE_INDEXER_QUEUE_CONN_STR`.
 ;; default is queues/common
-;ISSUE_INDEXER_QUEUE_DIR = queues/common
+;ISSUE_INDEXER_QUEUE_DIR = queues/common; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 ;;
 ;; When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string.
 ;; When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of
 ;; the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`.
-;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
+;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 ;;
 ;; Batch queue number, default is 20
-;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20
+;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Repository Indexer settings
@@ -1197,7 +1197,7 @@ PATH =
 ;REPO_INDEXER_EXCLUDE =
 ;;
 ;;
-;UPDATE_BUFFER_LEN = 20
+;UPDATE_BUFFER_LEN = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 ;MAX_FILE_SIZE = 1048576
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -356,10 +356,10 @@ relation to port exhaustion.
 - `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch
 - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch.
 - The next 4 configuration values are deprecated and should be set in `queue.issue_indexer` however are kept for backwards compatibility:
-- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`.
-- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. (Previously this was `indexers/issues.queue`.)
-- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`.
-- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number.
+- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`. **DEPRECATED** use settings in `[queue.issue_indexer]`.
+- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. **DEPRECATED** use settings in `[queue.issue_indexer]`.
+- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. **DEPRECATED** use settings in `[queue.issue_indexer]`.
+- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number. **DEPRECATED** use settings in `[queue.issue_indexer]`.
 
 - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
 - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`.
@@ -370,7 +370,7 @@ relation to port exhaustion.
 - `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files.
 - `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`.
 - `REPO_INDEXER_EXCLUDE_VENDORED`: **true**: Exclude vendored files from index.
-- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
+- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. **DEPRECATED** use settings in `[queue.issue_indexer]`.
 - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
 - `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to zero to never timeout.
 

integrations/api_issue_test.go

@@ -25,9 +25,10 @@ func TestAPIListIssues(t *testing.T) {
 
 	session := loginUser(t, owner.Name)
 	token := getTokenForLoggedInUser(t, session)
-	req := NewRequestf(t, "GET", "/api/v1/repos/%s/%s/issues?state=all&token=%s",
-		owner.Name, repo.Name, token)
-	resp := session.MakeRequest(t, req, http.StatusOK)
+	link, _ := url.Parse(fmt.Sprintf("/api/v1/repos/%s/%s/issues", owner.Name, repo.Name))
+
+	link.RawQuery = url.Values{"token": {token}, "state": {"all"}}.Encode()
+	resp := session.MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK)
 	var apiIssues []*api.Issue
 	DecodeJSON(t, resp, &apiIssues)
 	assert.Len(t, apiIssues, models.GetCount(t, &models.Issue{RepoID: repo.ID}))
@@ -36,15 +37,34 @@ func TestAPIListIssues(t *testing.T) {
 	}
 
 	// test milestone filter
-	req = NewRequestf(t, "GET", "/api/v1/repos/%s/%s/issues?state=all&type=all&milestones=ignore,milestone1,3,4&token=%s",
-		owner.Name, repo.Name, token)
-	resp = session.MakeRequest(t, req, http.StatusOK)
+	link.RawQuery = url.Values{"token": {token}, "state": {"all"}, "type": {"all"}, "milestones": {"ignore,milestone1,3,4"}}.Encode()
+	resp = session.MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK)
 	DecodeJSON(t, resp, &apiIssues)
 	if assert.Len(t, apiIssues, 2) {
 		assert.EqualValues(t, 3, apiIssues[0].Milestone.ID)
 		assert.EqualValues(t, 1, apiIssues[1].Milestone.ID)
 	}
 
+	link.RawQuery = url.Values{"token": {token}, "state": {"all"}, "created_by": {"user2"}}.Encode()
+	resp = session.MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK)
+	DecodeJSON(t, resp, &apiIssues)
+	if assert.Len(t, apiIssues, 1) {
+		assert.EqualValues(t, 5, apiIssues[0].ID)
+	}
+
+	link.RawQuery = url.Values{"token": {token}, "state": {"all"}, "assigned_by": {"user1"}}.Encode()
+	resp = session.MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK)
+	DecodeJSON(t, resp, &apiIssues)
+	if assert.Len(t, apiIssues, 1) {
+		assert.EqualValues(t, 1, apiIssues[0].ID)
+	}
+
+	link.RawQuery = url.Values{"token": {token}, "state": {"all"}, "mentioned_by": {"user4"}}.Encode()
+	resp = session.MakeRequest(t, NewRequest(t, "GET", link.String()), http.StatusOK)
+	DecodeJSON(t, resp, &apiIssues)
+	if assert.Len(t, apiIssues, 1) {
+		assert.EqualValues(t, 1, apiIssues[0].ID)
+	}
 }
 
 func TestAPICreateIssue(t *testing.T) {

models/fixtures/issue_user.yml

@@ -17,4 +17,4 @@
   uid: 4
   issue_id: 1
   is_read: false
-  is_mentioned: false
+  is_mentioned: true

models/migrations/migrations.go

@@ -317,6 +317,8 @@ var migrations = []Migration{
 	NewMigration("Add issue resource index table", addIssueResourceIndexTable),
 	// v183 -> v184
 	NewMigration("Create PushMirror table", createPushMirrorTable),
+	// v184 -> v185
+	NewMigration("Rename Task errors to message", renameTaskErrorsToMessage),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v184.go

@@ -0,0 +1,47 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"xorm.io/xorm"
+)
+
+func renameTaskErrorsToMessage(x *xorm.Engine) error {
+	type Task struct {
+		Errors string `xorm:"TEXT"` // if task failed, saved the error reason
+		Type   int
+		Status int `xorm:"index"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Sync2(new(Task)); err != nil {
+		return fmt.Errorf("error on Sync2: %v", err)
+	}
+
+	switch {
+	case setting.Database.UseMySQL:
+		if _, err := sess.Exec("ALTER TABLE `task` CHANGE errors message text"); err != nil {
+			return err
+		}
+	case setting.Database.UseMSSQL:
+		if _, err := sess.Exec("sp_rename 'task.errors', 'message', 'COLUMN'"); err != nil {
+			return err
+		}
+	default:
+		if _, err := sess.Exec("ALTER TABLE `task` RENAME COLUMN errors TO message"); err != nil {
+			return err
+		}
+	}
+	return sess.Commit()
+}

models/task.go

@@ -32,10 +32,16 @@ type Task struct {
 	StartTime      timeutil.TimeStamp
 	EndTime        timeutil.TimeStamp
 	PayloadContent string             `xorm:"TEXT"`
-	Errors         string             `xorm:"TEXT"` // if task failed, saved the error reason
+	Message        string             `xorm:"TEXT"` // if task failed, saved the error reason
 	Created        timeutil.TimeStamp `xorm:"created"`
 }
 
+// TranslatableMessage represents JSON struct that can be translated with a Locale
+type TranslatableMessage struct {
+	Format string
+	Args   []interface{} `json:"omitempty"`
+}
+
 // LoadRepo loads repository of the task
 func (task *Task) LoadRepo() error {
 	return task.loadRepo(x)

modules/git/repo_language_stats_nogogit.go

@@ -25,11 +25,7 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 	defer cancel()
 
 	writeID := func(id string) error {
-		_, err := batchStdinWriter.Write([]byte(id))
-		if err != nil {
-			return err
-		}
-		_, err = batchStdinWriter.Write([]byte{'\n'})
+		_, err := batchStdinWriter.Write([]byte(id + "\n"))
 		return err
 	}
 
@@ -85,10 +81,10 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			}
 
 			sizeToRead := size
-			discard := int64(0)
+			discard := int64(1)
 			if size > fileSizeLimit {
 				sizeToRead = fileSizeLimit
-				discard = size - fileSizeLimit
+				discard = size - fileSizeLimit + 1
 			}
 
 			_, err = contentBuf.ReadFrom(io.LimitReader(batchReader, sizeToRead))

modules/migrations/base/messenger.go

@@ -0,0 +1,11 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package base
+
+// Messenger is a formatting function similar to i18n.Tr
+type Messenger func(key string, args ...interface{})
+
+// NilMessenger represents an empty formatting function
+func NilMessenger(string, ...interface{}) {}

modules/migrations/dump.go

@@ -555,7 +555,7 @@ func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.Mi
 		return err
 	}
 
-	if err := migrateRepository(downloader, uploader, opts); err != nil {
+	if err := migrateRepository(downloader, uploader, opts, nil); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}
@@ -620,7 +620,7 @@ func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName
 	}
 	updateOptionsUnits(&migrateOpts, units)
 
-	if err = migrateRepository(downloader, uploader, migrateOpts); err != nil {
+	if err = migrateRepository(downloader, uploader, migrateOpts, nil); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}

modules/migrations/gitea_uploader_test.go

@@ -47,7 +47,7 @@ func TestGiteaUploadRepo(t *testing.T) {
 		PullRequests: true,
 		Private:      true,
 		Mirror:       false,
-	})
+	}, nil)
 	assert.NoError(t, err)
 
 	repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)

modules/migrations/migrate.go

@@ -99,7 +99,7 @@ func IsMigrateURLAllowed(remoteURL string, doer *models.User) error {
 }
 
 // MigrateRepository migrate repository according MigrateOptions
-func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions) (*models.Repository, error) {
+func MigrateRepository(ctx context.Context, doer *models.User, ownerName string, opts base.MigrateOptions, messenger base.Messenger) (*models.Repository, error) {
 	err := IsMigrateURLAllowed(opts.CloneAddr, doer)
 	if err != nil {
 		return nil, err
@@ -118,7 +118,7 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
 	var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
 	uploader.gitServiceType = opts.GitServiceType
 
-	if err := migrateRepository(downloader, uploader, opts); err != nil {
+	if err := migrateRepository(downloader, uploader, opts, messenger); err != nil {
 		if err1 := uploader.Rollback(); err1 != nil {
 			log.Error("rollback failed: %v", err1)
 		}
@@ -167,7 +167,11 @@ func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptio
 // migrateRepository will download information and then upload it to Uploader, this is a simple
 // process for small repository. For a big repository, save all the data to disk
 // before upload is better
-func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions) error {
+func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts base.MigrateOptions, messenger base.Messenger) error {
+	if messenger == nil {
+		messenger = base.NilMessenger
+	}
+
 	repo, err := downloader.GetRepoInfo()
 	if err != nil {
 		if !base.IsErrNotSupported(err) {
@@ -185,12 +189,14 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 	}
 
 	log.Trace("migrating git data from %s", repo.CloneURL)
+	messenger("repo.migrate.migrating_git")
 	if err = uploader.CreateRepo(repo, opts); err != nil {
 		return err
 	}
 	defer uploader.Close()
 
 	log.Trace("migrating topics")
+	messenger("repo.migrate.migrating_topics")
 	topics, err := downloader.GetTopics()
 	if err != nil {
 		if !base.IsErrNotSupported(err) {
@@ -206,6 +212,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.Milestones {
 		log.Trace("migrating milestones")
+		messenger("repo.migrate.migrating_milestones")
 		milestones, err := downloader.GetMilestones()
 		if err != nil {
 			if !base.IsErrNotSupported(err) {
@@ -229,6 +236,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.Labels {
 		log.Trace("migrating labels")
+		messenger("repo.migrate.migrating_labels")
 		labels, err := downloader.GetLabels()
 		if err != nil {
 			if !base.IsErrNotSupported(err) {
@@ -252,6 +260,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.Releases {
 		log.Trace("migrating releases")
+		messenger("repo.migrate.migrating_releases")
 		releases, err := downloader.GetReleases()
 		if err != nil {
 			if !base.IsErrNotSupported(err) {
@@ -285,6 +294,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.Issues {
 		log.Trace("migrating issues and comments")
+		messenger("repo.migrate.migrating_issues")
 		var issueBatchSize = uploader.MaxBatchInsertSize("issue")
 
 		for i := 1; ; i++ {
@@ -339,6 +349,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
 
 	if opts.PullRequests {
 		log.Trace("migrating pull requests and comments")
+		messenger("repo.migrate.migrating_pulls")
 		var prBatchSize = uploader.MaxBatchInsertSize("pullrequest")
 		for i := 1; ; i++ {
 			prs, isEnd, err := downloader.GetPullRequests(i, prBatchSize)