Merge branch 'main' into run-emoji-processor-on-whole-of-data

Author: 6543

MAINTAINERS

@@ -42,3 +42,4 @@ Norwin Roosen <[email protected]> (@noerw)
 Kyle Dumont <[email protected]> (@kdumontnu)
 Patrick Schratz <[email protected]> (@pat-s)
 Janis Estelmann <[email protected]> (@KN4CK3R)
+Steven Kriegler <[email protected]> (@justusbunsi)

custom/conf/app.example.ini

@@ -1155,20 +1155,20 @@ PATH =
 ;STARTUP_TIMEOUT = 30s
 ;;
 ;; Issue indexer queue, currently support: channel, levelqueue or redis, default is levelqueue (deprecated - use [queue.issue_indexer])
-;ISSUE_INDEXER_QUEUE_TYPE = levelqueue
+;ISSUE_INDEXER_QUEUE_TYPE = levelqueue; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 ;;
 ;; When ISSUE_INDEXER_QUEUE_TYPE is levelqueue, this will be the path where the queue will be saved.
 ;; This can be overridden by `ISSUE_INDEXER_QUEUE_CONN_STR`.
 ;; default is queues/common
-;ISSUE_INDEXER_QUEUE_DIR = queues/common
+;ISSUE_INDEXER_QUEUE_DIR = queues/common; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 ;;
 ;; When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string.
 ;; When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of
 ;; the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`.
-;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"
+;ISSUE_INDEXER_QUEUE_CONN_STR = "addrs=127.0.0.1:6379 db=0"; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 ;;
 ;; Batch queue number, default is 20
-;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20
+;ISSUE_INDEXER_QUEUE_BATCH_NUMBER = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;; Repository Indexer settings
@@ -1197,7 +1197,7 @@ PATH =
 ;REPO_INDEXER_EXCLUDE =
 ;;
 ;;
-;UPDATE_BUFFER_LEN = 20
+;UPDATE_BUFFER_LEN = 20; **DEPRECATED** use settings in `[queue.issue_indexer]`.
 ;MAX_FILE_SIZE = 1048576
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -270,6 +270,7 @@ The following configuration set `Content-Type: application/vnd.android.package-a
 - `SSH_AUTHORIZED_PRINCIPALS_ALLOW`: **off** or **username, email**: \[off, username, email, anything\]: Specify the principals values that users are allowed to use as principal. When set to `anything` no checks are done on the principal string. When set to `off` authorized principal are not allowed to be set.
 - `SSH_CREATE_AUTHORIZED_PRINCIPALS_FILE`: **false/true**: Gitea will create a authorized_principals file by default when it is not using the internal ssh server and `SSH_AUTHORIZED_PRINCIPALS_ALLOW` is not `off`.
 - `SSH_AUTHORIZED_PRINCIPALS_BACKUP`: **false/true**: Enable SSH Authorized Principals Backup when rewriting all keys, default is true if `SSH_AUTHORIZED_PRINCIPALS_ALLOW` is not `off`.
+- `SSH_AUTHORIZED_KEYS_COMMAND_TEMPLATE`: **{{.AppPath}} --config={{.CustomConf}} serv key-{{.Key.ID}}**: Set the template for the command to passed on authorized keys. Possible keys are: AppPath, AppWorkPath, CustomConf, CustomPath, Key - where Key is a `models.PublicKey` and the others are strings which are shellquoted.
 - `SSH_SERVER_CIPHERS`: **aes128-ctr, aes192-ctr, aes256-ctr, [email protected], arcfour256, arcfour128**: For the built-in SSH server, choose the ciphers to support for SSH connections, for system SSH this setting has no effect.
 - `SSH_SERVER_KEY_EXCHANGES`: **diffie-hellman-group1-sha1, diffie-hellman-group14-sha1, ecdh-sha2-nistp256, ecdh-sha2-nistp384, ecdh-sha2-nistp521, [email protected]**: For the built-in SSH server, choose the key exchange algorithms to support for SSH connections, for system SSH this setting has no effect.
 - `SSH_SERVER_MACS`: **[email protected], hmac-sha2-256, hmac-sha1, hmac-sha1-96**: For the built-in SSH server, choose the MACs to support for SSH connections, for system SSH this setting has no effect
@@ -355,10 +356,10 @@ relation to port exhaustion.
 - `ISSUE_INDEXER_NAME`: **gitea_issues**: Issue indexer name, available when ISSUE_INDEXER_TYPE is elasticsearch
 - `ISSUE_INDEXER_PATH`: **indexers/issues.bleve**: Index file used for issue search; available when ISSUE_INDEXER_TYPE is bleve and elasticsearch.
 - The next 4 configuration values are deprecated and should be set in `queue.issue_indexer` however are kept for backwards compatibility:
-- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`.
-- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. (Previously this was `indexers/issues.queue`.)
-- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`.
-- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number.
+- `ISSUE_INDEXER_QUEUE_TYPE`: **levelqueue**: Issue indexer queue, currently supports:`channel`, `levelqueue`, `redis`. **DEPRECATED** use settings in `[queue.issue_indexer]`.
+- `ISSUE_INDEXER_QUEUE_DIR`: **queues/common**: When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this will be the path where the queue will be saved. **DEPRECATED** use settings in `[queue.issue_indexer]`.
+- `ISSUE_INDEXER_QUEUE_CONN_STR`: **addrs=127.0.0.1:6379 db=0**: When `ISSUE_INDEXER_QUEUE_TYPE` is `redis`, this will store the redis connection string. When `ISSUE_INDEXER_QUEUE_TYPE` is `levelqueue`, this is a directory or additional options of the form `leveldb://path/to/db?option=value&....`, and overrides `ISSUE_INDEXER_QUEUE_DIR`. **DEPRECATED** use settings in `[queue.issue_indexer]`.
+- `ISSUE_INDEXER_QUEUE_BATCH_NUMBER`: **20**: Batch queue number. **DEPRECATED** use settings in `[queue.issue_indexer]`.
 
 - `REPO_INDEXER_ENABLED`: **false**: Enables code search (uses a lot of disk space, about 6 times more than the repository size).
 - `REPO_INDEXER_TYPE`: **bleve**: Code search engine type, could be `bleve` or `elasticsearch`.
@@ -369,7 +370,7 @@ relation to port exhaustion.
 - `REPO_INDEXER_INCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **include** in the index. Use `**.txt` to match any files with .txt extension. An empty list means include all files.
 - `REPO_INDEXER_EXCLUDE`: **empty**: A comma separated list of glob patterns (see https://github.com/gobwas/glob) to **exclude** from the index. Files that match this list will not be indexed, even if they match in `REPO_INDEXER_INCLUDE`.
 - `REPO_INDEXER_EXCLUDE_VENDORED`: **true**: Exclude vendored files from index.
-- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request.
+- `UPDATE_BUFFER_LEN`: **20**: Buffer length of index request. **DEPRECATED** use settings in `[queue.issue_indexer]`.
 - `MAX_FILE_SIZE`: **1048576**: Maximum size in bytes of files to be indexed.
 - `STARTUP_TIMEOUT`: **30s**: If the indexer takes longer than this timeout to start - fail. (This timeout will be added to the hammer time above for child processes - as bleve will not start until the previous parent is shutdown.) Set to zero to never timeout.
 

models/issue.go

@@ -1127,9 +1127,18 @@ func sortIssuesSession(sess *xorm.Session, sortType string, priorityRepoID int64
 		sess.Desc("issue.priority")
 	case "nearduedate":
 		// 253370764800 is 01/01/9999 @ 12:00am (UTC)
-		sess.OrderBy("CASE WHEN issue.deadline_unix = 0 THEN 253370764800 ELSE issue.deadline_unix END ASC")
+		sess.Join("LEFT", "milestone", "issue.milestone_id = milestone.id").
+			OrderBy("CASE " +
+				"WHEN issue.deadline_unix = 0 AND (milestone.deadline_unix = 0 OR milestone.deadline_unix IS NULL) THEN 253370764800 " +
+				"WHEN milestone.deadline_unix = 0 OR milestone.deadline_unix IS NULL THEN issue.deadline_unix " +
+				"WHEN milestone.deadline_unix < issue.deadline_unix OR issue.deadline_unix = 0 THEN milestone.deadline_unix " +
+				"ELSE issue.deadline_unix END ASC")
 	case "farduedate":
-		sess.Desc("issue.deadline_unix")
+		sess.Join("LEFT", "milestone", "issue.milestone_id = milestone.id").
+			OrderBy("CASE " +
+				"WHEN milestone.deadline_unix IS NULL THEN issue.deadline_unix " +
+				"WHEN milestone.deadline_unix < issue.deadline_unix OR issue.deadline_unix = 0 THEN milestone.deadline_unix " +
+				"ELSE issue.deadline_unix END DESC")
 	case "priorityrepo":
 		sess.OrderBy("CASE WHEN issue.repo_id = " + strconv.FormatInt(priorityRepoID, 10) + " THEN 1 ELSE 2 END, issue.created_unix DESC")
 	default:

models/issue_comment.go

@@ -762,6 +762,8 @@ func updateCommentInfos(e *xorm.Session, opts *CreateCommentOptions, comment *Co
 			}
 		}
 		fallthrough
+	case CommentTypeReview:
+		fallthrough
 	case CommentTypeComment:
 		if _, err = e.Exec("UPDATE `issue` SET num_comments=num_comments+1 WHERE id=?", opts.Issue.ID); err != nil {
 			return err

models/migrations/migrations.go

@@ -317,6 +317,8 @@ var migrations = []Migration{
 	NewMigration("Add issue resource index table", addIssueResourceIndexTable),
 	// v183 -> v184
 	NewMigration("Create PushMirror table", createPushMirrorTable),
+	// v184 -> v185
+	NewMigration("Rename Task errors to message", renameTaskErrorsToMessage),
 }
 
 // GetCurrentDBVersion returns the current db version

models/migrations/v180.go

@@ -5,9 +5,6 @@
 package migrations
 
 import (
-	"code.gitea.io/gitea/models"
-	"code.gitea.io/gitea/modules/migrations/base"
-	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/util"
 
 	jsoniter "github.com/json-iterator/go"
@@ -16,20 +13,38 @@ import (
 )
 
 func deleteMigrationCredentials(x *xorm.Engine) (err error) {
+	// Task represents a task
+	type Task struct {
+		ID             int64
+		DoerID         int64 `xorm:"index"` // operator
+		OwnerID        int64 `xorm:"index"` // repo owner id, when creating, the repoID maybe zero
+		RepoID         int64 `xorm:"index"`
+		Type           int
+		Status         int `xorm:"index"`
+		StartTime      int64
+		EndTime        int64
+		PayloadContent string `xorm:"TEXT"`
+		Errors         string `xorm:"TEXT"` // if task failed, saved the error reason
+		Created        int64  `xorm:"created"`
+	}
+
+	const TaskTypeMigrateRepo = 0
+	const TaskStatusStopped = 2
+
 	const batchSize = 100
 
 	// only match migration tasks, that are not pending or running
 	cond := builder.Eq{
-		"type": structs.TaskTypeMigrateRepo,
+		"type": TaskTypeMigrateRepo,
 	}.And(builder.Gte{
-		"status": structs.TaskStatusStopped,
+		"status": TaskStatusStopped,
 	})
 
 	sess := x.NewSession()
 	defer sess.Close()
 
 	for start := 0; ; start += batchSize {
-		tasks := make([]*models.Task, 0, batchSize)
+		tasks := make([]*Task, 0, batchSize)
 		if err = sess.Limit(batchSize, start).Where(cond, 0).Find(&tasks); err != nil {
 			return
 		}
@@ -55,7 +70,41 @@ func deleteMigrationCredentials(x *xorm.Engine) (err error) {
 }
 
 func removeCredentials(payload string) (string, error) {
-	var opts base.MigrateOptions
+	// MigrateOptions defines the way a repository gets migrated
+	// this is for internal usage by migrations module and func who interact with it
+	type MigrateOptions struct {
+		// required: true
+		CloneAddr             string `json:"clone_addr" binding:"Required"`
+		CloneAddrEncrypted    string `json:"clone_addr_encrypted,omitempty"`
+		AuthUsername          string `json:"auth_username"`
+		AuthPassword          string `json:"-"`
+		AuthPasswordEncrypted string `json:"auth_password_encrypted,omitempty"`
+		AuthToken             string `json:"-"`
+		AuthTokenEncrypted    string `json:"auth_token_encrypted,omitempty"`
+		// required: true
+		UID int `json:"uid" binding:"Required"`
+		// required: true
+		RepoName        string `json:"repo_name" binding:"Required"`
+		Mirror          bool   `json:"mirror"`
+		LFS             bool   `json:"lfs"`
+		LFSEndpoint     string `json:"lfs_endpoint"`
+		Private         bool   `json:"private"`
+		Description     string `json:"description"`
+		OriginalURL     string
+		GitServiceType  int
+		Wiki            bool
+		Issues          bool
+		Milestones      bool
+		Labels          bool
+		Releases        bool
+		Comments        bool
+		PullRequests    bool
+		ReleaseAssets   bool
+		MigrateToRepoID int64
+		MirrorInterval  string `json:"mirror_interval"`
+	}
+
+	var opts MigrateOptions
 	json := jsoniter.ConfigCompatibleWithStandardLibrary
 	err := json.Unmarshal([]byte(payload), &opts)
 	if err != nil {

models/migrations/v184.go

@@ -0,0 +1,47 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+	"fmt"
+
+	"code.gitea.io/gitea/modules/setting"
+
+	"xorm.io/xorm"
+)
+
+func renameTaskErrorsToMessage(x *xorm.Engine) error {
+	type Task struct {
+		Errors string `xorm:"TEXT"` // if task failed, saved the error reason
+		Type   int
+		Status int `xorm:"index"`
+	}
+
+	sess := x.NewSession()
+	defer sess.Close()
+	if err := sess.Begin(); err != nil {
+		return err
+	}
+
+	if err := sess.Sync2(new(Task)); err != nil {
+		return fmt.Errorf("error on Sync2: %v", err)
+	}
+
+	switch {
+	case setting.Database.UseMySQL:
+		if _, err := sess.Exec("ALTER TABLE `task` CHANGE errors message text"); err != nil {
+			return err
+		}
+	case setting.Database.UseMSSQL:
+		if _, err := sess.Exec("sp_rename 'task.errors', 'message', 'COLUMN'"); err != nil {
+			return err
+		}
+	default:
+		if _, err := sess.Exec("ALTER TABLE `task` RENAME COLUMN errors TO message"); err != nil {
+			return err
+		}
+	}
+	return sess.Commit()
+}

models/notification.go

@@ -74,6 +74,7 @@ type FindNotificationOptions struct {
 	RepoID            int64
 	IssueID           int64
 	Status            []NotificationStatus
+	Source            []NotificationSource
 	UpdatedAfterUnix  int64
 	UpdatedBeforeUnix int64
 }
@@ -93,6 +94,9 @@ func (opts *FindNotificationOptions) ToCond() builder.Cond {
 	if len(opts.Status) > 0 {
 		cond = cond.And(builder.In("notification.status", opts.Status))
 	}
+	if len(opts.Source) > 0 {
+		cond = cond.And(builder.In("notification.source", opts.Source))
+	}
 	if opts.UpdatedAfterUnix != 0 {
 		cond = cond.And(builder.Gte{"notification.updated_unix": opts.UpdatedAfterUnix})
 	}
@@ -111,13 +115,13 @@ func (opts *FindNotificationOptions) ToSession(e Engine) *xorm.Session {
 	return sess
 }
 
-func getNotifications(e Engine, options FindNotificationOptions) (nl NotificationList, err error) {
+func getNotifications(e Engine, options *FindNotificationOptions) (nl NotificationList, err error) {
 	err = options.ToSession(e).OrderBy("notification.updated_unix DESC").Find(&nl)
 	return
 }
 
 // GetNotifications returns all notifications that fit to the given options.
-func GetNotifications(opts FindNotificationOptions) (NotificationList, error) {
+func GetNotifications(opts *FindNotificationOptions) (NotificationList, error) {
 	return getNotifications(x, opts)
 }
 

models/review.go

@@ -347,7 +347,7 @@ func IsContentEmptyErr(err error) bool {
 }
 
 // SubmitReview creates a review out of the existing pending review or creates a new one if no pending review exist
-func SubmitReview(doer *User, issue *Issue, reviewType ReviewType, content, commitID string, stale bool) (*Review, *Comment, error) {
+func SubmitReview(doer *User, issue *Issue, reviewType ReviewType, content, commitID string, stale bool, attachmentUUIDs []string) (*Review, *Comment, error) {
 	sess := x.NewSession()
 	defer sess.Close()
 	if err := sess.Begin(); err != nil {
@@ -419,12 +419,13 @@ func SubmitReview(doer *User, issue *Issue, reviewType ReviewType, content, comm
 	}
 
 	comm, err := createComment(sess, &CreateCommentOptions{
-		Type:     CommentTypeReview,
-		Doer:     doer,
-		Content:  review.Content,
-		Issue:    issue,
-		Repo:     issue.Repo,
-		ReviewID: review.ID,
+		Type:        CommentTypeReview,
+		Doer:        doer,
+		Content:     review.Content,
+		Issue:       issue,
+		Repo:        issue.Repo,
+		ReviewID:    review.ID,
+		Attachments: attachmentUUIDs,
 	})
 	if err != nil || comm == nil {
 		return nil, nil, err

models/ssh_key.go

@@ -38,7 +38,6 @@ import (
 
 const (
 	tplCommentPrefix = `# gitea public key`
-	tplCommand       = "%s --config=%s serv key-%d"
 	tplPublicKey     = tplCommentPrefix + "\n" + `command=%s,no-port-forwarding,no-X11-forwarding,no-agent-forwarding,no-pty %s` + "\n"
 
 	authorizedPrincipalsFile = "authorized_principals"
@@ -88,7 +87,16 @@ func (key *PublicKey) OmitEmail() string {
 
 // AuthorizedString returns formatted public key string for authorized_keys file.
 func (key *PublicKey) AuthorizedString() string {
-	return fmt.Sprintf(tplPublicKey, util.ShellEscape(fmt.Sprintf(tplCommand, util.ShellEscape(setting.AppPath), util.ShellEscape(setting.CustomConf), key.ID)), key.Content)
+	sb := &strings.Builder{}
+	_ = setting.SSH.AuthorizedKeysCommandTemplateTemplate.Execute(sb, map[string]interface{}{
+		"AppPath":     util.ShellEscape(setting.AppPath),
+		"AppWorkPath": util.ShellEscape(setting.AppWorkPath),
+		"CustomConf":  util.ShellEscape(setting.CustomConf),
+		"CustomPath":  util.ShellEscape(setting.CustomPath),
+		"Key":         key,
+	})
+
+	return fmt.Sprintf(tplPublicKey, util.ShellEscape(sb.String()), key.Content)
 }
 
 func extractTypeFromBase64Key(key string) (string, error) {

models/task.go

@@ -32,10 +32,16 @@ type Task struct {
 	StartTime      timeutil.TimeStamp
 	EndTime        timeutil.TimeStamp
 	PayloadContent string             `xorm:"TEXT"`
-	Errors         string             `xorm:"TEXT"` // if task failed, saved the error reason
+	Message        string             `xorm:"TEXT"` // if task failed, saved the error reason
 	Created        timeutil.TimeStamp `xorm:"created"`
 }
 
+// TranslatableMessage represents JSON struct that can be translated with a Locale
+type TranslatableMessage struct {
+	Format string
+	Args   []interface{} `json:"omitempty"`
+}
+
 // LoadRepo loads repository of the task
 func (task *Task) LoadRepo() error {
 	return task.loadRepo(x)

models/token.go

@@ -57,9 +57,15 @@ func GetAccessTokenBySHA(token string) (*AccessToken, error) {
 	if token == "" {
 		return nil, ErrAccessTokenEmpty{}
 	}
-	if len(token) < 8 {
+	// A token is defined as being SHA1 sum these are 40 hexadecimal bytes long
+	if len(token) != 40 {
 		return nil, ErrAccessTokenNotExist{token}
 	}
+	for _, x := range []byte(token) {
+		if x < '0' || (x > '9' && x < 'a') || x > 'f' {
+			return nil, ErrAccessTokenNotExist{token}
+		}
+	}
 	var tokens []AccessToken
 	lastEight := token[len(token)-8:]
 	err := x.Table(&AccessToken{}).Where("token_last_eight = ?", lastEight).Find(&tokens)