Merge branch 'main' into refactor-migration-test

Author: techknowlogick

.drone.yml

@@ -65,7 +65,7 @@ steps:
 
   - name: checks-backend
     pull: always
-    image: golang:1.16
+    image: golang:1.17
     commands:
       - make checks-backend
     depends_on: [lint-backend]
@@ -93,7 +93,7 @@ steps:
     depends_on: [checks-backend]
 
   - name: build-backend-arm64
-    image: golang:1.16
+    image: golang:1.17
     environment:
       GO111MODULE: on
       GOPROXY: off
@@ -106,7 +106,7 @@ steps:
     depends_on: [checks-backend]
 
   - name: build-backend-windows
-    image: golang:1.16
+    image: golang:1.17
     environment:
       GO111MODULE: on
       GOPROXY: off
@@ -118,7 +118,7 @@ steps:
     depends_on: [checks-backend]
 
   - name: build-backend-386
-    image: golang:1.16
+    image: golang:1.17
     environment:
       GO111MODULE: on
       GOPROXY: off
@@ -193,7 +193,7 @@ steps:
 
   - name: build
     pull: always
-    image: golang:1.16
+    image: golang:1.17
     commands:
       - make backend
     environment:
@@ -208,7 +208,7 @@ steps:
       - git update-ref refs/heads/tag_test ${DRONE_COMMIT_SHA}
 
   - name: unit-test
-    image: golang:1.16
+    image: golang:1.17
     commands:
       - make unit-test-coverage test-check
     environment:
@@ -218,7 +218,7 @@ steps:
         from_secret: github_read_token
 
   - name: unit-test-race
-    image: golang:1.16
+    image: golang:1.17
     commands:
       - make test-backend
     environment:
@@ -230,7 +230,7 @@ steps:
 
   - name: unit-test-gogit
     pull: always
-    image: golang:1.16
+    image: golang:1.17
     commands:
       - make unit-test-coverage test-check
     environment:
@@ -277,7 +277,7 @@ steps:
       - build
 
   - name: generate-coverage
-    image: golang:1.16
+    image: golang:1.17
     commands:
       - make coverage
     environment:
@@ -351,7 +351,7 @@ steps:
 
   - name: build
     pull: always
-    image: golang:1.16
+    image: golang:1.17
     commands:
       - make backend
     environment:
@@ -463,7 +463,7 @@ trigger:
 
 steps:
   - name: download
-    image: golang:1.16
+    image: golang:1.17
     commands:
       - timeout -s ABRT 40m make generate-license generate-gitignore
 

.eslintrc

@@ -125,6 +125,7 @@ rules:
   import/no-deprecated: [0]
   import/no-dynamic-require: [0]
   import/no-extraneous-dependencies: [2]
+  import/no-import-module-exports: [0]
   import/no-internal-modules: [0]
   import/no-mutable-exports: [2]
   import/no-named-as-default-member: [0]
@@ -133,6 +134,7 @@ rules:
   import/no-named-export: [0]
   import/no-namespace: [0]
   import/no-nodejs-modules: [0]
+  import/no-relative-packages: [0]
   import/no-relative-parent-imports: [0]
   import/no-restricted-paths: [0]
   import/no-self-import: [2]
@@ -365,6 +367,7 @@ rules:
   unicorn/no-abusive-eslint-disable: [0]
   unicorn/no-array-for-each: [0]
   unicorn/no-array-instanceof: [0]
+  unicorn/no-array-method-this-argument: [2]
   unicorn/no-array-push-push: [2]
   unicorn/no-console-spaces: [0]
   unicorn/no-document-cookie: [2]
@@ -385,6 +388,8 @@ rules:
   unicorn/no-unreadable-array-destructuring: [0]
   unicorn/no-unsafe-regex: [0]
   unicorn/no-unused-properties: [2]
+  unicorn/no-useless-length-check: [2]
+  unicorn/no-useless-spread: [2]
   unicorn/no-useless-undefined: [0]
   unicorn/no-zero-fractions: [2]
   unicorn/number-literal-case: [0]
@@ -395,6 +400,7 @@ rules:
   unicorn/prefer-array-flat: [2]
   unicorn/prefer-array-index-of: [2]
   unicorn/prefer-array-some: [2]
+  unicorn/prefer-at: [0]
   unicorn/prefer-dataset: [2]
   unicorn/prefer-date-now: [2]
   unicorn/prefer-default-parameters: [0]
@@ -408,7 +414,10 @@ rules:
   unicorn/prefer-node-protocol: [0]
   unicorn/prefer-node-remove: [0]
   unicorn/prefer-number-properties: [0]
+  unicorn/prefer-object-from-entries: [2]
+  unicorn/prefer-object-has-own: [0]
   unicorn/prefer-optional-catch-binding: [2]
+  unicorn/prefer-prototype-methods: [0]
   unicorn/prefer-query-selector: [0]
   unicorn/prefer-reflect-apply: [0]
   unicorn/prefer-regexp-test: [2]
@@ -420,9 +429,13 @@ rules:
   unicorn/prefer-switch: [0]
   unicorn/prefer-ternary: [0]
   unicorn/prefer-text-content: [2]
+  unicorn/prefer-top-level-await: [0]
   unicorn/prefer-trim-start-end: [2]
   unicorn/prefer-type-error: [0]
   unicorn/prevent-abbreviations: [0]
+  unicorn/require-array-join-separator: [2]
+  unicorn/require-number-to-fixed-digits-argument: [2]
+  unicorn/require-post-message-target-origin: [0]
   unicorn/string-content: [0]
   unicorn/throw-new-error: [2]
   use-isnan: [2]

Dockerfile

@@ -1,7 +1,7 @@
 
 ###################################
 #Build stage
-FROM golang:1.16-alpine3.13 AS build-env
+FROM golang:1.17-alpine3.13 AS build-env
 
 ARG GOPROXY
 ENV GOPROXY ${GOPROXY:-direct}

Dockerfile.rootless

@@ -1,7 +1,7 @@
 
 ###################################
 #Build stage
-FROM golang:1.16-alpine3.13 AS build-env
+FROM golang:1.17-alpine3.13 AS build-env
 
 ARG GOPROXY
 ENV GOPROXY ${GOPROXY:-direct}

Makefile

@@ -24,7 +24,7 @@ SHASUM ?= shasum -a 256
 HAS_GO = $(shell hash $(GO) > /dev/null 2>&1 && echo "GO" || echo "NOGO" )
 COMMA := ,
 
-XGO_VERSION := go-1.16.x
+XGO_VERSION := go-1.17.x
 MIN_GO_VERSION := 001016000
 MIN_NODE_VERSION := 012017000
 

build/generate-svg.js

@@ -1,5 +1,5 @@
 import fastGlob from 'fast-glob';
-import {optimize, extendDefaultPlugins} from 'svgo';
+import {optimize} from 'svgo';
 import {resolve, parse, dirname} from 'path';
 import fs from 'fs';
 import {fileURLToPath} from 'url';
@@ -26,18 +26,13 @@ async function processFile(file, {prefix, fullName} = {}) {
   }
 
   const {data} = optimize(await readFile(file, 'utf8'), {
-    plugins: extendDefaultPlugins([
-      'removeXMLNS',
-      'removeDimensions',
-      {
-        name: 'addClassesToSVGElement',
-        params: {classNames: ['svg', name]},
-      },
-      {
-        name: 'addAttributesToSVGElement',
-        params: {attributes: [{'width': '16'}, {'height': '16'}, {'aria-hidden': 'true'}]},
-      },
-    ]),
+    plugins: [
+      {name: 'preset-default'},
+      {name: 'removeXMLNS'},
+      {name: 'removeDimensions'},
+      {name: 'addClassesToSVGElement', params: {classNames: ['svg', name]}},
+      {name: 'addAttributesToSVGElement', params: {attributes: [{'width': '16'}, {'height': '16'}, {'aria-hidden': 'true'}]}},
+    ],
   });
   await writeFile(resolve(outputDir, `${name}.svg`), data);
 }

custom/conf/app.example.ini

@@ -378,6 +378,10 @@ INTERNAL_TOKEN=
 ;;
 ;; Validate against https://haveibeenpwned.com/Passwords to see if a password has been exposed
 ;PASSWORD_CHECK_PWN = false
+;;
+;; Cache successful token hashes. API tokens are stored in the DB as pbkdf2 hashes however, this means that there is a potentially significant hashing load when there are multiple API operations.
+;; This cache will store the successfully hashed tokens in a LRU cache as a balance between performance and security.
+;SUCCESSFUL_TOKENS_CACHE_SIZE = 20
 
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;

docs/config.yaml

@@ -20,7 +20,7 @@ params:
   website: https://docs.gitea.io
   version: 1.14.6
   minGoVersion: 1.16
-  goVersion: 1.16
+  goVersion: 1.17
   minNodeVersion: 12.17
 
 outputs:

docs/content/doc/advanced/config-cheat-sheet.en-us.md

@@ -441,6 +441,7 @@ relation to port exhaustion.
     - spec - use one or more special characters as ``!"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~``
     - off - do not check password complexity
 - `PASSWORD_CHECK_PWN`: **false**: Check [HaveIBeenPwned](https://haveibeenpwned.com/Passwords) to see if a password has been exposed.
+- `SUCCESSFUL_TOKENS_CACHE_SIZE`: **20**: Cache successful token hashes. API tokens are stored in the DB as pbkdf2 hashes however, this means that there is a potentially significant hashing load when there are multiple API operations. This cache will store the successfully hashed tokens in a LRU cache as a balance between performance and security. 
 
 ## OpenID (`openid`)
 

jest.config.js

@@ -1,10 +1,9 @@
 export default {
+  rootDir: 'web_src',
   setupFilesAfterEnv: ['jest-extended'],
+  testEnvironment: 'jsdom',
+  testMatch: ['<rootDir>/**/*.test.js'],
   testTimeout: 20000,
-  rootDir: 'web_src',
-  testMatch: [
-    '<rootDir>/**/*.test.js',
-  ],
   transform: {},
   verbose: false,
 };

models/models.go

@@ -17,6 +17,7 @@ import (
 
 	// Needed for the MySQL driver
 	_ "github.com/go-sql-driver/mysql"
+	lru "github.com/hashicorp/golang-lru"
 	"xorm.io/xorm"
 	"xorm.io/xorm/names"
 	"xorm.io/xorm/schemas"
@@ -234,6 +235,15 @@ func NewEngine(ctx context.Context, migrateFunc func(*xorm.Engine) error) (err e
 		return fmt.Errorf("sync database struct error: %v", err)
 	}
 
+	if setting.SuccessfulTokensCacheSize > 0 {
+		successfulAccessTokenCache, err = lru.New(setting.SuccessfulTokensCacheSize)
+		if err != nil {
+			return fmt.Errorf("unable to allocate AccessToken cache: %v", err)
+		}
+	} else {
+		successfulAccessTokenCache = nil
+	}
+
 	return nil
 }
 

models/token.go

@@ -14,8 +14,11 @@ import (
 	"code.gitea.io/gitea/modules/util"
 
 	gouuid "github.com/google/uuid"
+	lru "github.com/hashicorp/golang-lru"
 )
 
+var successfulAccessTokenCache *lru.Cache
+
 // AccessToken represents a personal access token.
 type AccessToken struct {
 	ID             int64 `xorm:"pk autoincr"`
@@ -52,6 +55,21 @@ func NewAccessToken(t *AccessToken) error {
 	return err
 }
 
+func getAccessTokenIDFromCache(token string) int64 {
+	if successfulAccessTokenCache == nil {
+		return 0
+	}
+	tInterface, ok := successfulAccessTokenCache.Get(token)
+	if !ok {
+		return 0
+	}
+	t, ok := tInterface.(int64)
+	if !ok {
+		return 0
+	}
+	return t
+}
+
 // GetAccessTokenBySHA returns access token by given token value
 func GetAccessTokenBySHA(token string) (*AccessToken, error) {
 	if token == "" {
@@ -66,17 +84,38 @@ func GetAccessTokenBySHA(token string) (*AccessToken, error) {
 			return nil, ErrAccessTokenNotExist{token}
 		}
 	}
-	var tokens []AccessToken
+
 	lastEight := token[len(token)-8:]
+
+	if id := getAccessTokenIDFromCache(token); id > 0 {
+		token := &AccessToken{
+			TokenLastEight: lastEight,
+		}
+		// Re-get the token from the db in case it has been deleted in the intervening period
+		has, err := x.ID(id).Get(token)
+		if err != nil {
+			return nil, err
+		}
+		if has {
+			return token, nil
+		}
+		successfulAccessTokenCache.Remove(token)
+	}
+
+	var tokens []AccessToken
 	err := x.Table(&AccessToken{}).Where("token_last_eight = ?", lastEight).Find(&tokens)
 	if err != nil {
 		return nil, err
 	} else if len(tokens) == 0 {
 		return nil, ErrAccessTokenNotExist{token}
 	}
+
 	for _, t := range tokens {
 		tempHash := hashToken(token, t.TokenSalt)
 		if subtle.ConstantTimeCompare([]byte(t.TokenHash), []byte(tempHash)) == 1 {
+			if successfulAccessTokenCache != nil {
+				successfulAccessTokenCache.Add(token, t.ID)
+			}
 			return &t, nil
 		}
 	}

modules/setting/setting.go

@@ -189,6 +189,7 @@ var (
 	PasswordComplexity                 []string
 	PasswordHashAlgo                   string
 	PasswordCheckPwn                   bool
+	SuccessfulTokensCacheSize          int
 
 	// UI settings
 	UI = struct {
@@ -840,6 +841,7 @@ func NewContext() {
 	PasswordHashAlgo = sec.Key("PASSWORD_HASH_ALGO").MustString("pbkdf2")
 	CSRFCookieHTTPOnly = sec.Key("CSRF_COOKIE_HTTP_ONLY").MustBool(true)
 	PasswordCheckPwn = sec.Key("PASSWORD_CHECK_PWN").MustBool(false)
+	SuccessfulTokensCacheSize = sec.Key("SUCCESSFUL_TOKENS_CACHE_SIZE").MustInt(20)
 
 	InternalToken = loadInternalToken(sec)
 

options/locale/locale_ja-JP.ini

@@ -2432,6 +2432,7 @@ auths.allowed_domains=許可するドメイン
 auths.allowed_domains_helper=すべてのドメインを許可する場合は空のままにします。 複数のドメインはカンマ(',')で区切ります。
 auths.skip_tls_verify=TLS検証を省略
 auths.force_smtps=強制的にSMTPSにする
+auths.force_smtps_helper=ポート465ではSMTPSが常に使用されます。 これを指定すると、他のポートでもSMTPSの使用を強制します。 (指定しない場合は、ホストがサポートしていればSTARTTLSが使用されます。)
 auths.helo_hostname=HELOホストネーム
 auths.helo_hostname_helper=HELOで送られるホスト名。現在のホスト名で送信する場合は空白にします。
 auths.disable_helo=HELOを無効にする