Provide self-registering storage system (#12978)

* Provide self-registering storage system

Signed-off-by: Andrew Thornton <[email protected]>

* More simplification

Signed-off-by: Andrew Thornton <[email protected]>

* Remove old strings from setting

Signed-off-by: Andrew Thornton <[email protected]>

* oops attachments not attachment

Signed-off-by: Andrew Thornton <[email protected]>

Co-authored-by: Lunny Xiao <[email protected]>

Author: zeripath

cmd/migrate_storage.go

@@ -32,8 +32,8 @@ var CmdMigrateStorage = cli.Command{
 		},
 		cli.StringFlag{
 			Name:  "storage, s",
-			Value: setting.LocalStorageType,
-			Usage: "New storage type, local or minio",
+			Value: "",
+			Usage: "New storage type: local (default) or minio",
 		},
 		cli.StringFlag{
 			Name:  "path, p",
@@ -107,31 +107,40 @@ func runMigrateStorage(ctx *cli.Context) error {
 		return err
 	}
 
+	goCtx := context.Background()
+
 	if err := storage.Init(); err != nil {
 		return err
 	}
 
 	var dstStorage storage.ObjectStorage
 	var err error
 	switch strings.ToLower(ctx.String("storage")) {
-	case setting.LocalStorageType:
+	case "":
+		fallthrough
+	case string(storage.LocalStorageType):
 		p := ctx.String("path")
 		if p == "" {
 			log.Fatal("Path must be given when storage is loal")
 			return nil
 		}
-		dstStorage, err = storage.NewLocalStorage(p)
-	case setting.MinioStorageType:
+		dstStorage, err = storage.NewLocalStorage(
+			goCtx,
+			storage.LocalStorageConfig{
+				Path: p,
+			})
+	case string(storage.MinioStorageType):
 		dstStorage, err = storage.NewMinioStorage(
-			context.Background(),
-			ctx.String("minio-endpoint"),
-			ctx.String("minio-access-key-id"),
-			ctx.String("minio-secret-access-key"),
-			ctx.String("minio-bucket"),
-			ctx.String("minio-location"),
-			ctx.String("minio-base-path"),
-			ctx.Bool("minio-use-ssl"),
-		)
+			goCtx,
+			storage.MinioStorageConfig{
+				Endpoint:        ctx.String("minio-endpoint"),
+				AccessKeyID:     ctx.String("minio-access-key-id"),
+				SecretAccessKey: ctx.String("minio-secret-access-key"),
+				Bucket:          ctx.String("minio-bucket"),
+				Location:        ctx.String("minio-location"),
+				BasePath:        ctx.String("minio-base-path"),
+				UseSSL:          ctx.Bool("minio-use-ssl"),
+			})
 	default:
 		return fmt.Errorf("Unsupported attachments storage type: %s", ctx.String("storage"))
 	}

models/unit_tests.go

@@ -67,10 +67,8 @@ func MainTest(m *testing.M, pathToGiteaRoot string) {
 	if err != nil {
 		fatalTestError("url.Parse: %v\n", err)
 	}
-	setting.Attachment.Storage.Type = setting.LocalStorageType
 	setting.Attachment.Storage.Path = filepath.Join(setting.AppDataPath, "attachments")
 
-	setting.LFS.Storage.Type = setting.LocalStorageType
 	setting.LFS.Storage.Path = filepath.Join(setting.AppDataPath, "lfs")
 	if err = storage.Init(); err != nil {
 		fatalTestError("storage.Init: %v\n", err)

modules/setting/attachment.go

@@ -4,12 +4,6 @@
 
 package setting
 
-import (
-	"path/filepath"
-
-	"code.gitea.io/gitea/modules/log"
-)
-
 var (
 	// Attachment settings
 	Attachment = struct {
@@ -20,7 +14,6 @@ var (
 		Enabled      bool
 	}{
 		Storage: Storage{
-			Type:        LocalStorageType,
 			ServeDirect: false,
 		},
 		AllowedTypes: "image/jpeg,image/png,application/zip,application/gzip",
@@ -32,37 +25,9 @@ var (
 
 func newAttachmentService() {
 	sec := Cfg.Section("attachment")
-	Attachment.Storage.Type = sec.Key("STORAGE_TYPE").MustString("")
-	if Attachment.Storage.Type == "" {
-		Attachment.Storage.Type = "default"
-	}
+	storageType := sec.Key("STORAGE_TYPE").MustString("")
 
-	if Attachment.Storage.Type != LocalStorageType && Attachment.Storage.Type != MinioStorageType {
-		storage, ok := storages[Attachment.Storage.Type]
-		if !ok {
-			log.Fatal("Failed to get attachment storage type: %s", Attachment.Storage.Type)
-		}
-		Attachment.Storage = storage
-	}
-
-	// Override
-	Attachment.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(Attachment.ServeDirect)
-
-	switch Attachment.Storage.Type {
-	case LocalStorageType:
-		Attachment.Path = sec.Key("PATH").MustString(filepath.Join(AppDataPath, "attachments"))
-		if !filepath.IsAbs(Attachment.Path) {
-			Attachment.Path = filepath.Join(AppWorkPath, Attachment.Path)
-		}
-	case MinioStorageType:
-		Attachment.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString(Attachment.Minio.Endpoint)
-		Attachment.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString(Attachment.Minio.AccessKeyID)
-		Attachment.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString(Attachment.Minio.SecretAccessKey)
-		Attachment.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString(Attachment.Minio.Bucket)
-		Attachment.Minio.Location = sec.Key("MINIO_LOCATION").MustString(Attachment.Minio.Location)
-		Attachment.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(Attachment.Minio.UseSSL)
-		Attachment.Minio.BasePath = sec.Key("MINIO_BASE_PATH").MustString("attachments/")
-	}
+	Attachment.Storage = getStorage("attachments", storageType, sec)
 
 	Attachment.AllowedTypes = sec.Key("ALLOWED_TYPES").MustString(".docx,.gif,.gz,.jpeg,.jpg,.log,.pdf,.png,.pptx,.txt,.xlsx,.zip")
 	Attachment.MaxSize = sec.Key("MAX_SIZE").MustInt64(4)

modules/setting/lfs.go

@@ -37,40 +37,15 @@ func newLFSService() {
 	}
 
 	lfsSec := Cfg.Section("lfs")
-	LFS.Storage.Type = lfsSec.Key("STORAGE_TYPE").MustString("")
-	if LFS.Storage.Type == "" {
-		LFS.Storage.Type = "default"
-	}
-
-	if LFS.Storage.Type != LocalStorageType && LFS.Storage.Type != MinioStorageType {
-		storage, ok := storages[LFS.Storage.Type]
-		if !ok {
-			log.Fatal("Failed to get lfs storage type: %s", LFS.Storage.Type)
-		}
-		LFS.Storage = storage
-	}
+	storageType := lfsSec.Key("STORAGE_TYPE").MustString("")
 
-	// Override
-	LFS.ServeDirect = lfsSec.Key("SERVE_DIRECT").MustBool(LFS.ServeDirect)
-	switch LFS.Storage.Type {
-	case LocalStorageType:
-		// keep compatible
-		LFS.Path = sec.Key("LFS_CONTENT_PATH").MustString(filepath.Join(AppDataPath, "lfs"))
-		LFS.Path = lfsSec.Key("PATH").MustString(LFS.Path)
-		if !filepath.IsAbs(LFS.Path) {
-			LFS.Path = filepath.Join(AppWorkPath, LFS.Path)
-		}
+	// Specifically default PATH to LFS_CONTENT_PATH
+	lfsSec.Key("PATH").MustString(
+		sec.Key("LFS_CONTENT_PATH").String())
 
-	case MinioStorageType:
-		LFS.Minio.Endpoint = lfsSec.Key("MINIO_ENDPOINT").MustString(LFS.Minio.Endpoint)
-		LFS.Minio.AccessKeyID = lfsSec.Key("MINIO_ACCESS_KEY_ID").MustString(LFS.Minio.AccessKeyID)
-		LFS.Minio.SecretAccessKey = lfsSec.Key("MINIO_SECRET_ACCESS_KEY").MustString(LFS.Minio.SecretAccessKey)
-		LFS.Minio.Bucket = lfsSec.Key("MINIO_BUCKET").MustString(LFS.Minio.Bucket)
-		LFS.Minio.Location = lfsSec.Key("MINIO_LOCATION").MustString(LFS.Minio.Location)
-		LFS.Minio.UseSSL = lfsSec.Key("MINIO_USE_SSL").MustBool(LFS.Minio.UseSSL)
-		LFS.Minio.BasePath = lfsSec.Key("MINIO_BASE_PATH").MustString("lfs/")
-	}
+	LFS.Storage = getStorage("lfs", storageType, lfsSec)
 
+	// Rest of LFS service settings
 	if LFS.LocksPagingNum == 0 {
 		LFS.LocksPagingNum = 50
 	}

modules/setting/setting.go

@@ -804,7 +804,6 @@ func NewContext() {
 		}
 	}
 
-	newStorageService()
 	newAttachmentService()
 	newLFSService()
 

modules/setting/storage.go

@@ -5,65 +5,77 @@
 package setting
 
 import (
-	"strings"
+	"path/filepath"
+	"reflect"
 
-	"code.gitea.io/gitea/modules/log"
 	ini "gopkg.in/ini.v1"
 )
 
-// enumerate all storage types
-const (
-	LocalStorageType = "local"
-	MinioStorageType = "minio"
-)
-
 // Storage represents configuration of storages
 type Storage struct {
 	Type        string
 	Path        string
+	Section     *ini.Section
 	ServeDirect bool
-	Minio       struct {
-		Endpoint        string
-		AccessKeyID     string
-		SecretAccessKey string
-		UseSSL          bool
-		Bucket          string
-		Location        string
-		BasePath        string
+}
+
+// MapTo implements the Mappable interface
+func (s *Storage) MapTo(v interface{}) error {
+	pathValue := reflect.ValueOf(v).FieldByName("Path")
+	if pathValue.IsValid() && pathValue.Kind() == reflect.String {
+		pathValue.SetString(s.Path)
+	}
+	if s.Section != nil {
+		return s.Section.MapTo(v)
 	}
+	return nil
 }
 
-var (
-	storages = make(map[string]Storage)
-)
+func getStorage(name, typ string, overrides ...*ini.Section) Storage {
+	sectionName := "storage"
+	if len(name) > 0 {
+		sectionName = sectionName + "." + typ
+	}
+	sec := Cfg.Section(sectionName)
+
+	if len(overrides) == 0 {
+		overrides = []*ini.Section{
+			Cfg.Section(sectionName + "." + name),
+		}
+	}
 
-func getStorage(sec *ini.Section) Storage {
 	var storage Storage
-	storage.Type = sec.Key("STORAGE_TYPE").MustString(LocalStorageType)
+
+	storage.Type = sec.Key("STORAGE_TYPE").MustString("")
 	storage.ServeDirect = sec.Key("SERVE_DIRECT").MustBool(false)
-	switch storage.Type {
-	case LocalStorageType:
-	case MinioStorageType:
-		storage.Minio.Endpoint = sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
-		storage.Minio.AccessKeyID = sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
-		storage.Minio.SecretAccessKey = sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
-		storage.Minio.Bucket = sec.Key("MINIO_BUCKET").MustString("gitea")
-		storage.Minio.Location = sec.Key("MINIO_LOCATION").MustString("us-east-1")
-		storage.Minio.UseSSL = sec.Key("MINIO_USE_SSL").MustBool(false)
-	}
-	return storage
-}
 
-func newStorageService() {
-	sec := Cfg.Section("storage")
-	storages["default"] = getStorage(sec)
+	// Global Defaults
+	sec.Key("MINIO_ENDPOINT").MustString("localhost:9000")
+	sec.Key("MINIO_ACCESS_KEY_ID").MustString("")
+	sec.Key("MINIO_SECRET_ACCESS_KEY").MustString("")
+	sec.Key("MINIO_BUCKET").MustString("gitea")
+	sec.Key("MINIO_LOCATION").MustString("us-east-1")
+	sec.Key("MINIO_USE_SSL").MustBool(false)
+
+	storage.Section = sec
 
-	for _, sec := range Cfg.Section("storage").ChildSections() {
-		name := strings.TrimPrefix(sec.Name(), "storage.")
-		if name == "default" || name == LocalStorageType || name == MinioStorageType {
-			log.Error("storage name %s is system reserved!", name)
-			continue
+	for _, override := range overrides {
+		for _, key := range storage.Section.Keys() {
+			if !override.HasKey(key.Name()) {
+				_, _ = override.NewKey(key.Name(), key.Value())
+			}
 		}
-		storages[name] = getStorage(sec)
+		storage.ServeDirect = override.Key("SERVE_DIRECT").MustBool(false)
+		storage.Section = override
 	}
+
+	// Specific defaults
+	storage.Path = storage.Section.Key("PATH").MustString(filepath.Join(AppDataPath, name))
+	if !filepath.IsAbs(storage.Path) {
+		storage.Path = filepath.Join(AppWorkPath, storage.Path)
+		storage.Section.Key("PATH").SetValue(storage.Path)
+	}
+	storage.Section.Key("MINIO_BASE_PATH").MustString(name + "/")
+
+	return storage
 }

modules/storage/helper.go

@@ -0,0 +1,65 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package storage
+
+import (
+	"encoding/json"
+	"reflect"
+)
+
+// Mappable represents an interface that can MapTo another interface
+type Mappable interface {
+	MapTo(v interface{}) error
+}
+
+// toConfig will attempt to convert a given configuration cfg into the provided exemplar type.
+//
+// It will tolerate the cfg being passed as a []byte or string of a json representation of the
+// exemplar or the correct type of the exemplar itself
+func toConfig(exemplar, cfg interface{}) (interface{}, error) {
+
+	// First of all check if we've got the same type as the exemplar - if so it's all fine.
+	if reflect.TypeOf(cfg).AssignableTo(reflect.TypeOf(exemplar)) {
+		return cfg, nil
+	}
+
+	// Now if not - does it provide a MapTo function we can try?
+	if mappable, ok := cfg.(Mappable); ok {
+		newVal := reflect.New(reflect.TypeOf(exemplar))
+		if err := mappable.MapTo(newVal.Interface()); err == nil {
+			return newVal.Elem().Interface(), nil
+		}
+		// MapTo has failed us ... let's try the json route ...
+	}
+
+	// OK we've been passed a byte array right?
+	configBytes, ok := cfg.([]byte)
+	if !ok {
+		// oh ... it's a string then?
+		var configStr string
+
+		configStr, ok = cfg.(string)
+		configBytes = []byte(configStr)
+	}
+	if !ok {
+		// hmm ... can we marshal it to json?
+		var err error
+
+		configBytes, err = json.Marshal(cfg)
+		ok = (err == nil)
+	}
+	if !ok {
+		// no ... we've tried hard enough at this point - throw an error!
+		return nil, ErrInvalidConfiguration{cfg: cfg}
+	}
+
+	// OK unmarshal the byte array into a new copy of the exemplar
+	newVal := reflect.New(reflect.TypeOf(exemplar))
+	if err := json.Unmarshal(configBytes, newVal.Interface()); err != nil {
+		// If we can't unmarshal it then return an error!
+		return nil, ErrInvalidConfiguration{cfg: cfg, err: err}
+	}
+	return newVal.Elem().Interface(), nil
+}