Skip to content

Commit 236c645

Browse files
wxiaoguangwxiaoguang
authored
Refactor "Content" for file uploading (#25851)
Before: the concept "Content string" is used everywhere. It has some problems: 1. Sometimes it means "base64 encoded content", sometimes it means "raw binary content" 2. It doesn't work with large files, eg: uploading a 1G LFS file would make Gitea process OOM This PR does the refactoring: use "ContentReader" / "ContentBase64" instead of "Content" This PR is not breaking because the key in API JSON is still "content": `` ContentBase64 string `json:"content"` ``
1 parent 265a288 commit 236c645

15 files changed

+103
-80
lines changed

‎modules/structs/repo_file.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ type CreateFileOptions struct {
2626
FileOptions
2727
// content must be base64 encoded
2828
// required: true
29-
Content string `json:"content"`
29+
ContentBase64 string `json:"content"`
3030
}
3131

3232
// Branch returns branch name
@@ -54,7 +54,7 @@ type UpdateFileOptions struct {
5454
DeleteFileOptions
5555
// content must be base64 encoded
5656
// required: true
57-
Content string `json:"content"`
57+
ContentBase64 string `json:"content"`
5858
// from_path (optional) is the path of the original file which will be moved/renamed to the path in the URL
5959
FromPath string `json:"from_path" binding:"MaxSize(500)"`
6060
}
@@ -74,7 +74,7 @@ type ChangeFileOperation struct {
7474
// required: true
7575
Path string `json:"path" binding:"Required;MaxSize(500)"`
7676
// new or updated file content, must be base64 encoded
77-
Content string `json:"content"`
77+
ContentBase64 string `json:"content"`
7878
// sha is the SHA for the file that already exists, required for update or delete
7979
SHA string `json:"sha"`
8080
// old path of the file to move

‎routers/api/v1/repo/file.go

Lines changed: 39 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,14 @@ func canReadFiles(r *context.Repository) bool {
408408
return r.Permission.CanRead(unit.TypeCode)
409409
}
410410

411+
func base64Reader(s string) (io.Reader, error) {
412+
b, err := base64.StdEncoding.DecodeString(s)
413+
if err != nil {
414+
return nil, err
415+
}
416+
return bytes.NewReader(b), nil
417+
}
418+
411419
// ChangeFiles handles API call for modifying multiple files
412420
func ChangeFiles(ctx *context.APIContext) {
413421
// swagger:operation POST /repos/{owner}/{repo}/contents repository repoChangeFiles
@@ -449,14 +457,19 @@ func ChangeFiles(ctx *context.APIContext) {
449457
apiOpts.BranchName = ctx.Repo.Repository.DefaultBranch
450458
}
451459

452-
files := []*files_service.ChangeRepoFile{}
460+
var files []*files_service.ChangeRepoFile
453461
for _, file := range apiOpts.Files {
462+
contentReader, err := base64Reader(file.ContentBase64)
463+
if err != nil {
464+
ctx.Error(http.StatusUnprocessableEntity, "Invalid base64 content", err)
465+
return
466+
}
454467
changeRepoFile := &files_service.ChangeRepoFile{
455-
Operation: file.Operation,
456-
TreePath: file.Path,
457-
FromTreePath: file.FromPath,
458-
Content: file.Content,
459-
SHA: file.SHA,
468+
Operation: file.Operation,
469+
TreePath: file.Path,
470+
FromTreePath: file.FromPath,
471+
ContentReader: contentReader,
472+
SHA: file.SHA,
460473
}
461474
files = append(files, changeRepoFile)
462475
}
@@ -544,12 +557,18 @@ func CreateFile(ctx *context.APIContext) {
544557
apiOpts.BranchName = ctx.Repo.Repository.DefaultBranch
545558
}
546559

560+
contentReader, err := base64Reader(apiOpts.ContentBase64)
561+
if err != nil {
562+
ctx.Error(http.StatusUnprocessableEntity, "Invalid base64 content", err)
563+
return
564+
}
565+
547566
opts := &files_service.ChangeRepoFilesOptions{
548567
Files: []*files_service.ChangeRepoFile{
549568
{
550-
Operation: "create",
551-
TreePath: ctx.Params("*"),
552-
Content: apiOpts.Content,
569+
Operation: "create",
570+
TreePath: ctx.Params("*"),
571+
ContentReader: contentReader,
553572
},
554573
},
555574
Message: apiOpts.Message,
@@ -636,14 +655,20 @@ func UpdateFile(ctx *context.APIContext) {
636655
apiOpts.BranchName = ctx.Repo.Repository.DefaultBranch
637656
}
638657

658+
contentReader, err := base64Reader(apiOpts.ContentBase64)
659+
if err != nil {
660+
ctx.Error(http.StatusUnprocessableEntity, "Invalid base64 content", err)
661+
return
662+
}
663+
639664
opts := &files_service.ChangeRepoFilesOptions{
640665
Files: []*files_service.ChangeRepoFile{
641666
{
642-
Operation: "update",
643-
Content: apiOpts.Content,
644-
SHA: apiOpts.SHA,
645-
FromTreePath: apiOpts.FromPath,
646-
TreePath: ctx.Params("*"),
667+
Operation: "update",
668+
ContentReader: contentReader,
669+
SHA: apiOpts.SHA,
670+
FromTreePath: apiOpts.FromPath,
671+
TreePath: ctx.Params("*"),
647672
},
648673
},
649674
Message: apiOpts.Message,
@@ -709,14 +734,6 @@ func createOrUpdateFiles(ctx *context.APIContext, opts *files_service.ChangeRepo
709734
}
710735
}
711736

712-
for _, file := range opts.Files {
713-
content, err := base64.StdEncoding.DecodeString(file.Content)
714-
if err != nil {
715-
return nil, err
716-
}
717-
file.Content = string(content)
718-
}
719-
720737
return files_service.ChangeRepoFiles(ctx, ctx.Repo.Repository, ctx.Doer, opts)
721738
}
722739

‎routers/web/repo/editor.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -284,10 +284,10 @@ func editFilePost(ctx *context.Context, form forms.EditRepoFileForm, isNewFile b
284284
Message: message,
285285
Files: []*files_service.ChangeRepoFile{
286286
{
287-
Operation: operation,
288-
FromTreePath: ctx.Repo.TreePath,
289-
TreePath: form.TreePath,
290-
Content: strings.ReplaceAll(form.Content, "\r", ""),
287+
Operation: operation,
288+
FromTreePath: ctx.Repo.TreePath,
289+
TreePath: form.TreePath,
290+
ContentReader: strings.NewReader(strings.ReplaceAll(form.Content, "\r", "")),
291291
},
292292
},
293293
Signoff: form.Signoff,

‎services/repository/files/update.go

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ package files
66
import (
77
"context"
88
"fmt"
9+
"io"
910
"path"
1011
"strings"
1112
"time"
@@ -35,12 +36,12 @@ type CommitDateOptions struct {
3536
}
3637

3738
type ChangeRepoFile struct {
38-
Operation string
39-
TreePath string
40-
FromTreePath string
41-
Content string
42-
SHA string
43-
Options *RepoFileOptions
39+
Operation string
40+
TreePath string
41+
FromTreePath string
42+
ContentReader io.Reader
43+
SHA string
44+
Options *RepoFileOptions
4445
}
4546

4647
// ChangeRepoFilesOptions holds the repository files update options
@@ -387,7 +388,7 @@ func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file
387388
}
388389
}
389390

390-
treeObjectContent := file.Content
391+
treeObjectContentReader := file.ContentReader
391392
var lfsMetaObject *git_model.LFSMetaObject
392393
if setting.LFS.StartServer && hasOldBranch {
393394
// Check there is no way this can return multiple infos
@@ -402,17 +403,17 @@ func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file
402403

403404
if filename2attribute2info[file.Options.treePath] != nil && filename2attribute2info[file.Options.treePath]["filter"] == "lfs" {
404405
// OK so we are supposed to LFS this data!
405-
pointer, err := lfs.GeneratePointer(strings.NewReader(file.Content))
406+
pointer, err := lfs.GeneratePointer(treeObjectContentReader)
406407
if err != nil {
407408
return err
408409
}
409410
lfsMetaObject = &git_model.LFSMetaObject{Pointer: pointer, RepositoryID: repoID}
410-
treeObjectContent = pointer.StringContent()
411+
treeObjectContentReader = strings.NewReader(pointer.StringContent())
411412
}
412413
}
413414

414415
// Add the object to the database
415-
objectHash, err := t.HashObject(strings.NewReader(treeObjectContent))
416+
objectHash, err := t.HashObject(treeObjectContentReader)
416417
if err != nil {
417418
return err
418419
}
@@ -439,7 +440,7 @@ func CreateOrUpdateFile(ctx context.Context, t *TemporaryUploadRepository, file
439440
return err
440441
}
441442
if !exist {
442-
if err := contentStore.Put(lfsMetaObject.Pointer, strings.NewReader(file.Content)); err != nil {
443+
if err := contentStore.Put(lfsMetaObject.Pointer, file.ContentReader); err != nil {
443444
if _, err2 := git_model.RemoveLFSMetaObjectByOid(ctx, repoID, lfsMetaObject.Oid); err2 != nil {
444445
return fmt.Errorf("unable to remove failed inserted LFS object %s: %v (Prev Error: %w)", lfsMetaObject.Oid, err2, err)
445446
}

‎templates/swagger/v1_json.tmpl

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎tests/integration/actions_trigger_test.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package integration
55

66
import (
77
"net/url"
8+
"strings"
89
"testing"
910
"time"
1011

@@ -64,9 +65,9 @@ func TestPullRequestTargetEvent(t *testing.T) {
6465
addWorkflowToBaseResp, err := files_service.ChangeRepoFiles(git.DefaultContext, baseRepo, user2, &files_service.ChangeRepoFilesOptions{
6566
Files: []*files_service.ChangeRepoFile{
6667
{
67-
Operation: "create",
68-
TreePath: ".gitea/workflows/pr.yml",
69-
Content: "name: test\non: pull_request_target\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - run: echo helloworld\n",
68+
Operation: "create",
69+
TreePath: ".gitea/workflows/pr.yml",
70+
ContentReader: strings.NewReader("name: test\non: pull_request_target\njobs:\n test:\n runs-on: ubuntu-latest\n steps:\n - run: echo helloworld\n"),
7071
},
7172
},
7273
Message: "add workflow",
@@ -92,9 +93,9 @@ func TestPullRequestTargetEvent(t *testing.T) {
9293
addFileToForkedResp, err := files_service.ChangeRepoFiles(git.DefaultContext, forkedRepo, user3, &files_service.ChangeRepoFilesOptions{
9394
Files: []*files_service.ChangeRepoFile{
9495
{
95-
Operation: "create",
96-
TreePath: "file_1.txt",
97-
Content: "file1",
96+
Operation: "create",
97+
TreePath: "file_1.txt",
98+
ContentReader: strings.NewReader("file1"),
9899
},
99100
},
100101
Message: "add file1",

‎tests/integration/api_repo_file_create_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ func getCreateFileOptions() api.CreateFileOptions {
4646
Committer: time.Unix(978307190, 0),
4747
},
4848
},
49-
Content: contentEncoded,
49+
ContentBase64: contentEncoded,
5050
}
5151
}
5252

‎tests/integration/api_repo_file_helpers.go

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@
44
package integration
55

66
import (
7+
"strings"
8+
79
repo_model "code.gitea.io/gitea/models/repo"
810
user_model "code.gitea.io/gitea/models/user"
911
"code.gitea.io/gitea/modules/git"
@@ -15,9 +17,9 @@ func createFileInBranch(user *user_model.User, repo *repo_model.Repository, tree
1517
opts := &files_service.ChangeRepoFilesOptions{
1618
Files: []*files_service.ChangeRepoFile{
1719
{
18-
Operation: "create",
19-
TreePath: treePath,
20-
Content: content,
20+
Operation: "create",
21+
TreePath: treePath,
22+
ContentReader: strings.NewReader(content),
2123
},
2224
},
2325
OldBranch: branchName,

‎tests/integration/api_repo_file_update_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ func getUpdateFileOptions() *api.UpdateFileOptions {
4444
},
4545
SHA: "103ff9234cefeee5ec5361d22b49fbb04d385885",
4646
},
47-
Content: contentEncoded,
47+
ContentBase64: contentEncoded,
4848
}
4949
}
5050

‎tests/integration/api_repo_files_change_test.go

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -44,13 +44,13 @@ func getChangeFilesOptions() *api.ChangeFilesOptions {
4444
},
4545
Files: []*api.ChangeFileOperation{
4646
{
47-
Operation: "create",
48-
Content: newContentEncoded,
47+
Operation: "create",
48+
ContentBase64: newContentEncoded,
4949
},
5050
{
51-
Operation: "update",
52-
Content: updateContentEncoded,
53-
SHA: "103ff9234cefeee5ec5361d22b49fbb04d385885",
51+
Operation: "update",
52+
ContentBase64: updateContentEncoded,
53+
SHA: "103ff9234cefeee5ec5361d22b49fbb04d385885",
5454
},
5555
{
5656
Operation: "delete",

‎tests/integration/empty_repo_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ func TestEmptyRepoAddFileByAPI(t *testing.T) {
125125
NewBranchName: "new_branch",
126126
Message: "init",
127127
},
128-
Content: base64.StdEncoding.EncodeToString([]byte("newly-added-api-file")),
128+
ContentBase64: base64.StdEncoding.EncodeToString([]byte("newly-added-api-file")),
129129
})
130130

131131
resp := MakeRequest(t, req, http.StatusCreated)

‎tests/integration/gpg_git_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -337,7 +337,7 @@ func crudActionCreateFile(t *testing.T, ctx APITestContext, user *user_model.Use
337337
Email: user.Email,
338338
},
339339
},
340-
Content: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("This is new text for %s", path))),
340+
ContentBase64: base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("This is new text for %s", path))),
341341
}, callback...)
342342
}
343343

‎tests/integration/pull_merge_test.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -370,9 +370,9 @@ func TestConflictChecking(t *testing.T) {
370370
_, err = files_service.ChangeRepoFiles(git.DefaultContext, baseRepo, user, &files_service.ChangeRepoFilesOptions{
371371
Files: []*files_service.ChangeRepoFile{
372372
{
373-
Operation: "create",
374-
TreePath: "important_file",
375-
Content: "Just a non-important file",
373+
Operation: "create",
374+
TreePath: "important_file",
375+
ContentReader: strings.NewReader("Just a non-important file"),
376376
},
377377
},
378378
Message: "Add a important file",
@@ -385,9 +385,9 @@ func TestConflictChecking(t *testing.T) {
385385
_, err = files_service.ChangeRepoFiles(git.DefaultContext, baseRepo, user, &files_service.ChangeRepoFilesOptions{
386386
Files: []*files_service.ChangeRepoFile{
387387
{
388-
Operation: "create",
389-
TreePath: "important_file",
390-
Content: "Not the same content :P",
388+
Operation: "create",
389+
TreePath: "important_file",
390+
ContentReader: strings.NewReader("Not the same content :P"),
391391
},
392392
},
393393
Message: "Add a important file",

0 commit comments

Comments
 (0)