Skip to content

Commit 5e6a008

Browse files
authored
Add basic repository lfs management (#7199)
This PR adds basic repository LFS management UI including the ability to find all possible pointers within the repository. Locks are not managed at present but would be addable through some simple additions. * Add basic repository lfs management * add auto-associate function * Add functionality to find commits with this lfs file * Add link to find commits on the lfs file view * Adjust commit view to state the likely branch causing the commit * Only read Oid from database
1 parent af8957b commit 5e6a008

File tree

20 files changed

+1150
-136
lines changed

20 files changed

+1150
-136
lines changed

models/lfs.go

Lines changed: 77 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@ import (
88
"io"
99

1010
"code.gitea.io/gitea/modules/timeutil"
11+
12+
"xorm.io/builder"
1113
)
1214

1315
// LFSMetaObject stores metadata for LFS tracked files.
@@ -106,19 +108,91 @@ func (repo *Repository) GetLFSMetaObjectByOid(oid string) (*LFSMetaObject, error
106108

107109
// RemoveLFSMetaObjectByOid removes a LFSMetaObject entry from database by its OID.
108110
// It may return ErrLFSObjectNotExist or a database error.
109-
func (repo *Repository) RemoveLFSMetaObjectByOid(oid string) error {
111+
func (repo *Repository) RemoveLFSMetaObjectByOid(oid string) (int64, error) {
110112
if len(oid) == 0 {
111-
return ErrLFSObjectNotExist
113+
return 0, ErrLFSObjectNotExist
112114
}
113115

114116
sess := x.NewSession()
115117
defer sess.Close()
116118
if err := sess.Begin(); err != nil {
117-
return err
119+
return -1, err
118120
}
119121

120122
m := &LFSMetaObject{Oid: oid, RepositoryID: repo.ID}
121123
if _, err := sess.Delete(m); err != nil {
124+
return -1, err
125+
}
126+
127+
count, err := sess.Count(&LFSMetaObject{Oid: oid})
128+
if err != nil {
129+
return count, err
130+
}
131+
132+
return count, sess.Commit()
133+
}
134+
135+
// GetLFSMetaObjects returns all LFSMetaObjects associated with a repository
136+
func (repo *Repository) GetLFSMetaObjects(page, pageSize int) ([]*LFSMetaObject, error) {
137+
sess := x.NewSession()
138+
defer sess.Close()
139+
140+
if page >= 0 && pageSize > 0 {
141+
start := 0
142+
if page > 0 {
143+
start = (page - 1) * pageSize
144+
}
145+
sess.Limit(pageSize, start)
146+
}
147+
lfsObjects := make([]*LFSMetaObject, 0, pageSize)
148+
return lfsObjects, sess.Find(&lfsObjects, &LFSMetaObject{RepositoryID: repo.ID})
149+
}
150+
151+
// CountLFSMetaObjects returns a count of all LFSMetaObjects associated with a repository
152+
func (repo *Repository) CountLFSMetaObjects() (int64, error) {
153+
return x.Count(&LFSMetaObject{RepositoryID: repo.ID})
154+
}
155+
156+
// LFSObjectAccessible checks if a provided Oid is accessible to the user
157+
func LFSObjectAccessible(user *User, oid string) (bool, error) {
158+
if user.IsAdmin {
159+
count, err := x.Count(&LFSMetaObject{Oid: oid})
160+
return (count > 0), err
161+
}
162+
cond := accessibleRepositoryCondition(user.ID)
163+
count, err := x.Where(cond).Join("INNER", "repository", "`lfs_meta_object`.repository_id = `repository`.id").Count(&LFSMetaObject{Oid: oid})
164+
return (count > 0), err
165+
}
166+
167+
// LFSAutoAssociate auto associates accessible LFSMetaObjects
168+
func LFSAutoAssociate(metas []*LFSMetaObject, user *User, repoID int64) error {
169+
sess := x.NewSession()
170+
defer sess.Close()
171+
if err := sess.Begin(); err != nil {
172+
return err
173+
}
174+
175+
oids := make([]interface{}, len(metas))
176+
oidMap := make(map[string]*LFSMetaObject, len(metas))
177+
for i, meta := range metas {
178+
oids[i] = meta.Oid
179+
oidMap[meta.Oid] = meta
180+
}
181+
182+
cond := builder.NewCond()
183+
if !user.IsAdmin {
184+
cond = builder.In("`lfs_meta_object`.repository_id",
185+
builder.Select("`repository`.id").From("repository").Where(accessibleRepositoryCondition(user.ID)))
186+
}
187+
newMetas := make([]*LFSMetaObject, 0, len(metas))
188+
if err := sess.Cols("oid").Where(cond).In("oid", oids...).GroupBy("oid").Find(&newMetas); err != nil {
189+
return err
190+
}
191+
for i := range newMetas {
192+
newMetas[i].Size = oidMap[newMetas[i].Oid].Size
193+
newMetas[i].RepositoryID = repoID
194+
}
195+
if _, err := sess.InsertMulti(newMetas); err != nil {
122196
return err
123197
}
124198

models/repo_list.go

Lines changed: 26 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -176,28 +176,7 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) {
176176
if opts.Private {
177177
if !opts.UserIsAdmin && opts.UserID != 0 && opts.UserID != opts.OwnerID {
178178
// OK we're in the context of a User
179-
// We should be Either
180-
cond = cond.And(builder.Or(
181-
// 1. Be able to see all non-private repositories that either:
182-
cond.And(
183-
builder.Eq{"is_private": false},
184-
builder.Or(
185-
// A. Aren't in organisations __OR__
186-
builder.NotIn("owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"type": UserTypeOrganization})),
187-
// B. Isn't a private organisation. (Limited is OK because we're logged in)
188-
builder.NotIn("owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"visibility": structs.VisibleTypePrivate}))),
189-
),
190-
// 2. Be able to see all repositories that we have access to
191-
builder.In("id", builder.Select("repo_id").
192-
From("`access`").
193-
Where(builder.And(
194-
builder.Eq{"user_id": opts.UserID},
195-
builder.Gt{"mode": int(AccessModeNone)}))),
196-
// 3. Be able to see all repositories that we are in a team
197-
builder.In("id", builder.Select("`team_repo`.repo_id").
198-
From("team_repo").
199-
Where(builder.Eq{"`team_user`.uid": opts.UserID}).
200-
Join("INNER", "team_user", "`team_user`.team_id = `team_repo`.team_id"))))
179+
cond = cond.And(accessibleRepositoryCondition(opts.UserID))
201180
}
202181
} else {
203182
// Not looking at private organisations
@@ -316,6 +295,31 @@ func SearchRepository(opts *SearchRepoOptions) (RepositoryList, int64, error) {
316295
return repos, count, nil
317296
}
318297

298+
// accessibleRepositoryCondition takes a user a returns a condition for checking if a repository is accessible
299+
func accessibleRepositoryCondition(userID int64) builder.Cond {
300+
return builder.Or(
301+
// 1. Be able to see all non-private repositories that either:
302+
builder.And(
303+
builder.Eq{"`repository`.is_private": false},
304+
builder.Or(
305+
// A. Aren't in organisations __OR__
306+
builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"type": UserTypeOrganization})),
307+
// B. Isn't a private organisation. (Limited is OK because we're logged in)
308+
builder.NotIn("`repository`.owner_id", builder.Select("id").From("`user`").Where(builder.Eq{"visibility": structs.VisibleTypePrivate}))),
309+
),
310+
// 2. Be able to see all repositories that we have access to
311+
builder.In("`repository`.id", builder.Select("repo_id").
312+
From("`access`").
313+
Where(builder.And(
314+
builder.Eq{"user_id": userID},
315+
builder.Gt{"mode": int(AccessModeNone)}))),
316+
// 3. Be able to see all repositories that we are in a team
317+
builder.In("`repository`.id", builder.Select("`team_repo`.repo_id").
318+
From("team_repo").
319+
Where(builder.Eq{"`team_user`.uid": userID}).
320+
Join("INNER", "team_user", "`team_user`.team_id = `team_repo`.team_id")))
321+
}
322+
319323
// SearchRepositoryByName takes keyword and part of repository name to search,
320324
// it returns results in given range and number of total results.
321325
func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, error) {

modules/git/pipeline/catfile.go

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
// Copyright 2019 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package pipeline
6+
7+
import (
8+
"bufio"
9+
"bytes"
10+
"fmt"
11+
"io"
12+
"strconv"
13+
"strings"
14+
"sync"
15+
16+
"code.gitea.io/gitea/modules/git"
17+
"code.gitea.io/gitea/modules/log"
18+
)
19+
20+
// CatFileBatchCheck runs cat-file with --batch-check
21+
func CatFileBatchCheck(shasToCheckReader *io.PipeReader, catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
22+
defer wg.Done()
23+
defer shasToCheckReader.Close()
24+
defer catFileCheckWriter.Close()
25+
26+
stderr := new(bytes.Buffer)
27+
var errbuf strings.Builder
28+
cmd := git.NewCommand("cat-file", "--batch-check")
29+
if err := cmd.RunInDirFullPipeline(tmpBasePath, catFileCheckWriter, stderr, shasToCheckReader); err != nil {
30+
_ = catFileCheckWriter.CloseWithError(fmt.Errorf("git cat-file --batch-check [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
31+
}
32+
}
33+
34+
// CatFileBatchCheckAllObjects runs cat-file with --batch-check --batch-all
35+
func CatFileBatchCheckAllObjects(catFileCheckWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string, errChan chan<- error) {
36+
defer wg.Done()
37+
defer catFileCheckWriter.Close()
38+
39+
stderr := new(bytes.Buffer)
40+
var errbuf strings.Builder
41+
cmd := git.NewCommand("cat-file", "--batch-check", "--batch-all-objects")
42+
if err := cmd.RunInDirPipeline(tmpBasePath, catFileCheckWriter, stderr); err != nil {
43+
log.Error("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String())
44+
err = fmt.Errorf("git cat-file --batch-check --batch-all-object [%s]: %v - %s", tmpBasePath, err, errbuf.String())
45+
_ = catFileCheckWriter.CloseWithError(err)
46+
errChan <- err
47+
}
48+
}
49+
50+
// CatFileBatch runs cat-file --batch
51+
func CatFileBatch(shasToBatchReader *io.PipeReader, catFileBatchWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
52+
defer wg.Done()
53+
defer shasToBatchReader.Close()
54+
defer catFileBatchWriter.Close()
55+
56+
stderr := new(bytes.Buffer)
57+
var errbuf strings.Builder
58+
if err := git.NewCommand("cat-file", "--batch").RunInDirFullPipeline(tmpBasePath, catFileBatchWriter, stderr, shasToBatchReader); err != nil {
59+
_ = shasToBatchReader.CloseWithError(fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
60+
}
61+
}
62+
63+
// BlobsLessThan1024FromCatFileBatchCheck reads a pipeline from cat-file --batch-check and returns the blobs <1024 in size
64+
func BlobsLessThan1024FromCatFileBatchCheck(catFileCheckReader *io.PipeReader, shasToBatchWriter *io.PipeWriter, wg *sync.WaitGroup) {
65+
defer wg.Done()
66+
defer catFileCheckReader.Close()
67+
scanner := bufio.NewScanner(catFileCheckReader)
68+
defer func() {
69+
_ = shasToBatchWriter.CloseWithError(scanner.Err())
70+
}()
71+
for scanner.Scan() {
72+
line := scanner.Text()
73+
if len(line) == 0 {
74+
continue
75+
}
76+
fields := strings.Split(line, " ")
77+
if len(fields) < 3 || fields[1] != "blob" {
78+
continue
79+
}
80+
size, _ := strconv.Atoi(fields[2])
81+
if size > 1024 {
82+
continue
83+
}
84+
toWrite := []byte(fields[0] + "\n")
85+
for len(toWrite) > 0 {
86+
n, err := shasToBatchWriter.Write(toWrite)
87+
if err != nil {
88+
_ = catFileCheckReader.CloseWithError(err)
89+
break
90+
}
91+
toWrite = toWrite[n:]
92+
}
93+
}
94+
}

modules/git/pipeline/namerev.go

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
// Copyright 2019 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package pipeline
6+
7+
import (
8+
"bytes"
9+
"fmt"
10+
"io"
11+
"strings"
12+
"sync"
13+
14+
"code.gitea.io/gitea/modules/git"
15+
)
16+
17+
// NameRevStdin runs name-rev --stdin
18+
func NameRevStdin(shasToNameReader *io.PipeReader, nameRevStdinWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath string) {
19+
defer wg.Done()
20+
defer shasToNameReader.Close()
21+
defer nameRevStdinWriter.Close()
22+
23+
stderr := new(bytes.Buffer)
24+
var errbuf strings.Builder
25+
if err := git.NewCommand("name-rev", "--stdin", "--name-only", "--always").RunInDirFullPipeline(tmpBasePath, nameRevStdinWriter, stderr, shasToNameReader); err != nil {
26+
_ = shasToNameReader.CloseWithError(fmt.Errorf("git name-rev [%s]: %v - %s", tmpBasePath, err, errbuf.String()))
27+
}
28+
}

modules/git/pipeline/revlist.go

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
// Copyright 2019 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package pipeline
6+
7+
import (
8+
"bufio"
9+
"bytes"
10+
"fmt"
11+
"io"
12+
"strings"
13+
"sync"
14+
15+
"code.gitea.io/gitea/modules/git"
16+
"code.gitea.io/gitea/modules/log"
17+
)
18+
19+
// RevListAllObjects runs rev-list --objects --all and writes to a pipewriter
20+
func RevListAllObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, basePath string, errChan chan<- error) {
21+
defer wg.Done()
22+
defer revListWriter.Close()
23+
24+
stderr := new(bytes.Buffer)
25+
var errbuf strings.Builder
26+
cmd := git.NewCommand("rev-list", "--objects", "--all")
27+
if err := cmd.RunInDirPipeline(basePath, revListWriter, stderr); err != nil {
28+
log.Error("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String())
29+
err = fmt.Errorf("git rev-list --objects --all [%s]: %v - %s", basePath, err, errbuf.String())
30+
_ = revListWriter.CloseWithError(err)
31+
errChan <- err
32+
}
33+
}
34+
35+
// RevListObjects run rev-list --objects from headSHA to baseSHA
36+
func RevListObjects(revListWriter *io.PipeWriter, wg *sync.WaitGroup, tmpBasePath, headSHA, baseSHA string, errChan chan<- error) {
37+
defer wg.Done()
38+
defer revListWriter.Close()
39+
stderr := new(bytes.Buffer)
40+
var errbuf strings.Builder
41+
cmd := git.NewCommand("rev-list", "--objects", headSHA, "--not", baseSHA)
42+
if err := cmd.RunInDirPipeline(tmpBasePath, revListWriter, stderr); err != nil {
43+
log.Error("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())
44+
errChan <- fmt.Errorf("git rev-list [%s]: %v - %s", tmpBasePath, err, errbuf.String())
45+
}
46+
}
47+
48+
// BlobsFromRevListObjects reads a RevListAllObjects and only selects blobs
49+
func BlobsFromRevListObjects(revListReader *io.PipeReader, shasToCheckWriter *io.PipeWriter, wg *sync.WaitGroup) {
50+
defer wg.Done()
51+
defer revListReader.Close()
52+
scanner := bufio.NewScanner(revListReader)
53+
defer func() {
54+
_ = shasToCheckWriter.CloseWithError(scanner.Err())
55+
}()
56+
for scanner.Scan() {
57+
line := scanner.Text()
58+
if len(line) == 0 {
59+
continue
60+
}
61+
fields := strings.Split(line, " ")
62+
if len(fields) < 2 || len(fields[1]) == 0 {
63+
continue
64+
}
65+
toWrite := []byte(fields[0] + "\n")
66+
for len(toWrite) > 0 {
67+
n, err := shasToCheckWriter.Write(toWrite)
68+
if err != nil {
69+
_ = revListReader.CloseWithError(err)
70+
break
71+
}
72+
toWrite = toWrite[n:]
73+
}
74+
}
75+
}

modules/git/repo.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,11 @@ func OpenRepository(repoPath string) (*Repository, error) {
117117
}, nil
118118
}
119119

120+
// GoGitRepo gets the go-git repo representation
121+
func (repo *Repository) GoGitRepo() *gogit.Repository {
122+
return repo.gogitRepo
123+
}
124+
120125
// IsEmpty Check if repository is empty.
121126
func (repo *Repository) IsEmpty() (bool, error) {
122127
var errbuf strings.Builder

modules/lfs/server.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -332,7 +332,7 @@ func PutHandler(ctx *context.Context) {
332332
if err := contentStore.Put(meta, bodyReader); err != nil {
333333
ctx.Resp.WriteHeader(500)
334334
fmt.Fprintf(ctx.Resp, `{"message":"%s"}`, err)
335-
if err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
335+
if _, err = repository.RemoveLFSMetaObjectByOid(rv.Oid); err != nil {
336336
log.Error("RemoveLFSMetaObjectByOid: %v", err)
337337
}
338338
return

modules/repofiles/update.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -385,7 +385,7 @@ func CreateOrUpdateRepoFile(repo *models.Repository, doer *models.User, opts *Up
385385
contentStore := &lfs.ContentStore{BasePath: setting.LFS.ContentPath}
386386
if !contentStore.Exists(lfsMetaObject) {
387387
if err := contentStore.Put(lfsMetaObject, strings.NewReader(opts.Content)); err != nil {
388-
if err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
388+
if _, err2 := repo.RemoveLFSMetaObjectByOid(lfsMetaObject.Oid); err2 != nil {
389389
return nil, fmt.Errorf("Error whilst removing failed inserted LFS object %s: %v (Prev Error: %v)", lfsMetaObject.Oid, err2, err)
390390
}
391391
return nil, err

0 commit comments

Comments
 (0)