Skip to content

Commit bdbca39

Browse files
technoweenietechnoweenie
committed
each command should open and close a gitscanner just once
1 parent bbe552a commit bdbca39

File tree

10 files changed

+116
-50
lines changed

10 files changed

+116
-50
lines changed

‎commands/command_clone.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"path/filepath"
77
"strings"
88

9+
"github.com/git-lfs/git-lfs/lfs"
910
"github.com/git-lfs/git-lfs/localstorage"
1011
"github.com/git-lfs/git-lfs/subprocess"
1112

@@ -70,18 +71,18 @@ func cloneCommand(cmd *cobra.Command, args []string) {
7071

7172
includeArg, excludeArg := getIncludeExcludeArgs(cmd)
7273
include, exclude := determineIncludeExcludePaths(cfg, includeArg, excludeArg)
74+
gitscanner := lfs.NewGitScanner()
75+
defer gitscanner.Close()
7376
if cloneFlags.NoCheckout || cloneFlags.Bare {
7477
// If --no-checkout or --bare then we shouldn't check out, just fetch instead
75-
fetchRef("HEAD", include, exclude)
78+
fetchRef(gitscanner, "HEAD", include, exclude)
7679
} else {
77-
pull(include, exclude)
78-
80+
pull(gitscanner, include, exclude)
7981
err := postCloneSubmodules(args)
8082
if err != nil {
8183
Exit("Error performing 'git lfs pull' for submodules: %v", err)
8284
}
8385
}
84-
8586
}
8687

8788
func postCloneSubmodules(args []string) error {

‎commands/command_fetch.go

Lines changed: 28 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ func fetchCommand(cmd *cobra.Command, args []string) {
6161
}
6262

6363
success := true
64+
gitscanner := lfs.NewGitScanner()
65+
defer gitscanner.Close()
66+
6467
include, exclude := getIncludeExcludeArgs(cmd)
6568

6669
if fetchAllArg {
@@ -73,20 +76,20 @@ func fetchCommand(cmd *cobra.Command, args []string) {
7376
if len(cfg.FetchIncludePaths()) > 0 || len(cfg.FetchExcludePaths()) > 0 {
7477
Print("Ignoring global include / exclude paths to fulfil --all")
7578
}
76-
success = fetchAll()
79+
success = fetchAll(gitscanner)
7780

7881
} else { // !all
7982
includePaths, excludePaths := determineIncludeExcludePaths(cfg, include, exclude)
8083

8184
// Fetch refs sequentially per arg order; duplicates in later refs will be ignored
8285
for _, ref := range refs {
8386
Print("Fetching %v", ref.Name)
84-
s := fetchRef(ref.Sha, includePaths, excludePaths)
87+
s := fetchRef(gitscanner, ref.Sha, includePaths, excludePaths)
8588
success = success && s
8689
}
8790

8891
if fetchRecentArg || cfg.FetchPruneConfig().FetchRecentAlways {
89-
s := fetchRecent(refs, includePaths, excludePaths)
92+
s := fetchRecent(gitscanner, refs, includePaths, excludePaths)
9093
success = success && s
9194
}
9295
}
@@ -103,13 +106,17 @@ func fetchCommand(cmd *cobra.Command, args []string) {
103106
}
104107
}
105108

106-
func pointersToFetchForRef(ref string) ([]*lfs.WrappedPointer, error) {
107-
return lfs.ScanTree(ref)
109+
func pointersToFetchForRef(gitscanner *lfs.GitScanner, ref string) ([]*lfs.WrappedPointer, error) {
110+
pointerCh, err := gitscanner.ScanTree(ref)
111+
if err != nil {
112+
return nil, err
113+
}
114+
return collectPointers(pointerCh)
108115
}
109116

110-
func fetchRefToChan(ref string, include, exclude []string) chan *lfs.WrappedPointer {
117+
func fetchRefToChan(gitscanner *lfs.GitScanner, ref string, include, exclude []string) chan *lfs.WrappedPointer {
111118
c := make(chan *lfs.WrappedPointer)
112-
pointers, err := pointersToFetchForRef(ref)
119+
pointers, err := pointersToFetchForRef(gitscanner, ref)
113120
if err != nil {
114121
Panic(err, "Could not scan for Git LFS files")
115122
}
@@ -120,8 +127,8 @@ func fetchRefToChan(ref string, include, exclude []string) chan *lfs.WrappedPoin
120127
}
121128

122129
// Fetch all binaries for a given ref (that we don't have already)
123-
func fetchRef(ref string, include, exclude []string) bool {
124-
pointers, err := pointersToFetchForRef(ref)
130+
func fetchRef(gitscanner *lfs.GitScanner, ref string, include, exclude []string) bool {
131+
pointers, err := pointersToFetchForRef(gitscanner, ref)
125132
if err != nil {
126133
Panic(err, "Could not scan for Git LFS files")
127134
}
@@ -130,16 +137,20 @@ func fetchRef(ref string, include, exclude []string) bool {
130137

131138
// Fetch all previous versions of objects from since to ref (not including final state at ref)
132139
// So this will fetch all the '-' sides of the diff from since to ref
133-
func fetchPreviousVersions(ref string, since time.Time, include, exclude []string) bool {
134-
pointers, err := lfs.ScanPreviousVersions(ref, since)
140+
func fetchPreviousVersions(gitscanner *lfs.GitScanner, ref string, since time.Time, include, exclude []string) bool {
141+
pointerCh, err := gitscanner.ScanPreviousVersions(ref, since)
142+
if err != nil {
143+
ExitWithError(err)
144+
}
145+
pointers, err := collectPointers(pointerCh)
135146
if err != nil {
136147
Panic(err, "Could not scan for Git LFS previous versions")
137148
}
138149
return fetchPointers(pointers, include, exclude)
139150
}
140151

141152
// Fetch recent objects based on config
142-
func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) bool {
153+
func fetchRecent(gitscanner *lfs.GitScanner, alreadyFetchedRefs []*git.Ref, include, exclude []string) bool {
143154
fetchconf := cfg.FetchPruneConfig()
144155

145156
if fetchconf.FetchRecentRefsDays == 0 && fetchconf.FetchRecentCommitsDays == 0 {
@@ -169,7 +180,7 @@ func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) bool
169180
} else {
170181
uniqueRefShas[ref.Sha] = ref.Name
171182
Print("Fetching %v", ref.Name)
172-
k := fetchRef(ref.Sha, include, exclude)
183+
k := fetchRef(gitscanner, ref.Sha, include, exclude)
173184
ok = ok && k
174185
}
175186
}
@@ -185,27 +196,26 @@ func fetchRecent(alreadyFetchedRefs []*git.Ref, include, exclude []string) bool
185196
}
186197
Print("Fetching changes within %v days of %v", fetchconf.FetchRecentCommitsDays, refName)
187198
commitsSince := summ.CommitDate.AddDate(0, 0, -fetchconf.FetchRecentCommitsDays)
188-
k := fetchPreviousVersions(commit, commitsSince, include, exclude)
199+
k := fetchPreviousVersions(gitscanner, commit, commitsSince, include, exclude)
189200
ok = ok && k
190201
}
191202

192203
}
193204
return ok
194205
}
195206

196-
func fetchAll() bool {
197-
pointers := scanAll()
207+
func fetchAll(gitscanner *lfs.GitScanner) bool {
208+
pointers := scanAll(gitscanner)
198209
Print("Fetching objects...")
199210
return fetchPointers(pointers, nil, nil)
200211
}
201212

202-
func scanAll() []*lfs.WrappedPointer {
213+
func scanAll(gitscanner *lfs.GitScanner) []*lfs.WrappedPointer {
203214
// This could be a long process so use the chan version & report progress
204215
Print("Scanning for all objects ever referenced...")
205216
spinner := progress.NewSpinner()
206217
var numObjs int64
207218

208-
gitscanner := lfs.NewGitScanner()
209219
pointerCh, err := gitscanner.ScanAll()
210220
if err != nil {
211221
Panic(err, "Could not scan for Git LFS files")

‎commands/command_fsck.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ func doFsck() (bool, error) {
3030
pointerIndex := make(map[string]string)
3131

3232
gitscanner := lfs.NewGitScanner()
33+
defer gitscanner.Close()
3334
pointerCh, err := gitscanner.ScanRefWithDeleted(ref.Sha)
3435
if err != nil {
3536
return false, err

‎commands/command_pre_push.go

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,11 @@ func prePushCommand(cmd *cobra.Command, args []string) {
5555
ctx := newUploadContext(prePushDryRun)
5656

5757
gitscanner := lfs.NewGitScanner()
58-
gitscanner.RemoteForPush(cfg.CurrentRemote)
58+
if err := gitscanner.RemoteForPush(cfg.CurrentRemote); err != nil {
59+
ExitWithError(err)
60+
}
61+
62+
defer gitscanner.Close()
5963

6064
// We can be passed multiple lines of refs
6165
scanner := bufio.NewScanner(os.Stdin)

‎commands/command_prune.go

Lines changed: 16 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,14 @@ func prune(fetchPruneConfig config.FetchPruneConfig, verifyRemote, dryRun, verbo
8080
// Now find files to be retained from many sources
8181
retainChan := make(chan string, 100)
8282

83-
go pruneTaskGetRetainedCurrentAndRecentRefs(fetchPruneConfig, retainChan, errorChan, &taskwait)
84-
go pruneTaskGetRetainedUnpushed(fetchPruneConfig, retainChan, errorChan, &taskwait)
85-
go pruneTaskGetRetainedWorktree(retainChan, errorChan, &taskwait)
83+
gitscanner := lfs.NewGitScanner()
84+
defer gitscanner.Close()
85+
go pruneTaskGetRetainedCurrentAndRecentRefs(gitscanner, fetchPruneConfig, retainChan, errorChan, &taskwait)
86+
go pruneTaskGetRetainedUnpushed(gitscanner, fetchPruneConfig, retainChan, errorChan, &taskwait)
87+
go pruneTaskGetRetainedWorktree(gitscanner, retainChan, errorChan, &taskwait)
8688
if verifyRemote {
8789
reachableObjects = tools.NewStringSetWithCapacity(100)
88-
go pruneTaskGetReachableObjects(&reachableObjects, errorChan, &taskwait)
90+
go pruneTaskGetReachableObjects(gitscanner, &reachableObjects, errorChan, &taskwait)
8991
}
9092

9193
// Now collect all the retained objects, on separate wait
@@ -299,10 +301,9 @@ func pruneTaskGetLocalObjects(outLocalObjects *[]localstorage.Object, progChan P
299301
}
300302

301303
// Background task, must call waitg.Done() once at end
302-
func pruneTaskGetRetainedAtRef(ref string, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
304+
func pruneTaskGetRetainedAtRef(gitscanner *lfs.GitScanner, ref string, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
303305
defer waitg.Done()
304306

305-
gitscanner := lfs.NewGitScanner()
306307
refchan, err := gitscanner.ScanRef(ref)
307308
if err != nil {
308309
errorChan <- err
@@ -319,10 +320,9 @@ func pruneTaskGetRetainedAtRef(ref string, retainChan chan string, errorChan cha
319320
}
320321

321322
// Background task, must call waitg.Done() once at end
322-
func pruneTaskGetPreviousVersionsOfRef(ref string, since time.Time, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
323+
func pruneTaskGetPreviousVersionsOfRef(gitscanner *lfs.GitScanner, ref string, since time.Time, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
323324
defer waitg.Done()
324325

325-
gitscanner := lfs.NewGitScanner()
326326
refchan, err := gitscanner.ScanPreviousVersions(ref, since)
327327
if err != nil {
328328
errorChan <- err
@@ -339,7 +339,7 @@ func pruneTaskGetPreviousVersionsOfRef(ref string, since time.Time, retainChan c
339339
}
340340

341341
// Background task, must call waitg.Done() once at end
342-
func pruneTaskGetRetainedCurrentAndRecentRefs(fetchconf config.FetchPruneConfig, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
342+
func pruneTaskGetRetainedCurrentAndRecentRefs(gitscanner *lfs.GitScanner, fetchconf config.FetchPruneConfig, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
343343
defer waitg.Done()
344344

345345
// We actually increment the waitg in this func since we kick off sub-goroutines
@@ -353,7 +353,7 @@ func pruneTaskGetRetainedCurrentAndRecentRefs(fetchconf config.FetchPruneConfig,
353353
}
354354
commits.Add(ref.Sha)
355355
waitg.Add(1)
356-
go pruneTaskGetRetainedAtRef(ref.Sha, retainChan, errorChan, waitg)
356+
go pruneTaskGetRetainedAtRef(gitscanner, ref.Sha, retainChan, errorChan, waitg)
357357

358358
// Now recent
359359
if fetchconf.FetchRecentRefsDays > 0 {
@@ -369,7 +369,7 @@ func pruneTaskGetRetainedCurrentAndRecentRefs(fetchconf config.FetchPruneConfig,
369369
if commits.Add(ref.Sha) {
370370
// A new commit
371371
waitg.Add(1)
372-
go pruneTaskGetRetainedAtRef(ref.Sha, retainChan, errorChan, waitg)
372+
go pruneTaskGetRetainedAtRef(gitscanner, ref.Sha, retainChan, errorChan, waitg)
373373
}
374374
}
375375
}
@@ -387,16 +387,15 @@ func pruneTaskGetRetainedCurrentAndRecentRefs(fetchconf config.FetchPruneConfig,
387387
}
388388
commitsSince := summ.CommitDate.AddDate(0, 0, -pruneCommitDays)
389389
waitg.Add(1)
390-
go pruneTaskGetPreviousVersionsOfRef(commit, commitsSince, retainChan, errorChan, waitg)
390+
go pruneTaskGetPreviousVersionsOfRef(gitscanner, commit, commitsSince, retainChan, errorChan, waitg)
391391
}
392392
}
393393
}
394394

395395
// Background task, must call waitg.Done() once at end
396-
func pruneTaskGetRetainedUnpushed(fetchconf config.FetchPruneConfig, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
396+
func pruneTaskGetRetainedUnpushed(gitscanner *lfs.GitScanner, fetchconf config.FetchPruneConfig, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
397397
defer waitg.Done()
398398

399-
gitscanner := lfs.NewGitScanner()
400399
refchan, err := gitscanner.ScanUnpushed(fetchconf.PruneRemoteName)
401400
if err != nil {
402401
errorChan <- err
@@ -413,7 +412,7 @@ func pruneTaskGetRetainedUnpushed(fetchconf config.FetchPruneConfig, retainChan
413412
}
414413

415414
// Background task, must call waitg.Done() once at end
416-
func pruneTaskGetRetainedWorktree(retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
415+
func pruneTaskGetRetainedWorktree(gitscanner *lfs.GitScanner, retainChan chan string, errorChan chan error, waitg *sync.WaitGroup) {
417416
defer waitg.Done()
418417

419418
// Retain other worktree HEADs too
@@ -438,17 +437,16 @@ func pruneTaskGetRetainedWorktree(retainChan chan string, errorChan chan error,
438437
// Worktree is on a different commit
439438
waitg.Add(1)
440439
// Don't need to 'cd' to worktree since we share same repo
441-
go pruneTaskGetRetainedAtRef(ref.Sha, retainChan, errorChan, waitg)
440+
go pruneTaskGetRetainedAtRef(gitscanner, ref.Sha, retainChan, errorChan, waitg)
442441
}
443442
}
444443

445444
}
446445

447446
// Background task, must call waitg.Done() once at end
448-
func pruneTaskGetReachableObjects(outObjectSet *tools.StringSet, errorChan chan error, waitg *sync.WaitGroup) {
447+
func pruneTaskGetReachableObjects(gitscanner *lfs.GitScanner, outObjectSet *tools.StringSet, errorChan chan error, waitg *sync.WaitGroup) {
449448
defer waitg.Done()
450449

451-
gitscanner := lfs.NewGitScanner()
452450
pointerchan, err := gitscanner.ScanAll()
453451
if err != nil {
454452
errorChan <- fmt.Errorf("Error scanning for reachable objects: %v", err)

‎commands/command_pull.go

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"fmt"
55

66
"github.com/git-lfs/git-lfs/git"
7+
"github.com/git-lfs/git-lfs/lfs"
78
"github.com/spf13/cobra"
89
)
910

@@ -27,20 +28,20 @@ func pullCommand(cmd *cobra.Command, args []string) {
2728
}
2829

2930
includeArg, excludeArg := getIncludeExcludeArgs(cmd)
30-
pull(determineIncludeExcludePaths(cfg, includeArg, excludeArg))
31-
31+
include, exclude := determineIncludeExcludePaths(cfg, includeArg, excludeArg)
32+
gitscanner := lfs.NewGitScanner()
33+
defer gitscanner.Close()
34+
pull(gitscanner, include, exclude)
3235
}
3336

34-
func pull(includePaths, excludePaths []string) {
35-
37+
func pull(gitscanner *lfs.GitScanner, includePaths, excludePaths []string) {
3638
ref, err := git.CurrentRef()
3739
if err != nil {
3840
Panic(err, "Could not pull")
3941
}
4042

41-
c := fetchRefToChan(ref.Sha, includePaths, excludePaths)
43+
c := fetchRefToChan(gitscanner, ref.Sha, includePaths, excludePaths)
4244
checkoutFromFetchChan(includePaths, excludePaths, c)
43-
4445
}
4546

4647
func init() {

‎commands/command_push.go

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,10 @@ func uploadsBetweenRefAndRemote(ctx *uploadContext, refnames []string) {
2222
tracerx.Printf("Upload refs %v to remote %v", refnames, cfg.CurrentRemote)
2323

2424
gitscanner := lfs.NewGitScanner()
25-
gitscanner.RemoteForPush(cfg.CurrentRemote)
25+
if err := gitscanner.RemoteForPush(cfg.CurrentRemote); err != nil {
26+
ExitWithError(err)
27+
}
28+
defer gitscanner.Close()
2629

2730
refs, err := refsByNames(refnames)
2831
if err != nil {

‎commands/command_status.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ func statusCommand(cmd *cobra.Command, args []string) {
1919
ref, _ := git.CurrentRef()
2020

2121
gitscanner := lfs.NewGitScanner()
22+
defer gitscanner.Close()
2223

2324
scanIndexAt := "HEAD"
2425
if ref == nil {

‎commands/pointers.go

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package commands
2+
3+
import "github.com/git-lfs/git-lfs/lfs"
4+
5+
func collectPointers(pointerCh *lfs.PointerChannelWrapper) ([]*lfs.WrappedPointer, error) {
6+
var pointers []*lfs.WrappedPointer
7+
for p := range pointerCh.Results {
8+
pointers = append(pointers, p)
9+
}
10+
return pointers, pointerCh.Wait()
11+
}

0 commit comments

Comments
 (0)