Fix partial cloning a repo (#18373) (#18377)

* Fix partial cloning a repo (#18373)

- Backport from: #18373
- Backport isn't 1-1, because the frontport had a refactor in that area,
which v1.16 doesn't have.

* Include diff & use copy

* Add partial clone test

* patch

* Apply suggestions from code review

* globalArgs first

* avoid copy but make GlobalCMDArgs append first

* please linter

Co-authored-by: Lunny Xiao <[email protected]>
Co-authored-by: 6543 <[email protected]>

Author: 3 people

integrations/git_helper_for_declarative_test.go

@@ -123,6 +123,17 @@ func doGitClone(dstLocalPath string, u *url.URL) func(*testing.T) {
 	}
 }
 
+func doPartialGitClone(dstLocalPath string, u *url.URL) func(*testing.T) {
+	return func(t *testing.T) {
+		assert.NoError(t, git.CloneWithArgs(context.Background(), u.String(), dstLocalPath, allowLFSFilters(), git.CloneRepoOptions{
+			Filter: "blob:none",
+		}))
+		exist, err := util.IsExist(filepath.Join(dstLocalPath, "README.md"))
+		assert.NoError(t, err)
+		assert.True(t, exist)
+	}
+}
+
 func doGitCloneFail(u *url.URL) func(*testing.T) {
 	return func(t *testing.T) {
 		tmpDir, err := os.MkdirTemp("", "doGitCloneFail")

integrations/git_test.go

@@ -69,6 +69,12 @@ func testGit(t *testing.T, u *url.URL) {
 
 		t.Run("Clone", doGitClone(dstPath, u))
 
+		dstPath2, err := os.MkdirTemp("", httpContext.Reponame)
+		assert.NoError(t, err)
+		defer util.RemoveAll(dstPath2)
+
+		t.Run("Partial Clone", doPartialGitClone(dstPath2, u))
+
 		little, big := standardCommitAndPushTest(t, dstPath)
 		littleLFS, bigLFS := lfsCommitAndPushTest(t, dstPath)
 		rawTest(t, &httpContext, little, big, littleLFS, bigLFS)

modules/git/diff.go

@@ -59,27 +59,28 @@ func GetRepoRawDiffForFile(repo *Repository, startCommit, endCommit string, diff
 	ctx, _, finished := process.GetManager().AddContext(repo.Ctx, fmt.Sprintf("GetRawDiffForFile: [repo_path: %s]", repo.Path))
 	defer finished()
 
-	var cmd *exec.Cmd
+	cmd := exec.CommandContext(ctx, GitExecutable, GlobalCommandArgs...)
+
 	switch diffType {
 	case RawDiffNormal:
 		if len(startCommit) != 0 {
-			cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"diff", "-M", startCommit, endCommit}, fileArgs...)...)
+			cmd.Args = append(cmd.Args, append([]string{"diff", "-M", startCommit, endCommit}, fileArgs...)...)
 		} else if commit.ParentCount() == 0 {
-			cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"show", endCommit}, fileArgs...)...)
+			cmd.Args = append(cmd.Args, append([]string{"show", endCommit}, fileArgs...)...)
 		} else {
 			c, _ := commit.Parent(0)
-			cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"diff", "-M", c.ID.String(), endCommit}, fileArgs...)...)
+			cmd.Args = append(cmd.Args, append([]string{"diff", "-M", c.ID.String(), endCommit}, fileArgs...)...)
 		}
 	case RawDiffPatch:
 		if len(startCommit) != 0 {
 			query := fmt.Sprintf("%s...%s", endCommit, startCommit)
-			cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"format-patch", "--no-signature", "--stdout", "--root", query}, fileArgs...)...)
+			cmd.Args = append(cmd.Args, append([]string{"format-patch", "--no-signature", "--stdout", "--root", query}, fileArgs...)...)
 		} else if commit.ParentCount() == 0 {
-			cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"format-patch", "--no-signature", "--stdout", "--root", endCommit}, fileArgs...)...)
+			cmd.Args = append(cmd.Args, append([]string{"format-patch", "--no-signature", "--stdout", "--root", endCommit}, fileArgs...)...)
 		} else {
 			c, _ := commit.Parent(0)
 			query := fmt.Sprintf("%s...%s", endCommit, c.ID.String())
-			cmd = exec.CommandContext(ctx, GitExecutable, append([]string{"format-patch", "--no-signature", "--stdout", query}, fileArgs...)...)
+			cmd.Args = append(cmd.Args, append([]string{"format-patch", "--no-signature", "--stdout", query}, fileArgs...)...)
 		}
 	default:
 		return fmt.Errorf("invalid diffType: %s", diffType)

modules/git/repo.go

@@ -101,6 +101,7 @@ type CloneRepoOptions struct {
 	Shared     bool
 	NoCheckout bool
 	Depth      int
+	Filter     string
 }
 
 // Clone clones original repository to target path.
@@ -141,7 +142,9 @@ func CloneWithArgs(ctx context.Context, from, to string, args []string, opts Clo
 	if opts.Depth > 0 {
 		cmd.AddArguments("--depth", strconv.Itoa(opts.Depth))
 	}
-
+	if opts.Filter != "" {
+		cmd.AddArguments("--filter", opts.Filter)
+	}
 	if len(opts.Branch) > 0 {
 		cmd.AddArguments("-b", opts.Branch)
 	}

routers/web/repo/http.go

@@ -491,7 +491,10 @@ func serviceRPC(h serviceHandler, service string) {
 	defer finished()
 
 	var stderr bytes.Buffer
-	cmd := exec.CommandContext(ctx, git.GitExecutable, service, "--stateless-rpc", h.dir)
+	args := make([]string, len(git.GlobalCommandArgs))
+	copy(args, git.GlobalCommandArgs)
+	args = append(args, []string{service, "--stateless-rpc", h.dir}...)
+	cmd := exec.CommandContext(ctx, git.GitExecutable, args...)
 	cmd.Dir = h.dir
 	cmd.Env = append(os.Environ(), h.environ...)
 	cmd.Stdout = h.w