Inherit submodules from template repository content

modules/git/batch_reader.go

@@ -242,15 +242,15 @@ func BinToHex(objectFormat ObjectFormat, sha, out []byte) []byte {
 	return out
 }
 
-// ParseTreeLine reads an entry from a tree in a cat-file --batch stream
+// ParseCatFileTreeLine reads an entry from a tree in a cat-file --batch stream
 // This carefully avoids allocations - except where fnameBuf is too small.
 // It is recommended therefore to pass in an fnameBuf large enough to avoid almost all allocations
 //
 // Each line is composed of:
 // <mode-in-ascii-dropping-initial-zeros> SP <fname> NUL <binary HASH>
 //
 // We don't attempt to convert the raw HASH to save a lot of time
-func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
+func ParseCatFileTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBuf, shaBuf []byte) (mode, fname, sha []byte, n int, err error) {
 	var readBytes []byte
 
 	// Read the Mode & fname
@@ -260,7 +260,7 @@ func ParseTreeLine(objectFormat ObjectFormat, rd *bufio.Reader, modeBuf, fnameBu
 	}
 	idx := bytes.IndexByte(readBytes, ' ')
 	if idx < 0 {
-		log.Debug("missing space in readBytes ParseTreeLine: %s", readBytes)
+		log.Debug("missing space in readBytes ParseCatFileTreeLine: %s", readBytes)
 		return mode, fname, sha, n, &ErrNotExist{}
 	}
 

modules/git/parse_nogogit.go

@@ -23,68 +23,79 @@ func ParseTreeEntries(data []byte) ([]*TreeEntry, error) {
 
 var sepSpace = []byte{' '}
 
-func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
+func parseLsTreeLine(line []byte) (*TreeEntry, error) {
+	// expect line to be of the form:
+	// <mode> <type> <sha> <space-padded-size>\t<filename>
+	// <mode> <type> <sha>\t<filename>
+
 	var err error
+	posTab := bytes.IndexByte(line, '\t')
+	if posTab == -1 {
+		return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
+	}
+
+	entry := new(TreeEntry)
+
+	entryAttrs := line[:posTab]
+	entryName := line[posTab+1:]
+
+	entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
+	_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
+	entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
+	if len(entryAttrs) > 0 {
+		entrySize := entryAttrs // the last field is the space-padded-size
+		entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
+		entry.sized = true
+	}
+
+	switch string(entryMode) {
+	case "100644":
+		entry.entryMode = EntryModeBlob
+	case "100755":
+		entry.entryMode = EntryModeExec
+	case "120000":
+		entry.entryMode = EntryModeSymlink
+	case "160000":
+		entry.entryMode = EntryModeCommit
+	case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
+		entry.entryMode = EntryModeTree
+	default:
+		return nil, fmt.Errorf("unknown type: %v", string(entryMode))
+	}
+
+	entry.ID, err = NewIDFromString(string(entryObjectID))
+	if err != nil {
+		return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
+	}
+
+	if len(entryName) > 0 && entryName[0] == '"' {
+		entry.name, err = strconv.Unquote(string(entryName))
+		if err != nil {
+			return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
+		}
+	} else {
+		entry.name = string(entryName)
+	}
+	return entry, nil
+}
+
+// parseTreeEntries FIXME this function's design is not right, it should make the caller read all data into memory
+func parseTreeEntries(data []byte, ptree *Tree) ([]*TreeEntry, error) {
 	entries := make([]*TreeEntry, 0, bytes.Count(data, []byte{'\n'})+1)
 	for pos := 0; pos < len(data); {
-		// expect line to be of the form:
-		// <mode> <type> <sha> <space-padded-size>\t<filename>
-		// <mode> <type> <sha>\t<filename>
 		posEnd := bytes.IndexByte(data[pos:], '\n')
 		if posEnd == -1 {
 			posEnd = len(data)
 		} else {
 			posEnd += pos
 		}
-		line := data[pos:posEnd]
-		posTab := bytes.IndexByte(line, '\t')
-		if posTab == -1 {
-			return nil, fmt.Errorf("invalid ls-tree output (no tab): %q", line)
-		}
-
-		entry := new(TreeEntry)
-		entry.ptree = ptree
 
-		entryAttrs := line[:posTab]
-		entryName := line[posTab+1:]
-
-		entryMode, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
-		_ /* entryType */, entryAttrs, _ = bytes.Cut(entryAttrs, sepSpace) // the type is not used, the mode is enough to determine the type
-		entryObjectID, entryAttrs, _ := bytes.Cut(entryAttrs, sepSpace)
-		if len(entryAttrs) > 0 {
-			entrySize := entryAttrs // the last field is the space-padded-size
-			entry.size, _ = strconv.ParseInt(strings.TrimSpace(string(entrySize)), 10, 64)
-			entry.sized = true
-		}
-
-		switch string(entryMode) {
-		case "100644":
-			entry.entryMode = EntryModeBlob
-		case "100755":
-			entry.entryMode = EntryModeExec
-		case "120000":
-			entry.entryMode = EntryModeSymlink
-		case "160000":
-			entry.entryMode = EntryModeCommit
-		case "040000", "040755": // git uses 040000 for tree object, but some users may get 040755 for unknown reasons
-			entry.entryMode = EntryModeTree
-		default:
-			return nil, fmt.Errorf("unknown type: %v", string(entryMode))
-		}
-
-		entry.ID, err = NewIDFromString(string(entryObjectID))
+		line := data[pos:posEnd]
+		entry, err := parseLsTreeLine(line)
 		if err != nil {
-			return nil, fmt.Errorf("invalid ls-tree output (invalid object id): %q, err: %w", line, err)
-		}
-
-		if len(entryName) > 0 && entryName[0] == '"' {
-			entry.name, err = strconv.Unquote(string(entryName))
-			if err != nil {
-				return nil, fmt.Errorf("invalid ls-tree output (invalid name): %q, err: %w", line, err)
-			}
-		} else {
-			entry.name = string(entryName)
+			return nil, err
 		}
+		entry.ptree = ptree
 
 		pos = posEnd + 1
 		entries = append(entries, entry)
@@ -100,7 +111,7 @@ func catBatchParseTreeEntries(objectFormat ObjectFormat, ptree *Tree, rd *bufio.
 
 loop:
 	for sz > 0 {
-		mode, fname, sha, count, err := ParseTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf)
+		mode, fname, sha, count, err := ParseCatFileTreeLine(objectFormat, rd, modeBuf, fnameBuf, shaBuf)
 		if err != nil {
 			if err == io.EOF {
 				break loop

modules/git/pipeline/lfs_nogogit.go

@@ -114,7 +114,7 @@ func FindLFSFile(repo *git.Repository, objectID git.ObjectID) ([]*LFSResult, err
 			case "tree":
 				var n int64
 				for n < size {
-					mode, fname, binObjectID, count, err := git.ParseTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
+					mode, fname, binObjectID, count, err := git.ParseCatFileTreeLine(objectID.Type(), batchReader, modeBuf, fnameBuf, workingShaBuf)
 					if err != nil {
 						return nil, err
 					}

modules/git/submodule.go

@@ -0,0 +1,63 @@
+package git
+
+import (
+	"bufio"
+	"context"
+	"fmt"
+	"os"
+
+	"code.gitea.io/gitea/modules/log"
+)
+
+type TemplateSubmoduleCommit struct {
+	Path   string
+	Commit string
+}
+
+// GetTemplateSubmoduleCommits returns a list of submodules paths and their commits from a repository
+// This function is only for generating new repos based on existing template, the template couldn't be too large.
+func GetTemplateSubmoduleCommits(ctx context.Context, repoPath string) (submoduleCommits []TemplateSubmoduleCommit, _ error) {
+	stdoutReader, stdoutWriter, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+	opts := &RunOpts{
+		Dir:    repoPath,
+		Stdout: stdoutWriter,
+		PipelineFunc: func(ctx context.Context, cancel context.CancelFunc) error {
+			_ = stdoutWriter.Close()
+			defer stdoutReader.Close()
+
+			scanner := bufio.NewScanner(stdoutReader)
+			for scanner.Scan() {
+				entry, err := parseLsTreeLine(scanner.Bytes())
+				if err != nil {
+					cancel()
+					return err
+				}
+				if entry.IsSubModule() {
+					submoduleCommits = append(submoduleCommits, TemplateSubmoduleCommit{Path: entry.Name(), Commit: entry.ID.String()})
+				}
+			}
+			return scanner.Err()
+		},
+	}
+	err = NewCommand(ctx, "ls-tree", "-r", "--", "HEAD").Run(opts)
+	if err != nil {
+		return nil, fmt.Errorf("GetTemplateSubmoduleCommits: error running git ls-tree: %v", err)
+	}
+	return submoduleCommits, nil
+}
+
+// AddTemplateSubmoduleIndexes Adds the given submodules to the git index.
+// It is only for generating new repos based on existing template, requires the .gitmodules file to be already present in the work dir.
+func AddTemplateSubmoduleIndexes(ctx context.Context, repoPath string, submodules []TemplateSubmoduleCommit) error {
+	for _, submodule := range submodules {
+		cmd := NewCommand(ctx, "update-index", "--add", "--cacheinfo", "160000").AddDynamicArguments(submodule.Commit, submodule.Path)
+		if stdout, _, err := cmd.RunStdString(&RunOpts{Dir: repoPath}); err != nil {
+			log.Error("Unable to add %s as submodule to repo %s: stdout %s\nError: %v", submodule.Path, repoPath, stdout, err)
+			return err
+		}
+	}
+	return nil
+}

modules/git/submodule_test.go

@@ -0,0 +1,23 @@
+package git
+
+import (
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestRepository_GetSubmoduleCommits(t *testing.T) {
+	testRepoPath := filepath.Join(testReposDir, "repo4_submodules")
+	submodules, err := GetTemplateSubmoduleCommits(DefaultContext, testRepoPath)
+	require.NoError(t, err)
+
+	assert.EqualValues(t, len(submodules), 2)
+
+	assert.EqualValues(t, submodules[0].Path, "<°)))><")
+	assert.EqualValues(t, submodules[0].Commit, "d2932de67963f23d43e1c7ecf20173e92ee6c43c")
+
+	assert.EqualValues(t, submodules[1].Path, "libtest")
+	assert.EqualValues(t, submodules[1].Commit, "1234567890123456789012345678901234567890")
+}

modules/git/tests/repos/repo4_submodules/HEAD

@@ -0,0 +1 @@
+ref: refs/heads/master

modules/git/tests/repos/repo4_submodules/config

@@ -0,0 +1,4 @@
+[core]
+	repositoryformatversion = 0
+	filemode = true
+	bare = true

modules/git/tests/repos/repo4_submodules/objects/e1/e59caba97193d48862d6809912043871f37437

@@ -0,0 +1,2 @@
+x
[
+Â0EýÎ*æ_é$MÑ5tifBk Iŕ¹
7æk~ÞÃ9ܘ—åÜ ü¦ð.jÖÈ	ÅOÚäÉ"zÂ`ß#IirF…µÍ¹ÀØ$%¹Âçò|4)°¯?t¼É=”Ë
:K¦ï­#[$D¿¯û¿^˜…¡®Ó’y½HU/f?G

modules/git/tests/repos/repo4_submodules/refs/heads/master

@@ -0,0 +1 @@
+e1e59caba97193d48862d6809912043871f37437

modules/git/tree.go

@@ -17,7 +17,7 @@ func NewTree(repo *Repository, id ObjectID) *Tree {
 	}
 }
 
-// SubTree get a sub tree by the sub dir path
+// SubTree get a subtree by the sub dir path
 func (t *Tree) SubTree(rpath string) (*Tree, error) {
 	if len(rpath) == 0 {
 		return t, nil
@@ -63,7 +63,7 @@ func (repo *Repository) LsTree(ref string, filenames ...string) ([]string, error
 	return filelist, err
 }
 
-// GetTreePathLatestCommitID returns the latest commit of a tree path
+// GetTreePathLatestCommit returns the latest commit of a tree path
 func (repo *Repository) GetTreePathLatestCommit(refName, treePath string) (*Commit, error) {
 	stdout, _, err := NewCommand(repo.Ctx, "rev-list", "-1").
 		AddDynamicArguments(refName).AddDashesAndList(treePath).