Skip to content

Commit 15a5c10

Browse files
jolheiserguillep2k
authored andcommitted
Variable expansion in repository templates (#9163)
* Start expansion Signed-off-by: jolheiser <[email protected]> * _template rather than .template Signed-off-by: jolheiser <[email protected]> * Use ioutil Signed-off-by: jolheiser <[email protected]> * Add descriptions to mapping * Start globbing Signed-off-by: jolheiser <[email protected]> * Tune globbing Signed-off-by: jolheiser <[email protected]> * Re-arrange imports Signed-off-by: jolheiser <[email protected]> * Don't expand git hooks Signed-off-by: jolheiser <[email protected]> * Add glob tests for .giteatemplate Signed-off-by: jolheiser <[email protected]> * Parse globs separately so they can be tested more easily Signed-off-by: jolheiser <[email protected]> * Change template location and add docs Signed-off-by: jolheiser <[email protected]> * nit Signed-off-by: jolheiser <[email protected]> * Update docs/content/doc/features/gitea-directory.md Co-Authored-By: guillep2k <[email protected]> * Update docs/content/doc/features/gitea-directory.md Co-Authored-By: guillep2k <[email protected]> * Add upper-lower case match Signed-off-by: jolheiser <[email protected]> * Nits Signed-off-by: jolheiser <[email protected]> * Update models/repo_generate.go Co-Authored-By: guillep2k <[email protected]>
1 parent c9d50bc commit 15a5c10

File tree

5 files changed

+298
-51
lines changed

5 files changed

+298
-51
lines changed
Lines changed: 56 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,56 @@
1+
---
2+
date: "2019-11-28:00:00+02:00"
3+
title: "The .gitea Directory"
4+
slug: "gitea-directory"
5+
weight: 40
6+
toc: true
7+
draft: false
8+
menu:
9+
sidebar:
10+
parent: "features"
11+
name: "The .gitea Directory"
12+
weight: 50
13+
identifier: "gitea-directory"
14+
---
15+
16+
# The .gitea directory
17+
Gitea repositories can include a `.gitea` directory at their base which will store settings/configurations for certain features.
18+
19+
## Templates
20+
Gitea includes template repositories, and one feature implemented with them is auto-expansion of specific variables within your template files.
21+
To tell Gitea which files to expand, you must include a `template` file inside the `.gitea` directory of the template repository.
22+
Gitea uses [gobwas/glob](https://github.com/gobwas/glob) for its glob syntax. It closely resembles a traditional `.gitignore`, however there may be slight differences.
23+
24+
### Example `.gitea/template` file
25+
All paths are relative to the base of the repository
26+
```gitignore
27+
# All .go files, anywhere in the repository
28+
**.go
29+
30+
# All text files in the text directory
31+
text/*.txt
32+
33+
# A specific file
34+
a/b/c/d.json
35+
36+
# Batch files in both upper or lower case can be matched
37+
**.[bB][aA][tT]
38+
```
39+
**NOTE:** The `template` file will be removed from the `.gitea` directory when a repository is generated from the template.
40+
41+
### Variable Expansion
42+
In any file matched by the above globs, certain variables will be expanded.
43+
All variables must be of the form `$VAR` or `${VAR}`. To escape an expansion, use a double `$$`, such as `$$VAR` or `$${VAR}`
44+
45+
| Variable | Expands To |
46+
|----------------------|-----------------------------------------------------|
47+
| REPO_NAME | The name of the generated repository |
48+
| TEMPLATE_NAME | The name of the template repository |
49+
| REPO_DESCRIPTION | The description of the generated repository |
50+
| TEMPLATE_DESCRIPTION | The description of the template repository |
51+
| REPO_LINK | The URL to the generated repository |
52+
| TEMPLATE_LINK | The URL to the template repository |
53+
| REPO_HTTPS_URL | The HTTP(S) clone link for the generated repository |
54+
| TEMPLATE_HTTPS_URL | The HTTP(S) clone link for the template repository |
55+
| REPO_SSH_URL | The SSH clone link for the generated repository |
56+
| TEMPLATE_SSH_URL | The SSH clone link for the template repository |

models/repo.go

Lines changed: 0 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1361,54 +1361,6 @@ func prepareRepoCommit(e Engine, repo *Repository, tmpDir, repoPath string, opts
13611361
return nil
13621362
}
13631363

1364-
func generateRepoCommit(e Engine, repo, templateRepo *Repository, tmpDir string) error {
1365-
commitTimeStr := time.Now().Format(time.RFC3339)
1366-
authorSig := repo.Owner.NewGitSig()
1367-
1368-
// Because this may call hooks we should pass in the environment
1369-
env := append(os.Environ(),
1370-
"GIT_AUTHOR_NAME="+authorSig.Name,
1371-
"GIT_AUTHOR_EMAIL="+authorSig.Email,
1372-
"GIT_AUTHOR_DATE="+commitTimeStr,
1373-
"GIT_COMMITTER_NAME="+authorSig.Name,
1374-
"GIT_COMMITTER_EMAIL="+authorSig.Email,
1375-
"GIT_COMMITTER_DATE="+commitTimeStr,
1376-
)
1377-
1378-
// Clone to temporary path and do the init commit.
1379-
templateRepoPath := templateRepo.repoPath(e)
1380-
_, stderr, err := process.GetManager().ExecDirEnv(
1381-
-1, "",
1382-
fmt.Sprintf("generateRepoCommit(git clone): %s", templateRepoPath),
1383-
env,
1384-
git.GitExecutable, "clone", "--depth", "1", templateRepoPath, tmpDir,
1385-
)
1386-
if err != nil {
1387-
return fmt.Errorf("git clone: %v - %s", err, stderr)
1388-
}
1389-
1390-
if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
1391-
return fmt.Errorf("remove git dir: %v", err)
1392-
}
1393-
1394-
if err := git.InitRepository(tmpDir, false); err != nil {
1395-
return err
1396-
}
1397-
1398-
repoPath := repo.repoPath(e)
1399-
_, stderr, err = process.GetManager().ExecDirEnv(
1400-
-1, tmpDir,
1401-
fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath),
1402-
env,
1403-
git.GitExecutable, "remote", "add", "origin", repoPath,
1404-
)
1405-
if err != nil {
1406-
return fmt.Errorf("git remote add: %v - %s", err, stderr)
1407-
}
1408-
1409-
return initRepoCommit(tmpDir, repo.Owner)
1410-
}
1411-
14121364
func checkInitRepository(repoPath string) (err error) {
14131365
// Somehow the directory could exist.
14141366
if com.IsExist(repoPath) {

models/repo_generate.go

Lines changed: 181 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,20 @@ package models
66

77
import (
88
"fmt"
9+
"io/ioutil"
910
"os"
11+
"path"
1012
"path/filepath"
1113
"strconv"
1214
"strings"
1315
"time"
1416

1517
"code.gitea.io/gitea/modules/git"
1618
"code.gitea.io/gitea/modules/log"
19+
"code.gitea.io/gitea/modules/process"
20+
"code.gitea.io/gitea/modules/util"
1721

22+
"github.com/gobwas/glob"
1823
"github.com/unknwon/com"
1924
)
2025

@@ -36,8 +41,148 @@ func (gro GenerateRepoOptions) IsValid() bool {
3641
return gro.GitContent || gro.Topics || gro.GitHooks || gro.Webhooks || gro.Avatar || gro.IssueLabels // or other items as they are added
3742
}
3843

44+
// GiteaTemplate holds information about a .gitea/template file
45+
type GiteaTemplate struct {
46+
Path string
47+
Content []byte
48+
49+
globs []glob.Glob
50+
}
51+
52+
// Globs parses the .gitea/template globs or returns them if they were already parsed
53+
func (gt GiteaTemplate) Globs() []glob.Glob {
54+
if gt.globs != nil {
55+
return gt.globs
56+
}
57+
58+
gt.globs = make([]glob.Glob, 0)
59+
lines := strings.Split(string(util.NormalizeEOL(gt.Content)), "\n")
60+
for _, line := range lines {
61+
line = strings.TrimSpace(line)
62+
if line == "" || strings.HasPrefix(line, "#") {
63+
continue
64+
}
65+
g, err := glob.Compile(line, '/')
66+
if err != nil {
67+
log.Info("Invalid glob expression '%s' (skipped): %v", line, err)
68+
continue
69+
}
70+
gt.globs = append(gt.globs, g)
71+
}
72+
return gt.globs
73+
}
74+
75+
func checkGiteaTemplate(tmpDir string) (*GiteaTemplate, error) {
76+
gtPath := filepath.Join(tmpDir, ".gitea", "template")
77+
if _, err := os.Stat(gtPath); os.IsNotExist(err) {
78+
return nil, nil
79+
} else if err != nil {
80+
return nil, err
81+
}
82+
83+
content, err := ioutil.ReadFile(gtPath)
84+
if err != nil {
85+
return nil, err
86+
}
87+
88+
gt := &GiteaTemplate{
89+
Path: gtPath,
90+
Content: content,
91+
}
92+
93+
return gt, nil
94+
}
95+
96+
func generateRepoCommit(e Engine, repo, templateRepo, generateRepo *Repository, tmpDir string) error {
97+
commitTimeStr := time.Now().Format(time.RFC3339)
98+
authorSig := repo.Owner.NewGitSig()
99+
100+
// Because this may call hooks we should pass in the environment
101+
env := append(os.Environ(),
102+
"GIT_AUTHOR_NAME="+authorSig.Name,
103+
"GIT_AUTHOR_EMAIL="+authorSig.Email,
104+
"GIT_AUTHOR_DATE="+commitTimeStr,
105+
"GIT_COMMITTER_NAME="+authorSig.Name,
106+
"GIT_COMMITTER_EMAIL="+authorSig.Email,
107+
"GIT_COMMITTER_DATE="+commitTimeStr,
108+
)
109+
110+
// Clone to temporary path and do the init commit.
111+
templateRepoPath := templateRepo.repoPath(e)
112+
if err := git.Clone(templateRepoPath, tmpDir, git.CloneRepoOptions{
113+
Depth: 1,
114+
}); err != nil {
115+
return fmt.Errorf("git clone: %v", err)
116+
}
117+
118+
if err := os.RemoveAll(path.Join(tmpDir, ".git")); err != nil {
119+
return fmt.Errorf("remove git dir: %v", err)
120+
}
121+
122+
// Variable expansion
123+
gt, err := checkGiteaTemplate(tmpDir)
124+
if err != nil {
125+
return fmt.Errorf("checkGiteaTemplate: %v", err)
126+
}
127+
128+
if err := os.Remove(gt.Path); err != nil {
129+
return fmt.Errorf("remove .giteatemplate: %v", err)
130+
}
131+
132+
// Avoid walking tree if there are no globs
133+
if len(gt.Globs()) > 0 {
134+
tmpDirSlash := strings.TrimSuffix(filepath.ToSlash(tmpDir), "/") + "/"
135+
if err := filepath.Walk(tmpDirSlash, func(path string, info os.FileInfo, walkErr error) error {
136+
if walkErr != nil {
137+
return walkErr
138+
}
139+
140+
if info.IsDir() {
141+
return nil
142+
}
143+
144+
base := strings.TrimPrefix(filepath.ToSlash(path), tmpDirSlash)
145+
for _, g := range gt.Globs() {
146+
if g.Match(base) {
147+
content, err := ioutil.ReadFile(path)
148+
if err != nil {
149+
return err
150+
}
151+
152+
if err := ioutil.WriteFile(path,
153+
[]byte(generateExpansion(string(content), templateRepo, generateRepo)),
154+
0644); err != nil {
155+
return err
156+
}
157+
break
158+
}
159+
}
160+
return nil
161+
}); err != nil {
162+
return err
163+
}
164+
}
165+
166+
if err := git.InitRepository(tmpDir, false); err != nil {
167+
return err
168+
}
169+
170+
repoPath := repo.repoPath(e)
171+
_, stderr, err := process.GetManager().ExecDirEnv(
172+
-1, tmpDir,
173+
fmt.Sprintf("generateRepoCommit(git remote add): %s", repoPath),
174+
env,
175+
git.GitExecutable, "remote", "add", "origin", repoPath,
176+
)
177+
if err != nil {
178+
return fmt.Errorf("git remote add: %v - %s", err, stderr)
179+
}
180+
181+
return initRepoCommit(tmpDir, repo.Owner)
182+
}
183+
39184
// generateRepository initializes repository from template
40-
func generateRepository(e Engine, repo, templateRepo *Repository) (err error) {
185+
func generateRepository(e Engine, repo, templateRepo, generateRepo *Repository) (err error) {
41186
tmpDir := filepath.Join(os.TempDir(), "gitea-"+repo.Name+"-"+com.ToStr(time.Now().Nanosecond()))
42187

43188
if err := os.MkdirAll(tmpDir, os.ModePerm); err != nil {
@@ -50,7 +195,7 @@ func generateRepository(e Engine, repo, templateRepo *Repository) (err error) {
50195
}
51196
}()
52197

53-
if err = generateRepoCommit(e, repo, templateRepo, tmpDir); err != nil {
198+
if err = generateRepoCommit(e, repo, templateRepo, generateRepo, tmpDir); err != nil {
54199
return fmt.Errorf("generateRepoCommit: %v", err)
55200
}
56201

@@ -95,7 +240,7 @@ func GenerateRepository(ctx DBContext, doer, owner *User, templateRepo *Reposito
95240

96241
// GenerateGitContent generates git content from a template repository
97242
func GenerateGitContent(ctx DBContext, templateRepo, generateRepo *Repository) error {
98-
if err := generateRepository(ctx.e, generateRepo, templateRepo); err != nil {
243+
if err := generateRepository(ctx.e, generateRepo, templateRepo, generateRepo); err != nil {
99244
return err
100245
}
101246

@@ -210,3 +355,36 @@ func GenerateIssueLabels(ctx DBContext, templateRepo, generateRepo *Repository)
210355
}
211356
return nil
212357
}
358+
359+
func generateExpansion(src string, templateRepo, generateRepo *Repository) string {
360+
return os.Expand(src, func(key string) string {
361+
switch key {
362+
case "REPO_NAME":
363+
return generateRepo.Name
364+
case "TEMPLATE_NAME":
365+
return templateRepo.Name
366+
case "REPO_DESCRIPTION":
367+
return generateRepo.Description
368+
case "TEMPLATE_DESCRIPTION":
369+
return templateRepo.Description
370+
case "REPO_OWNER":
371+
return generateRepo.MustOwnerName()
372+
case "TEMPLATE_OWNER":
373+
return templateRepo.MustOwnerName()
374+
case "REPO_LINK":
375+
return generateRepo.Link()
376+
case "TEMPLATE_LINK":
377+
return templateRepo.Link()
378+
case "REPO_HTTPS_URL":
379+
return generateRepo.CloneLink().HTTPS
380+
case "TEMPLATE_HTTPS_URL":
381+
return templateRepo.CloneLink().HTTPS
382+
case "REPO_SSH_URL":
383+
return generateRepo.CloneLink().SSH
384+
case "TEMPLATE_SSH_URL":
385+
return templateRepo.CloneLink().SSH
386+
default:
387+
return key
388+
}
389+
})
390+
}

models/repo_generate_test.go

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
// Copyright 2019 The Gitea Authors. All rights reserved.
2+
// Use of this source code is governed by a MIT-style
3+
// license that can be found in the LICENSE file.
4+
5+
package models
6+
7+
import (
8+
"testing"
9+
10+
"github.com/stretchr/testify/assert"
11+
)
12+
13+
var giteaTemplate = []byte(`
14+
# Header
15+
16+
# All .go files
17+
**.go
18+
19+
# All text files in /text/
20+
text/*.txt
21+
22+
# All files in modules folders
23+
**/modules/*
24+
`)
25+
26+
func TestGiteaTemplate(t *testing.T) {
27+
gt := GiteaTemplate{Content: giteaTemplate}
28+
assert.Equal(t, len(gt.Globs()), 3)
29+
30+
tt := []struct {
31+
Path string
32+
Match bool
33+
}{
34+
{Path: "main.go", Match: true},
35+
{Path: "a/b/c/d/e.go", Match: true},
36+
{Path: "main.txt", Match: false},
37+
{Path: "a/b.txt", Match: false},
38+
{Path: "text/a.txt", Match: true},
39+
{Path: "text/b.txt", Match: true},
40+
{Path: "text/c.json", Match: false},
41+
{Path: "a/b/c/modules/README.md", Match: true},
42+
{Path: "a/b/c/modules/d/README.md", Match: false},
43+
}
44+
45+
for _, tc := range tt {
46+
t.Run(tc.Path, func(t *testing.T) {
47+
match := false
48+
for _, g := range gt.Globs() {
49+
if g.Match(tc.Path) {
50+
match = true
51+
break
52+
}
53+
}
54+
assert.Equal(t, tc.Match, match)
55+
})
56+
}
57+
}

0 commit comments

Comments
 (0)