get language from .gitattributes file

modules/git/repo_attribute.go

@@ -5,8 +5,13 @@
 package git
 
 import (
+	"bufio"
 	"bytes"
 	"fmt"
+	"io"
+	"strings"
+
+	"github.com/gobwas/glob"
 )
 
 // CheckAttributeOpts represents the possible options to CheckAttribute
@@ -80,3 +85,163 @@ func (repo *Repository) CheckAttribute(opts CheckAttributeOpts) (map[string]map[
 
 	return name2attribute2info, nil
 }
+
+// AttrCheckResultType result type of AttrCheckResult
+type AttrCheckResultType int
+
+const (
+	// AttrCheckResultTypeUnspecified the attribute is not defined for the path
+	AttrCheckResultTypeUnspecified AttrCheckResultType = iota
+	// AttrCheckResultTypeUnset the attribute is defined as false
+	AttrCheckResultTypeUnset
+	// AttrCheckResultTypeSet the attribute is defined as true
+	AttrCheckResultTypeSet
+	// AttrCheckResultTypeValue a value has been assigned to the attribute
+	AttrCheckResultTypeValue
+)
+
+// AttrCheckResult the result of CheckAttributeFile
+type AttrCheckResult struct {
+	typ  AttrCheckResultType
+	data string
+}
+
+// IsSet if the attribute is defined as true
+func (r *AttrCheckResult) IsSet() bool {
+	return r.typ == AttrCheckResultTypeSet
+}
+
+// Value get the value of AttrCheckResult
+func (r *AttrCheckResult) Value() string {
+	if r.typ != AttrCheckResultTypeValue {
+		return ""
+	}
+	return r.data
+}
+
+// AttrChecker Attribute checker
+// format attr: partens
+type AttrChecker map[string][]*attrCheckerItem
+
+type attrCheckerItem struct {
+	pattern glob.Glob
+	rs      *AttrCheckResult
+}
+
+// LoadAttrbutCheckerFromCommit load AttrChecker from a commit
+func LoadAttrbutCheckerFromCommit(commit *Commit) (AttrChecker, error) {
+	gitAttrEntry, err := commit.GetTreeEntryByPath("/.gitattributes")
+	if err != nil {
+		if !IsErrNotExist(err) {
+			return nil, err
+		}
+		return nil, nil
+	}
+	if gitAttrEntry.IsDir() {
+		return nil, nil
+	}
+
+	blob := gitAttrEntry.Blob()
+	dataRc, err := blob.DataAsync()
+	if err != nil {
+		return nil, err
+	}
+	defer dataRc.Close()
+	gitAttr := make([]byte, 1024)
+	n, _ := dataRc.Read(gitAttr)
+	gitAttr = gitAttr[:n]
+
+	return LoadAttrbutCheckerFromReader(bytes.NewReader(gitAttr))
+}
+
+// LoadAttrbutCheckerFromReader load AttrChecker from content reader
+func LoadAttrbutCheckerFromReader(r io.Reader) (AttrChecker, error) {
+	cheker := make(AttrChecker)
+
+	readr := bufio.NewScanner(r)
+	for readr.Scan() {
+		t := readr.Text()
+		// format: pattern attr1 attr2 ...
+		if len(t) == 0 {
+			continue
+		}
+
+		t = strings.TrimLeft(t, " \t\r\n")
+		if strings.HasPrefix(t, "#") {
+			continue
+		}
+
+		splits := strings.Split(t, " ")
+		if len(splits) < 2 {
+			continue
+		}
+
+		// to let `/AAA/*.txt` can match `AAA/bb.txt`, have to
+		// remove first / if exit
+		splits[0] = strings.TrimPrefix(splits[0], "/")
+
+		// get parten
+		g, err := glob.Compile(splits[0], '/')
+		if err != nil {
+			return nil, err
+		}
+
+		check := func(attr string) (string, *AttrCheckResult) {
+			// one attr may has three status:
+			// set: XXX
+			// unset: -XXX
+			// value: XXX=VVV
+			if kv := strings.SplitN(attr, "=", 2); len(kv) == 2 {
+				return kv[0], &AttrCheckResult{
+					typ:  AttrCheckResultTypeValue,
+					data: kv[1],
+				}
+			}
+			typ := AttrCheckResultTypeSet
+			if strings.HasPrefix(attr, "-") {
+				attr = attr[1:]
+				typ = AttrCheckResultTypeUnset
+			}
+			return attr, &AttrCheckResult{typ: typ}
+		}
+
+		// check attrs
+		attrs := splits[1:]
+		for _, tmp := range attrs {
+			attr, rs := check(tmp)
+			v, ok := cheker[attr]
+			if !ok {
+				v = make([]*attrCheckerItem, 0, 5)
+			}
+
+			v = append(v, &attrCheckerItem{
+				pattern: g,
+				rs:      rs,
+			})
+			cheker[attr] = v
+		}
+	}
+
+	return cheker, nil
+}
+
+// Check check an git attr
+func (c AttrChecker) Check(requestAttr, path string) *AttrCheckResult {
+	if c == nil {
+		return nil
+	}
+
+	v, ok := c[requestAttr]
+	if !ok {
+		return &AttrCheckResult{typ: AttrCheckResultTypeUnspecified}
+	}
+
+	for _, item := range v {
+		if !item.pattern.Match(path) {
+			continue
+		}
+		return item.rs
+	}
+
+	return &AttrCheckResult{typ: AttrCheckResultTypeUnspecified}
+}

modules/git/repo_attribute_test.go

@@ -0,0 +1,81 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package git
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestCheckAttributeFile(t *testing.T) {
+	testContent := `*.txt text eol=lf
+  # test command
+  /vendor/** -text -eol linguist-vendored
+/tools/**/*.py linguist-vendored
+`
+	r := strings.NewReader(testContent)
+	checker, err := LoadAttrbutCheckerFromReader(r)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.NotEmpty(t, checker) {
+		return
+	}
+
+	tests := []struct {
+		want        *AttrCheckResult
+		content     string
+		requestAttr string
+		path        string
+	}{
+		{
+			want: &AttrCheckResult{
+				typ:  AttrCheckResultTypeValue,
+				data: "lf",
+			},
+			path:        "aa.txt",
+			requestAttr: "eol",
+		},
+		{
+			want: &AttrCheckResult{
+				typ:  AttrCheckResultTypeUnset,
+				data: "",
+			},
+			path:        "vendor/aa.txt",
+			requestAttr: "eol",
+		},
+		{
+			want: &AttrCheckResult{
+				typ:  AttrCheckResultTypeUnspecified,
+				data: "",
+			},
+			path:        "aa.png",
+			requestAttr: "text",
+		},
+		{
+			want: &AttrCheckResult{
+				typ:  AttrCheckResultTypeSet,
+				data: "",
+			},
+			path:        "vendor/bbb/aa.json",
+			requestAttr: "linguist-vendored",
+		},
+		// TODO: glob tools/**/*.py should match it, but can't ...
+		// {
+		// 	want: &AttrCheckResult{
+		// 		typ:  AttrCheckResultTypeSet,
+		// 		data: "",
+		// 	},
+		// 	path:        "tools/aa.py",
+		// 	requestAttr: "linguist-vendored",
+		// },
+	}
+	for _, tt := range tests {
+		got := checker.Check(tt.requestAttr, tt.path)
+		assert.Equal(t, tt.want, got)
+	}
+}

modules/git/repo_language_stats_gogit.go

@@ -20,7 +20,7 @@ import (
 )
 
 // GetLanguageStats calculates language stats for git repository at specified commit
-func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
+func (repo *Repository) GetLanguageStats(commitID string, preCheck func(path string) (string, bool)) (map[string]int64, error) {
 	r, err := git.PlainOpen(repo.Path)
 	if err != nil {
 		return nil, err
@@ -57,9 +57,18 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			return nil
 		}
 
-		// TODO: Use .gitattributes file for linguist overrides
+		language := ""
+		skip := false
+		if preCheck != nil {
+			language, skip = preCheck(f.Name)
+			if skip {
+				return nil
+			}
+		}
 
-		language := analyze.GetCodeLanguage(f.Name, content)
+		if len(language) == 0 {
+			language = analyze.GetCodeLanguage(f.Name, content)
+		}
 		if language == enry.OtherLanguage || language == "" {
 			return nil
 		}

modules/git/repo_language_stats_nogogit.go

@@ -19,7 +19,7 @@ import (
 )
 
 // GetLanguageStats calculates language stats for git repository at specified commit
-func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, error) {
+func (repo *Repository) GetLanguageStats(commitID string, preCheck func(path string) (string, bool)) (map[string]int64, error) {
 	// We will feed the commit IDs in order into cat-file --batch, followed by blobs as necessary.
 	// so let's create a batch stdin and stdout
 
@@ -128,10 +128,22 @@ func (repo *Repository) GetLanguageStats(commitID string) (map[string]int64, err
 			continue
 		}
 
-		// TODO: Use .gitattributes file for linguist overrides
-		// FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary?
-		// - eg. do the all the detection tests using filename first before reading content.
-		language := analyze.GetCodeLanguage(f.Name(), content)
+		// Use .gitattributes file for linguist overrides
+		language := ""
+		skip := false
+		if preCheck != nil {
+			language, skip = preCheck(f.Name())
+			if skip {
+				continue
+			}
+		}
+
+		if len(language) == 0 {
+			// FIXME: Why can't we split this and the IsGenerated tests to avoid reading the blob unless absolutely necessary?
+			// - eg. do the all the detection tests using filename first before reading content.
+			language = analyze.GetCodeLanguage(f.Name(), content)
+		}
+
 		if language == enry.OtherLanguage || language == "" {
 			continue
 		}

modules/indexer/stats/db.go

@@ -47,8 +47,34 @@ func (db *DBIndexer) Index(id int64) error {
 		return nil
 	}
 
+	commit, err := gitRepo.GetCommit(commitID)
+	if err != nil {
+		return err
+	}
+
+	attrChecker, err := git.LoadAttrbutCheckerFromCommit(commit)
+	if err != nil {
+		return err
+	}
+
 	// Calculate and save language statistics to database
-	stats, err := gitRepo.GetLanguageStats(commitID)
+	stats, err := gitRepo.GetLanguageStats(commitID, func(path string) (string, bool) {
+		// get language follow linguist rulers
+		// linguist-language=<lang> attribute to an language
+		// linguist-vendored attribute to vendor or un-vendor paths
+
+		if attrChecker == nil {
+			return "", false
+		}
+
+		r := attrChecker.Check("linguist-vendored", path)
+		if r.IsSet() || r.Value() == "true" {
+			return "", true
+		}
+
+		r = attrChecker.Check("linguist-language", path)
+		return r.Value(), false
+	})
 	if err != nil {
 		log.Error("Unable to get language stats for ID %s for defaultbranch %s in %s. Error: %v", commitID, repo.DefaultBranch, repo.RepoPath(), err)
 		return err