Skip to content

Commit 67d9437

Browse files
authored
Merge pull request #17227 from smowton/smowton/fix/baseline-vs-nonroot-vendor-dirs
Go / configure-baseline: account for multiple vendor directories and the `CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS` setting
2 parents 18b99ff + f13f19d commit 67d9437

File tree

20 files changed

+130
-19
lines changed

20 files changed

+130
-19
lines changed

go/BUILD.bazel

+1
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ codeql_pkg_files(
4747
"//go/extractor/cli/go-autobuilder",
4848
"//go/extractor/cli/go-bootstrap",
4949
"//go/extractor/cli/go-build-runner",
50+
"//go/extractor/cli/go-configure-baseline",
5051
"//go/extractor/cli/go-extractor",
5152
"//go/extractor/cli/go-gen-dbscheme",
5253
"//go/extractor/cli/go-tokenizer",

go/codeql-tools/baseline-config-empty.json

-3
This file was deleted.

go/codeql-tools/baseline-config-vendor.json

-5
This file was deleted.
+3-5
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
@echo off
2-
if exist vendor\modules.txt (
3-
type "%CODEQL_EXTRACTOR_GO_ROOT%\tools\baseline-config-vendor.json"
4-
) else (
5-
type "%CODEQL_EXTRACTOR_GO_ROOT%\tools\baseline-config-empty.json"
6-
)
2+
3+
type NUL && "%CODEQL_EXTRACTOR_GO_ROOT%/tools/%CODEQL_PLATFORM%/go-configure-baseline.exe"
4+
exit /b %ERRORLEVEL%

go/codeql-tools/configure-baseline.sh

+1-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,3 @@
11
#!/bin/sh
22

3-
if [ -f vendor/modules.txt ]; then
4-
cat "$CODEQL_EXTRACTOR_GO_ROOT/tools/baseline-config-vendor.json"
5-
else
6-
cat "$CODEQL_EXTRACTOR_GO_ROOT/tools/baseline-config-empty.json"
7-
fi
3+
"$CODEQL_EXTRACTOR_GO_ROOT/tools/$CODEQL_PLATFORM/go-configure-baseline"

go/extractor/cli/go-configure-baseline/BUILD.bazel

+18
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package main
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/github/codeql-go/extractor/configurebaseline"
7+
)
8+
9+
func main() {
10+
jsonResult, err := configurebaseline.GetConfigBaselineAsJSON(".")
11+
if err != nil {
12+
panic(err)
13+
} else {
14+
fmt.Println(string(jsonResult))
15+
}
16+
}

go/extractor/configurebaseline/BUILD.bazel

+11
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
package configurebaseline
2+
3+
import (
4+
"encoding/json"
5+
"io/fs"
6+
"os"
7+
"path"
8+
"path/filepath"
9+
10+
"github.com/github/codeql-go/extractor/util"
11+
)
12+
13+
func fileExists(path string) bool {
14+
stat, err := os.Stat(path)
15+
return err == nil && stat.Mode().IsRegular()
16+
}
17+
18+
// Decides if `dirPath` is a vendor directory by testing whether it is called `vendor`
19+
// and contains a `modules.txt` file.
20+
func isGolangVendorDirectory(dirPath string) bool {
21+
return filepath.Base(dirPath) == "vendor" && fileExists(filepath.Join(dirPath, "modules.txt"))
22+
}
23+
24+
type BaselineConfig struct {
25+
PathsIgnore []string `json:"paths-ignore"`
26+
}
27+
28+
func GetConfigBaselineAsJSON(rootDir string) ([]byte, error) {
29+
vendorDirs := make([]string, 0)
30+
31+
if util.IsVendorDirExtractionEnabled() {
32+
// The user wants vendor directories scanned; emit an empty report.
33+
} else {
34+
filepath.WalkDir(rootDir, func(dirPath string, d fs.DirEntry, err error) error {
35+
if err != nil {
36+
// Ignore any unreadable paths -- if this script can't see it, very likely
37+
// it will not be extracted either.
38+
return nil
39+
}
40+
if isGolangVendorDirectory(dirPath) {
41+
// Note that CodeQL expects a forward-slash-separated path, even on Windows.
42+
vendorDirs = append(vendorDirs, path.Join(filepath.ToSlash(dirPath), "**"))
43+
return filepath.SkipDir
44+
} else {
45+
return nil
46+
}
47+
})
48+
}
49+
50+
outputStruct := BaselineConfig{PathsIgnore: vendorDirs}
51+
return json.Marshal(outputStruct)
52+
}

go/extractor/extractor.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ func ExtractWithFlags(buildFlags []string, patterns []string) error {
199199

200200
// If CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS is "true", we extract `vendor` directories;
201201
// otherwise (the default) is to exclude them from extraction
202-
includeVendor := os.Getenv("CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS") == "true"
202+
includeVendor := util.IsVendorDirExtractionEnabled()
203203
if !includeVendor {
204204
excludedDirs = append(excludedDirs, "vendor")
205205
}

go/extractor/util/BUILD.bazel

+1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
+9
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
package util
2+
3+
import (
4+
"os"
5+
)
6+
7+
func IsVendorDirExtractionEnabled() bool {
8+
return os.Getenv("CODEQL_EXTRACTOR_GO_EXTRACT_VENDOR_DIRS") == "true"
9+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
package abc

go/ql/integration-tests/all-platforms/go/configure-baseline/src/a/vendor/modules.txt

Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
package abc

go/ql/integration-tests/all-platforms/go/configure-baseline/src/b/vendor/modules.txt

Whitespace-only changes.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
package abc
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
package abc
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
import os.path
2+
import json
3+
4+
def test(codeql, go):
5+
codeql.database.init(source_root="src")
6+
baseline_info_path = os.path.join("test-db", "baseline-info.json")
7+
with open(baseline_info_path, "r") as f:
8+
baseline_info = json.load(f)
9+
assert set(baseline_info["languages"]["go"]["files"]) == set(["root.go", "c/vendor/cvendor.go"]), "Expected root.go and cvendor.go in baseline"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
category: fix
3+
---
4+
* Golang vendor directories not at the root of a repository are now correctly excluded from the baseline Go file count. This means code coverage information will be more accurate.

0 commit comments

Comments
 (0)