Skip to content

Commit 8b88238

Browse files
DmitriyLewenDmitriyLewen
authored
fix: use --file-patterns flag for all post analyzers (#7365)
1 parent e8c32de commit 8b88238

File tree

19 files changed

+100
-43
lines changed

19 files changed

+100
-43
lines changed

‎pkg/fanal/analyzer/analyzer.go

+29-22
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ type AnalyzerGroup struct {
131131
logger *log.Logger
132132
analyzers []analyzer
133133
postAnalyzers []PostAnalyzer
134-
filePatterns map[Type][]*regexp.Regexp
134+
filePatterns map[Type]FilePatterns
135135
detectionPriority types.DetectionPriority
136136
}
137137

@@ -149,8 +149,20 @@ type AnalysisInput struct {
149149
}
150150

151151
type PostAnalysisInput struct {
152-
FS fs.FS
153-
Options AnalysisOptions
152+
FS fs.FS
153+
FilePatterns FilePatterns
154+
Options AnalysisOptions
155+
}
156+
157+
type FilePatterns []*regexp.Regexp
158+
159+
func (f FilePatterns) Match(filePath string) bool {
160+
for _, pattern := range f {
161+
if pattern.MatchString(filePath) {
162+
return true
163+
}
164+
}
165+
return false
154166
}
155167

156168
type AnalysisOptions struct {
@@ -333,7 +345,7 @@ func NewAnalyzerGroup(opts AnalyzerOptions) (AnalyzerGroup, error) {
333345

334346
group := AnalyzerGroup{
335347
logger: log.WithPrefix("analyzer"),
336-
filePatterns: make(map[Type][]*regexp.Regexp),
348+
filePatterns: make(map[Type]FilePatterns),
337349
detectionPriority: opts.DetectionPriority,
338350
}
339351
for _, p := range opts.FilePatterns {
@@ -349,10 +361,6 @@ func NewAnalyzerGroup(opts AnalyzerOptions) (AnalyzerGroup, error) {
349361
return group, xerrors.Errorf("invalid file regexp (%s): %w", p, err)
350362
}
351363

352-
if _, ok := group.filePatterns[Type(fileType)]; !ok {
353-
group.filePatterns[Type(fileType)] = []*regexp.Regexp{}
354-
}
355-
356364
group.filePatterns[Type(fileType)] = append(group.filePatterns[Type(fileType)], r)
357365
}
358366

@@ -422,7 +430,7 @@ func (ag AnalyzerGroup) AnalyzeFile(ctx context.Context, wg *sync.WaitGroup, lim
422430
continue
423431
}
424432

425-
if !ag.filePatternMatch(a.Type(), cleanPath) && !a.Required(cleanPath, info) {
433+
if !ag.filePatterns[a.Type()].Match(cleanPath) && !a.Required(cleanPath, info) {
426434
continue
427435
}
428436
rc, err := opener()
@@ -468,7 +476,7 @@ func (ag AnalyzerGroup) RequiredPostAnalyzers(filePath string, info os.FileInfo)
468476
}
469477
var postAnalyzerTypes []Type
470478
for _, a := range ag.postAnalyzers {
471-
if ag.filePatternMatch(a.Type(), filePath) || a.Required(filePath, info) {
479+
if ag.filePatterns[a.Type()].Match(filePath) || a.Required(filePath, info) {
472480
postAnalyzerTypes = append(postAnalyzerTypes, a.Type())
473481
}
474482
}
@@ -479,7 +487,8 @@ func (ag AnalyzerGroup) RequiredPostAnalyzers(filePath string, info os.FileInfo)
479487
// and passes it to the respective post-analyzer.
480488
// The obtained results are merged into the "result".
481489
// This function may be called concurrently and must be thread-safe.
482-
func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeFS, result *AnalysisResult, opts AnalysisOptions) error {
490+
func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeFS, result *AnalysisResult,
491+
opts AnalysisOptions) error {
483492
for _, a := range ag.postAnalyzers {
484493
fsys, ok := compositeFS.Get(a.Type())
485494
if !ok {
@@ -510,8 +519,9 @@ func (ag AnalyzerGroup) PostAnalyze(ctx context.Context, compositeFS *CompositeF
510519
}
511520

512521
res, err := a.PostAnalyze(ctx, PostAnalysisInput{
513-
FS: filteredFS,
514-
Options: opts,
522+
FS: filteredFS,
523+
FilePatterns: ag.filePatterns[a.Type()],
524+
Options: opts,
515525
})
516526
if err != nil {
517527
return xerrors.Errorf("post analysis error: %w", err)
@@ -526,15 +536,6 @@ func (ag AnalyzerGroup) PostAnalyzerFS() (*CompositeFS, error) {
526536
return NewCompositeFS()
527537
}
528538

529-
func (ag AnalyzerGroup) filePatternMatch(analyzerType Type, filePath string) bool {
530-
for _, pattern := range ag.filePatterns[analyzerType] {
531-
if pattern.MatchString(filePath) {
532-
return true
533-
}
534-
}
535-
return false
536-
}
537-
538539
// StaticPaths collects static paths from all enabled analyzers
539540
// It returns the collected paths and a boolean indicating if all enabled analyzers implement StaticPathAnalyzer
540541
func (ag AnalyzerGroup) StaticPaths(disabled []Type) ([]string, bool) {
@@ -546,6 +547,12 @@ func (ag AnalyzerGroup) StaticPaths(disabled []Type) ([]string, bool) {
546547
continue
547548
}
548549

550+
// We can't be sure that the file pattern uses a static path.
551+
// So we don't need to use `StaticPath` logic if any enabled analyzer has a file pattern.
552+
if _, ok := ag.filePatterns[a.Type()]; ok {
553+
return nil, false
554+
}
555+
549556
// If any analyzer doesn't implement StaticPathAnalyzer, return false
550557
staticPathAnalyzer, ok := a.(StaticPathAnalyzer)
551558
if !ok {

‎pkg/fanal/analyzer/analyzer_test.go

+20-3
Original file line numberDiff line numberDiff line change
@@ -561,6 +561,7 @@ func TestAnalyzerGroup_PostAnalyze(t *testing.T) {
561561
name string
562562
dir string
563563
analyzerType analyzer.Type
564+
filePatterns []string
564565
want *analyzer.AnalysisResult
565566
}{
566567
{
@@ -584,11 +585,25 @@ func TestAnalyzerGroup_PostAnalyze(t *testing.T) {
584585
},
585586
},
586587
{
587-
name: "poetry files with invalid file",
588-
dir: "testdata/post-apps/poetry/",
588+
name: "poetry files with file from pattern and invalid file",
589+
dir: "testdata/post-apps/poetry/",
590+
filePatterns: []string{
591+
"poetry:poetry-pattern.lock",
592+
},
589593
analyzerType: analyzer.TypePoetry,
590594
want: &analyzer.AnalysisResult{
591595
Applications: []types.Application{
596+
{
597+
Type: types.Poetry,
598+
FilePath: "testdata/post-apps/poetry/happy/poetry-pattern.lock",
599+
Packages: types.Packages{
600+
{
601+
ID: "[email protected]",
602+
Name: "certifi",
603+
Version: "2022.12.7",
604+
},
605+
},
606+
},
592607
{
593608
Type: types.Poetry,
594609
FilePath: "testdata/post-apps/poetry/happy/poetry.lock",
@@ -606,7 +621,9 @@ func TestAnalyzerGroup_PostAnalyze(t *testing.T) {
606621
}
607622
for _, tt := range tests {
608623
t.Run(tt.name, func(t *testing.T) {
609-
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{})
624+
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
625+
FilePatterns: tt.filePatterns,
626+
})
610627
require.NoError(t, err)
611628

612629
// Create a virtual filesystem

‎pkg/fanal/analyzer/language/c/conan/conan.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func newConanLockAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, er
4545

4646
func (a conanLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) {
4747
required := func(filePath string, d fs.DirEntry) bool {
48-
// we need all file got from `a.Required` function (conan.lock files) and from file-patterns.
48+
// Parse all required files: `conan.lock` (from a.Required func) + input.FilePatterns.Match()
4949
return true
5050
}
5151

‎pkg/fanal/analyzer/language/dart/pub/pubspec.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ func (a pubSpecLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostA
5555
}
5656

5757
required := func(path string, d fs.DirEntry) bool {
58-
return filepath.Base(path) == types.PubSpecLock
58+
// Parse all required files: `pubspec.lock` (from a.Required func) + input.FilePatterns.Match()
59+
return true
5960
}
6061

6162
err = fsutils.WalkDir(input.FS, ".", required, func(path string, _ fs.DirEntry, r io.Reader) error {

‎pkg/fanal/analyzer/language/dotnet/nuget/nuget.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,8 @@ func (a *nugetLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.Pos
5959
a.logger.Debug("The nuget packages directory couldn't be found. License search disabled")
6060
}
6161

62-
// We saved only config and lock files in the FS,
63-
// so we need to parse all saved files
6462
required := func(path string, d fs.DirEntry) bool {
63+
// Parse all required files: `packages.lock.json`, `packages.config` (from a.Required func) + input.FilePatterns.Match()
6564
return true
6665
}
6766

‎pkg/fanal/analyzer/language/golang/mod/mod.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func (a *gomodAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys
6868
var apps []types.Application
6969

7070
required := func(path string, d fs.DirEntry) bool {
71-
return filepath.Base(path) == types.GoMod
71+
return filepath.Base(path) == types.GoMod || input.FilePatterns.Match(path)
7272
}
7373

7474
err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, _ io.Reader) error {

‎pkg/fanal/analyzer/language/java/gradle/lockfile.go

+2-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,8 @@ func (a gradleLockAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn
4949
}
5050

5151
required := func(path string, d fs.DirEntry) bool {
52-
return a.Required(path, nil)
52+
// Parse all required files: `*gradle.lockfile` (from a.Required func) + input.FilePatterns.Match()
53+
return true
5354
}
5455

5556
var apps []types.Application

‎pkg/fanal/analyzer/language/julia/pkg/pkg.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ func (a juliaAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysi
5454
var apps []types.Application
5555

5656
required := func(path string, d fs.DirEntry) bool {
57-
return filepath.Base(path) == types.JuliaManifest
57+
return filepath.Base(path) == types.JuliaManifest || input.FilePatterns.Match(path)
5858
}
5959

6060
err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error {

‎pkg/fanal/analyzer/language/nodejs/npm/npm.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ func newNpmLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, e
4747
func (a npmLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) {
4848
// Parse package-lock.json
4949
required := func(path string, _ fs.DirEntry) bool {
50-
return filepath.Base(path) == types.NpmPkgLock
50+
return filepath.Base(path) == types.NpmPkgLock || input.FilePatterns.Match(path)
5151
}
5252

5353
var apps []types.Application

‎pkg/fanal/analyzer/language/nodejs/pnpm/pnpm.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func (a pnpmAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis
4545
var apps []types.Application
4646

4747
required := func(path string, d fs.DirEntry) bool {
48-
return filepath.Base(path) == types.PnpmLock
48+
return filepath.Base(path) == types.PnpmLock || input.FilePatterns.Match(path)
4949
}
5050

5151
err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error {

‎pkg/fanal/analyzer/language/nodejs/yarn/yarn.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ func (a yarnAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysis
7171
var apps []types.Application
7272

7373
required := func(path string, d fs.DirEntry) bool {
74-
return filepath.Base(path) == types.YarnLock
74+
return filepath.Base(path) == types.YarnLock || input.FilePatterns.Match(path)
7575
}
7676

7777
err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error {

‎pkg/fanal/analyzer/language/php/composer/composer.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ func (a composerAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnal
4747
var apps []types.Application
4848

4949
required := func(path string, d fs.DirEntry) bool {
50-
return filepath.Base(path) == types.ComposerLock
50+
return filepath.Base(path) == types.ComposerLock || input.FilePatterns.Match(path)
5151
}
5252

5353
err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error {

‎pkg/fanal/analyzer/language/python/packaging/packaging.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ func (a packagingAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAna
6363
var apps []types.Application
6464

6565
required := func(path string, _ fs.DirEntry) bool {
66-
return filepath.Base(path) == "METADATA" || isEggFile(path)
66+
return filepath.Base(path) == "METADATA" || isEggFile(path) || input.FilePatterns.Match(path)
6767
}
6868

6969
err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error {

‎pkg/fanal/analyzer/language/python/pip/pip.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -59,8 +59,8 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn
5959
a.logger.Warn("Unable to find python `site-packages` directory. License detection is skipped.", log.Err(err))
6060
}
6161

62-
// We only saved the `requirements.txt` files
6362
required := func(_ string, _ fs.DirEntry) bool {
63+
// Parse all required files: `conan.lock` (from a.Required func) + input.FilePatterns.Match()
6464
return true
6565
}
6666

‎pkg/fanal/analyzer/language/python/poetry/poetry.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ func (a poetryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalys
4545
var apps []types.Application
4646

4747
required := func(path string, d fs.DirEntry) bool {
48-
return filepath.Base(path) == types.PoetryLock
48+
return filepath.Base(path) == types.PoetryLock || input.FilePatterns.Match(path)
4949
}
5050

5151
err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error {

‎pkg/fanal/analyzer/language/python/uv/uv.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,9 @@ func NewUvAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
3737

3838
func (a *uvAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysisInput) (*analyzer.AnalysisResult, error) {
3939
var apps []types.Application
40-
required := func(path string, d fs.DirEntry) bool {
41-
return filepath.Base(path) == types.UvLock
40+
required := func(_ string, _ fs.DirEntry) bool {
41+
// Parse all required files: `uv.lock` (from a.Required func) + input.FilePatterns.Match()
42+
return true
4243
}
4344

4445
err := fsutils.WalkDir(input.FS, ".", required, func(path string, d fs.DirEntry, r io.Reader) error {

‎pkg/fanal/analyzer/language/rust/cargo/cargo.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ func (a cargoAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAnalysi
5757
var apps []types.Application
5858

5959
required := func(path string, d fs.DirEntry) bool {
60-
return filepath.Base(path) == types.CargoLock
60+
return filepath.Base(path) == types.CargoLock || input.FilePatterns.Match(path)
6161
}
6262

6363
err := fsutils.WalkDir(input.FS, ".", required, func(filePath string, d fs.DirEntry, r io.Reader) error {

‎pkg/fanal/analyzer/testdata/post-apps/poetry/happy/poetry-pattern.lock

+18
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
# This file is automatically @generated by Poetry and should not be changed by hand.
2+
3+
[[package]]
4+
name = "certifi"
5+
version = "2022.12.7"
6+
description = "Python package for providing Mozilla's CA Bundle."
7+
category = "main"
8+
optional = true
9+
python-versions = ">=3.6"
10+
files = [
11+
{file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"},
12+
{file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"},
13+
]
14+
15+
[metadata]
16+
lock-version = "2.0"
17+
python-versions = "^3.9"
18+
content-hash = "0ee6cb4bc2d84091d5dcb0a0110a65f244987ed427933b2f49949195e3ef69c7"

‎pkg/fanal/artifact/local/fs_test.go

+14-1
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ func TestArtifact_Inspect(t *testing.T) {
9898
analyzer.TypeAlpine,
9999
analyzer.TypeApk,
100100
analyzer.TypePip,
101+
analyzer.TypeNpmPkgLock,
101102
},
102103
},
103104
wantBlobs: []cachetest.WantBlob{
@@ -2468,6 +2469,7 @@ func TestAnalyzerGroup_StaticPaths(t *testing.T) {
24682469
tests := []struct {
24692470
name string
24702471
disabledAnalyzers []analyzer.Type
2472+
filePatterns []string
24712473
want []string
24722474
wantAllStatic bool
24732475
}{
@@ -2480,6 +2482,15 @@ func TestAnalyzerGroup_StaticPaths(t *testing.T) {
24802482
},
24812483
wantAllStatic: true,
24822484
},
2485+
{
2486+
name: "all analyzers implement StaticPathAnalyzer, but there is file pattern",
2487+
disabledAnalyzers: append(analyzer.TypeConfigFiles, analyzer.TypePip, analyzer.TypeSecret),
2488+
filePatterns: []string{
2489+
"alpine:etc/alpine-release-custom",
2490+
},
2491+
want: []string{},
2492+
wantAllStatic: false,
2493+
},
24832494
{
24842495
name: "some analyzers don't implement StaticPathAnalyzer",
24852496
want: []string{},
@@ -2505,7 +2516,9 @@ func TestAnalyzerGroup_StaticPaths(t *testing.T) {
25052516
for _, tt := range tests {
25062517
t.Run(tt.name, func(t *testing.T) {
25072518
// Create a new analyzer group
2508-
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{})
2519+
a, err := analyzer.NewAnalyzerGroup(analyzer.AnalyzerOptions{
2520+
FilePatterns: tt.filePatterns,
2521+
})
25092522
require.NoError(t, err)
25102523

25112524
// Get static paths

0 commit comments

Comments
 (0)