feat(python): use minimum version for pip packages (#7348)

Author: afdesk

docs/docs/coverage/language/python.md

@@ -23,7 +23,7 @@ The following table provides an outline of the features Trivy offers.
 
 | Package manager | File             | Transitive dependencies | Dev dependencies | [Dependency graph][dependency-graph] | Position | [Detection Priority][detection-priority] |
 |-----------------|------------------|:-----------------------:|:----------------:|:------------------------------------:|:--------:|:----------------------------------------:|
-| pip             | requirements.txt |            -            |     Include      |                  -                   |    ✓     |                    -                     |
+| pip             | requirements.txt |            -            |     Include      |                  -                   |    ✓     |                    ✓                     |
 | Pipenv          | Pipfile.lock     |            ✓            |     Include      |                  -                   |    ✓     |                Not needed                |
 | Poetry          | poetry.lock      |            ✓            |     Exclude      |                  ✓                   |    -     |                Not needed                |
 
@@ -42,8 +42,17 @@ Trivy parses your files generated by package managers in filesystem/repository s
 ### pip
 
 #### Dependency detection
-Trivy only parses [version specifiers](https://packaging.python.org/en/latest/specifications/version-specifiers/#id5) with `==` comparison operator and without `.*`.
-To convert unsupported version specifiers - use the `pip  freeze` command.
+By default, Trivy only parses [version specifiers](https://packaging.python.org/en/latest/specifications/version-specifiers/#id5) with `==` comparison operator and without `.*`.
+
+Using the [--detection-priority comprehensive](#detection-priority) option ensures that the tool establishes a minimum version, which is particularly useful in scenarios where identifying the exact version is challenging. 
+In such case Trivy parses specifiers `>=`,`~=` and a trailing `.*`.
+
+```
+keyring >= 4.1.1            # Minimum version 4.1.1
+Mopidy-Dirble ~= 1.1        # Minimum version 1.1
+python-gitlab==2.0.*        # Minimum version 2.0.0
+```
+Also, there is a way to convert unsupported version specifiers - use the `pip  freeze` command.
 
 ```bash
 $ cat requirements.txt 

pkg/dependency/parser/python/pip/parse.go

@@ -25,14 +25,29 @@ const (
 )
 
 type Parser struct {
-	logger *log.Logger
+	logger        *log.Logger
+	useMinVersion bool
 }
 
-func NewParser() *Parser {
+func NewParser(useMinVersion bool) *Parser {
 	return &Parser{
-		logger: log.WithPrefix("pip"),
+		logger:        log.WithPrefix("pip"),
+		useMinVersion: useMinVersion,
 	}
 }
+func (p *Parser) splitLine(line string) []string {
+	separators := []string{"~=", ">=", "=="}
+	// Without useMinVersion check only `==`
+	if !p.useMinVersion {
+		separators = []string{"=="}
+	}
+	for _, sep := range separators {
+		if result := strings.Split(line, sep); len(result) == 2 {
+			return result
+		}
+	}
+	return nil
+}
 
 func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependency, error) {
 	// `requirements.txt` can use byte order marks (BOM)
@@ -53,10 +68,14 @@ func (p *Parser) Parse(r xio.ReadSeekerAt) ([]ftypes.Package, []ftypes.Dependenc
 		line = rStripByKey(line, commentMarker)
 		line = rStripByKey(line, endColon)
 		line = rStripByKey(line, hashMarker)
-		s := strings.Split(line, "==")
+
+		s := p.splitLine(line)
 		if len(s) != 2 {
 			continue
 		}
+		if p.useMinVersion && strings.HasSuffix(s[1], ".*") {
+			s[1] = strings.TrimSuffix(s[1], "*") + "0"
+		}
 
 		if !isValidName(s[0]) || !isValidVersion(s[1]) {
 			p.logger.Debug("Invalid package name/version in requirements.txt.", log.String("line", text))

pkg/dependency/parser/python/pip/parse_test.go

@@ -12,9 +12,10 @@ import (
 
 func TestParse(t *testing.T) {
 	tests := []struct {
-		name     string
-		filePath string
-		want     []ftypes.Package
+		name          string
+		filePath      string
+		useMinVersion bool
+		want          []ftypes.Package
 	}{
 		{
 			name:     "happy path",
@@ -66,14 +67,20 @@ func TestParse(t *testing.T) {
 			filePath: "testdata/requirements_with_templating_engine.txt",
 			want:     nil,
 		},
+		{
+			name:          "compatible versions",
+			filePath:      "testdata/requirements_compatible.txt",
+			useMinVersion: true,
+			want:          requirementsCompatibleVersions,
+		},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			f, err := os.Open(tt.filePath)
 			require.NoError(t, err)
 
-			got, _, err := NewParser().Parse(f)
+			got, _, err := NewParser(tt.useMinVersion).Parse(f)
 			require.NoError(t, err)
 
 			assert.Equal(t, tt.want, got)

pkg/dependency/parser/python/pip/parse_testcase.go

@@ -3,6 +3,38 @@ package pip
 import ftypes "github.com/aquasecurity/trivy/pkg/fanal/types"
 
 var (
+	requirementsCompatibleVersions = []ftypes.Package{
+		{
+			Name:    "keyring",
+			Version: "4.1.1",
+			Locations: []ftypes.Location{
+				{
+					StartLine: 1,
+					EndLine:   1,
+				},
+			},
+		},
+		{
+			Name:    "Mopidy-Dirble",
+			Version: "1.1",
+			Locations: []ftypes.Location{
+				{
+					StartLine: 2,
+					EndLine:   2,
+				},
+			},
+		},
+		{
+			Name:    "python-gitlab",
+			Version: "2.0.0",
+			Locations: []ftypes.Location{
+				{
+					StartLine: 3,
+					EndLine:   3,
+				},
+			},
+		},
+	}
 	requirementsFlask = []ftypes.Package{
 		{
 			Name:    "click",

pkg/dependency/parser/python/pip/testdata/requirements_compatible.txt

@@ -0,0 +1,5 @@
+keyring >= 4.1.1            # Minimum version 4.1.1
+Mopidy-Dirble ~= 1.1        # Compatible release. Same as >= 1.1, == 1.*
+python-gitlab==2.0.*
+django==5.*.*               # this dep should be skipped
+django==4.*.1

pkg/fanal/analyzer/language/python/pip/pip.go

@@ -38,14 +38,16 @@ var pythonExecNames = []string{
 }
 
 type pipLibraryAnalyzer struct {
-	logger         *log.Logger
-	metadataParser packaging.Parser
+	logger            *log.Logger
+	metadataParser    packaging.Parser
+	detectionPriority types.DetectionPriority
 }
 
-func newPipLibraryAnalyzer(_ analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
+func newPipLibraryAnalyzer(opts analyzer.AnalyzerOptions) (analyzer.PostAnalyzer, error) {
 	return pipLibraryAnalyzer{
-		logger:         log.WithPrefix("pip"),
-		metadataParser: *packaging.NewParser(),
+		logger:            log.WithPrefix("pip"),
+		metadataParser:    *packaging.NewParser(),
+		detectionPriority: opts.DetectionPriority,
 	}, nil
 }
 
@@ -62,8 +64,10 @@ func (a pipLibraryAnalyzer) PostAnalyze(_ context.Context, input analyzer.PostAn
 		return true
 	}
 
+	useMinVersion := a.detectionPriority == types.PriorityComprehensive
+
 	if err = fsutils.WalkDir(input.FS, ".", required, func(pathPath string, d fs.DirEntry, r io.Reader) error {
-		app, err := language.Parse(types.Pip, pathPath, r, pip.NewParser())
+		app, err := language.Parse(types.Pip, pathPath, r, pip.NewParser(useMinVersion))
 		if err != nil {
 			return xerrors.Errorf("unable to parse requirements.txt: %w", err)
 		}