Fix license detection bugs

* Top level packages were inconsistant because of license plugin post
  processing running after the package plugin post processing.

* Adjust license clues/false-postives/bad matches heuristics.

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

Author: AyanSinhaMahapatra

etc/scripts/licenses/buildrules-template.txt

@@ -12,7 +12,9 @@ is_license_clue: yes
 referenced_filenames:
 notes: 
 ---
-
+The only changes we've made are to
+accommodate the licenses we use, which are GPLv3 and LGPLv3 (or later)
+whereas the Linux kernel uses GPLv2.
 ----------------------------------------
 license_expression: 
 relevance: 100

src/licensedcode/detection.py

@@ -25,6 +25,7 @@
 from commoncode.text import python_safe_name
 from licensedcode.cache import get_index
 from licensedcode.cache import get_cache
+from licensedcode.cache import build_spdx_license_expression
 from licensedcode.match import LicenseMatch
 from licensedcode.match import set_matched_lines
 from licensedcode.models import UnDetectedRule
@@ -105,6 +106,7 @@ class DetectionCategory(Enum):
     EXTRA_WORDS = 'extra-words'
     UNKNOWN_MATCH = 'unknown-match'
     LICENSE_CLUES = 'license-clues'
+    LOW_QUALITY_MATCHES = 'license-clues'
     IMPERFECT_COVERAGE = 'imperfect-match-coverage'
     FALSE_POSITVE = 'possible-false-positive'
     UNDETECTED_LICENSE = 'undetected-license'
@@ -730,6 +732,10 @@ def collect_license_detections(codebase, include_license_clues=True):
                     )
                     if not detection_is_same:
                         package["declared_license_expression"] = license_expression
+                        package["declared_license_expression_spdx"] = str(build_spdx_license_expression(
+                            license_expression=license_expression,
+                            licensing=get_cache().licensing,
+                        ))
                         modified = True
 
                 other_license_detections = package["other_license_detections"]
@@ -741,6 +747,10 @@ def collect_license_detections(codebase, include_license_clues=True):
                     )
                     if not detection_is_same:
                         package["other_license_expression"] = license_expression
+                        package["other_license_expression_spdx"] = str(build_spdx_license_expression(
+                            license_expression=license_expression,
+                            licensing=get_cache().licensing,
+                        ))
                         modified = True
 
             if modified:
@@ -753,6 +763,30 @@ def collect_license_detections(codebase, include_license_clues=True):
                 )
                 all_license_detections.extend(package_license_detection_objects)
 
+    if has_packages and has_licenses:
+        for package in getattr(codebase.attributes, 'packages', []):
+            license_expression_package = package["declared_license_expression"]
+            if not license_expression:
+                continue
+
+            resource_paths = package["datafile_paths"]
+            if len(resource_paths) == 1:
+                resource_path = resource_paths[0]
+            else:
+                #TODO: implement the correct consistency check
+                # based on which datafile path the license came from
+                resource_path = resource_paths[0]
+            resource = codebase.get_resource(path=resource_path)
+            resource_packages = getattr(resource, 'package_data', None)
+            if not resource_packages or len(resource_packages) > 1:
+                continue
+            
+            resource_package = resource_packages[0]
+            if license_expression_package != resource_package["declared_license_expression"]:
+                package["license_detections"] = resource_package["license_detections"]
+                package["declared_license_expression"] = resource_package["declared_license_expression"]
+                package["declared_license_expression_spdx"] = resource_package["declared_license_expression_spdx"]
+
     return all_license_detections
 
 
@@ -1107,7 +1141,7 @@ def has_correct_license_clue_matches(license_matches):
     return is_correct_detection(license_matches) and all(match.rule.is_license_clue for match in license_matches)
 
 
-def is_license_clues(license_matches):
+def is_low_quality_matches(license_matches):
     """
     Return True if the license_matches are not part of a correct
     license detection and are mere license clues.
@@ -1329,6 +1363,14 @@ def get_detected_license_expression(
         detection_log.append(DetectionRule.LICENSE_CLUES.value)
         return detection_log, combined_expression
 
+    elif analysis == DetectionCategory.LOW_QUALITY_MATCHES.value:
+        if TRACE_ANALYSIS:
+            logger_debug(f'analysis {DetectionCategory.LICENSE_CLUES.value}')
+        # TODO: we are temporarily returning these as license clues, and not
+        # in detections but ideally we should return synthetic unknowns for these
+        detection_log.append(DetectionRule.LOW_QUALITY_MATCHES.value)
+        return detection_log, combined_expression
+
     else:
         if TRACE_ANALYSIS:
             logger_debug(f'analysis not-combined')
@@ -1510,8 +1552,8 @@ def analyze_detection(license_matches, package_license=False):
     elif has_unknown_matches(license_matches=license_matches):
         return DetectionCategory.UNKNOWN_MATCH.value
 
-    elif is_license_clues(license_matches=license_matches):
-        return DetectionCategory.LICENSE_CLUES.value
+    elif not package_license and is_low_quality_matches(license_matches=license_matches):
+        return DetectionCategory.LOW_QUALITY_MATCHES.value
 
     # Case where at least one of the matches have `match_coverage`
     # below IMPERFECT_MATCH_COVERAGE_THR
@@ -1644,19 +1686,24 @@ def process_detections(detections, licensing=Licensing()):
 
         for detection in detections:
             if detection.license_expression == None:
-                license_keys = licensing.license_keys(expression=detection.license_expression)
-                if all(
-                    key in detected_license_keys
-                    for key in license_keys
-                ):
-                    detection.license_expression = str(combine_expressions(
+                if has_correct_license_clue_matches(detection.matches):
+                    continue
+
+                license_expression = str(combine_expressions(
                         expressions=[
                             match.rule.license_expression
                             for match in detection.matches
                         ],
                         unique=True,
                         licensing=licensing,
                     ))
+                license_keys = licensing.license_keys(expression=license_expression)
+
+                if all(
+                    key in detected_license_keys
+                    for key in license_keys
+                ):
+                    detection.license_expression = license_expression
                     detection.detection_log.append(DetectionRule.NOT_LICENSE_CLUES.value)
                     detection.identifier = detection.identifier_with_expression
 

tests/licensedcode/data/licenses_reference_reporting/license-reference-works-with-clues.expected.json

@@ -12,15 +12,6 @@
       "detection_count": 1,
       "detection_log": []
     },
-    {
-      "identifier": "bsd_simplified-7517fbd6-3fa4-e9f9-2167-c65251d77656",
-      "license_expression": "bsd-simplified",
-      "detection_count": 1,
-      "detection_log": [
-        "license-clues",
-        "not-license-clues-as-more-detections-present"
-      ]
-    },
     {
       "identifier": "bzip2_libbzip_2010-7158bcb2-a4d7-9815-17d2-1b1d0a6d5de2",
       "license_expression": "bzip2-libbzip-2010",
@@ -1194,8 +1185,8 @@
     {
       "path": "python.LICENSE",
       "type": "file",
-      "detected_license_expression": "python AND (other-copyleft AND gpl-1.0-plus) AND (python AND python-cwi) AND bzip2-libbzip-2010 AND sleepycat AND bsd-simplified AND bsd-new AND openssl-ssleay AND openssl AND ssleay-windows AND tcl",
-      "detected_license_expression_spdx": "Python-2.0 AND (LicenseRef-scancode-other-copyleft AND GPL-1.0-or-later) AND (Python-2.0 AND LicenseRef-scancode-python-cwi) AND bzip2-1.0.6 AND Sleepycat AND BSD-2-Clause AND BSD-3-Clause AND OpenSSL AND LicenseRef-scancode-openssl AND LicenseRef-scancode-ssleay-windows AND TCL",
+      "detected_license_expression": "python AND (other-copyleft AND gpl-1.0-plus) AND (python AND python-cwi) AND bzip2-libbzip-2010 AND sleepycat AND bsd-new AND openssl-ssleay AND openssl AND ssleay-windows AND tcl",
+      "detected_license_expression_spdx": "Python-2.0 AND (LicenseRef-scancode-other-copyleft AND GPL-1.0-or-later) AND (Python-2.0 AND LicenseRef-scancode-python-cwi) AND bzip2-1.0.6 AND Sleepycat AND BSD-3-Clause AND OpenSSL AND LicenseRef-scancode-openssl AND LicenseRef-scancode-ssleay-windows AND TCL",
       "license_detections": [
         {
           "license_expression": "python",
@@ -1431,29 +1422,6 @@
           ],
           "identifier": "sleepycat-a7cd8833-ecc2-8ade-54d7-392befcce801"
         },
-        {
-          "license_expression": "bsd-simplified",
-          "matches": [
-            {
-              "score": 33.71,
-              "start_line": 358,
-              "end_line": 363,
-              "matched_length": 59,
-              "match_coverage": 33.71,
-              "matcher": "3-seq",
-              "license_expression": "bsd-simplified",
-              "rule_identifier": "bsd-simplified_242.RULE",
-              "rule_relevance": 100,
-              "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-simplified_242.RULE",
-              "matched_text": "INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS\n * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN\n * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)\n * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF\n * THE POSSIBILITY OF SUCH DAMAGE."
-            }
-          ],
-          "detection_log": [
-            "license-clues",
-            "not-license-clues-as-more-detections-present"
-          ],
-          "identifier": "bsd_simplified-7517fbd6-3fa4-e9f9-2167-c65251d77656"
-        },
         {
           "license_expression": "bsd-new",
           "matches": [
@@ -1653,7 +1621,21 @@
           "identifier": "tcl-75d8de8c-9cf0-d604-4b99-e03436ebfcd3"
         }
       ],
-      "license_clues": [],
+      "license_clues": [
+        {
+          "score": 33.71,
+          "start_line": 358,
+          "end_line": 363,
+          "matched_length": 59,
+          "match_coverage": 33.71,
+          "matcher": "3-seq",
+          "license_expression": "bsd-simplified",
+          "rule_identifier": "bsd-simplified_242.RULE",
+          "rule_relevance": 100,
+          "rule_url": "https://github.com/nexB/scancode-toolkit/tree/develop/src/licensedcode/data/rules/bsd-simplified_242.RULE",
+          "matched_text": "INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF\n * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS\n * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN\n * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)\n * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF\n * THE POSSIBILITY OF SUCH DAMAGE."
+        }
+      ],
       "percentage_of_license_text": 83.64,
       "scan_errors": []
     }

tests/packagedcode/data/debian/copyright/debian-2019-11-15/main/m/mariadb-10.3/stable_copyright-detailed.expected.yml

@@ -545,7 +545,7 @@ other_license_detections:
            2. Redistributions in binary form must the following disclaimer in
              the documentation and/or other materials provided with the
              distribution.
-    identifier:
+    identifier: bsd_new-7c8321ea-5f82-974c-692b-936bcaabf520
   - license_expression: bsd-new
     matches:
       - score: '100.0'

tests/packagedcode/data/debian/copyright/debian-2019-11-15/main/p/perl/copyright-detailed.expected.yml

@@ -3337,7 +3337,7 @@ other_license_detections:
         matched_text: |
           license notice, but it is assumed that it
           is licensed under the same terms as
-    identifier:
+    identifier: artistic_perl_1_0_or_gpl_1_0_plus-f8a67153-d3ca-59da-be8b-68b1535b0862
   - license_expression: artistic-perl-1.0 OR gpl-1.0-plus
     matches:
       - score: '100.0'