Rebase of dss branch with upstream yelp detect secrets master

Use base.disable_flag_text functionality

Strict camelCase

Author: justineyster

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": "test_data/.*|tests/.*|^.secrets.baseline$",
     "lines": null
   },
-  "generated_at": "2019-08-13T20:44:51Z",
+  "generated_at": "2020-01-13T18:12:22Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -41,56 +41,56 @@
         "hashed_secret": "daefe0b4345a654580dcad25c7c11ff4c944a8c0",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 44,
+        "line_number": 47,
         "type": "Private Key"
       },
       {
         "hashed_secret": "f0778f3e140a61d5bbbed5430773e52af2f5fba4",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 45,
+        "line_number": 48,
         "type": "Private Key"
       },
       {
         "hashed_secret": "27c6929aef41ae2bcadac15ca6abcaff72cda9cd",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 46,
+        "line_number": 49,
         "type": "Private Key"
       },
       {
         "hashed_secret": "4ada9713ec27066b2ffe0b7bd9c9c8d635dc4ab2",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 47,
+        "line_number": 50,
         "type": "Private Key"
       },
       {
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 48,
+        "line_number": 51,
         "type": "Private Key"
       },
       {
         "hashed_secret": "be4fc4886bd949b369d5e092eb87494f12e57e5b",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 49,
+        "line_number": 52,
         "type": "Private Key"
       },
       {
         "hashed_secret": "9279619d0c9a9529b0b223e3b809f4df24b8ba8b",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 50,
+        "line_number": 53,
         "type": "Private Key"
       },
       {
         "hashed_secret": "11200d1bf5e1eb358b5d823c443347d97e982a85",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 51,
+        "line_number": 54,
         "type": "Private Key"
       }
     ],
@@ -236,7 +236,7 @@
         "hashed_secret": "1348b145fa1a555461c1b790a2f66614781091e9",
         "is_secret": false,
         "is_verified": false,
-        "line_number": 799,
+        "line_number": 849,
         "type": "Private Key"
       }
     ]

.travis.yml

@@ -26,6 +26,8 @@ deploy:
     branch: mvp
 matrix:
     include:
+        - env: TOXENV=py27
+          python: 2.7
         - env: TOXENV=py35
           python: 3.5
         - env: TOXENV=py36

README.md

@@ -106,7 +106,7 @@ The current heuristic searches we implement out of the box include:
 
 * **ArtifactoryDetector**: checks to see if Artifactory credentials are present.
 
-* **GHDetector**: checks to see if GitHub credentials are present.
+* **GheDetector**: checks to see if GitHub credentials are present.
 
 ## IBM versioning and rebase guide
 

detect_secrets/__init__.py

@@ -1 +1 @@
-VERSION = '0.12.5-ibm.6-mvp'
+VERSION = '0.13.0-ibm.7-dss'

detect_secrets/core/audit.py

@@ -25,7 +25,7 @@ class SecretNotFoundOnSpecifiedLineError(Exception):
     def __init__(self, line):
         super(SecretNotFoundOnSpecifiedLineError, self).__init__(
             'ERROR: Secret not found on line {}!\n'.format(line)
-            + 'Try recreating or updating your baseline to fix this issue.',
+            + 'Try recreating your baseline to fix this issue.',
         )
 
 
@@ -206,7 +206,7 @@ def compare_baselines(old_baseline_filename, new_baseline_filename):
         except SecretNotFoundOnSpecifiedLineError:
             decision = _get_user_decision(prompt_secret_decision=False)
 
-        if decision == 'q':  # pragma: no cover
+        if decision == 'q':
             print('Quitting...')
             break
 
@@ -562,7 +562,7 @@ def _print_context(  # pragma: no cover
         raise error_obj
 
 
-def _get_user_decision(prompt_secret_decision=True, can_step_back=False):  # pragma: no cover
+def _get_user_decision(prompt_secret_decision=True, can_step_back=False):
     """
     :type prompt_secret_decision: bool
     :param prompt_secret_decision: if False, won't ask to label secret.
@@ -574,7 +574,7 @@ def _get_user_decision(prompt_secret_decision=True, can_step_back=False):  # pra
         allowable_user_input.append('b')
 
     user_input = None
-    while user_input not in allowable_user_input:  # pragma: no cover
+    while user_input not in allowable_user_input:
         if user_input:
             print('Invalid input.')
 

detect_secrets/core/baseline.py

@@ -3,9 +3,11 @@
 import re
 import subprocess
 
+from detect_secrets import util
 from detect_secrets.core.log import get_logger
 from detect_secrets.core.secrets_collection import SecretsCollection
 
+
 log = get_logger(format_string='%(message)s')
 
 
@@ -43,13 +45,16 @@ def initialize(
 
     :type should_scan_all_files: bool
     :type output_raw: bool
+    :type output_verified_false: bool
     :rtype: SecretsCollection
     """
     output = SecretsCollection(
         plugins,
         custom_plugin_paths=custom_plugin_paths,
         exclude_files=exclude_files_regex,
         exclude_lines=exclude_lines_regex,
+        word_list_file=word_list_file,
+        word_list_hash=word_list_hash,
         output_raw=output_raw,
         output_verified_false=output_verified_false,
     )
@@ -294,8 +299,8 @@ def _get_git_tracked_files(rootdir='.'):
             git_files = subprocess.check_output(
                 [
                     'git',
+                    '-C', rootdir,
                     'ls-files',
-                    rootdir,
                 ],
                 stderr=fnull,
             )

detect_secrets/core/potential_secret.py

@@ -58,15 +58,7 @@ def __init__(
         self.verified_result = None
         self.other_factors = {}
 
-        # If two PotentialSecrets have the same values for these fields,
-        # they are considered equal. Note that line numbers aren't included
-        # in this, because line numbers are subject to change.
-        self.fields_to_compare = ['filename', 'secret_hash', 'type']
-
-    def set_secret(self, secret):
-        self.secret_hash = self.hash_secret(secret)
-
-        # Note: Originally, we never wanted to keep the secret value in memory,
+        # NOTE: Originally, we never wanted to keep the secret value in memory,
         #       after finding it in the codebase. However, to support verifiable
         #       secrets (and avoid the pain of re-scanning again), we need to
         #       keep the plaintext in memory as such.
@@ -76,11 +68,16 @@ def set_secret(self, secret):
         #       in the repository.
         self.secret_value = secret
         self.output_raw = output_raw
+
         # If two PotentialSecrets have the same values for these fields,
         # they are considered equal. Note that line numbers aren't included
         # in this, because line numbers are subject to change.
         self.fields_to_compare = ['filename', 'secret_hash', 'type']
 
+    def set_secret(self, secret):
+        self.secret_hash = self.hash_secret(secret)
+        self.secret_value = secret
+
     @staticmethod
     def hash_secret(secret):
         """This offers a way to coherently test this class,

detect_secrets/core/secrets_collection.py

@@ -21,28 +21,26 @@ def __init__(
         custom_plugin_paths=None,
         exclude_files=None,
         exclude_lines=None,
+        word_list_file=None,
+        word_list_hash=None,
         output_raw=False,
         output_verified_false=False,
     ):
         """
         :type plugins: tuple of detect_secrets.plugins.base.BasePlugin
         :param plugins: rules to determine whether a string is a secret
-
-        :type custom_plugin_paths: Tuple[str]|None
-        :param custom_plugin_paths: possibly empty tuple of paths that have custom plugins.
-
         :type exclude_files: str|None
         :param exclude_files: optional regex for ignored paths.
-
         :type exclude_lines: str|None
         :param exclude_lines: optional regex for ignored lines.
-
-        :type version: str
-        :param version: version of detect-secrets that SecretsCollection
-            is valid at.
-
+        :type word_list_file: str|None
+        :param word_list_file: optional word list file for ignoring certain words.
+        :type word_list_hash: str|None
+        :param word_list_hash: optional iterated sha1 hash of the words in the word list.
         :type output_raw: bool|None
-        :param output_raw: whether or not to output the raw, unhashed secret
+        :param output_raw: whether or not to output the raw, unhashed secret.
+        :type output_verified_false: bool
+        :param output_verified_false: whether to output secrets that fail verification.
         """
         self.data = {}
         self.version = VERSION
@@ -51,14 +49,16 @@ def __init__(
         self.custom_plugin_paths = custom_plugin_paths or ()
         self.exclude_files = exclude_files
         self.exclude_lines = exclude_lines
-<<<<<<< HEAD
+
+        self.output_raw = output_raw
+        self.output_verified_false = output_verified_false
+        self.version = VERSION
+
         self.word_list_file = word_list_file
         self.word_list_hash = word_list_hash
-=======
+        self.version = VERSION
         self.output_raw = output_raw
         self.output_verified_false = output_verified_false
-        self.version = VERSION
->>>>>>> Option to output raw secret
 
     @classmethod
     def load_baseline_from_string(cls, string):
@@ -347,10 +347,12 @@ def _extract_secrets_from_file(self, f, filename):
             log.info('Checking file: %s', filename)
 
             for results, plugin in self._results_accumulator(filename):
-                results.update(plugin.analyze(
-                    f, filename, self.output_raw,
-                    self.output_verified_false,
-                ))
+                results.update(
+                    plugin.analyze(
+                        f, filename, self.output_raw,
+                        self.output_verified_false,
+                    ),
+                )
                 f.seek(0)
 
         except UnicodeDecodeError: