Fix non-ignored test, check all tests are executed (#4262)

* put test_distributions in separate job

* 🎨 sort

* add arviz compat job

* fix ignored test

* document

* document

* noop

Author: MarcoGorelli

.github/workflows/pytest.yml

@@ -21,12 +21,13 @@ jobs:
             --ignore=pymc3/tests/test_distributions_timeseries.py
             --ignore=pymc3/tests/test_examples.py
             --ignore=pymc3/tests/test_gp.py
+            --ignore=pymc3/tests/test_mixture.py
             --ignore=pymc3/tests/test_parallel_sampling.py
             --ignore=pymc3/tests/test_posteriors.py
             --ignore=pymc3/tests/test_quadpotential.py
             --ignore=pymc3/tests/test_random.py
             --ignore=pymc3/tests/test_sampling.py
-            --ignore=pymc3/tests/test_shape_handling
+            --ignore=pymc3/tests/test_shape_handling.py
             --ignore=pymc3/tests/test_shared.py
             --ignore=pymc3/tests/test_smc.py
             --ignore=pymc3/tests/test_updates.py
@@ -44,6 +45,7 @@ jobs:
           - |
             pymc3/tests/test_examples.py
             pymc3/tests/test_gp.py
+            pymc3/tests/test_mixture.py
             pymc3/tests/test_posteriors.py
             pymc3/tests/test_quadpotential.py
           - |

.pre-commit-config.yaml

@@ -31,14 +31,21 @@ repos:
 - repo: local
   hooks:
     - id: watermark
-      name: Check notebooks have watermark (see Jupyter style guide from PyMC3 Wiki)
-      types: [jupyter]
+      args: [--negate, --multiline]
       entry: '%load_ext watermark.*%watermark -n -u -v -iv -w'
       language: pygrep
-      args: [--negate, --multiline]
       minimum_pre_commit_version: 2.8.0
+      name: Check notebooks have watermark (see Jupyter style guide from PyMC3 Wiki)
+      types: [jupyter]
     - id: check-toc
+      entry: python scripts/check_toc_is_complete.py
+      language: python
       name: Check all notebooks appear in table of contents
+      pass_filenames: false
       types: [jupyter]
-      entry: python scripts/check_toc_is_complete.py
+    - id: check-no-tests-are-ignored
+      entry: python scripts/check_all_tests_are_covered.py
+      files: ^\.github/workflows/pytest\.yml$
       language: python
+      name: Check no tests are ignored
+      pass_filenames: false

scripts/check_all_tests_are_covered.py

@@ -0,0 +1,24 @@
+"""
+In .github/workflows/pytest.yml, tests are split between multiple jobs.
+
+Here, we check that the jobs ignored by the first job actually end up getting
+run by the other jobs.
+This is intended to be used as a pre-commit hook, see `.pre-commit-config.yaml`.
+You can run it manually with `pre-commit run check-no-tests-are-ignored --all`.
+"""
+
+from pathlib import Path
+
+import re
+
+if __name__ == "__main__":
+    pytest_ci_job = Path(".github") / "workflows/pytest.yml"
+    txt = pytest_ci_job.read_text()
+    ignored_tests = set(re.findall(r"(?<=--ignore=)(pymc3/tests.*\.py)", txt))
+    non_ignored_tests = set(re.findall(r"(?<!--ignore=)(pymc3/tests.*\.py)", txt))
+    assert (
+        ignored_tests <= non_ignored_tests
+    ), f"The following tests are ignored by the first job but not run by the others: {ignored_tests.difference(non_ignored_tests)}"
+    assert (
+        ignored_tests >= non_ignored_tests
+    ), f"The following tests are run by multiple jobs: {non_ignored_tests.difference(ignored_tests)}"