chore: test docs check script (#368)

* chore: Add checking script for tests and docs

* chore: Fix issues discovered by tests-docs-check

* chore: Add tests-docs-checker to pre-commit hook

* docs: generate docs

* docs: Mention tests-docs-check in contributing guide

Author: pnevyk

.README/rules/no-flow-fix-me-comments.md

@@ -19,4 +19,4 @@ This rule takes an optional RegExp that comments a text RegExp that makes the su
 }
 ```
 
-<!-- assertions no-flow-fix-me-comments -->
+<!-- assertions noFlowFixMeComments -->

.README/rules/require-types-at-top.md

@@ -11,4 +11,4 @@ The rule has a string option:
 
 The default value is `"always"`.
 
-<!-- assertions require-types-at-top -->
+<!-- assertions requireTypesAtTop -->

.README/rules/valid-syntax.md

@@ -3,3 +3,5 @@
 **Deprecated** Babylon (the Babel parser) v6.10.0 fixes parsing of the invalid syntax this plugin warned against.
 
 Checks for simple Flow syntax errors.
+
+<!-- assertions validSyntax -->

CONTRIBUTING.md

@@ -6,6 +6,7 @@
 
 When making a commit, the following Pre-Commit hooks run:
 
+* tests and docs checks
 * tests
 * lint
 * commit message validation (see "Commit Messages" below)

README.md

@@ -1308,7 +1308,44 @@ This rule takes an optional RegExp that comments a text RegExp that makes the su
 }
 ```
 
-<!-- assertions no-flow-fix-me-comments -->
+The following patterns are considered problems:
+
+```js
+// $FlowFixMe I am doing something evil here
+const text = 'HELLO';
+// Message: $FlowFixMe is treated as `any` and should be fixed.
+
+// Options: ["TODO [0-9]+"]
+// $FlowFixMe I am doing something evil here
+const text = 'HELLO';
+// Message: $FlowFixMe is treated as `any` and should be fixed. Fix it or match `/TODO [0-9]+/`.
+
+// Options: ["TODO [0-9]+"]
+// $FlowFixMe TODO abc 47 I am doing something evil here
+const text = 'HELLO';
+// Message: $FlowFixMe is treated as `any` and should be fixed. Fix it or match `/TODO [0-9]+/`.
+
+// $$FlowFixMeProps I am doing something evil here
+const text = 'HELLO';
+// Message: $FlowFixMe is treated as `any` and should be fixed.
+
+// Options: ["TODO [0-9]+"]
+// $FlowFixMeProps I am doing something evil here
+const text = 'HELLO';
+// Message: $FlowFixMe is treated as `any` and should be fixed. Fix it or match `/TODO [0-9]+/`.
+```
+
+The following patterns are not considered problems:
+
+```js
+const text = 'HELLO';
+
+// Options: ["TODO [0-9]+"]
+// $FlowFixMe TODO 48
+const text = 'HELLO';
+```
+
+
 
 <a name="eslint-plugin-flowtype-rules-no-mutable-array"></a>
 ### <code>no-mutable-array</code>
@@ -2494,7 +2531,65 @@ The rule has a string option:
 
 The default value is `"always"`.
 
-<!-- assertions require-types-at-top -->
+The following patterns are considered problems:
+
+```js
+const foo = 3;
+type Foo = number;
+// Message: All type declaration should be at the top of the file, after any import declarations.
+
+const foo = 3;
+opaque type Foo = number;
+// Message: All type declaration should be at the top of the file, after any import declarations.
+
+const foo = 3;
+export type Foo = number;
+// Message: All type declaration should be at the top of the file, after any import declarations.
+
+const foo = 3;
+export opaque type Foo = number;
+// Message: All type declaration should be at the top of the file, after any import declarations.
+
+const foo = 3;
+type Foo = number | string;
+// Message: All type declaration should be at the top of the file, after any import declarations.
+
+import bar from "./bar";
+const foo = 3;
+type Foo = number;
+// Message: All type declaration should be at the top of the file, after any import declarations.
+```
+
+The following patterns are not considered problems:
+
+```js
+type Foo = number;
+const foo = 3;
+
+opaque type Foo = number;
+const foo = 3;
+
+export type Foo = number;
+const foo = 3;
+
+export opaque type Foo = number;
+const foo = 3;
+
+type Foo = number;
+const foo = 3;
+
+import bar from "./bar";
+type Foo = number;
+
+type Foo = number;
+import bar from "./bar";
+
+// Options: ["never"]
+const foo = 3;
+type Foo = number;
+```
+
+
 
 <a name="eslint-plugin-flowtype-rules-require-valid-file-annotation"></a>
 ### <code>require-valid-file-annotation</code>
@@ -4658,3 +4753,13 @@ declare var A: Y
 
 Checks for simple Flow syntax errors.
 
+The following patterns are not considered problems:
+
+```js
+function x(foo: string = "1") {}
+
+function x(foo: Type = bar()) {}
+```
+
+
+

bin/testsAndDocsCheck.js

@@ -0,0 +1,174 @@
+/**
+ * This script checks if there is test suite and documentation for every rule and if they are correctly included in corresponding "index" files.
+ *
+ * Performed checks:
+ *
+ *   - tests
+ *     - file `/tests/rules/assertions/<rule>.js` exists
+ *     - rule is included in `reportingRules` variable in `/tests/rules/index.js`
+ *
+ *   - docs
+ *     - file `/.README/rules/<rule>.md` exists
+ *     - file `/.README/rules/<rule>.md` contains correct assertions placeholder (`<!-- assertions ... -->`)
+ *     - rule is included in gitdown directive in `/.README/README.md`
+ *     - rules in `/.README/README.md` are alphabetically sorted
+ */
+
+import path from 'path';
+import fs from 'fs';
+import glob from 'glob';
+import _ from 'lodash';
+
+// returns Array<[camelCase, kebab-case]>
+const getRules = () => {
+  const rulesFiles = glob.sync(path.resolve(__dirname, '../src/rules/*.js'));
+
+  const rulesNames = rulesFiles.map((file) => {
+    return path.basename(file, '.js');
+  }).map((name) => {
+    return [name, _.kebabCase(name)];
+  });
+
+  return rulesNames;
+};
+
+const isFile = (filepath) => {
+  try {
+    return fs.statSync(filepath).isFile();
+  } catch (error) {
+    return false;
+  }
+};
+
+const windows = (array, size) => {
+  const output = [];
+
+  for (let ii = 0; ii < array.length - size + 1; ii++) {
+    output.push(array.slice(ii, ii + size));
+  }
+
+  return output;
+};
+
+const getTestIndexRules = () => {
+  const content = fs.readFileSync(path.resolve(__dirname, '../tests/rules/index.js'), 'utf-8');
+
+  const result = content.split('\n').reduce((acc, line) => {
+    if (acc.inRulesArray) {
+      if (line === '];') {
+        acc.inRulesArray = false;
+      } else {
+        acc.rules.push(line.replace(/^\s*'([^']+)',?$/, '$1'));
+      }
+    } else if (line === 'const reportingRules = [') {
+      acc.inRulesArray = true;
+    }
+
+    return acc;
+  }, {
+    inRulesArray: false,
+    rules: []
+  });
+
+  const rules = result.rules;
+
+  if (rules.length === 0) {
+    throw new Error('Tests checker is broken - it could not extract rules from test index file.');
+  }
+
+  return rules;
+};
+
+const getDocIndexRules = () => {
+  const content = fs.readFileSync(path.resolve(__dirname, '../.README/README.md'), 'utf-8');
+
+  const rules = content.split('\n').map((line) => {
+    const match = /^{"gitdown": "include", "file": "([^"]+)"}$/.exec(line);
+
+    if (match === null) {
+      return null;
+    } else {
+      return match[1].replace('./rules/', '').replace('.md', '');
+    }
+  }).filter((rule) => {
+    return rule !== null;
+  });
+
+  if (rules.length === 0) {
+    throw new Error('Docs checker is broken - it could not extract rules from docs index file.');
+  }
+
+  return rules;
+};
+
+const hasCorrectAssertions = (docPath, name) => {
+  const content = fs.readFileSync(docPath, 'utf-8');
+
+  const match = /<!-- assertions ([a-zA-Z]+) -->/.exec(content);
+
+  if (match === null) {
+    return false;
+  } else {
+    return match[1] === name;
+  }
+};
+
+const checkTests = (rulesNames) => {
+  const testIndexRules = getTestIndexRules();
+
+  const invalid = rulesNames.filter((names) => {
+    const testExists = isFile(path.resolve(__dirname, '../tests/rules/assertions', names[0] + '.js'));
+    const inIndex = testIndexRules.indexOf(names[1]) !== -1;
+
+    return !(testExists && inIndex);
+  });
+
+  if (invalid.length > 0) {
+    const invalidList = invalid.map((names) => {
+      return names[0];
+    }).join(', ');
+
+    throw new Error(
+      'Tests checker encountered an error in: ' + invalidList + '. ' +
+      'Make sure that for every rule you created test suite and included the rule name in `tests/rules/index.js` file.'
+    );
+  }
+};
+
+const checkDocs = (rulesNames) => {
+  const docIndexRules = getDocIndexRules();
+
+  const sorted = windows(docIndexRules, 2).every((chunk) => {
+    return chunk[0] < chunk[1];
+  });
+
+  if (!sorted) {
+    throw new Error('Rules are not alphabetically sorted in `.README/README.md` file.');
+  }
+
+  const invalid = rulesNames.filter((names) => {
+    const docPath = path.resolve(__dirname, '../.README/rules', names[1] + '.md');
+    const docExists = isFile(docPath);
+    const inIndex = docIndexRules.indexOf(names[1]) !== -1;
+    const hasAssertions = docExists ? hasCorrectAssertions(docPath, names[0]) : false;
+
+    return !(docExists && inIndex && hasAssertions);
+  });
+
+  if (invalid.length > 0) {
+    const invalidList = invalid.map((names) => {
+      return names[0];
+    }).join(', ');
+
+    throw new Error(
+      'Docs checker encountered an error in: ' + invalidList + '. ' +
+      'Make sure that for every rule you created documentation file with assertions placeholder in camelCase ' +
+      'and included the file path in `.README/README.md` file.'
+    );
+  }
+};
+
+const rulesList = getRules();
+
+checkTests(rulesList);
+checkDocs(rulesList);

package.json

@@ -43,7 +43,7 @@
   "husky": {
     "hooks": {
       "post-commit": "npm run create-readme && git add README.md && git commit -m 'docs: generate docs' --no-verify",
-      "pre-commit": "npm run lint && npm run test && npm run build && npm run format-json"
+      "pre-commit": "npm run tests-docs-check && npm run lint && npm run test && npm run build && npm run format-json"
     }
   },
   "repository": {
@@ -55,6 +55,7 @@
     "create-readme": "gitdown ./.README/README.md --output-file ./README.md && npm run documentation-add-assertions",
     "documentation-add-assertions": "babel-node ./bin/readmeAssertions",
     "format-json": "jsonlint --sort-keys --in-place --indent '  ' ./src/configs/recommended.json && echo '' >> ./src/configs/recommended.json",
+    "tests-docs-check": "babel-node ./bin/testsAndDocsCheck",
     "lint": "eslint ./src ./tests",
     "test": "mocha --compilers js:babel-register ./tests/rules/index.js"
   },

tests/rules/index.js

@@ -19,6 +19,7 @@ const reportingRules = [
   'generic-spacing',
   'newline-after-flow-annotation',
   'no-dupe-keys',
+  'no-existential-type',
   'no-flow-fix-me-comments',
   'no-mutable-array',
   'no-primitive-constructor-types',