benchmark each module in their own node process

.eslintignore

@@ -1,3 +1,3 @@
 **/node_modules
 docs/dist
-compiled/spectypes/build
+cases/spectypes/build

.eslintrc.json

@@ -1,5 +1,8 @@
 {
   "extends": "./node_modules/gts/",
+  "rules": {
+    "node/no-unpublished-import": "off"
+  },
   "env": {
     "jest": true
   }

.gitignore

@@ -80,7 +80,7 @@ typings/
 # nuxt.js build output
 .nuxt
 
-# react / gatsby 
+# react / gatsby
 public/
 
 # vuepress build output
@@ -99,3 +99,6 @@ public/
 
 # sourcemaps
 docs/dist/app.js.map
+
+# spectype build artifacts
+cases/spectypes/build

README.md

@@ -1,5 +1,9 @@
 # 📊 Benchmark Comparison of Packages with Runtime Validation and TypeScript Support
 
+- - - -
+**⚡⚠ Benchmark results have changed after switching to isolated node processes for each benchmarked package, see [#864](https://github.com/moltar/typescript-runtime-type-benchmarks/issues/864) ⚠⚡**
+- - - -
+
 ## Benchmark Results
 
 [![Fastest Packages - click to view details](docs/results/preview.svg)](https://moltar.github.io/typescript-runtime-type-benchmarks)
@@ -74,3 +78,10 @@ function isMyDataValid(data: any) {
 // `res` is now type casted to the right type
 const res = isMyDataValid(data)
 ```
+
+## Local Development
+
+* `npm run start` - run benchmarks for all modules
+* `npm run start run zod myzod valita` - run benchmarks only for a few selected modules
+* `npm run docs:serve` - result viewer
+* `npm run test` - run tests on all modules

benchmarks/helpers/main.ts

@@ -1,5 +1,5 @@
 import { add, complete, cycle, suite } from 'benny';
-import { readFileSync, writeFileSync } from 'fs';
+import { readFileSync, writeFileSync, existsSync, unlinkSync } from 'fs';
 import { join } from 'path';
 import { writePreviewGraph } from './graph';
 import { getRegisteredBenchmarks } from './register';
@@ -10,22 +10,24 @@ const NODE_VERSION = process.env.NODE_VERSION || process.version;
 const NODE_VERSION_FOR_PREVIEW = 17;
 const TEST_PREVIEW_GENERATION = false;
 
-export async function main() {
+/**
+ * Run all registered benchmarks and append the results to a file.
+ */
+export async function runAllBenchmarks() {
   if (TEST_PREVIEW_GENERATION) {
-    // just generate the preview without using benchmark data from a previous run
+    // during development: generate the preview using benchmark data from a previous run
     const allResults: BenchmarkResult[] = JSON.parse(
       readFileSync(join(DOCS_DIR, 'results', 'node-17.json')).toString()
     ).results;
 
     await writePreviewGraph({
-      filename: join(DOCS_DIR, 'results', 'preview.svg'),
+      filename: previewSvgFilename(),
       values: allResults,
     });
 
     return;
   }
 
-  const majorVersion = getNodeMajorVersion();
   const allResults: BenchmarkResult[] = [];
 
   for (const [benchmark, benchmarks] of getRegisteredBenchmarks()) {
@@ -42,24 +44,40 @@ export async function main() {
     });
   }
 
-  writeFileSync(
-    join(DOCS_DIR, 'results', `node-${majorVersion}.json`),
+  // collect results of isolated benchmark runs into a single file
+  appendResults(allResults);
+}
 
-    JSON.stringify({
-      results: allResults,
-    }),
+/**
+ * Remove the results json file.
+ */
+export function deleteResults() {
+  const fileName = resultsJsonFilename();
 
-    { encoding: 'utf8' }
-  );
+  if (existsSync(fileName)) {
+    unlinkSync(fileName);
+  }
+}
+
+/**
+ * Generate the preview svg shown in the readme.
+ */
+export async function createPreviewGraph() {
+  const majorVersion = getNodeMajorVersion();
 
   if (majorVersion === NODE_VERSION_FOR_PREVIEW) {
+    const allResults: BenchmarkResult[] = JSON.parse(
+      readFileSync(resultsJsonFilename()).toString()
+    ).results;
+
     await writePreviewGraph({
-      filename: join(DOCS_DIR, 'results', 'preview.svg'),
+      filename: previewSvgFilename(),
       values: allResults,
     });
   }
 }
 
+// run a benchmark fn with benny
 async function runBenchmarks(name: string, cases: BenchmarkCase[]) {
   const fns = cases.map(c => add(c.moduleName, () => c.run()));
 
@@ -74,6 +92,52 @@ async function runBenchmarks(name: string, cases: BenchmarkCase[]) {
   );
 }
 
+// append results to an existing file or create a new one
+function appendResults(results: BenchmarkResult[]) {
+  const fileName = resultsJsonFilename();
+  const existingResults: BenchmarkResult[] = existsSync(fileName)
+    ? JSON.parse(readFileSync(fileName).toString()).results
+    : [];
+
+  // check that we're appending unique data
+  const getKey = ({
+    benchmark,
+    name,
+    nodeVersion,
+  }: BenchmarkResult): string => {
+    return JSON.stringify({ benchmark, name, nodeVersion });
+  };
+  const existingResultsIndex = new Set(existingResults.map(r => getKey(r)));
+
+  results.forEach(r => {
+    if (existingResultsIndex.has(getKey(r))) {
+      console.error('Result %s already exists in', getKey(r), fileName);
+
+      throw new Error('Duplicate result in result json file');
+    }
+  });
+
+  writeFileSync(
+    fileName,
+
+    JSON.stringify({
+      results: [...existingResults, ...results],
+    }),
+
+    { encoding: 'utf8' }
+  );
+}
+
+function resultsJsonFilename() {
+  const majorVersion = getNodeMajorVersion();
+
+  return join(DOCS_DIR, 'results', `node-${majorVersion}.json`);
+}
+
+function previewSvgFilename() {
+  return join(DOCS_DIR, 'results', 'preview.svg');
+}
+
 function getNodeMajorVersion() {
   let majorVersion = 0;
 

benchmarks/index.ts

@@ -1,4 +1,8 @@
-export { main } from './helpers/main';
+export {
+  runAllBenchmarks,
+  createPreviewGraph,
+  deleteResults,
+} from './helpers/main';
 export {
   addCase,
   AvailableBenchmarksIds,

cases/index.ts

@@ -1,28 +1,36 @@
-import './ajv';
-import './bueno';
-import './class-validator';
-import './computed-types';
-import './decoders';
-import './io-ts';
-import './jointz';
-import './json-decoder';
-import './marshal';
-import './mojotech-json-type-validation';
-import './myzod';
-import './ok-computer';
-import './purify-ts';
-import './rulr';
-import './runtypes';
-import './simple-runtypes';
-import './spectypes';
-import './superstruct';
-import './suretype';
-import './toi';
-import './tson';
-import './ts-interface-checker';
-import './ts-json-validator';
-import './ts-utils';
-import './typeofweb-schema';
-import './valita';
-import './yup';
-import './zod';
+export const cases = [
+  'ajv',
+  'bueno',
+  'class-validator',
+  'computed-types',
+  'decoders',
+  'io-ts',
+  'jointz',
+  'json-decoder',
+  'marshal',
+  'mojotech-json-type-validation',
+  'myzod',
+  'ok-computer',
+  'purify-ts',
+  'rulr',
+  'runtypes',
+  'simple-runtypes',
+  'spectypes',
+  'superstruct',
+  'suretype',
+  'toi',
+  'ts-interface-checker',
+  'ts-json-validator',
+  'ts-utils',
+  'tson',
+  'typeofweb-schema',
+  'valita',
+  'yup',
+  'zod',
+] as const;
+
+export type CaseName = typeof cases[number];
+
+export async function importCase(caseName: CaseName) {
+  await import('./' + caseName);
+}

index.ts

@@ -1,4 +1,71 @@
-import { main } from './benchmarks';
-import './cases';
+import * as childProcess from 'child_process';
+import * as benchmarks from './benchmarks';
+import * as cases from './cases';
 
-main();
+async function main() {
+  // a runtype lib would be handy here to check the passed command names ;)
+  const [command, ...args] = process.argv.slice(2);
+
+  switch (command) {
+    case undefined:
+    case 'run':
+      // run the given or all benchmarks, each in its own node process, see
+      // https://github.com/moltar/typescript-runtime-type-benchmarks/issues/864
+      {
+        console.log('Removing previous results');
+        benchmarks.deleteResults();
+
+        const caseNames = args.length ? args : cases.cases;
+
+        for (const c of caseNames) {
+          if (c === 'spectypes') {
+            // hack: manually run the spectypes compilation step - avoids
+            // having to run it before any other benchmark, esp when working
+            // locally and checking against a few selected ones.
+            childProcess.execSync('npm run compile:spectypes', {
+              stdio: 'inherit',
+            });
+          }
+
+          const cmd = [...process.argv.slice(0, 2), 'run-internal', c];
+
+          console.log('Executing "%s"', c);
+          childProcess.execSync(cmd.join(' '), {
+            stdio: 'inherit',
+          });
+        }
+      }
+      break;
+
+    case 'create-preview-svg':
+      // separate command, because preview generation needs the accumulated
+      // results from the benchmark runs
+      await benchmarks.createPreviewGraph();
+      break;
+
+    case 'run-internal':
+      // run the given benchmark(s) & append the results
+      {
+        const caseNames = args as cases.CaseName[];
+
+        for (const c of caseNames) {
+          console.log('Loading "%s"', c);
+
+          await cases.importCase(c);
+        }
+
+        await benchmarks.runAllBenchmarks();
+      }
+      break;
+
+    default:
+      console.error('unknown command:', command);
+
+      // eslint-disable-next-line no-process-exit
+      process.exit(1);
+  }
+}
+
+main().catch(e => {
+  throw e;
+});

package.json

@@ -11,7 +11,7 @@
   "scripts": {
     "lint": "gts check",
     "lint:fix": "gts fix",
-    "start": "npm run compile:spectypes && ts-node index.ts",
+    "start": "ts-node index.ts",
     "test:build": "npm run compile:spectypes && tsc --noEmit",
     "test": "npm run compile:spectypes && jest",
     "docs:serve": "serve docs",

start.sh

@@ -4,4 +4,5 @@ set -ex
 
 export NODE_VERSION="${NODE_VERSION:-$(node -v)}"
 
-npm start
+npm run start
+npm run start create-preview-svg