11import * as childProcess from 'child_process' ;
2+ import * as fs from 'fs' ;
23import * as path from 'path' ;
34import * as glob from 'glob' ;
45
6+ /**
7+ * The number of browsers we run the tests in.
8+ */
9+ const NUM_BROWSERS = 3 ;
10+
11+ /**
12+ * Assume that each test runs for 3s.
13+ */
14+ const ASSUMED_TEST_DURATION_SECONDS = 3 ;
15+
16+ /**
17+ * We keep the runtime of the detector if possible under 30min.
18+ */
19+ const MAX_TARGET_TEST_RUNTIME_SECONDS = 30 * 60 ;
20+
21+ /**
22+ * Running one test 50x is what we consider enough to detect flakiness.
23+ * Running one test 5x is the bare minimum
24+ */
25+ const MAX_PER_TEST_RUN_COUNT = 50 ;
26+ const MIN_PER_TEST_RUN_COUNT = 5 ;
27+
528async function run ( ) : Promise < void > {
629 let testPaths : string [ ] = [ ] ;
730
@@ -20,23 +43,8 @@ ${changedPaths.join('\n')}
2043 }
2144 }
2245
23- let runCount : number ;
24- if ( process . env . TEST_RUN_COUNT === 'AUTO' ) {
25- // No test paths detected: run everything 5x
26- runCount = 5 ;
27-
28- if ( testPaths . length > 0 ) {
29- // Run everything up to 100x, assuming that total runtime is less than 60min.
30- // We assume an average runtime of 3s per test, times 4 (for different browsers) = 12s per detected testPaths
31- // We want to keep overall runtime under 30min
32- const testCount = testPaths . length * 4 ;
33- const expectedRuntimePerTestPath = testCount * 3 ;
34- const expectedRuntime = Math . floor ( ( 30 * 60 ) / expectedRuntimePerTestPath ) ;
35- runCount = Math . min ( 50 , Math . max ( expectedRuntime , 5 ) ) ;
36- }
37- } else {
38- runCount = parseInt ( process . env . TEST_RUN_COUNT || '10' ) ;
39- }
46+ const repeatEachCount = getPerTestRunCount ( testPaths ) ;
47+ console . log ( `Running tests ${ repeatEachCount } ) ;
4048
4149 const cwd = path . join ( __dirname , '../' ) ;
4250
@@ -45,7 +53,7 @@ ${changedPaths.join('\n')}
4553 const cp = childProcess . spawn (
4654 `npx playwright test ${
4755 testPaths . length ? testPaths . join ( ' ' ) : './suites'
48- } --reporter='line' --repeat-each ${ runCount } ,
56+ } --reporter='line' --repeat-each ${ repeatEachCount } ,
4957 { shell : true , cwd } ,
5058 ) ;
5159
@@ -88,6 +96,33 @@ ${changedPaths.join('\n')}
8896 console . log ( `☑️ All tests passed.` ) ;
8997}
9098
99+ /**
100+ * Returns how many time one test should run based on the chosen mode and a bunch of heuristics
101+ */
102+ function getPerTestRunCount ( testPaths : string [ ] ) {
103+ if ( process . env . TEST_RUN_COUNT === 'AUTO' && testPaths . length > 0 ) {
104+ // Run everything up to 100x, assuming that total runtime is less than 60min.
105+ // We assume an average runtime of 3s per test, times 4 (for different browsers) = 12s per detected testPaths
106+ // We want to keep overall runtime under 30min
107+ const estimatedNumberOfTests = testPaths . map ( getApproximateNumberOfTests ) . reduce ( ( a , b ) => a + b ) ;
108+ console . log ( `Estimated number of tests: ${ estimatedNumberOfTests } ) ;
109+
110+ // tests are usually run against all browsers we test with, so let's assume this
111+ const testRunCount = estimatedNumberOfTests * NUM_BROWSERS ;
112+ console . log ( `Estimated test runs for one round: ${ testRunCount } ) ;
113+
114+ const estimatedTestRuntime = testRunCount * ASSUMED_TEST_DURATION_SECONDS ;
115+ console . log ( `Estimated test runtime: ${ estimatedTestRuntime } ) ;
116+
117+ const expectedPerTestRunCount = Math . floor ( MAX_TARGET_TEST_RUNTIME_SECONDS / estimatedTestRuntime ) ;
118+ console . log ( `Expected per-test run count: ${ expectedPerTestRunCount } ) ;
119+
120+ return Math . min ( MAX_PER_TEST_RUN_COUNT , Math . max ( expectedPerTestRunCount , MIN_PER_TEST_RUN_COUNT ) ) ;
121+ }
122+
123+ return parseInt ( process . env . TEST_RUN_COUNT || '5' ) ;
124+ }
125+
91126function getTestPaths ( ) : string [ ] {
92127 const paths = glob . sync ( 'suites/**/test.{ts,js}' , {
93128 cwd : path . join ( __dirname , '../' ) ,
@@ -111,4 +146,22 @@ function logError(error: unknown) {
111146 }
112147}
113148
149+ /**
150+ * Definitely not bulletproof way of getting the number of tests in a file :D
151+ * We simply match on `it(`, `test(`, etc and count the matches.
152+ *
153+ * Note: This test completely disregards parameterized tests (`it.each`, etc) or
154+ * skipped/disabled tests and other edge cases. It's just a rough estimate.
155+ */
156+ function getApproximateNumberOfTests ( testPath : string ) : number {
157+ try {
158+ const content = fs . readFileSync ( path . join ( process . cwd ( ) , testPath , 'test.ts' ) , 'utf-8' ) ;
159+ const matches = content . match ( / i t \( | t e s t \( | s e n t r y T e s t \( / g) ;
160+ return Math . max ( matches ? matches . length : 1 , 1 ) ;
161+ } catch ( e ) {
162+ console . error ( `Could not read file ${ testPath } ) ;
163+ return 1 ;
164+ }
165+ }
166+
114167run ( ) ;
0 commit comments