Skip to content

Commit aaac579

Browse files
committed
fix flaky test detector
1 parent 01bf194 commit aaac579

File tree

1 file changed

+10
-39
lines changed

1 file changed

+10
-39
lines changed

dev-packages/browser-integration-tests/scripts/detectFlakyTests.ts

Lines changed: 10 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,9 @@ import * as path from 'path';
44
import * as glob from 'glob';
55

66
/**
7-
* The number of browsers we run the tests in.
7+
* Assume that each test runs for 3s.
88
*/
9-
const NUM_BROWSERS = 3;
10-
11-
/**
12-
* Assume that each test runs for 2s.
13-
*/
14-
const ASSUMED_TEST_DURATION_SECONDS = 2;
9+
const ASSUMED_TEST_DURATION_SECONDS = 3;
1510

1611
/**
1712
* We keep the runtime of the detector if possible under 30min.
@@ -51,22 +46,12 @@ ${changedPaths.join('\n')}
5146
try {
5247
await new Promise<void>((resolve, reject) => {
5348
const cp = childProcess.spawn(
54-
`npx playwright test ${
55-
testPaths.length ? testPaths.join(' ') : './suites'
56-
} --reporter='line' --repeat-each ${repeatEachCount}`,
57-
{ shell: true, cwd },
49+
`npx playwright test ${testPaths.length ? testPaths.join(' ') : './suites'} --repeat-each ${repeatEachCount}`,
50+
{ shell: true, cwd, stdio: 'inherit' },
5851
);
5952

6053
let error: Error | undefined;
6154

62-
cp.stdout.on('data', data => {
63-
console.log(data ? (data as object).toString() : '');
64-
});
65-
66-
cp.stderr.on('data', data => {
67-
console.log(data ? (data as object).toString() : '');
68-
});
69-
7055
cp.on('error', e => {
7156
console.error(e);
7257
error = e;
@@ -107,15 +92,16 @@ function getPerTestRunCount(testPaths: string[]) {
10792
const estimatedNumberOfTests = testPaths.map(getApproximateNumberOfTests).reduce((a, b) => a + b);
10893
console.log(`Estimated number of tests: ${estimatedNumberOfTests}`);
10994

110-
// tests are usually run against all browsers we test with, so let's assume this
111-
const testRunCount = estimatedNumberOfTests * NUM_BROWSERS;
95+
const testRunCount = estimatedNumberOfTests;
11296
console.log(`Estimated test runs for one round: ${testRunCount}`);
11397

11498
const estimatedTestRuntime = testRunCount * ASSUMED_TEST_DURATION_SECONDS;
11599
console.log(`Estimated test runtime: ${estimatedTestRuntime}s`);
116100

117101
const expectedPerTestRunCount = Math.floor(MAX_TARGET_TEST_RUNTIME_SECONDS / estimatedTestRuntime);
118-
console.log(`Expected per-test run count: ${expectedPerTestRunCount}`);
102+
console.log(
103+
`Calculated # of repetitions: ${expectedPerTestRunCount} (min ${MIN_PER_TEST_RUN_COUNT}, max ${MAX_PER_TEST_RUN_COUNT})`,
104+
);
119105

120106
return Math.min(MAX_PER_TEST_RUN_COUNT, Math.max(expectedPerTestRunCount, MIN_PER_TEST_RUN_COUNT));
121107
}
@@ -128,22 +114,7 @@ function getTestPaths(): string[] {
128114
cwd: path.join(__dirname, '../'),
129115
});
130116

131-
return paths.map(p => path.dirname(p));
132-
}
133-
134-
function logError(error: unknown) {
135-
if (process.env.CI) {
136-
console.log('::group::Test failed');
137-
} else {
138-
console.error(' ⚠️ Test failed:');
139-
}
140-
141-
console.log((error as any).stdout);
142-
console.log((error as any).stderr);
143-
144-
if (process.env.CI) {
145-
console.log('::endgroup::');
146-
}
117+
return paths.map(p => `${path.dirname(p)}/`);
147118
}
148119

149120
/**
@@ -156,7 +127,7 @@ function logError(error: unknown) {
156127
function getApproximateNumberOfTests(testPath: string): number {
157128
try {
158129
const content = fs.readFileSync(path.join(process.cwd(), testPath, 'test.ts'), 'utf-8');
159-
const matches = content.match(/it\(|test\(|sentryTest\(/g);
130+
const matches = content.match(/sentryTest\(/g);
160131
return Math.max(matches ? matches.length : 1, 1);
161132
} catch (e) {
162133
console.error(`Could not read file ${testPath}`);

0 commit comments

Comments
 (0)