Skip to content

Commit 15d791a

Browse files
jsjoeiojsjoeio
authored
Merge pull request #3016 from cdr/jsjoeio/refactor-e2e
dev(testing): add jest-playwright and reduce flakiness of e2e tests
2 parents 4ff7338 + ad0f12e commit 15d791a

21 files changed

+735
-266
lines changed

‎.eslintrc.yaml

+6-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,12 @@ extends:
1515
- plugin:import/recommended
1616
- plugin:import/typescript
1717
- plugin:prettier/recommended
18-
- prettier # Removes eslint rules that conflict with prettier.
18+
# Recommended by jest-playwright
19+
# https://github.com/playwright-community/jest-playwright#globals
20+
- plugin:jest-playwright/recommended
21+
# Prettier should always be last
22+
# Removes eslint rules that conflict with prettier.
23+
- prettier
1924

2025
rules:
2126
# Sometimes you need to add args to implement a function signature even

‎.github/workflows/ci.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ jobs:
6363
- uses: microsoft/playwright-github-action@v1
6464
- name: Install dependencies and run end-to-end tests
6565
run: |
66-
./release-packages/code-server*-linux-amd64/bin/code-server &
66+
./release-packages/code-server*-linux-amd64/bin/code-server --log trace &
6767
yarn --frozen-lockfile
6868
yarn test:e2e
6969
- name: Upload test artifacts

‎ci/dev/test-e2e.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ main() {
1515
echo -e "\n"
1616
exit 1
1717
fi
18-
CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts
18+
CS_DISABLE_PLUGINS=true ./test/node_modules/.bin/jest "$@" --config ./test/jest.e2e.config.ts --runInBand
1919
}
2020

2121
main "$@"

‎ci/steps/test-e2e.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ set -euo pipefail
44
main() {
55
cd "$(dirname "$0")/../.."
66

7-
"./release-packages/code-server*-linux-amd64/bin/code-server" --home "$CODE_SERVER_ADDRESS"/healthz &
7+
"./release-packages/code-server*-linux-amd64/bin/code-server" &
88
yarn --frozen-lockfile
99
yarn test:e2e
1010
}

‎lib/vscode/src/vs/platform/extensionManagement/node/extensionDownloader.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ export class ExtensionsDownloader extends Disposable {
7676
private async cleanUp(): Promise<void> {
7777
try {
7878
if (!(await this.fileService.exists(this.extensionsDownloadDir))) {
79-
this.logService.trace('Extension VSIX downlads cache dir does not exist');
79+
this.logService.trace('Extension VSIX downloads cache dir does not exist');
8080
return;
8181
}
8282
const folderStat = await this.fileService.resolve(this.extensionsDownloadDir, { resolveMetadata: true });

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@
6161
"eslint-config-prettier": "^8.1.0",
6262
"eslint-import-resolver-alias": "^1.1.2",
6363
"eslint-plugin-import": "^2.18.2",
64+
"eslint-plugin-jest-playwright": "^0.2.1",
6465
"eslint-plugin-prettier": "^3.1.0",
6566
"istanbul-badges-readme": "^1.2.0",
6667
"leaked-handles": "^5.2.0",

‎test/e2e/browser.test.ts

+35
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
/// <reference types="jest-playwright-preset" />
2+
3+
// This test is for nothing more than to make sure
4+
// tests are running in multiple browsers
5+
describe("Browser gutcheck", () => {
6+
beforeEach(async () => {
7+
await jestPlaywright.resetBrowser({
8+
logger: {
9+
isEnabled: (name) => name === "browser",
10+
log: (name, severity, message, args) => console.log(`${name} ${message}`),
11+
},
12+
})
13+
})
14+
15+
test("should display correct browser based on userAgent", async () => {
16+
const displayNames = {
17+
chromium: "Chrome",
18+
firefox: "Firefox",
19+
webkit: "Safari",
20+
}
21+
const userAgent = await page.evaluate("navigator.userAgent")
22+
23+
if (browserName === "chromium") {
24+
expect(userAgent).toContain(displayNames[browserName])
25+
}
26+
27+
if (browserName === "firefox") {
28+
expect(userAgent).toContain(displayNames[browserName])
29+
}
30+
31+
if (browserName === "webkit") {
32+
expect(userAgent).toContain(displayNames[browserName])
33+
}
34+
})
35+
})

‎test/e2e/e2e.test.ts

-24
This file was deleted.

‎test/e2e/globalSetup.test.ts

+21
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
/// <reference types="jest-playwright-preset" />
2+
import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
3+
4+
// This test is to make sure the globalSetup works as expected
5+
// meaning globalSetup ran and stored the storageState in STORAGE
6+
describe("globalSetup", () => {
7+
beforeEach(async () => {
8+
// Create a new context with the saved storage state
9+
// so we don't have to logged in
10+
const storageState = JSON.parse(STORAGE) || {}
11+
await jestPlaywright.resetContext({
12+
storageState,
13+
})
14+
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
15+
})
16+
17+
it("should keep us logged in using the storageState", async () => {
18+
// Make sure the editor actually loaded
19+
expect(await page.isVisible("div.monaco-workbench"))
20+
})
21+
})

‎test/e2e/login.test.ts

+6-25
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,19 @@
1-
import { chromium, Page, Browser, BrowserContext } from "playwright"
1+
/// <reference types="jest-playwright-preset" />
22
import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
33

44
describe("login", () => {
5-
let browser: Browser
6-
let page: Page
7-
let context: BrowserContext
8-
9-
beforeAll(async () => {
10-
browser = await chromium.launch()
11-
context = await browser.newContext()
12-
})
13-
14-
afterAll(async () => {
15-
await browser.close()
16-
})
17-
185
beforeEach(async () => {
19-
page = await context.newPage()
20-
})
21-
22-
afterEach(async () => {
23-
await page.close()
24-
// Remove password from local storage
25-
await context.clearCookies()
6+
await jestPlaywright.resetBrowser()
7+
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
268
})
279

2810
it("should be able to login", async () => {
29-
await page.goto(CODE_SERVER_ADDRESS)
3011
// Type in password
3112
await page.fill(".password", PASSWORD)
3213
// Click the submit button and login
3314
await page.click(".submit")
34-
// See the editor
35-
const codeServerEditor = await page.isVisible(".monaco-workbench")
36-
expect(codeServerEditor).toBeTruthy()
15+
await page.waitForLoadState("networkidle")
16+
// Make sure the editor actually loaded
17+
expect(await page.isVisible("div.monaco-workbench"))
3718
})
3819
})

‎test/e2e/loginPage.test.ts

+20
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
/// <reference types="jest-playwright-preset" />
2+
3+
import { CODE_SERVER_ADDRESS } from "../utils/constants"
4+
5+
describe("login page", () => {
6+
beforeEach(async () => {
7+
await jestPlaywright.resetContext({
8+
logger: {
9+
isEnabled: (name, severity) => name === "browser",
10+
log: (name, severity, message, args) => console.log(`${name} ${message}`),
11+
},
12+
})
13+
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
14+
})
15+
16+
it("should see the login page", async () => {
17+
// It should send us to the login page
18+
expect(await page.title()).toBe("code-server login")
19+
})
20+
})

‎test/e2e/logout.test.ts

+16-31
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,20 @@
1-
import { chromium, Page, Browser, BrowserContext } from "playwright"
2-
import { CODE_SERVER_ADDRESS, PASSWORD, E2E_VIDEO_DIR } from "../utils/constants"
1+
/// <reference types="jest-playwright-preset" />
2+
import { CODE_SERVER_ADDRESS, PASSWORD } from "../utils/constants"
33

44
describe("logout", () => {
5-
let browser: Browser
6-
let page: Page
7-
let context: BrowserContext
8-
9-
beforeAll(async () => {
10-
browser = await chromium.launch()
11-
context = await browser.newContext({
12-
recordVideo: { dir: E2E_VIDEO_DIR },
13-
})
14-
})
15-
16-
afterAll(async () => {
17-
await browser.close()
18-
})
19-
205
beforeEach(async () => {
21-
page = await context.newPage()
22-
})
23-
24-
afterEach(async () => {
25-
await page.close()
26-
// Remove password from local storage
27-
await context.clearCookies()
6+
await jestPlaywright.resetBrowser()
7+
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
288
})
299

3010
it("should be able login and logout", async () => {
31-
await page.goto(CODE_SERVER_ADDRESS)
3211
// Type in password
3312
await page.fill(".password", PASSWORD)
3413
// Click the submit button and login
3514
await page.click(".submit")
36-
// See the editor
37-
const codeServerEditor = await page.isVisible(".monaco-workbench")
38-
expect(codeServerEditor).toBeTruthy()
15+
await page.waitForLoadState("networkidle")
16+
// Make sure the editor actually loaded
17+
expect(await page.isVisible("div.monaco-workbench"))
3918

4019
// Click the Application menu
4120
await page.click("[aria-label='Application Menu']")
@@ -45,10 +24,16 @@ describe("logout", () => {
4524
expect(await page.isVisible(logoutButton))
4625

4726
await page.hover(logoutButton)
48-
49-
await page.click(logoutButton)
50-
// it takes a couple seconds to navigate
27+
// TODO(@jsjoeio)
28+
// Look into how we're attaching the handlers for the logout feature
29+
// We need to see how it's done upstream and add logging to the
30+
// handlers themselves.
31+
// They may be attached too slowly, hence why we need this timeout
5132
await page.waitForTimeout(2000)
33+
34+
// Recommended by Playwright for async navigation
35+
// https://github.com/microsoft/playwright/issues/1987#issuecomment-620182151
36+
await Promise.all([page.waitForNavigation(), page.click(logoutButton)])
5237
const currentUrl = page.url()
5338
expect(currentUrl).toBe(`${CODE_SERVER_ADDRESS}/login`)
5439
})

‎test/e2e/openHelpAbout.test.ts

+7-60
Original file line numberDiff line numberDiff line change
@@ -1,71 +1,18 @@
1-
import { chromium, Page, Browser, BrowserContext, Cookie } from "playwright"
2-
import { hash } from "../../src/node/util"
3-
import { CODE_SERVER_ADDRESS, PASSWORD, STORAGE, E2E_VIDEO_DIR } from "../utils/constants"
4-
import { createCookieIfDoesntExist } from "../utils/helpers"
1+
/// <reference types="jest-playwright-preset" />
2+
import { CODE_SERVER_ADDRESS, STORAGE } from "../utils/constants"
53

64
describe("Open Help > About", () => {
7-
let browser: Browser
8-
let page: Page
9-
let context: BrowserContext
10-
11-
beforeAll(async () => {
12-
browser = await chromium.launch()
5+
beforeEach(async () => {
136
// Create a new context with the saved storage state
7+
// so we don't have to logged in
148
const storageState = JSON.parse(STORAGE) || {}
15-
16-
const cookieToStore = {
17-
sameSite: "Lax" as const,
18-
name: "key",
19-
value: hash(PASSWORD),
20-
domain: "localhost",
21-
path: "/",
22-
expires: -1,
23-
httpOnly: false,
24-
secure: false,
25-
}
26-
27-
// For some odd reason, the login method used in globalSetup.ts doesn't always work
28-
// I don't know if it's on playwright clearing our cookies by accident
29-
// or if it's our cookies disappearing.
30-
// This means we need an additional check to make sure we're logged in.
31-
// We do this by manually adding the cookie to the browser environment
32-
// if it's not there at the time the test starts
33-
const cookies: Cookie[] = storageState.cookies || []
34-
// If the cookie exists in cookies then
35-
// this will return the cookies with no changes
36-
// otherwise if it doesn't exist, it will create it
37-
// hence the name maybeUpdatedCookies
38-
//
39-
// TODO(@jsjoeio)
40-
// The playwright storage thing sometimes works and sometimes doesn't. We should investigate this further
41-
// at some point.
42-
// See discussion: https://github.com/cdr/code-server/pull/2648#discussion_r575434946
43-
44-
const maybeUpdatedCookies = createCookieIfDoesntExist(cookies, cookieToStore)
45-
46-
context = await browser.newContext({
47-
storageState: { cookies: maybeUpdatedCookies },
48-
recordVideo: { dir: E2E_VIDEO_DIR },
9+
await jestPlaywright.resetContext({
10+
storageState,
4911
})
50-
})
51-
52-
afterAll(async () => {
53-
// Remove password from local storage
54-
await context.clearCookies()
55-
56-
await context.close()
57-
await browser.close()
58-
})
59-
60-
beforeEach(async () => {
61-
page = await context.newPage()
12+
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "networkidle" })
6213
})
6314

6415
it("should see a 'Help' then 'About' button in the Application Menu that opens a dialog", async () => {
65-
// waitUntil: "domcontentloaded"
66-
// In case the page takes a long time to load
67-
await page.goto(CODE_SERVER_ADDRESS, { waitUntil: "domcontentloaded" })
68-
6916
// Make sure the editor actually loaded
7017
expect(await page.isVisible("div.monaco-workbench"))
7118

‎test/jest.e2e.config.ts

+18-1
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,28 @@
22
import type { Config } from "@jest/types"
33

44
const config: Config.InitialOptions = {
5+
preset: "jest-playwright-preset",
56
transform: {
67
"^.+\\.ts$": "<rootDir>/node_modules/ts-jest",
78
},
89
globalSetup: "<rootDir>/utils/globalSetup.ts",
9-
testEnvironment: "node",
10+
testEnvironmentOptions: {
11+
"jest-playwright": {
12+
// TODO(@jsjoeio) enable on webkit and firefox
13+
// waiting on next playwright release
14+
// - https://github.com/microsoft/playwright/issues/6009#event-4536210890
15+
// - https://github.com/microsoft/playwright/issues/6020
16+
browsers: ["chromium"],
17+
// If there's a page error, we don't exit
18+
// i.e. something logged in the console
19+
exitOnPageError: false,
20+
contextOptions: {
21+
recordVideo: {
22+
dir: "./test/e2e/videos",
23+
},
24+
},
25+
},
26+
},
1027
testPathIgnorePatterns: ["/node_modules/", "/lib/", "/out/", "test/unit"],
1128
testTimeout: 30000,
1229
modulePathIgnorePatterns: [

‎test/package.json

+4
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,13 @@
77
"@types/node-fetch": "^2.5.8",
88
"@types/supertest": "^2.0.10",
99
"jest": "^26.6.3",
10+
"jest-playwright-preset": "^1.5.1",
1011
"jsdom": "^16.4.0",
1112
"node-fetch": "^2.6.1",
1213
"playwright": "^1.8.0",
14+
"playwright-chromium": "^1.10.0",
15+
"playwright-firefox": "^1.10.0",
16+
"playwright-webkit": "^1.10.0",
1317
"supertest": "^6.1.1",
1418
"ts-jest": "^26.4.4"
1519
}

0 commit comments

Comments
 (0)