Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: run global setup in test tools (#38040)
Pavel Feldman12075443bcfa
diff --git a/.github/workflows/tests_mcp.yml b/.github/workflows/tests_mcp.ymlindex 1beebe5249..753bb7412e 100644--- a/.github/workflows/tests_mcp.yml+++ b/.github/workflows/tests_mcp.yml@@ -23,7 +23,6 @@ env:# Force terminal colors. @see https://www.npmjs.com/package/colorsFORCE_COLOR: 1ELECTRON_SKIP_BINARY_DOWNLOAD: 1- DEBUG_GIT_COMMIT_INFO: 1jobs:test_mcp:diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.ymlindex 2cb96d9ad6..304870c65d 100644--- a/.github/workflows/tests_primary.yml+++ b/.github/workflows/tests_primary.yml@@ -25,7 +25,6 @@ env:# Force terminal colors. @see https://www.npmjs.com/package/colorsFORCE_COLOR: 1ELECTRON_SKIP_BINARY_DOWNLOAD: 1- DEBUG_GIT_COMMIT_INFO: 1jobs:test_linux:@@ -177,7 +176,6 @@ jobs:runs-on: ubuntu-latestenv:PWTEST_BOT_NAME: "vscode-extension"- DEBUG_GIT_COMMIT_INFO: ""steps:- uses: actions/checkout@v5- uses: actions/setup-node@v5diff --git a/packages/playwright/src/mcp/test/streams.ts b/packages/playwright/src/mcp/test/streams.tsindex f78eb82e3f..4513411016 100644--- a/packages/playwright/src/mcp/test/streams.ts+++ b/packages/playwright/src/mcp/test/streams.ts@@ -16,20 +16,24 @@import { Writable } from 'stream';+import { stripAnsiEscapes } from '../../util';+import type { ProgressCallback } from '../sdk/server';export class StringWriteStream extends Writable {private _progress: ProgressCallback;+ private _prefix: string;- constructor(progress: ProgressCallback) {+ constructor(progress: ProgressCallback, stdio: 'stdout' | 'stderr') {super();this._progress = progress;+ this._prefix = stdio === 'stdout' ? '' : '[err] ';}override _write(chunk: any, encoding: any, callback: any) {- const text = chunk.toString();+ const text = stripAnsiEscapes(chunk.toString());// Progress wraps these as individual messages.- this._progress({ message: text.endsWith('\n') ? text.slice(0, -1) : text });+ this._progress({ message: `${this._prefix}${text.endsWith('\n') ? text.slice(0, -1) : text}` });callback();}}diff --git a/packages/playwright/src/mcp/test/testContext.ts b/packages/playwright/src/mcp/test/testContext.tsindex 60a2d16b8f..d71a41ff48 100644--- a/packages/playwright/src/mcp/test/testContext.ts+++ b/packages/playwright/src/mcp/test/testContext.ts@@ -138,45 +138,93 @@ export class TestContext {}async runSeedTest(seedFile: string, projectName: string, progress: ProgressCallback) {- const { screen } = this.createScreen(progress);+ await this.runWithGlobalSetup(async (testRunner, reporter) => {+ const result = await testRunner.runTests(reporter, {+ headed: !this.options?.headless,+ locations: ['/' + escapeRegExp(seedFile) + '/'],+ projects: [projectName],+ timeout: 0,+ workers: 1,+ pauseAtEnd: true,+ disableConfigReporters: true,+ failOnLoadErrors: true,+ });+ // Ideally, we should check that page was indeed created and browser mcp has kicked in.+ // However, that is handled in the upper layer, so hard to check here.+ if (result.status === 'passed' && !reporter.suite?.allTests().length)+ throw new Error('seed test not found.');++ if (result.status !== 'passed')+ throw new Error('Errors while running the seed test.');+ }, progress);+ }++ async runWithGlobalSetup(+ callback: (testRunner: TestRunner, reporter: ListReporter) => Promise<void>,+ progress: ProgressCallback): Promise<void> {+ const { screen, claimStdio, releaseStdio } = createScreen(progress);const configDir = this.configLocation.configDir;- const reporter = new ListReporter({ configDir, screen });const testRunner = await this.createTestRunner();- const result = await testRunner.runTests(reporter, {- headed: !this.options?.headless,- locations: ['/' + escapeRegExp(seedFile) + '/'],- projects: [projectName],- timeout: 0,- workers: 1,- pauseAtEnd: true,- disableConfigReporters: true,- failOnLoadErrors: true,- });+ claimStdio();+ try {+ const setupReporter = new ListReporter({ configDir, screen, includeTestId: true });+ const { status } = await testRunner.runGlobalSetup([setupReporter]);+ if (status !== 'passed')+ throw new Error('Failed to run global setup');+ } finally {+ releaseStdio();+ }- // Ideally, we should check that page was indeed created and browser mcp has kicked in.- // However, that is handled in the upper layer, so hard to check here.- if (result.status === 'passed' && !reporter.suite?.allTests().length)- throw new Error('seed test not found.');+ try {+ const reporter = new ListReporter({ configDir, screen, includeTestId: true });+ return await callback(testRunner, reporter);+ } finally {+ claimStdio();+ await testRunner.runGlobalTeardown().finally(() => {+ releaseStdio();+ });+ }+ }- if (result.status !== 'passed')- throw new Error('Errors while running the seed test.');+ async close() {}+}- createScreen(progress: ProgressCallback) {- const stream = new StringWriteStream(progress);- const screen = {- ...terminalScreen,- isTTY: false,- colors: noColors,- stdout: stream as unknown as NodeJS.WriteStream,- stderr: stream as unknown as NodeJS.WriteStream,+export function createScreen(progress: ProgressCallback) {+ const stdout = new StringWriteStream(progress, 'stdout');+ const stderr = new StringWriteStream(progress, 'stderr');++ const screen = {+ ...terminalScreen,+ isTTY: false,+ colors: noColors,+ stdout: stdout as unknown as NodeJS.WriteStream,+ stderr: stderr as unknown as NodeJS.WriteStream,+ };++ /* eslint-disable no-restricted-properties */+ const originalStdoutWrite = process.stdout.write;+ const originalStderrWrite = process.stderr.write;++ const claimStdio = () => {+ process.stdout.write = (chunk: string | Buffer) => {+ stdout.write(chunk);+ return true;};- return { screen, stream };- }+ process.stderr.write = (chunk: string | Buffer) => {+ stderr.write(chunk);+ return true;+ };+ };- async close() {- }+ const releaseStdio = () => {+ process.stdout.write = originalStdoutWrite;+ process.stderr.write = originalStderrWrite;+ };+ /* eslint-enable no-restricted-properties */++ return { screen, claimStdio, releaseStdio };}const bestPracticesMarkdown = `diff --git a/packages/playwright/src/mcp/test/testTools.ts b/packages/playwright/src/mcp/test/testTools.tsindex a3c86ed080..7a4f0bda8d 100644--- a/packages/playwright/src/mcp/test/testTools.ts+++ b/packages/playwright/src/mcp/test/testTools.ts@@ -15,9 +15,9 @@*/import { z } from '../sdk/bundle';-import ListReporter from '../../reporters/list';import ListModeReporter from '../../reporters/listModeReporter';import { defineTestTool } from './testTool';+import { createScreen } from './testContext';export const listTests = defineTestTool({schema: {@@ -29,7 +29,7 @@ export const listTests = defineTestTool({},handle: async (context, _, progress) => {- const { screen } = context.createScreen(progress);+ const { screen } = createScreen(progress);const reporter = new ListModeReporter({ screen, includeTestId: true });const testRunner = await context.createTestRunner();await testRunner.listTests(reporter, {});@@ -51,15 +51,13 @@ export const runTests = defineTestTool({},handle: async (context, params, progress) => {- const { screen } = context.createScreen(progress);- const configDir = context.configLocation.configDir;- const reporter = new ListReporter({ configDir, screen, includeTestId: true, prefixStdio: 'out' });- const testRunner = await context.createTestRunner();- await testRunner.runTests(reporter, {- locations: params.locations,- projects: params.projects,- disableConfigReporters: true,- });+ await context.runWithGlobalSetup(async (testRunner, reporter) => {+ await testRunner.runTests(reporter, {+ locations: params.locations,+ projects: params.projects,+ disableConfigReporters: true,+ });+ }, progress);return { content: [] };},@@ -80,19 +78,17 @@ export const debugTest = defineTestTool({},handle: async (context, params, progress) => {- const { screen } = context.createScreen(progress);- const configDir = context.configLocation.configDir;- const reporter = new ListReporter({ configDir, screen, includeTestId: true, prefixStdio: 'out' });- const testRunner = await context.createTestRunner();- await testRunner.runTests(reporter, {- headed: !context.options?.headless,- testIds: [params.test.id],- // For automatic recovery- timeout: 0,- workers: 1,- pauseOnError: true,- disableConfigReporters: true,- });+ await context.runWithGlobalSetup(async (testRunner, reporter) => {+ await testRunner.runTests(reporter, {+ headed: !context.options?.headless,+ testIds: [params.test.id],+ // For automatic recovery+ timeout: 0,+ workers: 1,+ pauseOnError: true,+ disableConfigReporters: true,+ });+ }, progress);return { content: [] };},diff --git a/packages/playwright/src/reporters/list.ts b/packages/playwright/src/reporters/list.tsindex 591883e3bf..b2b399e6eb 100644--- a/packages/playwright/src/reporters/list.ts+++ b/packages/playwright/src/reporters/list.ts@@ -38,12 +38,10 @@ class ListReporter extends TerminalReporter {private _stepIndex = new Map<TestStep, string>();private _needNewLine = false;private _printSteps: boolean;- private _prefixStdio?: string;- constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions & { prefixStdio?: string }) {+ constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions) {super(options);this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options?.printSteps);- this._prefixStdio = options?.prefixStdio;}override onBegin(suite: Suite) {@@ -164,10 +162,7 @@ class ListReporter extends TerminalReporter {return;const text = chunk.toString('utf-8');this._updateLineCountAndNewLineFlagForOutput(text);- if (this._prefixStdio)- stream.write(`[${stdio}] ${chunk}`);- else- stream.write(chunk);+ stream.write(chunk);}override onTestEnd(test: TestCase, result: TestResult) {diff --git a/tests/mcp/planner.spec.ts b/tests/mcp/planner.spec.tsindex 64684a56c2..1917660b52 100644--- a/tests/mcp/planner.spec.ts+++ b/tests/mcp/planner.spec.ts@@ -183,6 +183,7 @@ test('planner_setup_page (loading error)', async ({ startClient }) => {});test('planner_setup_page (wrong test location)', async ({ startClient }) => {+ await writeFiles({});const { client } = await startClient();expect(await client.callTool({name: 'planner_setup_page',@@ -196,6 +197,7 @@ test('planner_setup_page (wrong test location)', async ({ startClient }) => {});test('planner_setup_page (no test location)', async ({ startClient }) => {+ await writeFiles({});const { client } = await startClient();expect(await client.callTool({name: 'planner_setup_page',diff --git a/tests/mcp/seed-default-project.spec.ts b/tests/mcp/seed-default-project.spec.tsindex c0acdef706..beac69d682 100644--- a/tests/mcp/seed-default-project.spec.ts+++ b/tests/mcp/seed-default-project.spec.ts@@ -47,6 +47,23 @@ test('seed test runs in first top-level project by default', async ({ startClienexpect(fs.existsSync(path.join(baseDir, 'third', 'seed.spec.ts'))).toBe(false);});+test('respects provided seed test', async ({ startClient }) => {+ await writeFiles({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('test', async ({ page }) => {});+ `,+ });++ const { client } = await startClient();+ expect(await client.callTool({+ name: 'planner_setup_page',+ arguments: {+ seedFile: 'a.test.ts',+ },+ })).toHaveTextResponse(expect.stringContaining(`### Paused at end of test. ready for interaction`));+});+test('seed test runs in first top-level project with dependencies', async ({ startClient }) => {const baseDir = await writeFiles({'playwright.config.ts': `diff --git a/tests/mcp/test-debug.spec.ts b/tests/mcp/test-debug.spec.tsindex 0c3b031ef2..b0f1a87581 100644--- a/tests/mcp/test-debug.spec.ts+++ b/tests/mcp/test-debug.spec.ts@@ -323,7 +323,7 @@ test('test_debug w/ console.log in test', async ({ startClient }) => {},})).toHaveTextResponse(expect.stringContaining(`Running 1 test using 1 worker-[out] console.log+console.log[err] console.error### Paused on error:Error: expect(locator).toBeVisible() failed`));diff --git a/tests/mcp/test-setup.spec.ts b/tests/mcp/test-setup.spec.tsnew file mode 100644index 0000000000..455b1fcb38--- /dev/null+++ b/tests/mcp/test-setup.spec.ts@@ -0,0 +1,131 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { test, expect, writeFiles } from './fixtures';++test.use({ mcpServerType: 'test-mcp' });++const workspace = {+ 'playwright.config.ts': `+ module.exports = {+ projects: [{ name: 'chromium' }],+ webServer: {+ command: 'node web-server.js',+ stdout: 'pipe',+ stderr: 'pipe',+ wait: { stderr: /started/ }+ },+ globalSetup: 'global-setup.ts',+ globalTeardown: 'global-teardown.ts',+ };+ `,+ 'web-server.js': `+ console.log('web server started');+ console.error('web server started');+ `,+ 'global-setup.ts': `+ module.exports = async () => {+ console.log('global setup');+ console.error('global setup');+ };+ `,+ 'global-teardown.ts': `+ module.exports = async () => {+ console.log('global teardown');+ console.error('global teardown');+ };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('test', async ({ page }) => {+ console.log('test');+ console.error('test');+ });+ `+};++test('setup should run global setup and teardown', async ({ startClient }, { workerIndex }) => {+ await writeFiles(workspace);+ const { client } = await startClient();++ // Call planner_setup_page without specifying a project - should use first top-level project+ expect(await client.callTool({+ name: 'planner_setup_page',+ arguments: {+ seedFile: 'a.test.ts',+ },+ })).toHaveTextResponse(expect.stringContaining(`[WebServer] web server started+[err] [WebServer] web server started+global setup+[err] global setup++Running 1 test using 1 worker+test+[err] test+### Paused at end of test. ready for interaction`));+});++test('test_run should run global setup and teardown', async ({ startClient }) => {+ await writeFiles(workspace);++ const { client } = await startClient();+ expect(await client.callTool({+ name: 'test_run',+ arguments: {+ locations: ['a.test.ts'],+ projects: ['chromium'],+ },+ })).toHaveTextResponse(`[WebServer] web server started+[err] [WebServer] web server started+global setup+[err] global setup++Running 1 test using 1 worker+test+[err] test+ ok 1 [id=<ID>] [project=chromium] › a.test.ts:3:9 › test (XXms)+ 1 passed (XXms)+global teardown+[err] global teardown`);+});++test('test_debug should run global setup and teardown', async ({ startClient }) => {+ await writeFiles(workspace);++ const { client } = await startClient();+ const listResult = await client.callTool({+ name: 'test_list',+ });+ const [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/);++ expect(await client.callTool({+ name: 'test_debug',+ arguments: {+ test: { id, title: 'pass' },+ },+ })).toHaveTextResponse(`[WebServer] web server started+[err] [WebServer] web server started+global setup+[err] global setup++Running 1 test using 1 worker+test+[err] test+ ok 1 [id=<ID>] [project=chromium] › a.test.ts:3:9 › test (XXms)+ 1 passed (XXms)+global teardown+[err] global teardown`);+});