Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: run global setup in test tools (#38040)

Pavel Feldman12075443bcfa
diff --git a/.github/workflows/tests_mcp.yml b/.github/workflows/tests_mcp.yml
index 1beebe5249..753bb7412e 100644
--- a/.github/workflows/tests_mcp.yml
+++ b/.github/workflows/tests_mcp.yml
@@ -23,7 +23,6 @@ env:
# Force terminal colors. @see https://www.npmjs.com/package/colors
FORCE_COLOR: 1
ELECTRON_SKIP_BINARY_DOWNLOAD: 1
- DEBUG_GIT_COMMIT_INFO: 1
jobs:
test_mcp:
diff --git a/.github/workflows/tests_primary.yml b/.github/workflows/tests_primary.yml
index 2cb96d9ad6..304870c65d 100644
--- a/.github/workflows/tests_primary.yml
+++ b/.github/workflows/tests_primary.yml
@@ -25,7 +25,6 @@ env:
# Force terminal colors. @see https://www.npmjs.com/package/colors
FORCE_COLOR: 1
ELECTRON_SKIP_BINARY_DOWNLOAD: 1
- DEBUG_GIT_COMMIT_INFO: 1
jobs:
test_linux:
@@ -177,7 +176,6 @@ jobs:
runs-on: ubuntu-latest
env:
PWTEST_BOT_NAME: "vscode-extension"
- DEBUG_GIT_COMMIT_INFO: ""
steps:
- uses: actions/checkout@v5
- uses: actions/setup-node@v5
diff --git a/packages/playwright/src/mcp/test/streams.ts b/packages/playwright/src/mcp/test/streams.ts
index f78eb82e3f..4513411016 100644
--- a/packages/playwright/src/mcp/test/streams.ts
+++ b/packages/playwright/src/mcp/test/streams.ts
@@ -16,20 +16,24 @@
import { Writable } from 'stream';
+import { stripAnsiEscapes } from '../../util';
+
import type { ProgressCallback } from '../sdk/server';
export class StringWriteStream extends Writable {
private _progress: ProgressCallback;
+ private _prefix: string;
- constructor(progress: ProgressCallback) {
+ constructor(progress: ProgressCallback, stdio: 'stdout' | 'stderr') {
super();
this._progress = progress;
+ this._prefix = stdio === 'stdout' ? '' : '[err] ';
}
override _write(chunk: any, encoding: any, callback: any) {
- const text = chunk.toString();
+ const text = stripAnsiEscapes(chunk.toString());
// Progress wraps these as individual messages.
- this._progress({ message: text.endsWith('\n') ? text.slice(0, -1) : text });
+ this._progress({ message: `${this._prefix}${text.endsWith('\n') ? text.slice(0, -1) : text}` });
callback();
}
}
diff --git a/packages/playwright/src/mcp/test/testContext.ts b/packages/playwright/src/mcp/test/testContext.ts
index 60a2d16b8f..d71a41ff48 100644
--- a/packages/playwright/src/mcp/test/testContext.ts
+++ b/packages/playwright/src/mcp/test/testContext.ts
@@ -138,45 +138,93 @@ export class TestContext {
}
async runSeedTest(seedFile: string, projectName: string, progress: ProgressCallback) {
- const { screen } = this.createScreen(progress);
+ await this.runWithGlobalSetup(async (testRunner, reporter) => {
+ const result = await testRunner.runTests(reporter, {
+ headed: !this.options?.headless,
+ locations: ['/' + escapeRegExp(seedFile) + '/'],
+ projects: [projectName],
+ timeout: 0,
+ workers: 1,
+ pauseAtEnd: true,
+ disableConfigReporters: true,
+ failOnLoadErrors: true,
+ });
+ // Ideally, we should check that page was indeed created and browser mcp has kicked in.
+ // However, that is handled in the upper layer, so hard to check here.
+ if (result.status === 'passed' && !reporter.suite?.allTests().length)
+ throw new Error('seed test not found.');
+
+ if (result.status !== 'passed')
+ throw new Error('Errors while running the seed test.');
+ }, progress);
+ }
+
+ async runWithGlobalSetup(
+ callback: (testRunner: TestRunner, reporter: ListReporter) => Promise<void>,
+ progress: ProgressCallback): Promise<void> {
+ const { screen, claimStdio, releaseStdio } = createScreen(progress);
const configDir = this.configLocation.configDir;
- const reporter = new ListReporter({ configDir, screen });
const testRunner = await this.createTestRunner();
- const result = await testRunner.runTests(reporter, {
- headed: !this.options?.headless,
- locations: ['/' + escapeRegExp(seedFile) + '/'],
- projects: [projectName],
- timeout: 0,
- workers: 1,
- pauseAtEnd: true,
- disableConfigReporters: true,
- failOnLoadErrors: true,
- });
+ claimStdio();
+ try {
+ const setupReporter = new ListReporter({ configDir, screen, includeTestId: true });
+ const { status } = await testRunner.runGlobalSetup([setupReporter]);
+ if (status !== 'passed')
+ throw new Error('Failed to run global setup');
+ } finally {
+ releaseStdio();
+ }
- // Ideally, we should check that page was indeed created and browser mcp has kicked in.
- // However, that is handled in the upper layer, so hard to check here.
- if (result.status === 'passed' && !reporter.suite?.allTests().length)
- throw new Error('seed test not found.');
+ try {
+ const reporter = new ListReporter({ configDir, screen, includeTestId: true });
+ return await callback(testRunner, reporter);
+ } finally {
+ claimStdio();
+ await testRunner.runGlobalTeardown().finally(() => {
+ releaseStdio();
+ });
+ }
+ }
- if (result.status !== 'passed')
- throw new Error('Errors while running the seed test.');
+ async close() {
}
+}
- createScreen(progress: ProgressCallback) {
- const stream = new StringWriteStream(progress);
- const screen = {
- ...terminalScreen,
- isTTY: false,
- colors: noColors,
- stdout: stream as unknown as NodeJS.WriteStream,
- stderr: stream as unknown as NodeJS.WriteStream,
+export function createScreen(progress: ProgressCallback) {
+ const stdout = new StringWriteStream(progress, 'stdout');
+ const stderr = new StringWriteStream(progress, 'stderr');
+
+ const screen = {
+ ...terminalScreen,
+ isTTY: false,
+ colors: noColors,
+ stdout: stdout as unknown as NodeJS.WriteStream,
+ stderr: stderr as unknown as NodeJS.WriteStream,
+ };
+
+ /* eslint-disable no-restricted-properties */
+ const originalStdoutWrite = process.stdout.write;
+ const originalStderrWrite = process.stderr.write;
+
+ const claimStdio = () => {
+ process.stdout.write = (chunk: string | Buffer) => {
+ stdout.write(chunk);
+ return true;
};
- return { screen, stream };
- }
+ process.stderr.write = (chunk: string | Buffer) => {
+ stderr.write(chunk);
+ return true;
+ };
+ };
- async close() {
- }
+ const releaseStdio = () => {
+ process.stdout.write = originalStdoutWrite;
+ process.stderr.write = originalStderrWrite;
+ };
+ /* eslint-enable no-restricted-properties */
+
+ return { screen, claimStdio, releaseStdio };
}
const bestPracticesMarkdown = `
diff --git a/packages/playwright/src/mcp/test/testTools.ts b/packages/playwright/src/mcp/test/testTools.ts
index a3c86ed080..7a4f0bda8d 100644
--- a/packages/playwright/src/mcp/test/testTools.ts
+++ b/packages/playwright/src/mcp/test/testTools.ts
@@ -15,9 +15,9 @@
*/
import { z } from '../sdk/bundle';
-import ListReporter from '../../reporters/list';
import ListModeReporter from '../../reporters/listModeReporter';
import { defineTestTool } from './testTool';
+import { createScreen } from './testContext';
export const listTests = defineTestTool({
schema: {
@@ -29,7 +29,7 @@ export const listTests = defineTestTool({
},
handle: async (context, _, progress) => {
- const { screen } = context.createScreen(progress);
+ const { screen } = createScreen(progress);
const reporter = new ListModeReporter({ screen, includeTestId: true });
const testRunner = await context.createTestRunner();
await testRunner.listTests(reporter, {});
@@ -51,15 +51,13 @@ export const runTests = defineTestTool({
},
handle: async (context, params, progress) => {
- const { screen } = context.createScreen(progress);
- const configDir = context.configLocation.configDir;
- const reporter = new ListReporter({ configDir, screen, includeTestId: true, prefixStdio: 'out' });
- const testRunner = await context.createTestRunner();
- await testRunner.runTests(reporter, {
- locations: params.locations,
- projects: params.projects,
- disableConfigReporters: true,
- });
+ await context.runWithGlobalSetup(async (testRunner, reporter) => {
+ await testRunner.runTests(reporter, {
+ locations: params.locations,
+ projects: params.projects,
+ disableConfigReporters: true,
+ });
+ }, progress);
return { content: [] };
},
@@ -80,19 +78,17 @@ export const debugTest = defineTestTool({
},
handle: async (context, params, progress) => {
- const { screen } = context.createScreen(progress);
- const configDir = context.configLocation.configDir;
- const reporter = new ListReporter({ configDir, screen, includeTestId: true, prefixStdio: 'out' });
- const testRunner = await context.createTestRunner();
- await testRunner.runTests(reporter, {
- headed: !context.options?.headless,
- testIds: [params.test.id],
- // For automatic recovery
- timeout: 0,
- workers: 1,
- pauseOnError: true,
- disableConfigReporters: true,
- });
+ await context.runWithGlobalSetup(async (testRunner, reporter) => {
+ await testRunner.runTests(reporter, {
+ headed: !context.options?.headless,
+ testIds: [params.test.id],
+ // For automatic recovery
+ timeout: 0,
+ workers: 1,
+ pauseOnError: true,
+ disableConfigReporters: true,
+ });
+ }, progress);
return { content: [] };
},
diff --git a/packages/playwright/src/reporters/list.ts b/packages/playwright/src/reporters/list.ts
index 591883e3bf..b2b399e6eb 100644
--- a/packages/playwright/src/reporters/list.ts
+++ b/packages/playwright/src/reporters/list.ts
@@ -38,12 +38,10 @@ class ListReporter extends TerminalReporter {
private _stepIndex = new Map<TestStep, string>();
private _needNewLine = false;
private _printSteps: boolean;
- private _prefixStdio?: string;
- constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions & { prefixStdio?: string }) {
+ constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions) {
super(options);
this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options?.printSteps);
- this._prefixStdio = options?.prefixStdio;
}
override onBegin(suite: Suite) {
@@ -164,10 +162,7 @@ class ListReporter extends TerminalReporter {
return;
const text = chunk.toString('utf-8');
this._updateLineCountAndNewLineFlagForOutput(text);
- if (this._prefixStdio)
- stream.write(`[${stdio}] ${chunk}`);
- else
- stream.write(chunk);
+ stream.write(chunk);
}
override onTestEnd(test: TestCase, result: TestResult) {
diff --git a/tests/mcp/planner.spec.ts b/tests/mcp/planner.spec.ts
index 64684a56c2..1917660b52 100644
--- a/tests/mcp/planner.spec.ts
+++ b/tests/mcp/planner.spec.ts
@@ -183,6 +183,7 @@ test('planner_setup_page (loading error)', async ({ startClient }) => {
});
test('planner_setup_page (wrong test location)', async ({ startClient }) => {
+ await writeFiles({});
const { client } = await startClient();
expect(await client.callTool({
name: 'planner_setup_page',
@@ -196,6 +197,7 @@ test('planner_setup_page (wrong test location)', async ({ startClient }) => {
});
test('planner_setup_page (no test location)', async ({ startClient }) => {
+ await writeFiles({});
const { client } = await startClient();
expect(await client.callTool({
name: 'planner_setup_page',
diff --git a/tests/mcp/seed-default-project.spec.ts b/tests/mcp/seed-default-project.spec.ts
index c0acdef706..beac69d682 100644
--- a/tests/mcp/seed-default-project.spec.ts
+++ b/tests/mcp/seed-default-project.spec.ts
@@ -47,6 +47,23 @@ test('seed test runs in first top-level project by default', async ({ startClien
expect(fs.existsSync(path.join(baseDir, 'third', 'seed.spec.ts'))).toBe(false);
});
+test('respects provided seed test', async ({ startClient }) => {
+ await writeFiles({
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('test', async ({ page }) => {});
+ `,
+ });
+
+ const { client } = await startClient();
+ expect(await client.callTool({
+ name: 'planner_setup_page',
+ arguments: {
+ seedFile: 'a.test.ts',
+ },
+ })).toHaveTextResponse(expect.stringContaining(`### Paused at end of test. ready for interaction`));
+});
+
test('seed test runs in first top-level project with dependencies', async ({ startClient }) => {
const baseDir = await writeFiles({
'playwright.config.ts': `
diff --git a/tests/mcp/test-debug.spec.ts b/tests/mcp/test-debug.spec.ts
index 0c3b031ef2..b0f1a87581 100644
--- a/tests/mcp/test-debug.spec.ts
+++ b/tests/mcp/test-debug.spec.ts
@@ -323,7 +323,7 @@ test('test_debug w/ console.log in test', async ({ startClient }) => {
},
})).toHaveTextResponse(expect.stringContaining(`
Running 1 test using 1 worker
-[out] console.log
+console.log
[err] console.error
### Paused on error:
Error: expect(locator).toBeVisible() failed`));
diff --git a/tests/mcp/test-setup.spec.ts b/tests/mcp/test-setup.spec.ts
new file mode 100644
index 0000000000..455b1fcb38
--- /dev/null
+++ b/tests/mcp/test-setup.spec.ts
@@ -0,0 +1,131 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, writeFiles } from './fixtures';
+
+test.use({ mcpServerType: 'test-mcp' });
+
+const workspace = {
+ 'playwright.config.ts': `
+ module.exports = {
+ projects: [{ name: 'chromium' }],
+ webServer: {
+ command: 'node web-server.js',
+ stdout: 'pipe',
+ stderr: 'pipe',
+ wait: { stderr: /started/ }
+ },
+ globalSetup: 'global-setup.ts',
+ globalTeardown: 'global-teardown.ts',
+ };
+ `,
+ 'web-server.js': `
+ console.log('web server started');
+ console.error('web server started');
+ `,
+ 'global-setup.ts': `
+ module.exports = async () => {
+ console.log('global setup');
+ console.error('global setup');
+ };
+ `,
+ 'global-teardown.ts': `
+ module.exports = async () => {
+ console.log('global teardown');
+ console.error('global teardown');
+ };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('test', async ({ page }) => {
+ console.log('test');
+ console.error('test');
+ });
+ `
+};
+
+test('setup should run global setup and teardown', async ({ startClient }, { workerIndex }) => {
+ await writeFiles(workspace);
+ const { client } = await startClient();
+
+ // Call planner_setup_page without specifying a project - should use first top-level project
+ expect(await client.callTool({
+ name: 'planner_setup_page',
+ arguments: {
+ seedFile: 'a.test.ts',
+ },
+ })).toHaveTextResponse(expect.stringContaining(`[WebServer] web server started
+[err] [WebServer] web server started
+global setup
+[err] global setup
+
+Running 1 test using 1 worker
+test
+[err] test
+### Paused at end of test. ready for interaction`));
+});
+
+test('test_run should run global setup and teardown', async ({ startClient }) => {
+ await writeFiles(workspace);
+
+ const { client } = await startClient();
+ expect(await client.callTool({
+ name: 'test_run',
+ arguments: {
+ locations: ['a.test.ts'],
+ projects: ['chromium'],
+ },
+ })).toHaveTextResponse(`[WebServer] web server started
+[err] [WebServer] web server started
+global setup
+[err] global setup
+
+Running 1 test using 1 worker
+test
+[err] test
+ ok 1 [id=<ID>] [project=chromium] › a.test.ts:3:9 › test (XXms)
+ 1 passed (XXms)
+global teardown
+[err] global teardown`);
+});
+
+test('test_debug should run global setup and teardown', async ({ startClient }) => {
+ await writeFiles(workspace);
+
+ const { client } = await startClient();
+ const listResult = await client.callTool({
+ name: 'test_list',
+ });
+ const [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/);
+
+ expect(await client.callTool({
+ name: 'test_debug',
+ arguments: {
+ test: { id, title: 'pass' },
+ },
+ })).toHaveTextResponse(`[WebServer] web server started
+[err] [WebServer] web server started
+global setup
+[err] global setup
+
+Running 1 test using 1 worker
+test
+[err] test
+ ok 1 [id=<ID>] [project=chromium] › a.test.ts:3:9 › test (XXms)
+ 1 passed (XXms)
+global teardown
+[err] global teardown`);
+});