Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: split run tests into run and debug (#37246)

Pavel Feldman981bcd517bbc
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index bdaeda9832..7a78630475 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -240,7 +240,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
(testInfo as TestInfoImpl)._setDebugMode();
playwright._defaultContextOptions = _combinedContextOptions;
- playwright._defaultContextTimeout = process.env.PLAYWRIGHT_DEBUGGER_MCP ? 5000 : actionTimeout || 0;
+ playwright._defaultContextTimeout = process.env.PLAYWRIGHT_DEBUGGER_ENABLED ? 5000 : actionTimeout || 0;
playwright._defaultContextNavigationTimeout = navigationTimeout || 0;
await use();
playwright._defaultContextOptions = undefined;
diff --git a/packages/playwright/src/mcp/browser/backend.ts b/packages/playwright/src/mcp/browser/backend.ts
index cd815dc393..aadf7a48ef 100644
--- a/packages/playwright/src/mcp/browser/backend.ts
+++ b/packages/playwright/src/mcp/browser/backend.ts
@@ -73,7 +73,7 @@ const doneToolSchema = defineToolSchema({
});
export async function runBrowserBackendOnError(page: playwright.Page, message: () => string) {
- if (!process.env.PLAYWRIGHT_DEBUGGER_MCP)
+ if (!process.env.PLAYWRIGHT_DEBUGGER_ENABLED)
return;
const snapshot = await (page as PageEx)._snapshotForAI();
const introMessage = `### Paused on error:
@@ -84,5 +84,5 @@ ${snapshot}
### Task
Try recovering from the error prior to continuing, use following tools to recover: ${tools.map(tool => tool.schema.name).join(', ')}`;
- await runOnPauseBackendLoop(process.env.PLAYWRIGHT_DEBUGGER_MCP!, new BrowserBackend(page), introMessage);
+ await runOnPauseBackendLoop(process.env.PLAYWRIGHT_MDB_URL!, new BrowserBackend(page), introMessage);
}
diff --git a/packages/playwright/src/mcp/sdk/mdb.ts b/packages/playwright/src/mcp/sdk/mdb.ts
index 4c58863bde..1e48865454 100644
--- a/packages/playwright/src/mcp/sdk/mdb.ts
+++ b/packages/playwright/src/mcp/sdk/mdb.ts
@@ -153,7 +153,7 @@ export async function runMainBackend(backendFactory: mcpServer.ServerBackendFact
create: () => mdbBackend
};
const url = await startAsHttp(factory, { port: options?.port || 0 });
- process.env.PLAYWRIGHT_DEBUGGER_MCP = url;
+ process.env.PLAYWRIGHT_MDB_URL = url;
if (options?.port !== undefined)
return url;
diff --git a/packages/playwright/src/mcp/test/backend.ts b/packages/playwright/src/mcp/test/backend.ts
index cb0a1b0733..ca37bbcd18 100644
--- a/packages/playwright/src/mcp/test/backend.ts
+++ b/packages/playwright/src/mcp/test/backend.ts
@@ -16,8 +16,7 @@
import * as mcp from '../sdk/exports.js';
import { Context } from './context';
-import { listTests } from './listTests';
-import { runTests } from './runTests';
+import { listTests, runTests, debugTest } from './tools.js';
import { snapshot, pickLocator, evaluate } from '../browser/tools';
import type { ConfigLocation } from '../../common/config';
@@ -27,7 +26,7 @@ import type { Tool } from './tool';
export class TestServerBackend implements mcp.ServerBackend {
readonly name = 'Playwright';
readonly version = '0.0.1';
- private _tools: Tool<any>[] = [listTests, runTests];
+ private _tools: Tool<any>[] = [listTests, runTests, debugTest];
private _context: Context;
constructor(resolvedLocation: ConfigLocation, options?: { muteConsole?: boolean }) {
diff --git a/packages/playwright/src/mcp/test/listTests.ts b/packages/playwright/src/mcp/test/listTests.ts
deleted file mode 100644
index ab204ea914..0000000000
--- a/packages/playwright/src/mcp/test/listTests.ts
+++ /dev/null
@@ -1,78 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import path from 'path';
-
-import { z } from '../sdk/bundle';
-import { defineTool } from './tool.js';
-
-import type * as reporterTypes from 'playwright/types/testReporter';
-
-export const listTests = defineTool({
- schema: {
- name: 'playwright_test_list_tests',
- title: 'List tests',
- description: 'List tests',
- inputSchema: z.object({}),
- type: 'readOnly',
- },
-
- handle: async (context, params) => {
- const reporter = new ListModeReporter();
- const testRunner = await context.createTestRunner();
- await testRunner.listTests(reporter, {});
-
- if (reporter.hasErrors())
- throw new Error(reporter.content());
-
- return {
- content: [{ type: 'text', text: reporter.content() }],
- };
- },
-});
-
-class ListModeReporter implements reporterTypes.Reporter {
- private _lines: string[] = [];
- private _hasErrors = false;
-
- onBegin(config: reporterTypes.FullConfig, suite: reporterTypes.Suite): void {
- this._lines.push(`Listing tests:`);
- const tests = suite.allTests();
- const files = new Set<string>();
- for (const test of tests) {
- // root, project, file, ...describes, test
- const [, projectName, , ...titles] = test.titlePath();
- const location = `${path.relative(config.rootDir, test.location.file)}:${test.location.line}:${test.location.column}`;
- const projectTitle = projectName ? `[${projectName}] › ` : '';
- this._lines.push(` [id=${test.id}] ${projectTitle}${location} › ${titles.join(' › ')}`);
- files.add(test.location.file);
- }
- this._lines.push(`Total: ${tests.length} ${tests.length === 1 ? 'test' : 'tests'} in ${files.size} ${files.size === 1 ? 'file' : 'files'}`);
- }
-
- onError(error: reporterTypes.TestError) {
- this._hasErrors = true;
- this._lines.push(error.stack || error.message || error.value || '');
- }
-
- hasErrors(): boolean {
- return this._hasErrors;
- }
-
- content(): string {
- return this._lines.join('\n');
- }
-}
diff --git a/packages/playwright/src/mcp/test/runTests.ts b/packages/playwright/src/mcp/test/runTests.ts
deleted file mode 100644
index 3e9d6dc605..0000000000
--- a/packages/playwright/src/mcp/test/runTests.ts
+++ /dev/null
@@ -1,67 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import { noColors } from 'playwright-core/lib/utils';
-
-import { z } from '../sdk/bundle';
-import { terminalScreen } from '../../reporters/base';
-import ListReporter from '../../reporters/list';
-
-import { defineTool } from './tool';
-import { StringWriteStream } from './streams';
-
-export const runTests = defineTool({
- schema: {
- name: 'playwright_test_run_tests',
- title: 'Run tests',
- description: 'Run tests',
- inputSchema: z.object({
- tests: z.array(z.object({
- id: z.string().describe('Test ID to run.'),
- title: z.string().describe('Human readable test title for granting permission to run the test.'),
- })).optional().describe('Tests to run. All tests are run if not provided.'),
- }),
- type: 'readOnly',
- },
-
- handle: async (context, params) => {
- const stream = new StringWriteStream();
- const screen = {
- ...terminalScreen,
- isTTY: false,
- colors: noColors,
- stdout: stream as unknown as NodeJS.WriteStream,
- stderr: stream as unknown as NodeJS.WriteStream,
- };
- const configDir = context.configLocation.configDir;
- const reporter = new ListReporter({ configDir, screen });
- const testRunner = await context.createTestRunner();
- const result = await testRunner.runTests(reporter, {
- testIds: params.tests?.map(test => test.id),
- // For automatic recovery
- timeout: 0,
- workers: 1,
- });
-
- const text = stream.content();
- return {
- content: [
- { type: 'text', text },
- ],
- isError: result.status !== 'passed',
- };
- },
-});
diff --git a/packages/playwright/src/mcp/test/tools.ts b/packages/playwright/src/mcp/test/tools.ts
new file mode 100644
index 0000000000..bae91402fd
--- /dev/null
+++ b/packages/playwright/src/mcp/test/tools.ts
@@ -0,0 +1,137 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { noColors } from 'playwright-core/lib/utils';
+
+import { z } from '../sdk/bundle';
+import { terminalScreen } from '../../reporters/base';
+import ListReporter from '../../reporters/list';
+import ListModeReporter from '../../reporters/listModeReporter';
+
+import { defineTool } from './tool';
+import { StringWriteStream } from './streams';
+
+export const listTests = defineTool({
+ schema: {
+ name: 'playwright_test_list_tests',
+ title: 'List tests',
+ description: 'List tests',
+ inputSchema: z.object({}),
+ type: 'readOnly',
+ },
+
+ handle: async context => {
+ const { screen, stream } = createScreen();
+ const reporter = new ListModeReporter({ screen, includeTestId: true });
+ const testRunner = await context.createTestRunner();
+ await testRunner.listTests(reporter, {});
+
+ return {
+ content: [{ type: 'text', text: stream.content() }],
+ };
+ },
+});
+
+export const runTests = defineTool({
+ schema: {
+ name: 'playwright_test_run_tests',
+ title: 'Run tests',
+ description: 'Run tests',
+ inputSchema: z.object({
+ locations: z.array(z.string()).describe('Folder, file or location to run: "test/e2e" or "test/e2e/file.spec.ts" or "test/e2e/file.spec.ts:20"'),
+ projects: z.array(z.string()).optional().describe('Projects to run, projects from playwright.config.ts, by default runs all projects. Running with "chromium" is a good start'),
+ }),
+ type: 'readOnly',
+ },
+
+ handle: async (context, params) => {
+ const { screen, stream } = createScreen();
+ const configDir = context.configLocation.configDir;
+ const reporter = new ListReporter({ configDir, screen, includeTestId: true });
+ const testRunner = await context.createTestRunner();
+ const result = await testRunner.runTests(reporter, {
+ locations: params.locations,
+ projects: params.projects,
+ });
+
+ const text = stream.content();
+ return {
+ content: [
+ { type: 'text', text },
+ ],
+ isError: result.status !== 'passed',
+ };
+ },
+});
+
+export const debugTest = defineTool({
+ schema: {
+ name: 'playwright_test_debug_test',
+ title: 'Debug single test',
+ description: 'Debug single test',
+ inputSchema: z.object({
+ test: z.object({
+ id: z.string().describe('Test ID to debug.'),
+ title: z.string().describe('Human readable test title for granting permission to debug the test.'),
+ }),
+ }),
+ type: 'readOnly',
+ },
+
+ handle: async (context, params) => {
+ const stream = new StringWriteStream();
+ const screen = {
+ ...terminalScreen,
+ isTTY: false,
+ colors: noColors,
+ stdout: stream as unknown as NodeJS.WriteStream,
+ stderr: stream as unknown as NodeJS.WriteStream,
+ };
+ const configDir = context.configLocation.configDir;
+ const reporter = new ListReporter({ configDir, screen });
+ const testRunner = await context.createTestRunner();
+ process.env.PLAYWRIGHT_DEBUGGER_ENABLED = '1';
+ const result = await testRunner.runTests(reporter, {
+ headed: true,
+ testIds: [params.test.id],
+ // For automatic recovery
+ timeout: 0,
+ workers: 1,
+ }).finally(() => {
+ process.env.PLAYWRIGHT_DEBUGGER_ENABLED = undefined;
+ });
+
+ const text = stream.content();
+ return {
+ content: [
+ { type: 'text', text },
+ ],
+ isError: result.status !== 'passed',
+ };
+ },
+});
+
+function createScreen() {
+ const stream = new StringWriteStream();
+ const screen = {
+ ...terminalScreen,
+ isTTY: false,
+ colors: noColors,
+ stdout: stream as unknown as NodeJS.WriteStream,
+ stderr: stream as unknown as NodeJS.WriteStream,
+ };
+ return { screen, stream };
+}
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts
index 7211692770..c11657d69d 100644
--- a/packages/playwright/src/reporters/base.ts
+++ b/packages/playwright/src/reporters/base.ts
@@ -145,6 +145,7 @@ export const internalScreen: Screen = {
export type TerminalReporterOptions = {
screen?: TerminalScreen;
omitFailures?: boolean;
+ includeTestId?: boolean;
};
export class TerminalReporter implements ReporterV2 {
@@ -154,13 +155,13 @@ export class TerminalReporter implements ReporterV2 {
totalTestCount = 0;
result!: FullResult;
private fileDurations = new Map<string, { duration: number, workers: Set<number> }>();
- private _omitFailures: boolean;
+ private _options: TerminalReporterOptions;
private _fatalErrors: TestError[] = [];
private _failureCount: number = 0;
constructor(options: TerminalReporterOptions = {}) {
this.screen = options.screen ?? terminalScreen;
- this._omitFailures = options.omitFailures || false;
+ this._options = options;
}
version(): 'v2' {
@@ -312,7 +313,7 @@ export class TerminalReporter implements ReporterV2 {
epilogue(full: boolean) {
const summary = this.generateSummary();
const summaryMessage = this.generateSummaryMessage(summary);
- if (full && summary.failuresToPrint.length && !this._omitFailures)
+ if (full && summary.failuresToPrint.length && !this._options.omitFailures)
this._printFailures(summary.failuresToPrint);
this._printSlowTests();
this._printSummary(summaryMessage);
@@ -343,16 +344,16 @@ export class TerminalReporter implements ReporterV2 {
return test.outcome() === 'unexpected' && test.results.length <= test.retries;
}
- formatTestTitle(test: TestCase, step?: TestStep, omitLocation: boolean = false): string {
- return formatTestTitle(this.screen, this.config, test, step, omitLocation);
+ formatTestTitle(test: TestCase, step?: TestStep): string {
+ return formatTestTitle(this.screen, this.config, test, step, this._options);
}
formatTestHeader(test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error' } = {}): string {
- return formatTestHeader(this.screen, this.config, test, options);
+ return formatTestHeader(this.screen, this.config, test, { ...options, includeTestId: this._options.includeTestId });
}
formatFailure(test: TestCase, index?: number): string {
- return formatFailure(this.screen, this.config, test, index);
+ return formatFailure(this.screen, this.config, test, index, this._options);
}
formatError(error: TestError): ErrorDetails {
@@ -364,9 +365,9 @@ export class TerminalReporter implements ReporterV2 {
}
}
-export function formatFailure(screen: Screen, config: FullConfig, test: TestCase, index?: number): string {
+export function formatFailure(screen: Screen, config: FullConfig, test: TestCase, index?: number, options?: TerminalReporterOptions): string {
const lines: string[] = [];
- const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error' });
+ const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error', includeTestId: options?.includeTestId });
lines.push(screen.colors.red(header));
for (const result of test.results) {
const resultLines: string[] = [];
@@ -494,22 +495,20 @@ export function stepSuffix(step: TestStep | undefined) {
return stepTitles.map(t => t.split('\n')[0]).map(t => ' › ' + t).join('');
}
-function formatTestTitle(screen: Screen, config: FullConfig, test: TestCase, step?: TestStep, omitLocation: boolean = false): string {
+function formatTestTitle(screen: Screen, config: FullConfig, test: TestCase, step?: TestStep, options: { includeTestId?: boolean } = {}): string {
// root, project, file, ...describes, test
const [, projectName, , ...titles] = test.titlePath();
- let location;
- if (omitLocation)
- location = `${relativeTestPath(screen, config, test)}`;
- else
- location = `${relativeTestPath(screen, config, test)}:${test.location.line}:${test.location.column}`;
- const projectTitle = projectName ? `[${projectName}] › ` : '';
- const testTitle = `${projectTitle}${location} › ${titles.join(' › ')}`;
+ const location = `${relativeTestPath(screen, config, test)}:${test.location.line}:${test.location.column}`;
+ const testId = options.includeTestId ? `[id=${test.id}] ` : '';
+ const projectLabel = options.includeTestId ? `project=` : '';
+ const projectTitle = projectName ? `[${projectLabel}${projectName}] › ` : '';
+ const testTitle = `${testId}${projectTitle}${location} › ${titles.join(' › ')}`;
const extraTags = test.tags.filter(t => !testTitle.includes(t));
return `${testTitle}${stepSuffix(step)}${extraTags.length ? ' ' + extraTags.join(' ') : ''}`;
}
-function formatTestHeader(screen: Screen, config: FullConfig, test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error' } = {}): string {
- const title = formatTestTitle(screen, config, test);
+function formatTestHeader(screen: Screen, config: FullConfig, test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error', includeTestId?: boolean } = {}): string {
+ const title = formatTestTitle(screen, config, test, undefined, options);
const header = `${options.indent || ''}${options.index ? options.index + ') ' : ''}${title}`;
let fullHeader = header;
diff --git a/packages/playwright/src/reporters/listModeReporter.ts b/packages/playwright/src/reporters/listModeReporter.ts
index 2337067ed0..1732cd8db6 100644
--- a/packages/playwright/src/reporters/listModeReporter.ts
+++ b/packages/playwright/src/reporters/listModeReporter.ts
@@ -26,8 +26,10 @@ import type { ReporterV2 } from './reporterV2';
class ListModeReporter implements ReporterV2 {
private config!: FullConfig;
private screen: TerminalScreen;
+ private _options: { screen?: TerminalScreen, includeTestId?: boolean };
- constructor(options?: { screen?: TerminalScreen }) {
+ constructor(options: { screen?: TerminalScreen, includeTestId?: boolean } = {}) {
+ this._options = options;
this.screen = options?.screen ?? terminalScreen;
}
@@ -47,8 +49,10 @@ class ListModeReporter implements ReporterV2 {
// root, project, file, ...describes, test
const [, projectName, , ...titles] = test.titlePath();
const location = `${path.relative(this.config.rootDir, test.location.file)}:${test.location.line}:${test.location.column}`;
- const projectTitle = projectName ? `[${projectName}] › ` : '';
- this._writeLine(` ${projectTitle}${location} › ${titles.join(' › ')}`);
+ const testId = this._options.includeTestId ? `[id=${test.id}] ` : '';
+ const projectLabel = this._options.includeTestId ? `project=` : '';
+ const projectTitle = projectName ? `[${projectLabel}${projectName}] › ` : '';
+ this._writeLine(` ${testId}${projectTitle}${location} › ${titles.join(' › ')}`);
files.add(test.location.file);
}
this._writeLine(`Total: ${tests.length} ${tests.length === 1 ? 'test' : 'tests'} in ${files.size} ${files.size === 1 ? 'file' : 'files'}`);