Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: split run tests into run and debug (#37246)
Pavel Feldman981bcd517bbc
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex bdaeda9832..7a78630475 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -240,7 +240,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({(testInfo as TestInfoImpl)._setDebugMode();playwright._defaultContextOptions = _combinedContextOptions;- playwright._defaultContextTimeout = process.env.PLAYWRIGHT_DEBUGGER_MCP ? 5000 : actionTimeout || 0;+ playwright._defaultContextTimeout = process.env.PLAYWRIGHT_DEBUGGER_ENABLED ? 5000 : actionTimeout || 0;playwright._defaultContextNavigationTimeout = navigationTimeout || 0;await use();playwright._defaultContextOptions = undefined;diff --git a/packages/playwright/src/mcp/browser/backend.ts b/packages/playwright/src/mcp/browser/backend.tsindex cd815dc393..aadf7a48ef 100644--- a/packages/playwright/src/mcp/browser/backend.ts+++ b/packages/playwright/src/mcp/browser/backend.ts@@ -73,7 +73,7 @@ const doneToolSchema = defineToolSchema({});export async function runBrowserBackendOnError(page: playwright.Page, message: () => string) {- if (!process.env.PLAYWRIGHT_DEBUGGER_MCP)+ if (!process.env.PLAYWRIGHT_DEBUGGER_ENABLED)return;const snapshot = await (page as PageEx)._snapshotForAI();const introMessage = `### Paused on error:@@ -84,5 +84,5 @@ ${snapshot}### TaskTry recovering from the error prior to continuing, use following tools to recover: ${tools.map(tool => tool.schema.name).join(', ')}`;- await runOnPauseBackendLoop(process.env.PLAYWRIGHT_DEBUGGER_MCP!, new BrowserBackend(page), introMessage);+ await runOnPauseBackendLoop(process.env.PLAYWRIGHT_MDB_URL!, new BrowserBackend(page), introMessage);}diff --git a/packages/playwright/src/mcp/sdk/mdb.ts b/packages/playwright/src/mcp/sdk/mdb.tsindex 4c58863bde..1e48865454 100644--- a/packages/playwright/src/mcp/sdk/mdb.ts+++ b/packages/playwright/src/mcp/sdk/mdb.ts@@ -153,7 +153,7 @@ export async function runMainBackend(backendFactory: mcpServer.ServerBackendFactcreate: () => mdbBackend};const url = await startAsHttp(factory, { port: options?.port || 0 });- process.env.PLAYWRIGHT_DEBUGGER_MCP = url;+ process.env.PLAYWRIGHT_MDB_URL = url;if (options?.port !== undefined)return url;diff --git a/packages/playwright/src/mcp/test/backend.ts b/packages/playwright/src/mcp/test/backend.tsindex cb0a1b0733..ca37bbcd18 100644--- a/packages/playwright/src/mcp/test/backend.ts+++ b/packages/playwright/src/mcp/test/backend.ts@@ -16,8 +16,7 @@import * as mcp from '../sdk/exports.js';import { Context } from './context';-import { listTests } from './listTests';-import { runTests } from './runTests';+import { listTests, runTests, debugTest } from './tools.js';import { snapshot, pickLocator, evaluate } from '../browser/tools';import type { ConfigLocation } from '../../common/config';@@ -27,7 +26,7 @@ import type { Tool } from './tool';export class TestServerBackend implements mcp.ServerBackend {readonly name = 'Playwright';readonly version = '0.0.1';- private _tools: Tool<any>[] = [listTests, runTests];+ private _tools: Tool<any>[] = [listTests, runTests, debugTest];private _context: Context;constructor(resolvedLocation: ConfigLocation, options?: { muteConsole?: boolean }) {diff --git a/packages/playwright/src/mcp/test/listTests.ts b/packages/playwright/src/mcp/test/listTests.tsdeleted file mode 100644index ab204ea914..0000000000--- a/packages/playwright/src/mcp/test/listTests.ts+++ /dev/null@@ -1,78 +0,0 @@-/**- * Copyright (c) Microsoft Corporation.- *- * Licensed under the Apache License, Version 2.0 (the "License");- * you may not use this file except in compliance with the License.- * You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import path from 'path';--import { z } from '../sdk/bundle';-import { defineTool } from './tool.js';--import type * as reporterTypes from 'playwright/types/testReporter';--export const listTests = defineTool({- schema: {- name: 'playwright_test_list_tests',- title: 'List tests',- description: 'List tests',- inputSchema: z.object({}),- type: 'readOnly',- },-- handle: async (context, params) => {- const reporter = new ListModeReporter();- const testRunner = await context.createTestRunner();- await testRunner.listTests(reporter, {});-- if (reporter.hasErrors())- throw new Error(reporter.content());-- return {- content: [{ type: 'text', text: reporter.content() }],- };- },-});--class ListModeReporter implements reporterTypes.Reporter {- private _lines: string[] = [];- private _hasErrors = false;-- onBegin(config: reporterTypes.FullConfig, suite: reporterTypes.Suite): void {- this._lines.push(`Listing tests:`);- const tests = suite.allTests();- const files = new Set<string>();- for (const test of tests) {- // root, project, file, ...describes, test- const [, projectName, , ...titles] = test.titlePath();- const location = `${path.relative(config.rootDir, test.location.file)}:${test.location.line}:${test.location.column}`;- const projectTitle = projectName ? `[${projectName}] › ` : '';- this._lines.push(` [id=${test.id}] ${projectTitle}${location} › ${titles.join(' › ')}`);- files.add(test.location.file);- }- this._lines.push(`Total: ${tests.length} ${tests.length === 1 ? 'test' : 'tests'} in ${files.size} ${files.size === 1 ? 'file' : 'files'}`);- }-- onError(error: reporterTypes.TestError) {- this._hasErrors = true;- this._lines.push(error.stack || error.message || error.value || '');- }-- hasErrors(): boolean {- return this._hasErrors;- }-- content(): string {- return this._lines.join('\n');- }-}diff --git a/packages/playwright/src/mcp/test/runTests.ts b/packages/playwright/src/mcp/test/runTests.tsdeleted file mode 100644index 3e9d6dc605..0000000000--- a/packages/playwright/src/mcp/test/runTests.ts+++ /dev/null@@ -1,67 +0,0 @@-/**- * Copyright (c) Microsoft Corporation.- *- * Licensed under the Apache License, Version 2.0 (the "License");- * you may not use this file except in compliance with the License.- * You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import { noColors } from 'playwright-core/lib/utils';--import { z } from '../sdk/bundle';-import { terminalScreen } from '../../reporters/base';-import ListReporter from '../../reporters/list';--import { defineTool } from './tool';-import { StringWriteStream } from './streams';--export const runTests = defineTool({- schema: {- name: 'playwright_test_run_tests',- title: 'Run tests',- description: 'Run tests',- inputSchema: z.object({- tests: z.array(z.object({- id: z.string().describe('Test ID to run.'),- title: z.string().describe('Human readable test title for granting permission to run the test.'),- })).optional().describe('Tests to run. All tests are run if not provided.'),- }),- type: 'readOnly',- },-- handle: async (context, params) => {- const stream = new StringWriteStream();- const screen = {- ...terminalScreen,- isTTY: false,- colors: noColors,- stdout: stream as unknown as NodeJS.WriteStream,- stderr: stream as unknown as NodeJS.WriteStream,- };- const configDir = context.configLocation.configDir;- const reporter = new ListReporter({ configDir, screen });- const testRunner = await context.createTestRunner();- const result = await testRunner.runTests(reporter, {- testIds: params.tests?.map(test => test.id),- // For automatic recovery- timeout: 0,- workers: 1,- });-- const text = stream.content();- return {- content: [- { type: 'text', text },- ],- isError: result.status !== 'passed',- };- },-});diff --git a/packages/playwright/src/mcp/test/tools.ts b/packages/playwright/src/mcp/test/tools.tsnew file mode 100644index 0000000000..bae91402fd--- /dev/null+++ b/packages/playwright/src/mcp/test/tools.ts@@ -0,0 +1,137 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { noColors } from 'playwright-core/lib/utils';++import { z } from '../sdk/bundle';+import { terminalScreen } from '../../reporters/base';+import ListReporter from '../../reporters/list';+import ListModeReporter from '../../reporters/listModeReporter';++import { defineTool } from './tool';+import { StringWriteStream } from './streams';++export const listTests = defineTool({+ schema: {+ name: 'playwright_test_list_tests',+ title: 'List tests',+ description: 'List tests',+ inputSchema: z.object({}),+ type: 'readOnly',+ },++ handle: async context => {+ const { screen, stream } = createScreen();+ const reporter = new ListModeReporter({ screen, includeTestId: true });+ const testRunner = await context.createTestRunner();+ await testRunner.listTests(reporter, {});++ return {+ content: [{ type: 'text', text: stream.content() }],+ };+ },+});++export const runTests = defineTool({+ schema: {+ name: 'playwright_test_run_tests',+ title: 'Run tests',+ description: 'Run tests',+ inputSchema: z.object({+ locations: z.array(z.string()).describe('Folder, file or location to run: "test/e2e" or "test/e2e/file.spec.ts" or "test/e2e/file.spec.ts:20"'),+ projects: z.array(z.string()).optional().describe('Projects to run, projects from playwright.config.ts, by default runs all projects. Running with "chromium" is a good start'),+ }),+ type: 'readOnly',+ },++ handle: async (context, params) => {+ const { screen, stream } = createScreen();+ const configDir = context.configLocation.configDir;+ const reporter = new ListReporter({ configDir, screen, includeTestId: true });+ const testRunner = await context.createTestRunner();+ const result = await testRunner.runTests(reporter, {+ locations: params.locations,+ projects: params.projects,+ });++ const text = stream.content();+ return {+ content: [+ { type: 'text', text },+ ],+ isError: result.status !== 'passed',+ };+ },+});++export const debugTest = defineTool({+ schema: {+ name: 'playwright_test_debug_test',+ title: 'Debug single test',+ description: 'Debug single test',+ inputSchema: z.object({+ test: z.object({+ id: z.string().describe('Test ID to debug.'),+ title: z.string().describe('Human readable test title for granting permission to debug the test.'),+ }),+ }),+ type: 'readOnly',+ },++ handle: async (context, params) => {+ const stream = new StringWriteStream();+ const screen = {+ ...terminalScreen,+ isTTY: false,+ colors: noColors,+ stdout: stream as unknown as NodeJS.WriteStream,+ stderr: stream as unknown as NodeJS.WriteStream,+ };+ const configDir = context.configLocation.configDir;+ const reporter = new ListReporter({ configDir, screen });+ const testRunner = await context.createTestRunner();+ process.env.PLAYWRIGHT_DEBUGGER_ENABLED = '1';+ const result = await testRunner.runTests(reporter, {+ headed: true,+ testIds: [params.test.id],+ // For automatic recovery+ timeout: 0,+ workers: 1,+ }).finally(() => {+ process.env.PLAYWRIGHT_DEBUGGER_ENABLED = undefined;+ });++ const text = stream.content();+ return {+ content: [+ { type: 'text', text },+ ],+ isError: result.status !== 'passed',+ };+ },+});++function createScreen() {+ const stream = new StringWriteStream();+ const screen = {+ ...terminalScreen,+ isTTY: false,+ colors: noColors,+ stdout: stream as unknown as NodeJS.WriteStream,+ stderr: stream as unknown as NodeJS.WriteStream,+ };+ return { screen, stream };+}diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.tsindex 7211692770..c11657d69d 100644--- a/packages/playwright/src/reporters/base.ts+++ b/packages/playwright/src/reporters/base.ts@@ -145,6 +145,7 @@ export const internalScreen: Screen = {export type TerminalReporterOptions = {screen?: TerminalScreen;omitFailures?: boolean;+ includeTestId?: boolean;};export class TerminalReporter implements ReporterV2 {@@ -154,13 +155,13 @@ export class TerminalReporter implements ReporterV2 {totalTestCount = 0;result!: FullResult;private fileDurations = new Map<string, { duration: number, workers: Set<number> }>();- private _omitFailures: boolean;+ private _options: TerminalReporterOptions;private _fatalErrors: TestError[] = [];private _failureCount: number = 0;constructor(options: TerminalReporterOptions = {}) {this.screen = options.screen ?? terminalScreen;- this._omitFailures = options.omitFailures || false;+ this._options = options;}version(): 'v2' {@@ -312,7 +313,7 @@ export class TerminalReporter implements ReporterV2 {epilogue(full: boolean) {const summary = this.generateSummary();const summaryMessage = this.generateSummaryMessage(summary);- if (full && summary.failuresToPrint.length && !this._omitFailures)+ if (full && summary.failuresToPrint.length && !this._options.omitFailures)this._printFailures(summary.failuresToPrint);this._printSlowTests();this._printSummary(summaryMessage);@@ -343,16 +344,16 @@ export class TerminalReporter implements ReporterV2 {return test.outcome() === 'unexpected' && test.results.length <= test.retries;}- formatTestTitle(test: TestCase, step?: TestStep, omitLocation: boolean = false): string {- return formatTestTitle(this.screen, this.config, test, step, omitLocation);+ formatTestTitle(test: TestCase, step?: TestStep): string {+ return formatTestTitle(this.screen, this.config, test, step, this._options);}formatTestHeader(test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error' } = {}): string {- return formatTestHeader(this.screen, this.config, test, options);+ return formatTestHeader(this.screen, this.config, test, { ...options, includeTestId: this._options.includeTestId });}formatFailure(test: TestCase, index?: number): string {- return formatFailure(this.screen, this.config, test, index);+ return formatFailure(this.screen, this.config, test, index, this._options);}formatError(error: TestError): ErrorDetails {@@ -364,9 +365,9 @@ export class TerminalReporter implements ReporterV2 {}}-export function formatFailure(screen: Screen, config: FullConfig, test: TestCase, index?: number): string {+export function formatFailure(screen: Screen, config: FullConfig, test: TestCase, index?: number, options?: TerminalReporterOptions): string {const lines: string[] = [];- const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error' });+ const header = formatTestHeader(screen, config, test, { indent: ' ', index, mode: 'error', includeTestId: options?.includeTestId });lines.push(screen.colors.red(header));for (const result of test.results) {const resultLines: string[] = [];@@ -494,22 +495,20 @@ export function stepSuffix(step: TestStep | undefined) {return stepTitles.map(t => t.split('\n')[0]).map(t => ' › ' + t).join('');}-function formatTestTitle(screen: Screen, config: FullConfig, test: TestCase, step?: TestStep, omitLocation: boolean = false): string {+function formatTestTitle(screen: Screen, config: FullConfig, test: TestCase, step?: TestStep, options: { includeTestId?: boolean } = {}): string {// root, project, file, ...describes, testconst [, projectName, , ...titles] = test.titlePath();- let location;- if (omitLocation)- location = `${relativeTestPath(screen, config, test)}`;- else- location = `${relativeTestPath(screen, config, test)}:${test.location.line}:${test.location.column}`;- const projectTitle = projectName ? `[${projectName}] › ` : '';- const testTitle = `${projectTitle}${location} › ${titles.join(' › ')}`;+ const location = `${relativeTestPath(screen, config, test)}:${test.location.line}:${test.location.column}`;+ const testId = options.includeTestId ? `[id=${test.id}] ` : '';+ const projectLabel = options.includeTestId ? `project=` : '';+ const projectTitle = projectName ? `[${projectLabel}${projectName}] › ` : '';+ const testTitle = `${testId}${projectTitle}${location} › ${titles.join(' › ')}`;const extraTags = test.tags.filter(t => !testTitle.includes(t));return `${testTitle}${stepSuffix(step)}${extraTags.length ? ' ' + extraTags.join(' ') : ''}`;}-function formatTestHeader(screen: Screen, config: FullConfig, test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error' } = {}): string {- const title = formatTestTitle(screen, config, test);+function formatTestHeader(screen: Screen, config: FullConfig, test: TestCase, options: { indent?: string, index?: number, mode?: 'default' | 'error', includeTestId?: boolean } = {}): string {+ const title = formatTestTitle(screen, config, test, undefined, options);const header = `${options.indent || ''}${options.index ? options.index + ') ' : ''}${title}`;let fullHeader = header;diff --git a/packages/playwright/src/reporters/listModeReporter.ts b/packages/playwright/src/reporters/listModeReporter.tsindex 2337067ed0..1732cd8db6 100644--- a/packages/playwright/src/reporters/listModeReporter.ts+++ b/packages/playwright/src/reporters/listModeReporter.ts@@ -26,8 +26,10 @@ import type { ReporterV2 } from './reporterV2';class ListModeReporter implements ReporterV2 {private config!: FullConfig;private screen: TerminalScreen;+ private _options: { screen?: TerminalScreen, includeTestId?: boolean };- constructor(options?: { screen?: TerminalScreen }) {+ constructor(options: { screen?: TerminalScreen, includeTestId?: boolean } = {}) {+ this._options = options;this.screen = options?.screen ?? terminalScreen;}@@ -47,8 +49,10 @@ class ListModeReporter implements ReporterV2 {// root, project, file, ...describes, testconst [, projectName, , ...titles] = test.titlePath();const location = `${path.relative(this.config.rootDir, test.location.file)}:${test.location.line}:${test.location.column}`;- const projectTitle = projectName ? `[${projectName}] › ` : '';- this._writeLine(` ${projectTitle}${location} › ${titles.join(' › ')}`);+ const testId = this._options.includeTestId ? `[id=${test.id}] ` : '';+ const projectLabel = this._options.includeTestId ? `project=` : '';+ const projectTitle = projectName ? `[${projectLabel}${projectName}] › ` : '';+ this._writeLine(` ${testId}${projectTitle}${location} › ${titles.join(' › ')}`);files.add(test.location.file);}this._writeLine(`Total: ${tests.length} ${tests.length === 1 ? 'test' : 'tests'} in ${files.size} ${files.size === 1 ? 'file' : 'files'}`);