Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(test-mcp): fix test runner mcp issues, cover with tests (#37365)
Pavel Feldmane331e5665c06
diff --git a/.github/workflows/tests_mcp.yml b/.github/workflows/tests_mcp.ymlindex 5c447ceb99..3be6a59554 100644--- a/.github/workflows/tests_mcp.yml+++ b/.github/workflows/tests_mcp.yml@@ -43,4 +43,4 @@ jobs:- run: npm ci- run: npm run build- run: npx playwright install --with-deps- - run: npm run mcp-test+ - run: npm run test-mcpdiff --git a/package.json b/package.jsonindex e9b74e2430..02299a5b2e 100644--- a/package.json+++ b/package.json@@ -30,9 +30,9 @@"ttest": "node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright.config.ts","ct": "playwright test tests/components/test-all.spec.js --reporter=list","test": "playwright test --config=tests/library/playwright.config.ts",- "mcp-test": "playwright test --config=tests/mcp/playwright.config.ts",- "mcp-ctest": "playwright test --config=tests/mcp/playwright.config.ts --project=chrome",- "mcp-etest": "playwright test --config=tests/mcp/playwright.config.extension.ts",+ "test-mcp": "playwright test --config=tests/mcp/playwright.config.ts",+ "ctest-mcp": "playwright test --config=tests/mcp/playwright.config.ts --project=chrome",+ "etest-mcp": "playwright test --config=tests/mcp/playwright.config.extension.ts","eslint": "eslint --cache","tsc": "tsc -p . && tsc -p packages/html-reporter/","doc": "node utils/doclint/cli.js",diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.tsindex 9031be7eea..179ba048b6 100644--- a/packages/playwright/src/common/ipc.ts+++ b/packages/playwright/src/common/ipc.ts@@ -66,6 +66,7 @@ export type WorkerInitParams = {projectId: string;config: SerializedConfig;artifactsDir: string;+ pauseOnError: boolean;};export type TestBeginPayload = {diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.tsindex b9b681838e..efbc00e686 100644--- a/packages/playwright/src/index.ts+++ b/packages/playwright/src/index.ts@@ -240,7 +240,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({(testInfo as TestInfoImpl)._setDebugMode();playwright._defaultContextOptions = _combinedContextOptions;- playwright._defaultContextTimeout = process.env.PLAYWRIGHT_DEBUGGER_ENABLED ? 5000 : actionTimeout || 0;+ playwright._defaultContextTimeout = (testInfo as TestInfoImpl)._pauseOnError() ? 5000 : actionTimeout || 0;playwright._defaultContextNavigationTimeout = navigationTimeout || 0;await use();playwright._defaultContextOptions = undefined;@@ -291,6 +291,10 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({tracingGroupSteps.push(step);},onApiCallRecovery: (data, error, channelOwner, recoveryHandlers) => {+ const testInfo = currentTestInfo();+ if (!testInfo || !testInfo._pauseOnError())+ return;+const step = data.userData;if (!step)return;diff --git a/packages/playwright/src/mcp/sdk/mdb.ts b/packages/playwright/src/mcp/sdk/mdb.tsindex 7fbcc3d188..bf3108cac7 100644--- a/packages/playwright/src/mcp/sdk/mdb.ts+++ b/packages/playwright/src/mcp/sdk/mdb.ts@@ -162,7 +162,7 @@ export async function runMainBackend(backendFactory: mcpServer.ServerBackendFactawait mcpServer.connect(factory, new mcpBundle.StdioServerTransport(), false);}-export async function runOnPauseBackendLoop(mdbUrl: string, backend: ServerBackendOnPause, introMessage: string) {+export async function runOnPauseBackendLoop(backend: ServerBackendOnPause, introMessage: string) {const wrappedBackend = new OnceTimeServerBackendWrapper(backend);const factory = {@@ -178,7 +178,7 @@ export async function runOnPauseBackendLoop(mdbUrl: string, backend: ServerBackeconst client = new mcpBundle.Client({ name: 'Internal client', version: '0.0.0' });client.setRequestHandler(mcpBundle.PingRequestSchema, () => ({}));- const transport = new mcpBundle.StreamableHTTPClientTransport(new URL(mdbUrl));+ const transport = new mcpBundle.StreamableHTTPClientTransport(new URL(process.env.PLAYWRIGHT_DEBUGGER_MCP!));await client.connect(transport);const pushToolsResult = await client.callTool({diff --git a/packages/playwright/src/mcp/test/browserBackend.ts b/packages/playwright/src/mcp/test/browserBackend.tsindex 42ded6d876..4ec79ebe42 100644--- a/packages/playwright/src/mcp/test/browserBackend.ts+++ b/packages/playwright/src/mcp/test/browserBackend.ts@@ -16,6 +16,7 @@import * as mcp from '../sdk/exports';import * as mcpBundle from '../sdk/bundle';+import { currentTestInfo } from '../../common/globals';import { snapshot, pickLocator, evaluate } from './browserTools';import { stripAnsiEscapes } from '../../util';@@ -72,8 +73,10 @@ const doneToolSchema = mcp.defineToolSchema({});export async function runBrowserBackendOnError(page: playwright.Page, message: () => string) {- if (!process.env.PLAYWRIGHT_DEBUGGER_ENABLED)+ const testInfo = currentTestInfo();+ if (!testInfo || !testInfo._pauseOnError())return;+const snapshot = await (page as PageEx)._snapshotForAI();const introMessage = `### Paused on error:${stripAnsiEscapes(message())}@@ -83,5 +86,5 @@ ${snapshot}### TaskTry recovering from the error prior to continuing, use following tools to recover: ${tools.map(tool => tool.schema.name).join(', ')}`;- await mcp.runOnPauseBackendLoop(process.env.PLAYWRIGHT_MDB_URL!, new BrowserBackend(page), introMessage);+ await mcp.runOnPauseBackendLoop(new BrowserBackend(page), introMessage);}diff --git a/packages/playwright/src/mcp/test/testBackend.ts b/packages/playwright/src/mcp/test/testBackend.tsindex 0ef240a2ad..5625aeda55 100644--- a/packages/playwright/src/mcp/test/testBackend.ts+++ b/packages/playwright/src/mcp/test/testBackend.ts@@ -29,7 +29,7 @@ export class TestServerBackend implements mcp.ServerBackend {private _tools: TestTool<any>[] = [listTests, runTests, debugTest];private _context: TestContext;- constructor(resolvedLocation: ConfigLocation, options?: { muteConsole?: boolean }) {+ constructor(resolvedLocation: ConfigLocation, options?: { muteConsole?: boolean, headless?: boolean }) {this._context = new TestContext(resolvedLocation, options);}diff --git a/packages/playwright/src/mcp/test/testContext.ts b/packages/playwright/src/mcp/test/testContext.tsindex 11b5ab5d01..3209b9d3f5 100644--- a/packages/playwright/src/mcp/test/testContext.ts+++ b/packages/playwright/src/mcp/test/testContext.ts@@ -21,9 +21,9 @@ import type { ConfigLocation } from '../../common/config';export class TestContext {private _testRunner: TestRunner | undefined;readonly configLocation: ConfigLocation;- readonly options?: { muteConsole?: boolean };+ readonly options?: { muteConsole?: boolean, headless?: boolean };- constructor(configLocation: ConfigLocation, options?: { muteConsole?: boolean }) {+ constructor(configLocation: ConfigLocation, options?: { muteConsole?: boolean, headless?: boolean }) {this.configLocation = configLocation;this.options = options;}diff --git a/packages/playwright/src/mcp/test/testTools.ts b/packages/playwright/src/mcp/test/testTools.tsindex 08c74e982c..2d2929c9a3 100644--- a/packages/playwright/src/mcp/test/testTools.ts+++ b/packages/playwright/src/mcp/test/testTools.ts@@ -51,7 +51,7 @@ export const runTests = defineTestTool({title: 'Run tests',description: 'Run tests',inputSchema: z.object({- locations: z.array(z.string()).describe('Folder, file or location to run: "test/e2e" or "test/e2e/file.spec.ts" or "test/e2e/file.spec.ts:20"'),+ locations: z.array(z.string()).optional().describe('Folder, file or location to run: "test/e2e" or "test/e2e/file.spec.ts" or "test/e2e/file.spec.ts:20"'),projects: z.array(z.string()).optional().describe('Projects to run, projects from playwright.config.ts, by default runs all projects. Running with "chromium" is a good start'),}),type: 'readOnly',@@ -103,15 +103,13 @@ export const debugTest = defineTestTool({const configDir = context.configLocation.configDir;const reporter = new ListReporter({ configDir, screen });const testRunner = await context.createTestRunner();- process.env.PLAYWRIGHT_DEBUGGER_ENABLED = '1';const result = await testRunner.runTests(reporter, {- headed: true,+ headed: !context.options?.headless,testIds: [params.test.id],// For automatic recoverytimeout: 0,workers: 1,- }).finally(() => {- process.env.PLAYWRIGHT_DEBUGGER_ENABLED = undefined;+ pauseOnError: true,});const text = stream.content();diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex 62d02f04d8..ae41d56ab7 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -154,6 +154,7 @@ function addBrowserMCPServerCommand(program: Command) {function addTestMCPServerCommand(program: Command) {const command = program.command('run-test-mcp-server', { hidden: true });command.description('Interact with the test runner over MCP');+ command.option('--headless', 'run browser in headless mode, headed by default');command.option('-c, --config <file>', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`);command.option('--host <host>', 'host to bind server to. Default is localhost. Use 0.0.0.0 to bind to all interfaces.');command.option('--port <port>', 'port to listen on for SSE transport.');@@ -163,7 +164,7 @@ function addTestMCPServerCommand(program: Command) {name: 'Playwright Test Runner',nameInConfig: 'playwright-test-runner',version: packageJSON.version,- create: () => new TestServerBackend(resolvedLocation, { muteConsole: options.port === undefined }),+ create: () => new TestServerBackend(resolvedLocation, { muteConsole: options.port === undefined, headless: options.headless }),};const mdbUrl = await runMainBackend(backendFactory, { port: options.port === undefined ? undefined : +options.port });if (mdbUrl)diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex 388586fe96..5a2310e733 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -218,7 +218,7 @@ export class Dispatcher {_createWorker(testGroup: TestGroup, parallelIndex: number, loaderData: SerializedConfig) {const projectConfig = this._config.projects.find(p => p.id === testGroup.projectId)!;const outputDir = projectConfig.project.outputDir;- const worker = new WorkerHost(testGroup, parallelIndex, loaderData, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir);+ const worker = new WorkerHost(testGroup, parallelIndex, loaderData, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir, this._failureTracker.pauseOnError());const handleOutput = (params: TestOutputPayload) => {const chunk = chunkFromParams(params);if (worker.didFail()) {diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.tsindex dd8ce50587..499365deaa 100644--- a/packages/playwright/src/runner/failureTracker.ts+++ b/packages/playwright/src/runner/failureTracker.ts@@ -22,8 +22,10 @@ export class FailureTracker {private _failureCount = 0;private _hasWorkerErrors = false;private _rootSuite: Suite | undefined;+ private _pauseOnError: boolean;- constructor(private _config: FullConfigInternal) {+ constructor(private _config: FullConfigInternal, options?: { pauseOnError?: boolean }) {+ this._pauseOnError = options?.pauseOnError ?? false;}onRootSuite(rootSuite: Suite) {@@ -40,6 +42,10 @@ export class FailureTracker {this._hasWorkerErrors = true;}+ pauseOnError(): boolean {+ return this._pauseOnError;+ }+hasReachedMaxFailures() {return this.maxFailures() > 0 && this._failureCount >= this.maxFailures();}diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex ea4123be24..4c980f2d57 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -64,10 +64,10 @@ export class TestRun {projectFiles: Map<FullProjectInternal, string[]> = new Map();projectSuites: Map<FullProjectInternal, Suite[]> = new Map();- constructor(config: FullConfigInternal, reporter: InternalReporter) {+ constructor(config: FullConfigInternal, reporter: InternalReporter, options?: { pauseOnError?: boolean }) {this.config = config;this.reporter = reporter;- this.failureTracker = new FailureTracker(config);+ this.failureTracker = new FailureTracker(config, options);}}diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.tsindex 8255f67ae3..bb06ff33f7 100644--- a/packages/playwright/src/runner/testRunner.ts+++ b/packages/playwright/src/runner/testRunner.ts@@ -71,6 +71,7 @@ export type RunTestsParams = {projects?: string[];reuseContext?: boolean;connectWsEndpoint?: string;+ pauseOnError?: boolean;};type FullResultStatus = reporterTypes.FullResult['status'];@@ -335,7 +336,7 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }),...createRunTestsTasks(config),];- const testRun = new TestRun(config, reporter);+ const testRun = new TestRun(config, reporter, { pauseOnError: params.pauseOnError });const run = runTasks(testRun, tasks, 0, stop).then(async status => {this._testRun = undefined;return status;diff --git a/packages/playwright/src/runner/workerHost.ts b/packages/playwright/src/runner/workerHost.tsindex 9d69506330..bf5451672b 100644--- a/packages/playwright/src/runner/workerHost.ts+++ b/packages/playwright/src/runner/workerHost.ts@@ -36,7 +36,7 @@ export class WorkerHost extends ProcessHost {private _params: WorkerInitParams;private _didFail = false;- constructor(testGroup: TestGroup, parallelIndex: number, config: SerializedConfig, extraEnv: Record<string, string | undefined>, outputDir: string) {+ constructor(testGroup: TestGroup, parallelIndex: number, config: SerializedConfig, extraEnv: Record<string, string | undefined>, outputDir: string, pauseOnError: boolean) {const workerIndex = lastWorkerIndex++;super(require.resolve('../worker/workerMain.js'), `worker-${workerIndex}`, {...extraEnv,@@ -54,6 +54,7 @@ export class WorkerHost extends ProcessHost {projectId: testGroup.projectId,config,artifactsDir: path.join(outputDir, artifactsFolderName(workerIndex)),+ pauseOnError,};}diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.tsindex 8d93429136..93dda2e2ad 100644--- a/packages/playwright/src/worker/testInfo.ts+++ b/packages/playwright/src/worker/testInfo.ts@@ -119,6 +119,7 @@ export class TestInfoImpl implements TestInfo {readonly snapshotDir: string;errors: TestInfoErrorImpl[] = [];readonly _attachmentsPush: (...items: TestInfo['attachments']) => number;+ private _workerParams: WorkerInitParams;get error(): TestInfoErrorImpl | undefined {return this.errors[0];@@ -169,6 +170,7 @@ export class TestInfoImpl implements TestInfo {this._startWallTime = Date.now();this._requireFile = test?._requireFile ?? '';this._uniqueSymbol = Symbol('testInfoUniqueSymbol');+ this._workerParams = workerParams;this.repeatEachIndex = workerParams.repeatEachIndex;this.retry = retry;@@ -608,6 +610,10 @@ export class TestInfoImpl implements TestInfo {setTimeout(timeout: number) {this._timeoutManager.setTimeout(timeout);}++ _pauseOnError(): boolean {+ return this._workerParams.pauseOnError;+ }}export class TestStepInfoImpl implements TestStepInfo {diff --git a/tests/mcp/cdp.spec.ts b/tests/mcp/cdp.spec.tsindex fe209b7404..650dc57239 100644--- a/tests/mcp/cdp.spec.ts+++ b/tests/mcp/cdp.spec.ts@@ -15,7 +15,7 @@*/import { spawnSync } from 'child_process';-import { test, expect, programPath } from './fixtures';+import { test, expect, mcpServerPath } from './fixtures';test('cdp server', async ({ cdpServer, startClient, server }) => {await cdpServer.start();@@ -84,7 +84,7 @@ test('should throw connection error and allow re-connecting', async ({ cdpServertest('does not support --device', async () => {const result = spawnSync('node', [- ...programPath, '--device=Pixel 5', '--cdp-endpoint=http://localhost:1234',+ ...mcpServerPath, '--device=Pixel 5', '--cdp-endpoint=http://localhost:1234',]);expect(result.error).toBeUndefined();expect(result.status).toBe(1);diff --git a/tests/mcp/config.spec.ts b/tests/mcp/config.spec.tsindex b7109f9cd9..b132176175 100644--- a/tests/mcp/config.spec.ts+++ b/tests/mcp/config.spec.ts@@ -20,7 +20,7 @@ import { test, expect } from './fixtures';import { configFromCLIOptions } from '../../packages/playwright/lib/mcp/browser/config';import type { Config } from '../../packages/playwright/src/mcp/config';-test('config user data dir', async ({ startClient, server, mcpMode }, testInfo) => {+test('config user data dir', async ({ startClient, server }, testInfo) => {server.setContent('/', `<title>Title</title><body>Hello, world!</body>@@ -59,7 +59,7 @@ test('executable path', async ({ startClient, server }, testInfo) => {test.describe(() => {test.use({ mcpBrowser: '' });- test('browserName', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright-mcp/issues/458' } }, async ({ startClient, mcpMode }, testInfo) => {+ test('browserName', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright-mcp/issues/458' } }, async ({ startClient }, testInfo) => {const config: Config = {browser: {browserName: 'firefox',diff --git a/tests/mcp/device.spec.ts b/tests/mcp/device.spec.tsindex 5c0f733101..2207b5b25f 100644--- a/tests/mcp/device.spec.ts+++ b/tests/mcp/device.spec.ts@@ -16,7 +16,7 @@import { test, expect } from './fixtures';-test('--device should work', async ({ startClient, server, mcpMode }) => {+test('--device should work', async ({ startClient, server }) => {const { client } = await startClient({args: ['--device', 'iPhone 15'],});diff --git a/tests/mcp/files.spec.ts b/tests/mcp/files.spec.tsindex 5a1bb645d2..a632209524 100644--- a/tests/mcp/files.spec.ts+++ b/tests/mcp/files.spec.ts@@ -100,7 +100,7 @@ test('browser_file_upload', async ({ client, server }, testInfo) => {}});-test('clicking on download link emits download', async ({ startClient, server, mcpMode }, testInfo) => {+test('clicking on download link emits download', async ({ startClient, server }, testInfo) => {const { client } = await startClient({config: { outputDir: testInfo.outputPath('output') },});@@ -126,7 +126,7 @@ test('clicking on download link emits download', async ({ startClient, server, m});});-test('navigating to download link emits download', async ({ startClient, server, mcpBrowser, mcpMode }, testInfo) => {+test('navigating to download link emits download', async ({ startClient, server, mcpBrowser }, testInfo) => {const { client } = await startClient({config: { outputDir: testInfo.outputPath('output') },});diff --git a/tests/mcp/fixtures.ts b/tests/mcp/fixtures.tsindex 47dc2e7e7c..9045ca2c3d 100644--- a/tests/mcp/fixtures.ts+++ b/tests/mcp/fixtures.ts@@ -34,7 +34,7 @@ import type { ServerFixtures, ServerWorkerOptions } from '../config/serverFixturexport type TestOptions = {mcpArgs: string[] | undefined;mcpBrowser: string | undefined;- mcpMode: 'docker' | undefined;+ mcpServerType: 'mcp' | 'test-mcp';};type CDPServer = {@@ -75,25 +75,32 @@ export const test = serverTest.extend<TestFixtures & TestOptions, WorkerFixturesawait use(client);},- startClient: async ({ mcpHeadless, mcpBrowser, mcpMode, mcpArgs }, use, testInfo) => {+ startClient: async ({ mcpHeadless, mcpBrowser, mcpArgs, mcpServerType }, use, testInfo) => {const configDir = path.dirname(test.info().config.configFile!);const clients: Client[] = [];await use(async options => {const args: string[] = mcpArgs ?? [];- if (process.env.CI && process.platform === 'linux')- args.push('--no-sandbox');+if (mcpHeadless)args.push('--headless');- if (mcpBrowser)- args.push(`--browser=${mcpBrowser}`);++ if (mcpServerType === 'test-mcp') {+ args.push('--config', test.info().outputPath());+ } else {+ if (process.env.CI && process.platform === 'linux')+ args.push('--no-sandbox');+ if (mcpBrowser)+ args.push(`--browser=${mcpBrowser}`);+ if (options?.config) {+ const configFile = testInfo.outputPath('config.json');+ await fs.promises.writeFile(configFile, JSON.stringify(options.config, null, 2));+ args.push(`--config=${path.relative(configDir, configFile)}`);+ }+ }+if (options?.args)args.push(...options.args);- if (options?.config) {- const configFile = testInfo.outputPath('config.json');- await fs.promises.writeFile(configFile, JSON.stringify(options.config, null, 2));- args.push(`--config=${path.relative(configDir, configFile)}`);- }const client = new Client({ name: options?.clientName ?? 'test', version: '1.0.0' }, options?.roots ? { capabilities: { roots: {} } } : undefined);if (options?.roots) {@@ -105,7 +112,7 @@ export const test = serverTest.extend<TestFixtures & TestOptions, WorkerFixtures};});}- const { transport, stderr } = await createTransport(args, mcpMode, testInfo.outputPath('ms-playwright'));+ const { transport, stderr } = await createTransport(mcpServerType, args);let stderrBuffer = '';stderr?.on('data', data => {if (process.env.PWMCP_DEBUG)@@ -166,29 +173,18 @@ export const test = serverTest.extend<TestFixtures & TestOptions, WorkerFixturesmcpBrowser: ['chrome', { option: true }],- mcpMode: [undefined, { option: true }],+ mcpServerType: ['mcp', { option: true }],});-async function createTransport(args: string[], mcpMode: TestOptions['mcpMode'], profilesDir: string): Promise<{+async function createTransport(mcpServerType: TestOptions['mcpServerType'], args: string[]): Promise<{transport: Transport,stderr: Stream | null,}> {- if (mcpMode === 'docker') {- const dockerArgs = ['run', '--rm', '-i', '--network=host', '-v', `${test.info().project.outputDir}:/app/test-results`];- const transport = new StdioClientTransport({- command: 'docker',- args: [...dockerArgs, 'playwright-mcp-dev:latest', ...args],- });- return {- transport,- stderr: transport.stderr,- };- }-+ const profilesDir = test.info().outputPath('ms-playwright');const transport = new StdioClientTransport({command: 'node',- args: [...programPath, ...args],- cwd: path.dirname(test.info().config.configFile!),+ args: [...(mcpServerType === 'test-mcp' ? testMcpServerPath : mcpServerPath), ...args],+ cwd: test.info().outputPath(),stderr: 'pipe',env: {...process.env,@@ -226,6 +222,30 @@ export const expect = baseExpect.extend({message: () => ``,};},++ toHaveTextResponse(response: Response, value: any) {+ const text = response.content[0].text+ .replace(/\[id=[^\]]+\]/g, '[id=<ID>]')+ .replace(/\([\d\.]+m?s\)/g, '(XXms)')+ .replace(/[✓] /g, 'ok');++ const isNot = this.isNot;+ try {+ if (isNot)+ expect(text).not.toEqual(value);+ else+ expect(text).toEqual(value);+ } catch (e) {+ return {+ pass: isNot,+ message: () => e.message,+ };+ }+ return {+ pass: !isNot,+ message: () => ``,+ };+ },});export function formatOutput(output: string): string[] {@@ -277,4 +297,35 @@ function parseSections(text: string): Map<string, string> {return sections;}-export const programPath = [path.join(__dirname, '../../packages/playwright/cli.js'), 'run-mcp-server'];+export const mcpServerPath = [path.join(__dirname, '../../packages/playwright/cli.js'), 'run-mcp-server'];+export const testMcpServerPath = [path.join(__dirname, '../../packages/playwright-test/cli.js'), 'run-test-mcp-server'];++type Files = { [key: string]: string | Buffer };++export async function writeFiles(files: Files, options?: { update?: boolean }) {+ const baseDir = test.info().outputPath();++ if (!options?.update && !Object.keys(files).some(name => name.includes('package.json'))) {+ files = {+ ...files,+ 'package.json': `{ "name": "test-project" }`,+ };+ }++ if (!options?.update && !Object.keys(files).some(name => name.includes('tsconfig.json') || name.includes('jsconfig.json'))) {+ files = {+ ...files,+ 'tsconfig.json': `{}`,+ };+ }++ await Promise.all(Object.keys(files).map(async name => {+ const fullName = path.join(baseDir, name);+ if (files[name] === undefined)+ return;+ await fs.promises.mkdir(path.dirname(fullName), { recursive: true });+ await fs.promises.writeFile(fullName, files[name]);+ }));++ return baseDir;+}diff --git a/tests/mcp/headed.spec.ts b/tests/mcp/headed.spec.tsindex 1654a30a50..56fd4923c6 100644--- a/tests/mcp/headed.spec.ts+++ b/tests/mcp/headed.spec.ts@@ -20,7 +20,6 @@ for (const mcpHeadless of [false, true]) {test.describe(`mcpHeadless: ${mcpHeadless}`, () => {test.use({ mcpHeadless });test.skip(process.platform === 'linux', 'Auto-detection wont let this test run on linux');- test.skip(({ mcpMode, mcpHeadless }) => mcpMode === 'docker' && !mcpHeadless, 'Headed mode is not supported in docker');test('browser', async ({ client, server, mcpBrowser }) => {test.skip(!['chrome', 'msedge', 'chromium'].includes(mcpBrowser ?? ''), 'Only chrome is supported for this test');diff --git a/tests/mcp/http.spec.ts b/tests/mcp/http.spec.tsindex 83ed6b86f5..f533dd8dc8 100644--- a/tests/mcp/http.spec.ts+++ b/tests/mcp/http.spec.ts@@ -19,7 +19,7 @@ import fs from 'fs';import { ChildProcess, spawn } from 'child_process';import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';import { Client } from '@modelcontextprotocol/sdk/client/index.js';-import { test as baseTest, expect, programPath } from './fixtures';+import { test as baseTest, expect, mcpServerPath } from './fixtures';import type { Config } from '../../packages/playwright/src/mcp/config';import { ListRootsRequestSchema } from 'packages/playwright/lib/mcp/sdk/bundle';@@ -33,7 +33,7 @@ const test = baseTest.extend<{ serverEndpoint: (options?: { args?: string[], noPthrow new Error('Process already running');cp = spawn('node', [- ...programPath,+ ...mcpServerPath,...(options?.noPort ? [] : ['--port=0']),'--user-data-dir=' + userDataDir,...(mcpHeadless ? ['--headless'] : []),diff --git a/tests/mcp/launch.spec.ts b/tests/mcp/launch.spec.tsindex f7f0171aa9..03a637b998 100644--- a/tests/mcp/launch.spec.ts+++ b/tests/mcp/launch.spec.ts@@ -18,7 +18,7 @@ import fs from 'fs';import { test, expect, formatOutput } from './fixtures';-test('test reopen browser', async ({ startClient, server, mcpMode }) => {+test('test reopen browser', async ({ startClient, server }) => {const { client, stderr } = await startClient();await client.callTool({name: 'browser_navigate',diff --git a/tests/mcp/mdb.spec.ts b/tests/mcp/mdb.spec.tsindex 85229a7864..74f240385d 100644--- a/tests/mcp/mdb.spec.ts+++ b/tests/mcp/mdb.spec.ts@@ -170,12 +170,12 @@ class CLIBackend {if (name === 'cli_echo')return { content: [{ type: 'text', text: 'Echo: ' + (args?.message as string) }] };if (name === 'cli_pause_in_gdb') {- await runOnPauseBackendLoop(this.mdbUrlBox.mdbUrl!, new GDBBackend(), 'Paused on exception');+ await runOnPauseBackendLoop(new GDBBackend(), 'Paused on exception');return { content: [{ type: 'text', text: 'Done' }] };}if (name === 'cli_pause_in_gdb_twice') {- await runOnPauseBackendLoop(this.mdbUrlBox.mdbUrl!, new GDBBackend(), 'Paused on exception 1');- await runOnPauseBackendLoop(this.mdbUrlBox.mdbUrl!, new GDBBackend(), 'Paused on exception 2');+ await runOnPauseBackendLoop(new GDBBackend(), 'Paused on exception 1');+ await runOnPauseBackendLoop(new GDBBackend(), 'Paused on exception 2');return { content: [{ type: 'text', text: 'Done' }] };}throw new Error(`Unknown tool: ${name}`);diff --git a/tests/mcp/playwright.config.ts b/tests/mcp/playwright.config.tsindex 923a0f8034..7edcbb1687 100644--- a/tests/mcp/playwright.config.ts+++ b/tests/mcp/playwright.config.ts@@ -28,14 +28,6 @@ export default defineConfig<TestOptions>({projects: [{ name: 'chrome' },{ name: 'chromium', use: { mcpBrowser: 'chromium' } },- ...process.env.MCP_IN_DOCKER ? [{- name: 'chromium-docker',- grep: /browser_navigate|browser_click/,- use: {- mcpBrowser: 'chromium',- mcpMode: 'docker' as const- }- }] : [],{ name: 'firefox', use: { mcpBrowser: 'firefox' } },{ name: 'webkit', use: { mcpBrowser: 'webkit' } },... process.platform === 'win32' ? [{ name: 'msedge', use: { mcpBrowser: 'msedge' } }] : [],diff --git a/tests/mcp/sse.spec.ts b/tests/mcp/sse.spec.tsindex 5d8d706c0d..d67a24a98b 100644--- a/tests/mcp/sse.spec.ts+++ b/tests/mcp/sse.spec.ts@@ -19,7 +19,7 @@ import fs from 'fs';import { ChildProcess, spawn } from 'child_process';import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';import { Client } from '@modelcontextprotocol/sdk/client/index.js';-import { test as baseTest, expect, programPath } from './fixtures';+import { test as baseTest, expect, mcpServerPath } from './fixtures';import type { Config } from '../../packages/playwright/src/mcp/config';@@ -32,7 +32,7 @@ const test = baseTest.extend<{ serverEndpoint: (options?: { args?: string[], noPthrow new Error('Process already running');cp = spawn('node', [- ...programPath,+ ...mcpServerPath,...(options?.noPort ? [] : ['--port=0']),'--user-data-dir=' + userDataDir,...(mcpHeadless ? ['--headless'] : []),diff --git a/tests/mcp/test-tools.spec.ts b/tests/mcp/test-tools.spec.tsnew file mode 100644index 0000000000..f6a02cbf1c--- /dev/null+++ b/tests/mcp/test-tools.spec.ts@@ -0,0 +1,284 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import { test, expect, writeFiles, StartClient } from './fixtures';++test.use({ mcpServerType: 'test-mcp' });++test('playwright_test_list_tests', async ({ startClient }) => {+ await writeFiles({+ 'playwright.config.ts': `+ module.exports = { projects: [{ name: 'foo' }, {}] };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('example1', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ test('example2', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ `+ });++ const { client } = await startClient();+ expect(await client.callTool({+ name: 'playwright_test_list_tests',+ arguments: {},+ })).toHaveTextResponse(`Listing tests:+ [id=<ID>] [project=foo] › a.test.ts:3:11 › example1+ [id=<ID>] [project=foo] › a.test.ts:6:11 › example2+ [id=<ID>] a.test.ts:3:11 › example1+ [id=<ID>] a.test.ts:6:11 › example2+Total: 4 tests in 1 file+`);+});++test('playwright_test_run_tests', async ({ startClient }) => {+ await writeFiles({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('passes', () => {});+ test('fails', () => { expect(1).toBe(2); });+ test.describe('suite', () => {+ test('inner passes', () => {});+ test('inner fails', () => { expect(1).toBe(2); });+ });+ `,+ 'b.test.ts': `+ import { test, expect } from '@playwright/test';+ test('passes', () => {});+ test('fails', () => { expect(1).toBe(2); });+ `,+ 'c.test.ts': `+ import { test, expect } from '@playwright/test';+ test('passes', () => {});+ test.skip('skipped', () => {});+ `,+ });++ const { client } = await startClient();+ const response = await client.callTool({+ name: 'playwright_test_run_tests',+ });++ const text = response.content[0].text;++ expect(text).toContain(`3 failed`);+ expect(text).toContain(`1 skipped`);+ expect(text).toContain(`4 passed`);++ expect(text).toContain(`a.test.ts:3:11 › passes`);+ expect(text).toContain(`c.test.ts:3:11 › passes`);+ expect(text).toContain(`c.test.ts:4:12 › skipped`);+ expect(text).toContain(`b.test.ts:3:11 › passes`);+ expect(text).toContain(`a.test.ts:4:11 › fails`);+ expect(text).toContain(`b.test.ts:4:11 › fails`);+ expect(text).toContain(`a.test.ts:6:13 › suite › inner passes`);+ expect(text).toContain(`a.test.ts:7:13 › suite › inner fails`);++ expect(text).not.toContain(`../../test-results`);+});++test('playwright_test_run_tests filters', async ({ startClient }) => {+ await writeFiles({+ 'playwright.config.ts': `+ module.exports = { projects: [{ name: 'foo' }, { name: 'bar' }] };+ `,+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('example1', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ test('example2', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ `,+ 'b.test.ts': `+ import { test, expect } from '@playwright/test';+ test('example1', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ test('example2', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ `+ });++ const { client } = await startClient();+ expect(await client.callTool({+ name: 'playwright_test_run_tests',+ arguments: {+ locations: ['b.test.ts'],+ projects: ['foo'],+ },+ })).toHaveTextResponse(`+Running 2 tests using 1 worker++ ok 1 [id=<ID>] [project=foo] › b.test.ts:3:11 › example1 (XXms)+ ok 2 [id=<ID>] [project=foo] › b.test.ts:6:11 › example2 (XXms)++ 2 passed (XXms)+`);+});++test('playwright_test_debug_test (passed)', async ({ startClient }) => {+ await writeFiles({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('pass', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ `+ });++ const { client } = await startClient();+ const listResult = await client.callTool({+ name: 'playwright_test_list_tests',+ });+ const [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/);++ expect(await client.callTool({+ name: 'playwright_test_debug_test',+ arguments: {+ test: { id, title: 'pass' },+ },+ })).toHaveTextResponse(`+Running 1 test using 1 worker++ ok 1 a.test.ts:3:11 › pass (XXms)++ 1 passed (XXms)+`);+});++test('playwright_test_debug_test (pause/resume)', async ({ startClient }) => {+ const { client, id } = await prepareDebugTest(startClient);++ expect(await client.callTool({+ name: 'playwright_test_debug_test',+ arguments: {+ test: { id, title: 'fail' },+ },+ })).toHaveTextResponse(`### Paused on error:+expect(locator).toBeVisible() failed++Locator: getByRole('button', { name: 'Missing' })+Expected: visible+Error: element(s) not found+Timeout: 1000ms++Call log:+ - Expect "toBeVisible" with timeout 1000ms+ - waiting for getByRole('button', { name: 'Missing' })+++### Current page snapshot:+- button "Submit" [ref=e2]++### Task+Try recovering from the error prior to continuing, use following tools to recover: playwright_test_browser_snapshot, playwright_test_generate_locator, playwright_test_evaluate_on_pause`);++ expect(await client.callTool({+ name: 'playwright_test_run_tests',+ arguments: {+ locations: ['a.test.ts'],+ },+ })).toHaveTextResponse(expect.stringContaining(`1) [id=<ID>] a.test.ts:3:11 › fail`));+});++test('playwright_test_browser_snapshot', async ({ startClient }) => {+ const { client, id } = await prepareDebugTest(startClient);+ await client.callTool({+ name: 'playwright_test_debug_test',+ arguments: {+ test: { id, title: 'fail' },+ },+ });+ expect(await client.callTool({+ name: 'playwright_test_browser_snapshot',+ })).toHaveTextResponse(`- button \"Submit\" [ref=e2]`);+});++test('playwright_test_evaluate_on_pause', async ({ startClient }) => {+ const { client, id } = await prepareDebugTest(startClient);+ await client.callTool({+ name: 'playwright_test_debug_test',+ arguments: {+ test: { id, title: 'fail' },+ },+ });+ expect(await client.callTool({+ name: 'playwright_test_evaluate_on_pause',+ arguments: {+ function: '() => 21+21',+ },+ })).toHaveTextResponse(`42`);+});++test('playwright_test_evaluate_on_pause (with element)', async ({ startClient }) => {+ const { client, id } = await prepareDebugTest(startClient);+ await client.callTool({+ name: 'playwright_test_debug_test',+ arguments: {+ test: { id, title: 'fail' },+ },+ });+ expect(await client.callTool({+ name: 'playwright_test_evaluate_on_pause',+ arguments: {+ function: 'element => element.textContent',+ element: 'button',+ ref: 'e2',+ },+ })).toHaveTextResponse(`"Submit"`);+});++test('playwright_test_generate_locator', async ({ startClient }) => {+ const { client, id } = await prepareDebugTest(startClient);+ await client.callTool({+ name: 'playwright_test_debug_test',+ arguments: {+ test: { id, title: 'fail' },+ },+ });+ expect(await client.callTool({+ name: 'playwright_test_generate_locator',+ arguments: {+ element: 'button',+ ref: 'e2',+ },+ })).toHaveTextResponse(`getByRole('button', { name: 'Submit' })`);+});++async function prepareDebugTest(startClient: StartClient) {+ await writeFiles({+ 'a.test.ts': `+ import { test, expect } from '@playwright/test';+ test('fail', async ({ page }) => {+ await page.setContent('<button>Submit</button>');+ await expect(page.getByRole('button', { name: 'Missing' })).toBeVisible({ timeout: 1000 });+ });+ `+ });++ const { client } = await startClient();+ const listResult = await client.callTool({+ name: 'playwright_test_list_tests',+ });+ const [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/);+ return { client, id };+}diff --git a/tests/mcp/tracing.spec.ts b/tests/mcp/tracing.spec.tsindex 23bc94fee5..9a1d88a4ca 100644--- a/tests/mcp/tracing.spec.ts+++ b/tests/mcp/tracing.spec.ts@@ -18,7 +18,7 @@ import fs from 'fs';import path from 'path';import { test, expect } from './fixtures';-test('check that trace is saved with --save-trace', async ({ startClient, server, mcpMode }, testInfo) => {+test('check that trace is saved with --save-trace', async ({ startClient, server }, testInfo) => {const outputDir = testInfo.outputPath('output');const { client } = await startClient({@@ -36,7 +36,7 @@ test('check that trace is saved with --save-trace', async ({ startClient, serverexpect(file).toContain('traces');});-test('check that trace is saved with browser_start_tracing', async ({ startClient, server, mcpMode }, testInfo) => {+test('check that trace is saved with browser_start_tracing', async ({ startClient, server }, testInfo) => {const outputDir = testInfo.outputPath('output');const { client } = await startClient({ args: [`--output-dir=${outputDir}`, '--caps=tracing'] });