Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(test-mcp): fix test runner mcp issues, cover with tests (#37365)

Pavel Feldmane331e5665c06
diff --git a/.github/workflows/tests_mcp.yml b/.github/workflows/tests_mcp.yml
index 5c447ceb99..3be6a59554 100644
--- a/.github/workflows/tests_mcp.yml
+++ b/.github/workflows/tests_mcp.yml
@@ -43,4 +43,4 @@ jobs:
- run: npm ci
- run: npm run build
- run: npx playwright install --with-deps
- - run: npm run mcp-test
+ - run: npm run test-mcp
diff --git a/package.json b/package.json
index e9b74e2430..02299a5b2e 100644
--- a/package.json
+++ b/package.json
@@ -30,9 +30,9 @@
"ttest": "node ./tests/playwright-test/stable-test-runner/node_modules/@playwright/test/cli test --config=tests/playwright-test/playwright.config.ts",
"ct": "playwright test tests/components/test-all.spec.js --reporter=list",
"test": "playwright test --config=tests/library/playwright.config.ts",
- "mcp-test": "playwright test --config=tests/mcp/playwright.config.ts",
- "mcp-ctest": "playwright test --config=tests/mcp/playwright.config.ts --project=chrome",
- "mcp-etest": "playwright test --config=tests/mcp/playwright.config.extension.ts",
+ "test-mcp": "playwright test --config=tests/mcp/playwright.config.ts",
+ "ctest-mcp": "playwright test --config=tests/mcp/playwright.config.ts --project=chrome",
+ "etest-mcp": "playwright test --config=tests/mcp/playwright.config.extension.ts",
"eslint": "eslint --cache",
"tsc": "tsc -p . && tsc -p packages/html-reporter/",
"doc": "node utils/doclint/cli.js",
diff --git a/packages/playwright/src/common/ipc.ts b/packages/playwright/src/common/ipc.ts
index 9031be7eea..179ba048b6 100644
--- a/packages/playwright/src/common/ipc.ts
+++ b/packages/playwright/src/common/ipc.ts
@@ -66,6 +66,7 @@ export type WorkerInitParams = {
projectId: string;
config: SerializedConfig;
artifactsDir: string;
+ pauseOnError: boolean;
};
export type TestBeginPayload = {
diff --git a/packages/playwright/src/index.ts b/packages/playwright/src/index.ts
index b9b681838e..efbc00e686 100644
--- a/packages/playwright/src/index.ts
+++ b/packages/playwright/src/index.ts
@@ -240,7 +240,7 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
(testInfo as TestInfoImpl)._setDebugMode();
playwright._defaultContextOptions = _combinedContextOptions;
- playwright._defaultContextTimeout = process.env.PLAYWRIGHT_DEBUGGER_ENABLED ? 5000 : actionTimeout || 0;
+ playwright._defaultContextTimeout = (testInfo as TestInfoImpl)._pauseOnError() ? 5000 : actionTimeout || 0;
playwright._defaultContextNavigationTimeout = navigationTimeout || 0;
await use();
playwright._defaultContextOptions = undefined;
@@ -291,6 +291,10 @@ const playwrightFixtures: Fixtures<TestFixtures, WorkerFixtures> = ({
tracingGroupSteps.push(step);
},
onApiCallRecovery: (data, error, channelOwner, recoveryHandlers) => {
+ const testInfo = currentTestInfo();
+ if (!testInfo || !testInfo._pauseOnError())
+ return;
+
const step = data.userData;
if (!step)
return;
diff --git a/packages/playwright/src/mcp/sdk/mdb.ts b/packages/playwright/src/mcp/sdk/mdb.ts
index 7fbcc3d188..bf3108cac7 100644
--- a/packages/playwright/src/mcp/sdk/mdb.ts
+++ b/packages/playwright/src/mcp/sdk/mdb.ts
@@ -162,7 +162,7 @@ export async function runMainBackend(backendFactory: mcpServer.ServerBackendFact
await mcpServer.connect(factory, new mcpBundle.StdioServerTransport(), false);
}
-export async function runOnPauseBackendLoop(mdbUrl: string, backend: ServerBackendOnPause, introMessage: string) {
+export async function runOnPauseBackendLoop(backend: ServerBackendOnPause, introMessage: string) {
const wrappedBackend = new OnceTimeServerBackendWrapper(backend);
const factory = {
@@ -178,7 +178,7 @@ export async function runOnPauseBackendLoop(mdbUrl: string, backend: ServerBacke
const client = new mcpBundle.Client({ name: 'Internal client', version: '0.0.0' });
client.setRequestHandler(mcpBundle.PingRequestSchema, () => ({}));
- const transport = new mcpBundle.StreamableHTTPClientTransport(new URL(mdbUrl));
+ const transport = new mcpBundle.StreamableHTTPClientTransport(new URL(process.env.PLAYWRIGHT_DEBUGGER_MCP!));
await client.connect(transport);
const pushToolsResult = await client.callTool({
diff --git a/packages/playwright/src/mcp/test/browserBackend.ts b/packages/playwright/src/mcp/test/browserBackend.ts
index 42ded6d876..4ec79ebe42 100644
--- a/packages/playwright/src/mcp/test/browserBackend.ts
+++ b/packages/playwright/src/mcp/test/browserBackend.ts
@@ -16,6 +16,7 @@
import * as mcp from '../sdk/exports';
import * as mcpBundle from '../sdk/bundle';
+import { currentTestInfo } from '../../common/globals';
import { snapshot, pickLocator, evaluate } from './browserTools';
import { stripAnsiEscapes } from '../../util';
@@ -72,8 +73,10 @@ const doneToolSchema = mcp.defineToolSchema({
});
export async function runBrowserBackendOnError(page: playwright.Page, message: () => string) {
- if (!process.env.PLAYWRIGHT_DEBUGGER_ENABLED)
+ const testInfo = currentTestInfo();
+ if (!testInfo || !testInfo._pauseOnError())
return;
+
const snapshot = await (page as PageEx)._snapshotForAI();
const introMessage = `### Paused on error:
${stripAnsiEscapes(message())}
@@ -83,5 +86,5 @@ ${snapshot}
### Task
Try recovering from the error prior to continuing, use following tools to recover: ${tools.map(tool => tool.schema.name).join(', ')}`;
- await mcp.runOnPauseBackendLoop(process.env.PLAYWRIGHT_MDB_URL!, new BrowserBackend(page), introMessage);
+ await mcp.runOnPauseBackendLoop(new BrowserBackend(page), introMessage);
}
diff --git a/packages/playwright/src/mcp/test/testBackend.ts b/packages/playwright/src/mcp/test/testBackend.ts
index 0ef240a2ad..5625aeda55 100644
--- a/packages/playwright/src/mcp/test/testBackend.ts
+++ b/packages/playwright/src/mcp/test/testBackend.ts
@@ -29,7 +29,7 @@ export class TestServerBackend implements mcp.ServerBackend {
private _tools: TestTool<any>[] = [listTests, runTests, debugTest];
private _context: TestContext;
- constructor(resolvedLocation: ConfigLocation, options?: { muteConsole?: boolean }) {
+ constructor(resolvedLocation: ConfigLocation, options?: { muteConsole?: boolean, headless?: boolean }) {
this._context = new TestContext(resolvedLocation, options);
}
diff --git a/packages/playwright/src/mcp/test/testContext.ts b/packages/playwright/src/mcp/test/testContext.ts
index 11b5ab5d01..3209b9d3f5 100644
--- a/packages/playwright/src/mcp/test/testContext.ts
+++ b/packages/playwright/src/mcp/test/testContext.ts
@@ -21,9 +21,9 @@ import type { ConfigLocation } from '../../common/config';
export class TestContext {
private _testRunner: TestRunner | undefined;
readonly configLocation: ConfigLocation;
- readonly options?: { muteConsole?: boolean };
+ readonly options?: { muteConsole?: boolean, headless?: boolean };
- constructor(configLocation: ConfigLocation, options?: { muteConsole?: boolean }) {
+ constructor(configLocation: ConfigLocation, options?: { muteConsole?: boolean, headless?: boolean }) {
this.configLocation = configLocation;
this.options = options;
}
diff --git a/packages/playwright/src/mcp/test/testTools.ts b/packages/playwright/src/mcp/test/testTools.ts
index 08c74e982c..2d2929c9a3 100644
--- a/packages/playwright/src/mcp/test/testTools.ts
+++ b/packages/playwright/src/mcp/test/testTools.ts
@@ -51,7 +51,7 @@ export const runTests = defineTestTool({
title: 'Run tests',
description: 'Run tests',
inputSchema: z.object({
- locations: z.array(z.string()).describe('Folder, file or location to run: "test/e2e" or "test/e2e/file.spec.ts" or "test/e2e/file.spec.ts:20"'),
+ locations: z.array(z.string()).optional().describe('Folder, file or location to run: "test/e2e" or "test/e2e/file.spec.ts" or "test/e2e/file.spec.ts:20"'),
projects: z.array(z.string()).optional().describe('Projects to run, projects from playwright.config.ts, by default runs all projects. Running with "chromium" is a good start'),
}),
type: 'readOnly',
@@ -103,15 +103,13 @@ export const debugTest = defineTestTool({
const configDir = context.configLocation.configDir;
const reporter = new ListReporter({ configDir, screen });
const testRunner = await context.createTestRunner();
- process.env.PLAYWRIGHT_DEBUGGER_ENABLED = '1';
const result = await testRunner.runTests(reporter, {
- headed: true,
+ headed: !context.options?.headless,
testIds: [params.test.id],
// For automatic recovery
timeout: 0,
workers: 1,
- }).finally(() => {
- process.env.PLAYWRIGHT_DEBUGGER_ENABLED = undefined;
+ pauseOnError: true,
});
const text = stream.content();
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index 62d02f04d8..ae41d56ab7 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -154,6 +154,7 @@ function addBrowserMCPServerCommand(program: Command) {
function addTestMCPServerCommand(program: Command) {
const command = program.command('run-test-mcp-server', { hidden: true });
command.description('Interact with the test runner over MCP');
+ command.option('--headless', 'run browser in headless mode, headed by default');
command.option('-c, --config <file>', `Configuration file, or a test directory with optional "playwright.config.{m,c}?{js,ts}"`);
command.option('--host <host>', 'host to bind server to. Default is localhost. Use 0.0.0.0 to bind to all interfaces.');
command.option('--port <port>', 'port to listen on for SSE transport.');
@@ -163,7 +164,7 @@ function addTestMCPServerCommand(program: Command) {
name: 'Playwright Test Runner',
nameInConfig: 'playwright-test-runner',
version: packageJSON.version,
- create: () => new TestServerBackend(resolvedLocation, { muteConsole: options.port === undefined }),
+ create: () => new TestServerBackend(resolvedLocation, { muteConsole: options.port === undefined, headless: options.headless }),
};
const mdbUrl = await runMainBackend(backendFactory, { port: options.port === undefined ? undefined : +options.port });
if (mdbUrl)
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index 388586fe96..5a2310e733 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -218,7 +218,7 @@ export class Dispatcher {
_createWorker(testGroup: TestGroup, parallelIndex: number, loaderData: SerializedConfig) {
const projectConfig = this._config.projects.find(p => p.id === testGroup.projectId)!;
const outputDir = projectConfig.project.outputDir;
- const worker = new WorkerHost(testGroup, parallelIndex, loaderData, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir);
+ const worker = new WorkerHost(testGroup, parallelIndex, loaderData, this._extraEnvByProjectId.get(testGroup.projectId) || {}, outputDir, this._failureTracker.pauseOnError());
const handleOutput = (params: TestOutputPayload) => {
const chunk = chunkFromParams(params);
if (worker.didFail()) {
diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.ts
index dd8ce50587..499365deaa 100644
--- a/packages/playwright/src/runner/failureTracker.ts
+++ b/packages/playwright/src/runner/failureTracker.ts
@@ -22,8 +22,10 @@ export class FailureTracker {
private _failureCount = 0;
private _hasWorkerErrors = false;
private _rootSuite: Suite | undefined;
+ private _pauseOnError: boolean;
- constructor(private _config: FullConfigInternal) {
+ constructor(private _config: FullConfigInternal, options?: { pauseOnError?: boolean }) {
+ this._pauseOnError = options?.pauseOnError ?? false;
}
onRootSuite(rootSuite: Suite) {
@@ -40,6 +42,10 @@ export class FailureTracker {
this._hasWorkerErrors = true;
}
+ pauseOnError(): boolean {
+ return this._pauseOnError;
+ }
+
hasReachedMaxFailures() {
return this.maxFailures() > 0 && this._failureCount >= this.maxFailures();
}
diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts
index ea4123be24..4c980f2d57 100644
--- a/packages/playwright/src/runner/tasks.ts
+++ b/packages/playwright/src/runner/tasks.ts
@@ -64,10 +64,10 @@ export class TestRun {
projectFiles: Map<FullProjectInternal, string[]> = new Map();
projectSuites: Map<FullProjectInternal, Suite[]> = new Map();
- constructor(config: FullConfigInternal, reporter: InternalReporter) {
+ constructor(config: FullConfigInternal, reporter: InternalReporter, options?: { pauseOnError?: boolean }) {
this.config = config;
this.reporter = reporter;
- this.failureTracker = new FailureTracker(config);
+ this.failureTracker = new FailureTracker(config, options);
}
}
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index 8255f67ae3..bb06ff33f7 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -71,6 +71,7 @@ export type RunTestsParams = {
projects?: string[];
reuseContext?: boolean;
connectWsEndpoint?: string;
+ pauseOnError?: boolean;
};
type FullResultStatus = reporterTypes.FullResult['status'];
@@ -335,7 +336,7 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {
createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }),
...createRunTestsTasks(config),
];
- const testRun = new TestRun(config, reporter);
+ const testRun = new TestRun(config, reporter, { pauseOnError: params.pauseOnError });
const run = runTasks(testRun, tasks, 0, stop).then(async status => {
this._testRun = undefined;
return status;
diff --git a/packages/playwright/src/runner/workerHost.ts b/packages/playwright/src/runner/workerHost.ts
index 9d69506330..bf5451672b 100644
--- a/packages/playwright/src/runner/workerHost.ts
+++ b/packages/playwright/src/runner/workerHost.ts
@@ -36,7 +36,7 @@ export class WorkerHost extends ProcessHost {
private _params: WorkerInitParams;
private _didFail = false;
- constructor(testGroup: TestGroup, parallelIndex: number, config: SerializedConfig, extraEnv: Record<string, string | undefined>, outputDir: string) {
+ constructor(testGroup: TestGroup, parallelIndex: number, config: SerializedConfig, extraEnv: Record<string, string | undefined>, outputDir: string, pauseOnError: boolean) {
const workerIndex = lastWorkerIndex++;
super(require.resolve('../worker/workerMain.js'), `worker-${workerIndex}`, {
...extraEnv,
@@ -54,6 +54,7 @@ export class WorkerHost extends ProcessHost {
projectId: testGroup.projectId,
config,
artifactsDir: path.join(outputDir, artifactsFolderName(workerIndex)),
+ pauseOnError,
};
}
diff --git a/packages/playwright/src/worker/testInfo.ts b/packages/playwright/src/worker/testInfo.ts
index 8d93429136..93dda2e2ad 100644
--- a/packages/playwright/src/worker/testInfo.ts
+++ b/packages/playwright/src/worker/testInfo.ts
@@ -119,6 +119,7 @@ export class TestInfoImpl implements TestInfo {
readonly snapshotDir: string;
errors: TestInfoErrorImpl[] = [];
readonly _attachmentsPush: (...items: TestInfo['attachments']) => number;
+ private _workerParams: WorkerInitParams;
get error(): TestInfoErrorImpl | undefined {
return this.errors[0];
@@ -169,6 +170,7 @@ export class TestInfoImpl implements TestInfo {
this._startWallTime = Date.now();
this._requireFile = test?._requireFile ?? '';
this._uniqueSymbol = Symbol('testInfoUniqueSymbol');
+ this._workerParams = workerParams;
this.repeatEachIndex = workerParams.repeatEachIndex;
this.retry = retry;
@@ -608,6 +610,10 @@ export class TestInfoImpl implements TestInfo {
setTimeout(timeout: number) {
this._timeoutManager.setTimeout(timeout);
}
+
+ _pauseOnError(): boolean {
+ return this._workerParams.pauseOnError;
+ }
}
export class TestStepInfoImpl implements TestStepInfo {
diff --git a/tests/mcp/cdp.spec.ts b/tests/mcp/cdp.spec.ts
index fe209b7404..650dc57239 100644
--- a/tests/mcp/cdp.spec.ts
+++ b/tests/mcp/cdp.spec.ts
@@ -15,7 +15,7 @@
*/
import { spawnSync } from 'child_process';
-import { test, expect, programPath } from './fixtures';
+import { test, expect, mcpServerPath } from './fixtures';
test('cdp server', async ({ cdpServer, startClient, server }) => {
await cdpServer.start();
@@ -84,7 +84,7 @@ test('should throw connection error and allow re-connecting', async ({ cdpServer
test('does not support --device', async () => {
const result = spawnSync('node', [
- ...programPath, '--device=Pixel 5', '--cdp-endpoint=http://localhost:1234',
+ ...mcpServerPath, '--device=Pixel 5', '--cdp-endpoint=http://localhost:1234',
]);
expect(result.error).toBeUndefined();
expect(result.status).toBe(1);
diff --git a/tests/mcp/config.spec.ts b/tests/mcp/config.spec.ts
index b7109f9cd9..b132176175 100644
--- a/tests/mcp/config.spec.ts
+++ b/tests/mcp/config.spec.ts
@@ -20,7 +20,7 @@ import { test, expect } from './fixtures';
import { configFromCLIOptions } from '../../packages/playwright/lib/mcp/browser/config';
import type { Config } from '../../packages/playwright/src/mcp/config';
-test('config user data dir', async ({ startClient, server, mcpMode }, testInfo) => {
+test('config user data dir', async ({ startClient, server }, testInfo) => {
server.setContent('/', `
<title>Title</title>
<body>Hello, world!</body>
@@ -59,7 +59,7 @@ test('executable path', async ({ startClient, server }, testInfo) => {
test.describe(() => {
test.use({ mcpBrowser: '' });
- test('browserName', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright-mcp/issues/458' } }, async ({ startClient, mcpMode }, testInfo) => {
+ test('browserName', { annotation: { type: 'issue', description: 'https://github.com/microsoft/playwright-mcp/issues/458' } }, async ({ startClient }, testInfo) => {
const config: Config = {
browser: {
browserName: 'firefox',
diff --git a/tests/mcp/device.spec.ts b/tests/mcp/device.spec.ts
index 5c0f733101..2207b5b25f 100644
--- a/tests/mcp/device.spec.ts
+++ b/tests/mcp/device.spec.ts
@@ -16,7 +16,7 @@
import { test, expect } from './fixtures';
-test('--device should work', async ({ startClient, server, mcpMode }) => {
+test('--device should work', async ({ startClient, server }) => {
const { client } = await startClient({
args: ['--device', 'iPhone 15'],
});
diff --git a/tests/mcp/files.spec.ts b/tests/mcp/files.spec.ts
index 5a1bb645d2..a632209524 100644
--- a/tests/mcp/files.spec.ts
+++ b/tests/mcp/files.spec.ts
@@ -100,7 +100,7 @@ test('browser_file_upload', async ({ client, server }, testInfo) => {
}
});
-test('clicking on download link emits download', async ({ startClient, server, mcpMode }, testInfo) => {
+test('clicking on download link emits download', async ({ startClient, server }, testInfo) => {
const { client } = await startClient({
config: { outputDir: testInfo.outputPath('output') },
});
@@ -126,7 +126,7 @@ test('clicking on download link emits download', async ({ startClient, server, m
});
});
-test('navigating to download link emits download', async ({ startClient, server, mcpBrowser, mcpMode }, testInfo) => {
+test('navigating to download link emits download', async ({ startClient, server, mcpBrowser }, testInfo) => {
const { client } = await startClient({
config: { outputDir: testInfo.outputPath('output') },
});
diff --git a/tests/mcp/fixtures.ts b/tests/mcp/fixtures.ts
index 47dc2e7e7c..9045ca2c3d 100644
--- a/tests/mcp/fixtures.ts
+++ b/tests/mcp/fixtures.ts
@@ -34,7 +34,7 @@ import type { ServerFixtures, ServerWorkerOptions } from '../config/serverFixtur
export type TestOptions = {
mcpArgs: string[] | undefined;
mcpBrowser: string | undefined;
- mcpMode: 'docker' | undefined;
+ mcpServerType: 'mcp' | 'test-mcp';
};
type CDPServer = {
@@ -75,25 +75,32 @@ export const test = serverTest.extend<TestFixtures & TestOptions, WorkerFixtures
await use(client);
},
- startClient: async ({ mcpHeadless, mcpBrowser, mcpMode, mcpArgs }, use, testInfo) => {
+ startClient: async ({ mcpHeadless, mcpBrowser, mcpArgs, mcpServerType }, use, testInfo) => {
const configDir = path.dirname(test.info().config.configFile!);
const clients: Client[] = [];
await use(async options => {
const args: string[] = mcpArgs ?? [];
- if (process.env.CI && process.platform === 'linux')
- args.push('--no-sandbox');
+
if (mcpHeadless)
args.push('--headless');
- if (mcpBrowser)
- args.push(`--browser=${mcpBrowser}`);
+
+ if (mcpServerType === 'test-mcp') {
+ args.push('--config', test.info().outputPath());
+ } else {
+ if (process.env.CI && process.platform === 'linux')
+ args.push('--no-sandbox');
+ if (mcpBrowser)
+ args.push(`--browser=${mcpBrowser}`);
+ if (options?.config) {
+ const configFile = testInfo.outputPath('config.json');
+ await fs.promises.writeFile(configFile, JSON.stringify(options.config, null, 2));
+ args.push(`--config=${path.relative(configDir, configFile)}`);
+ }
+ }
+
if (options?.args)
args.push(...options.args);
- if (options?.config) {
- const configFile = testInfo.outputPath('config.json');
- await fs.promises.writeFile(configFile, JSON.stringify(options.config, null, 2));
- args.push(`--config=${path.relative(configDir, configFile)}`);
- }
const client = new Client({ name: options?.clientName ?? 'test', version: '1.0.0' }, options?.roots ? { capabilities: { roots: {} } } : undefined);
if (options?.roots) {
@@ -105,7 +112,7 @@ export const test = serverTest.extend<TestFixtures & TestOptions, WorkerFixtures
};
});
}
- const { transport, stderr } = await createTransport(args, mcpMode, testInfo.outputPath('ms-playwright'));
+ const { transport, stderr } = await createTransport(mcpServerType, args);
let stderrBuffer = '';
stderr?.on('data', data => {
if (process.env.PWMCP_DEBUG)
@@ -166,29 +173,18 @@ export const test = serverTest.extend<TestFixtures & TestOptions, WorkerFixtures
mcpBrowser: ['chrome', { option: true }],
- mcpMode: [undefined, { option: true }],
+ mcpServerType: ['mcp', { option: true }],
});
-async function createTransport(args: string[], mcpMode: TestOptions['mcpMode'], profilesDir: string): Promise<{
+async function createTransport(mcpServerType: TestOptions['mcpServerType'], args: string[]): Promise<{
transport: Transport,
stderr: Stream | null,
}> {
- if (mcpMode === 'docker') {
- const dockerArgs = ['run', '--rm', '-i', '--network=host', '-v', `${test.info().project.outputDir}:/app/test-results`];
- const transport = new StdioClientTransport({
- command: 'docker',
- args: [...dockerArgs, 'playwright-mcp-dev:latest', ...args],
- });
- return {
- transport,
- stderr: transport.stderr,
- };
- }
-
+ const profilesDir = test.info().outputPath('ms-playwright');
const transport = new StdioClientTransport({
command: 'node',
- args: [...programPath, ...args],
- cwd: path.dirname(test.info().config.configFile!),
+ args: [...(mcpServerType === 'test-mcp' ? testMcpServerPath : mcpServerPath), ...args],
+ cwd: test.info().outputPath(),
stderr: 'pipe',
env: {
...process.env,
@@ -226,6 +222,30 @@ export const expect = baseExpect.extend({
message: () => ``,
};
},
+
+ toHaveTextResponse(response: Response, value: any) {
+ const text = response.content[0].text
+ .replace(/\[id=[^\]]+\]/g, '[id=<ID>]')
+ .replace(/\([\d\.]+m?s\)/g, '(XXms)')
+ .replace(/[✓] /g, 'ok');
+
+ const isNot = this.isNot;
+ try {
+ if (isNot)
+ expect(text).not.toEqual(value);
+ else
+ expect(text).toEqual(value);
+ } catch (e) {
+ return {
+ pass: isNot,
+ message: () => e.message,
+ };
+ }
+ return {
+ pass: !isNot,
+ message: () => ``,
+ };
+ },
});
export function formatOutput(output: string): string[] {
@@ -277,4 +297,35 @@ function parseSections(text: string): Map<string, string> {
return sections;
}
-export const programPath = [path.join(__dirname, '../../packages/playwright/cli.js'), 'run-mcp-server'];
+export const mcpServerPath = [path.join(__dirname, '../../packages/playwright/cli.js'), 'run-mcp-server'];
+export const testMcpServerPath = [path.join(__dirname, '../../packages/playwright-test/cli.js'), 'run-test-mcp-server'];
+
+type Files = { [key: string]: string | Buffer };
+
+export async function writeFiles(files: Files, options?: { update?: boolean }) {
+ const baseDir = test.info().outputPath();
+
+ if (!options?.update && !Object.keys(files).some(name => name.includes('package.json'))) {
+ files = {
+ ...files,
+ 'package.json': `{ "name": "test-project" }`,
+ };
+ }
+
+ if (!options?.update && !Object.keys(files).some(name => name.includes('tsconfig.json') || name.includes('jsconfig.json'))) {
+ files = {
+ ...files,
+ 'tsconfig.json': `{}`,
+ };
+ }
+
+ await Promise.all(Object.keys(files).map(async name => {
+ const fullName = path.join(baseDir, name);
+ if (files[name] === undefined)
+ return;
+ await fs.promises.mkdir(path.dirname(fullName), { recursive: true });
+ await fs.promises.writeFile(fullName, files[name]);
+ }));
+
+ return baseDir;
+}
diff --git a/tests/mcp/headed.spec.ts b/tests/mcp/headed.spec.ts
index 1654a30a50..56fd4923c6 100644
--- a/tests/mcp/headed.spec.ts
+++ b/tests/mcp/headed.spec.ts
@@ -20,7 +20,6 @@ for (const mcpHeadless of [false, true]) {
test.describe(`mcpHeadless: ${mcpHeadless}`, () => {
test.use({ mcpHeadless });
test.skip(process.platform === 'linux', 'Auto-detection wont let this test run on linux');
- test.skip(({ mcpMode, mcpHeadless }) => mcpMode === 'docker' && !mcpHeadless, 'Headed mode is not supported in docker');
test('browser', async ({ client, server, mcpBrowser }) => {
test.skip(!['chrome', 'msedge', 'chromium'].includes(mcpBrowser ?? ''), 'Only chrome is supported for this test');
diff --git a/tests/mcp/http.spec.ts b/tests/mcp/http.spec.ts
index 83ed6b86f5..f533dd8dc8 100644
--- a/tests/mcp/http.spec.ts
+++ b/tests/mcp/http.spec.ts
@@ -19,7 +19,7 @@ import fs from 'fs';
import { ChildProcess, spawn } from 'child_process';
import { StreamableHTTPClientTransport } from '@modelcontextprotocol/sdk/client/streamableHttp.js';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
-import { test as baseTest, expect, programPath } from './fixtures';
+import { test as baseTest, expect, mcpServerPath } from './fixtures';
import type { Config } from '../../packages/playwright/src/mcp/config';
import { ListRootsRequestSchema } from 'packages/playwright/lib/mcp/sdk/bundle';
@@ -33,7 +33,7 @@ const test = baseTest.extend<{ serverEndpoint: (options?: { args?: string[], noP
throw new Error('Process already running');
cp = spawn('node', [
- ...programPath,
+ ...mcpServerPath,
...(options?.noPort ? [] : ['--port=0']),
'--user-data-dir=' + userDataDir,
...(mcpHeadless ? ['--headless'] : []),
diff --git a/tests/mcp/launch.spec.ts b/tests/mcp/launch.spec.ts
index f7f0171aa9..03a637b998 100644
--- a/tests/mcp/launch.spec.ts
+++ b/tests/mcp/launch.spec.ts
@@ -18,7 +18,7 @@ import fs from 'fs';
import { test, expect, formatOutput } from './fixtures';
-test('test reopen browser', async ({ startClient, server, mcpMode }) => {
+test('test reopen browser', async ({ startClient, server }) => {
const { client, stderr } = await startClient();
await client.callTool({
name: 'browser_navigate',
diff --git a/tests/mcp/mdb.spec.ts b/tests/mcp/mdb.spec.ts
index 85229a7864..74f240385d 100644
--- a/tests/mcp/mdb.spec.ts
+++ b/tests/mcp/mdb.spec.ts
@@ -170,12 +170,12 @@ class CLIBackend {
if (name === 'cli_echo')
return { content: [{ type: 'text', text: 'Echo: ' + (args?.message as string) }] };
if (name === 'cli_pause_in_gdb') {
- await runOnPauseBackendLoop(this.mdbUrlBox.mdbUrl!, new GDBBackend(), 'Paused on exception');
+ await runOnPauseBackendLoop(new GDBBackend(), 'Paused on exception');
return { content: [{ type: 'text', text: 'Done' }] };
}
if (name === 'cli_pause_in_gdb_twice') {
- await runOnPauseBackendLoop(this.mdbUrlBox.mdbUrl!, new GDBBackend(), 'Paused on exception 1');
- await runOnPauseBackendLoop(this.mdbUrlBox.mdbUrl!, new GDBBackend(), 'Paused on exception 2');
+ await runOnPauseBackendLoop(new GDBBackend(), 'Paused on exception 1');
+ await runOnPauseBackendLoop(new GDBBackend(), 'Paused on exception 2');
return { content: [{ type: 'text', text: 'Done' }] };
}
throw new Error(`Unknown tool: ${name}`);
diff --git a/tests/mcp/playwright.config.ts b/tests/mcp/playwright.config.ts
index 923a0f8034..7edcbb1687 100644
--- a/tests/mcp/playwright.config.ts
+++ b/tests/mcp/playwright.config.ts
@@ -28,14 +28,6 @@ export default defineConfig<TestOptions>({
projects: [
{ name: 'chrome' },
{ name: 'chromium', use: { mcpBrowser: 'chromium' } },
- ...process.env.MCP_IN_DOCKER ? [{
- name: 'chromium-docker',
- grep: /browser_navigate|browser_click/,
- use: {
- mcpBrowser: 'chromium',
- mcpMode: 'docker' as const
- }
- }] : [],
{ name: 'firefox', use: { mcpBrowser: 'firefox' } },
{ name: 'webkit', use: { mcpBrowser: 'webkit' } },
... process.platform === 'win32' ? [{ name: 'msedge', use: { mcpBrowser: 'msedge' } }] : [],
diff --git a/tests/mcp/sse.spec.ts b/tests/mcp/sse.spec.ts
index 5d8d706c0d..d67a24a98b 100644
--- a/tests/mcp/sse.spec.ts
+++ b/tests/mcp/sse.spec.ts
@@ -19,7 +19,7 @@ import fs from 'fs';
import { ChildProcess, spawn } from 'child_process';
import { SSEClientTransport } from '@modelcontextprotocol/sdk/client/sse.js';
import { Client } from '@modelcontextprotocol/sdk/client/index.js';
-import { test as baseTest, expect, programPath } from './fixtures';
+import { test as baseTest, expect, mcpServerPath } from './fixtures';
import type { Config } from '../../packages/playwright/src/mcp/config';
@@ -32,7 +32,7 @@ const test = baseTest.extend<{ serverEndpoint: (options?: { args?: string[], noP
throw new Error('Process already running');
cp = spawn('node', [
- ...programPath,
+ ...mcpServerPath,
...(options?.noPort ? [] : ['--port=0']),
'--user-data-dir=' + userDataDir,
...(mcpHeadless ? ['--headless'] : []),
diff --git a/tests/mcp/test-tools.spec.ts b/tests/mcp/test-tools.spec.ts
new file mode 100644
index 0000000000..f6a02cbf1c
--- /dev/null
+++ b/tests/mcp/test-tools.spec.ts
@@ -0,0 +1,284 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { test, expect, writeFiles, StartClient } from './fixtures';
+
+test.use({ mcpServerType: 'test-mcp' });
+
+test('playwright_test_list_tests', async ({ startClient }) => {
+ await writeFiles({
+ 'playwright.config.ts': `
+ module.exports = { projects: [{ name: 'foo' }, {}] };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('example1', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ test('example2', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ `
+ });
+
+ const { client } = await startClient();
+ expect(await client.callTool({
+ name: 'playwright_test_list_tests',
+ arguments: {},
+ })).toHaveTextResponse(`Listing tests:
+ [id=<ID>] [project=foo] › a.test.ts:3:11 › example1
+ [id=<ID>] [project=foo] › a.test.ts:6:11 › example2
+ [id=<ID>] a.test.ts:3:11 › example1
+ [id=<ID>] a.test.ts:6:11 › example2
+Total: 4 tests in 1 file
+`);
+});
+
+test('playwright_test_run_tests', async ({ startClient }) => {
+ await writeFiles({
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('passes', () => {});
+ test('fails', () => { expect(1).toBe(2); });
+ test.describe('suite', () => {
+ test('inner passes', () => {});
+ test('inner fails', () => { expect(1).toBe(2); });
+ });
+ `,
+ 'b.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('passes', () => {});
+ test('fails', () => { expect(1).toBe(2); });
+ `,
+ 'c.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('passes', () => {});
+ test.skip('skipped', () => {});
+ `,
+ });
+
+ const { client } = await startClient();
+ const response = await client.callTool({
+ name: 'playwright_test_run_tests',
+ });
+
+ const text = response.content[0].text;
+
+ expect(text).toContain(`3 failed`);
+ expect(text).toContain(`1 skipped`);
+ expect(text).toContain(`4 passed`);
+
+ expect(text).toContain(`a.test.ts:3:11 › passes`);
+ expect(text).toContain(`c.test.ts:3:11 › passes`);
+ expect(text).toContain(`c.test.ts:4:12 › skipped`);
+ expect(text).toContain(`b.test.ts:3:11 › passes`);
+ expect(text).toContain(`a.test.ts:4:11 › fails`);
+ expect(text).toContain(`b.test.ts:4:11 › fails`);
+ expect(text).toContain(`a.test.ts:6:13 › suite › inner passes`);
+ expect(text).toContain(`a.test.ts:7:13 › suite › inner fails`);
+
+ expect(text).not.toContain(`../../test-results`);
+});
+
+test('playwright_test_run_tests filters', async ({ startClient }) => {
+ await writeFiles({
+ 'playwright.config.ts': `
+ module.exports = { projects: [{ name: 'foo' }, { name: 'bar' }] };
+ `,
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('example1', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ test('example2', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ `,
+ 'b.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('example1', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ test('example2', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ `
+ });
+
+ const { client } = await startClient();
+ expect(await client.callTool({
+ name: 'playwright_test_run_tests',
+ arguments: {
+ locations: ['b.test.ts'],
+ projects: ['foo'],
+ },
+ })).toHaveTextResponse(`
+Running 2 tests using 1 worker
+
+ ok 1 [id=<ID>] [project=foo] › b.test.ts:3:11 › example1 (XXms)
+ ok 2 [id=<ID>] [project=foo] › b.test.ts:6:11 › example2 (XXms)
+
+ 2 passed (XXms)
+`);
+});
+
+test('playwright_test_debug_test (passed)', async ({ startClient }) => {
+ await writeFiles({
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('pass', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ `
+ });
+
+ const { client } = await startClient();
+ const listResult = await client.callTool({
+ name: 'playwright_test_list_tests',
+ });
+ const [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/);
+
+ expect(await client.callTool({
+ name: 'playwright_test_debug_test',
+ arguments: {
+ test: { id, title: 'pass' },
+ },
+ })).toHaveTextResponse(`
+Running 1 test using 1 worker
+
+ ok 1 a.test.ts:3:11 › pass (XXms)
+
+ 1 passed (XXms)
+`);
+});
+
+test('playwright_test_debug_test (pause/resume)', async ({ startClient }) => {
+ const { client, id } = await prepareDebugTest(startClient);
+
+ expect(await client.callTool({
+ name: 'playwright_test_debug_test',
+ arguments: {
+ test: { id, title: 'fail' },
+ },
+ })).toHaveTextResponse(`### Paused on error:
+expect(locator).toBeVisible() failed
+
+Locator: getByRole('button', { name: 'Missing' })
+Expected: visible
+Error: element(s) not found
+Timeout: 1000ms
+
+Call log:
+ - Expect "toBeVisible" with timeout 1000ms
+ - waiting for getByRole('button', { name: 'Missing' })
+
+
+### Current page snapshot:
+- button "Submit" [ref=e2]
+
+### Task
+Try recovering from the error prior to continuing, use following tools to recover: playwright_test_browser_snapshot, playwright_test_generate_locator, playwright_test_evaluate_on_pause`);
+
+ expect(await client.callTool({
+ name: 'playwright_test_run_tests',
+ arguments: {
+ locations: ['a.test.ts'],
+ },
+ })).toHaveTextResponse(expect.stringContaining(`1) [id=<ID>] a.test.ts:3:11 › fail`));
+});
+
+test('playwright_test_browser_snapshot', async ({ startClient }) => {
+ const { client, id } = await prepareDebugTest(startClient);
+ await client.callTool({
+ name: 'playwright_test_debug_test',
+ arguments: {
+ test: { id, title: 'fail' },
+ },
+ });
+ expect(await client.callTool({
+ name: 'playwright_test_browser_snapshot',
+ })).toHaveTextResponse(`- button \"Submit\" [ref=e2]`);
+});
+
+test('playwright_test_evaluate_on_pause', async ({ startClient }) => {
+ const { client, id } = await prepareDebugTest(startClient);
+ await client.callTool({
+ name: 'playwright_test_debug_test',
+ arguments: {
+ test: { id, title: 'fail' },
+ },
+ });
+ expect(await client.callTool({
+ name: 'playwright_test_evaluate_on_pause',
+ arguments: {
+ function: '() => 21+21',
+ },
+ })).toHaveTextResponse(`42`);
+});
+
+test('playwright_test_evaluate_on_pause (with element)', async ({ startClient }) => {
+ const { client, id } = await prepareDebugTest(startClient);
+ await client.callTool({
+ name: 'playwright_test_debug_test',
+ arguments: {
+ test: { id, title: 'fail' },
+ },
+ });
+ expect(await client.callTool({
+ name: 'playwright_test_evaluate_on_pause',
+ arguments: {
+ function: 'element => element.textContent',
+ element: 'button',
+ ref: 'e2',
+ },
+ })).toHaveTextResponse(`"Submit"`);
+});
+
+test('playwright_test_generate_locator', async ({ startClient }) => {
+ const { client, id } = await prepareDebugTest(startClient);
+ await client.callTool({
+ name: 'playwright_test_debug_test',
+ arguments: {
+ test: { id, title: 'fail' },
+ },
+ });
+ expect(await client.callTool({
+ name: 'playwright_test_generate_locator',
+ arguments: {
+ element: 'button',
+ ref: 'e2',
+ },
+ })).toHaveTextResponse(`getByRole('button', { name: 'Submit' })`);
+});
+
+async function prepareDebugTest(startClient: StartClient) {
+ await writeFiles({
+ 'a.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('fail', async ({ page }) => {
+ await page.setContent('<button>Submit</button>');
+ await expect(page.getByRole('button', { name: 'Missing' })).toBeVisible({ timeout: 1000 });
+ });
+ `
+ });
+
+ const { client } = await startClient();
+ const listResult = await client.callTool({
+ name: 'playwright_test_list_tests',
+ });
+ const [, id] = listResult.content[0].text.match(/\[id=([^\]]+)\]/);
+ return { client, id };
+}
diff --git a/tests/mcp/tracing.spec.ts b/tests/mcp/tracing.spec.ts
index 23bc94fee5..9a1d88a4ca 100644
--- a/tests/mcp/tracing.spec.ts
+++ b/tests/mcp/tracing.spec.ts
@@ -18,7 +18,7 @@ import fs from 'fs';
import path from 'path';
import { test, expect } from './fixtures';
-test('check that trace is saved with --save-trace', async ({ startClient, server, mcpMode }, testInfo) => {
+test('check that trace is saved with --save-trace', async ({ startClient, server }, testInfo) => {
const outputDir = testInfo.outputPath('output');
const { client } = await startClient({
@@ -36,7 +36,7 @@ test('check that trace is saved with --save-trace', async ({ startClient, server
expect(file).toContain('traces');
});
-test('check that trace is saved with browser_start_tracing', async ({ startClient, server, mcpMode }, testInfo) => {
+test('check that trace is saved with browser_start_tracing', async ({ startClient, server }, testInfo) => {
const outputDir = testInfo.outputPath('output');
const { client } = await startClient({ args: [`--output-dir=${outputDir}`, '--caps=tracing'] });