Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(mcp): add run tests command (#37047)

Pavel Feldman1fa832872438
diff --git a/packages/playwright-test-mcp/src/context.ts b/packages/playwright-test-mcp/src/context.ts
index d77955122e..aa20aac081 100644
--- a/packages/playwright-test-mcp/src/context.ts
+++ b/packages/playwright-test-mcp/src/context.ts
@@ -15,13 +15,14 @@
*/
import { TestRunner } from 'playwright/lib/runner/testRunner';
-import { resolveConfigLocation } from 'playwright/lib/common/configLoader';
+
+import type { ConfigLocation } from 'playwright/lib/common/config';
export class Context {
readonly testRunner: TestRunner;
- constructor(configFile: string | undefined) {
- this.testRunner = new TestRunner(resolveConfigLocation(configFile), {});
+ constructor(configLocation: ConfigLocation) {
+ this.testRunner = new TestRunner(configLocation, {});
}
async close() {
diff --git a/packages/playwright-test-mcp/src/program.ts b/packages/playwright-test-mcp/src/program.ts
index 228db419e1..8346317db0 100644
--- a/packages/playwright-test-mcp/src/program.ts
+++ b/packages/playwright-test-mcp/src/program.ts
@@ -14,7 +14,10 @@
* limitations under the License.
*/
+import path from 'path';
import { program } from 'commander';
+import { resolveConfigLocation } from 'playwright/lib/common/configLoader';
+
import { packageJSON } from './utils/package.js';
import * as mcpTransport from './mcp/transport.js';
import { TestServerBackend } from './testServerBackend.js';
@@ -26,7 +29,10 @@ program
.option('--host <host>', 'host to bind server to. Default is localhost. Use 0.0.0.0 to bind to all interfaces.')
.option('--port <port>', 'port to listen on for SSE transport.')
.action(async options => {
- const serverBackendFactory = () => new TestServerBackend(options.config);
+ const resolvedLocation = resolveConfigLocation(options.config);
+ // eslint-disable-next-line no-console
+ console.error('Test config: ', path.relative(process.cwd(), resolvedLocation.resolvedConfigFile ?? resolvedLocation.configDir));
+ const serverBackendFactory = () => new TestServerBackend(resolvedLocation);
await mcpTransport.start(serverBackendFactory, options);
});
diff --git a/packages/playwright-test-mcp/src/testServerBackend.ts b/packages/playwright-test-mcp/src/testServerBackend.ts
index 6ec1e63ee8..70b654f92b 100644
--- a/packages/playwright-test-mcp/src/testServerBackend.ts
+++ b/packages/playwright-test-mcp/src/testServerBackend.ts
@@ -17,19 +17,21 @@
import { packageJSON } from './utils/package';
import { Context } from './context';
import { toMcpTool } from './mcp/tool';
-import tools from './tools';
+import { listTests } from './tools/listTests';
+import { runTests } from './tools/runTests';
import type { Tool } from './tool';
import type * as mcpServer from './mcp/server';
+import type { ConfigLocation } from 'playwright/lib/common/config';
export class TestServerBackend implements mcpServer.ServerBackend {
readonly name = 'Playwright';
readonly version = packageJSON.version;
- private _tools: Tool<any>[] = tools;
+ private _tools: Tool<any>[] = [listTests, runTests];
private _context: Context;
- constructor(configFile: string | undefined) {
- this._context = new Context(configFile);
+ constructor(resolvedLocation: ConfigLocation) {
+ this._context = new Context(resolvedLocation);
}
async initialize() {
diff --git a/packages/playwright-test-mcp/src/tools.ts b/packages/playwright-test-mcp/src/tools/listTests.ts
similarity index 92%
rename from packages/playwright-test-mcp/src/tools.ts
rename to packages/playwright-test-mcp/src/tools/listTests.ts
index ade26414fd..1ebe85de60 100644
--- a/packages/playwright-test-mcp/src/tools.ts
+++ b/packages/playwright-test-mcp/src/tools/listTests.ts
@@ -16,11 +16,11 @@
import path from 'path';
import { z } from 'zod';
-import { defineTool } from './tool.js';
+import { defineTool } from '../tool.js';
import type * as reporterTypes from 'playwright/types/testReporter';
-const listTests = defineTool({
+export const listTests = defineTool({
schema: {
name: 'playwright_test_list_tests',
title: 'List tests',
@@ -55,7 +55,7 @@ class ListModeReporter implements reporterTypes.Reporter {
const [, projectName, , ...titles] = test.titlePath();
const location = `${path.relative(config.rootDir, test.location.file)}:${test.location.line}:${test.location.column}`;
const projectTitle = projectName ? `[${projectName}] › ` : '';
- this._lines.push(` ${projectTitle}${location} › ${titles.join(' › ')}`);
+ this._lines.push(` [id=${test.id}] ${projectTitle}${location} › ${titles.join(' › ')}`);
files.add(test.location.file);
}
this._lines.push(`Total: ${tests.length} ${tests.length === 1 ? 'test' : 'tests'} in ${files.size} ${files.size === 1 ? 'file' : 'files'}`);
@@ -74,5 +74,3 @@ class ListModeReporter implements reporterTypes.Reporter {
return this._lines.join('\n');
}
}
-
-export default [listTests];
diff --git a/packages/playwright-test-mcp/src/tools/runTests.ts b/packages/playwright-test-mcp/src/tools/runTests.ts
new file mode 100644
index 0000000000..fecefb8e49
--- /dev/null
+++ b/packages/playwright-test-mcp/src/tools/runTests.ts
@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { z } from 'zod';
+import { noColors } from 'playwright-core/lib/utils';
+import { terminalScreen } from 'playwright/lib/reporters/base';
+import ListReporter from 'playwright/lib/reporters/list';
+
+import { defineTool } from '../tool';
+import { StringWriteStream } from '../utils/streams';
+
+export const runTests = defineTool({
+ schema: {
+ name: 'playwright_test_run_tests',
+ title: 'Run tests',
+ description: 'Run tests',
+ inputSchema: z.object({
+ tests: z.array(z.object({
+ id: z.string().describe('Test ID to run.'),
+ title: z.string().describe('Human readable test title for granting permission to run the test.'),
+ })).optional().describe('Tests to run. All tests are run if not provided.'),
+ }),
+ type: 'readOnly',
+ },
+
+ handle: async (context, params) => {
+ const stream = new StringWriteStream();
+ const screen = {
+ ...terminalScreen,
+ isTTY: false,
+ colors: noColors,
+ stdout: stream as unknown as NodeJS.WriteStream,
+ stderr: stream as unknown as NodeJS.WriteStream,
+ };
+ const configDir = context.testRunner.configLocation.configDir;
+ const reporter = new ListReporter({ configDir, screen });
+ const result = await context.testRunner.runTests(reporter, {
+ testIds: params.tests?.map(test => test.id),
+ });
+ const text = stream.content();
+
+ return {
+ content: [
+ { type: 'text', text },
+ ],
+ isError: result.status !== 'passed',
+ };
+ },
+});
diff --git a/packages/playwright-test-mcp/src/utils/streams.ts b/packages/playwright-test-mcp/src/utils/streams.ts
new file mode 100644
index 0000000000..d02664e103
--- /dev/null
+++ b/packages/playwright-test-mcp/src/utils/streams.ts
@@ -0,0 +1,30 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Writable } from 'stream';
+
+export class StringWriteStream extends Writable {
+ private _chunks: string[] = [];
+
+ override _write(chunk: any, encoding: any, callback: any) {
+ this._chunks.push(chunk.toString());
+ callback();
+ }
+
+ content() {
+ return this._chunks.join('');
+ }
+}
diff --git a/packages/playwright/package.json b/packages/playwright/package.json
index 0c9cf50f94..9cdb939e7d 100644
--- a/packages/playwright/package.json
+++ b/packages/playwright/package.json
@@ -22,6 +22,8 @@
"./lib/common/configLoader": "./lib/common/configLoader.js",
"./lib/fsWatcher": "./lib/fsWatcher.js",
"./lib/program": "./lib/program.js",
+ "./lib/reporters/base": "./lib/reporters/base.js",
+ "./lib/reporters/list": "./lib/reporters/list.js",
"./lib/transform/babelBundle": "./lib/transform/babelBundle.js",
"./lib/transform/compilationCache": "./lib/transform/compilationCache.js",
"./lib/transform/esmLoader": "./lib/transform/esmLoader.js",
diff --git a/packages/playwright/src/reporters/base.ts b/packages/playwright/src/reporters/base.ts
index d8989e8852..0277a6dc3f 100644
--- a/packages/playwright/src/reporters/base.ts
+++ b/packages/playwright/src/reporters/base.ts
@@ -48,9 +48,9 @@ type TestSummary = {
export type CommonReporterOptions = {
configDir: string,
- _mode: 'list' | 'test' | 'merge',
- _isTestServer: boolean,
- _commandHash: string,
+ _mode?: 'list' | 'test' | 'merge',
+ _isTestServer?: boolean,
+ _commandHash?: string,
};
export type Screen = {
@@ -142,8 +142,13 @@ export const internalScreen: Screen = {
resolveFiles: 'rootDir',
};
+export type TerminalReporterOptions = {
+ screen?: TerminalScreen;
+ omitFailures?: boolean;
+};
+
export class TerminalReporter implements ReporterV2 {
- screen: TerminalScreen = terminalScreen;
+ screen: TerminalScreen;
config!: FullConfig;
suite!: Suite;
totalTestCount = 0;
@@ -153,7 +158,8 @@ export class TerminalReporter implements ReporterV2 {
private _fatalErrors: TestError[] = [];
private _failureCount: number = 0;
- constructor(options: { omitFailures?: boolean } = {}) {
+ constructor(options: TerminalReporterOptions = {}) {
+ this.screen = options.screen ?? terminalScreen;
this._omitFailures = options.omitFailures || false;
}
diff --git a/packages/playwright/src/reporters/list.ts b/packages/playwright/src/reporters/list.ts
index 8f5bf4ebd2..9a08476da1 100644
--- a/packages/playwright/src/reporters/list.ts
+++ b/packages/playwright/src/reporters/list.ts
@@ -22,7 +22,7 @@ import { stripAnsiEscapes } from '../util';
import type { ListReporterOptions } from '../../types/test';
import type { FullResult, Suite, TestCase, TestError, TestResult, TestStep } from '../../types/testReporter';
-import type { CommonReporterOptions } from './base';
+import type { CommonReporterOptions, TerminalReporterOptions } from './base';
// Allow it in the Visual Studio Code Terminal and the new Windows Terminal
const DOES_NOT_SUPPORT_UTF8_IN_TERMINAL = process.platform === 'win32' && process.env.TERM_PROGRAM !== 'vscode' && !process.env.WT_SESSION;
@@ -39,8 +39,8 @@ class ListReporter extends TerminalReporter {
private _needNewLine = false;
private _printSteps: boolean;
- constructor(options?: ListReporterOptions & CommonReporterOptions) {
- super();
+ constructor(options?: ListReporterOptions & CommonReporterOptions & TerminalReporterOptions) {
+ super(options);
this._printSteps = getAsBooleanFromENV('PLAYWRIGHT_LIST_PRINT_STEPS', options?.printSteps);
}
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index 1f10c72982..a2c1774263 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -83,7 +83,7 @@ export type RunTestsParams = {
type FullResultStatus = reporterTypes.FullResult['status'];
export class TestRunner extends EventEmitter<TestRunnerEventMap> {
- private _configLocation: ConfigLocation;
+ readonly configLocation: ConfigLocation;
private _configCLIOverrides: ConfigCLIOverrides;
private _watcher: Watcher;
@@ -103,7 +103,7 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {
constructor(configLocation: ConfigLocation, configCLIOverrides: ConfigCLIOverrides) {
super();
- this._configLocation = configLocation;
+ this.configLocation = configLocation;
this._configCLIOverrides = configCLIOverrides;
this._watcher = new Watcher(events => {
const collector = new Set<string>();
@@ -411,7 +411,7 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {
private async _loadConfig(overrides?: ConfigCLIOverrides): Promise<{ config: FullConfigInternal | null, error?: reporterTypes.TestError }> {
try {
- const config = await loadConfig(this._configLocation, overrides);
+ const config = await loadConfig(this.configLocation, overrides);
// Preserve plugin instances between setup and build.
if (!this._plugins) {
webServerPluginsForConfig(config).forEach(p => config.plugins.push({ factory: p }));