Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(test-mcp): handle dependent projects in tools (#37459)

Dmitry Gozman8a6a4522e836
diff --git a/packages/playwright/src/mcp/test/testTools.ts b/packages/playwright/src/mcp/test/testTools.ts
index 894e64fe0f..b5ed411b0c 100644
--- a/packages/playwright/src/mcp/test/testTools.ts
+++ b/packages/playwright/src/mcp/test/testTools.ts
@@ -23,6 +23,7 @@ import { z } from '../sdk/bundle';
import { terminalScreen } from '../../reporters/base';
import ListReporter from '../../reporters/list';
import ListModeReporter from '../../reporters/listModeReporter';
+import { findTopLevelProjects } from '../../runner/projectUtils';
import { defineTestTool } from './testTool';
import { StringWriteStream } from './streams';
@@ -68,6 +69,7 @@ export const runTests = defineTestTool({
const result = await testRunner.runTests(reporter, {
locations: params.locations,
projects: params.projects,
+ disableConfigReporters: true,
});
const text = stream.content();
@@ -106,6 +108,7 @@ export const debugTest = defineTestTool({
timeout: 0,
workers: 1,
pauseOnError: true,
+ disableConfigReporters: true,
});
const text = stream.content();
@@ -140,10 +143,11 @@ export const setupPage = defineTestTool({
if (!testLocation) {
testLocation = 'default.seed.spec.ts';
const config = await testRunner.loadConfig();
- const project = params.project ? config.projects.find(p => p.project.name === params.project) : config.projects[0];
+ const project = params.project ? config.projects.find(p => p.project.name === params.project) : findTopLevelProjects(config)[0];
const testDir = project?.project.testDir || configDir;
const seedFile = path.join(testDir, testLocation);
if (!fs.existsSync(seedFile)) {
+ await fs.promises.mkdir(path.dirname(seedFile), { recursive: true });
await fs.promises.writeFile(seedFile, `import { test, expect } from '@playwright/test';
test('seed', async ({ page }) => {});
@@ -158,6 +162,7 @@ test('seed', async ({ page }) => {});
timeout: 0,
workers: 1,
pauseAtEnd: true,
+ disableConfigReporters: true,
});
const text = stream.content();
diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.ts
index 7d64c2f3a5..d9db512cd9 100644
--- a/packages/playwright/src/runner/dispatcher.ts
+++ b/packages/playwright/src/runner/dispatcher.ts
@@ -224,7 +224,7 @@ export class Dispatcher {
extraEnv: this._extraEnvByProjectId.get(testGroup.projectId) || {},
outputDir,
pauseOnError: this._failureTracker.pauseOnError(),
- pauseAtEnd: this._failureTracker.pauseAtEnd(),
+ pauseAtEnd: this._failureTracker.pauseAtEnd(projectConfig),
});
const handleOutput = (params: TestOutputPayload) => {
const chunk = chunkFromParams(params);
diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.ts
index c2d709ec61..ef276fa509 100644
--- a/packages/playwright/src/runner/failureTracker.ts
+++ b/packages/playwright/src/runner/failureTracker.ts
@@ -15,13 +15,14 @@
*/
import type { TestResult } from '../../types/testReporter';
-import type { FullConfigInternal } from '../common/config';
+import type { FullConfigInternal, FullProjectInternal } from '../common/config';
import type { Suite, TestCase } from '../common/test';
export class FailureTracker {
private _failureCount = 0;
private _hasWorkerErrors = false;
private _rootSuite: Suite | undefined;
+ private _topLevelProjects: FullProjectInternal[] = [];
private _pauseOnError: boolean;
private _pauseAtEnd: boolean;
@@ -30,8 +31,9 @@ export class FailureTracker {
this._pauseAtEnd = options?.pauseAtEnd ?? false;
}
- onRootSuite(rootSuite: Suite) {
+ onRootSuite(rootSuite: Suite, topLevelProjects: FullProjectInternal[]) {
this._rootSuite = rootSuite;
+ this._topLevelProjects = topLevelProjects;
}
onTestEnd(test: TestCase, result: TestResult) {
@@ -48,8 +50,8 @@ export class FailureTracker {
return this._pauseOnError;
}
- pauseAtEnd(): boolean {
- return this._pauseAtEnd;
+ pauseAtEnd(inProject: FullProjectInternal): boolean {
+ return this._pauseAtEnd && this._topLevelProjects.includes(inProject);
}
hasReachedMaxFailures() {
diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.ts
index 05c5bba718..065cd0cc4a 100644
--- a/packages/playwright/src/runner/loadUtils.ts
+++ b/packages/playwright/src/runner/loadUtils.ts
@@ -120,7 +120,7 @@ export async function loadFileSuites(testRun: TestRun, mode: 'out-of-process' |
}
}
-export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean): Promise<Suite> {
+export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean): Promise<{ rootSuite: Suite, topLevelProjects: FullProjectInternal[] }> {
const config = testRun.config;
// Create root suite, where each child will be a project suite with cloned file suites inside it.
const rootSuite = new Suite('', 'root');
@@ -200,6 +200,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
if (config.postShardTestFilters.length)
filterTestsRemoveEmptySuites(rootSuite, test => config.postShardTestFilters.every(filter => filter(test)));
+ const topLevelProjects = [];
// Now prepend dependency projects without filtration.
{
// Filtering 'only' and sharding might have reduced the number of top-level projects.
@@ -210,10 +211,12 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], sho
for (const [project, level] of projectClosure.entries()) {
if (level === 'dependency')
rootSuite._prependSuite(buildProjectSuite(project, projectSuites.get(project)!));
+ else
+ topLevelProjects.push(project);
}
}
- return rootSuite;
+ return { rootSuite, topLevelProjects };
}
function createProjectSuite(project: FullProjectInternal, fileSuites: Suite[]): Suite {
diff --git a/packages/playwright/src/runner/projectUtils.ts b/packages/playwright/src/runner/projectUtils.ts
index b52503c50c..3ad639a995 100644
--- a/packages/playwright/src/runner/projectUtils.ts
+++ b/packages/playwright/src/runner/projectUtils.ts
@@ -23,7 +23,7 @@ import { minimatch } from 'playwright-core/lib/utilsBundle';
import { createFileMatcher } from '../util';
-import type { FullProjectInternal } from '../common/config';
+import type { FullConfigInternal, FullProjectInternal } from '../common/config';
const readFileAsync = promisify(fs.readFile);
@@ -115,6 +115,11 @@ export function buildProjectsClosure(projects: FullProjectInternal[], hasTests?:
return result;
}
+export function findTopLevelProjects(config: FullConfigInternal): FullProjectInternal[] {
+ const closure = buildProjectsClosure(config.projects);
+ return [...closure].filter(entry => entry[1] === 'top-level').map(entry => entry[0]);
+}
+
export function buildDependentProjects(forProjects: FullProjectInternal[], projects: FullProjectInternal[]): Set<FullProjectInternal> {
const reverseDeps = new Map<FullProjectInternal, FullProjectInternal[]>(projects.map(p => ([p, []])));
for (const project of projects) {
diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.ts
index c383d71360..7d6245ad87 100644
--- a/packages/playwright/src/runner/tasks.ts
+++ b/packages/playwright/src/runner/tasks.ts
@@ -63,6 +63,7 @@ export class TestRun {
readonly phases: Phase[] = [];
projectFiles: Map<FullProjectInternal, string[]> = new Map();
projectSuites: Map<FullProjectInternal, Suite[]> = new Map();
+ topLevelProjects: FullProjectInternal[] = [];
constructor(config: FullConfigInternal, reporter: InternalReporter, options?: { pauseOnError?: boolean, pauseAtEnd?: boolean }) {
this.config = config;
@@ -232,8 +233,9 @@ export function createListFilesTask(): Task<TestRun> {
return {
title: 'load tests',
setup: async (testRun, errors) => {
- testRun.rootSuite = await createRootSuite(testRun, errors, false);
- testRun.failureTracker.onRootSuite(testRun.rootSuite);
+ const { rootSuite, topLevelProjects } = await createRootSuite(testRun, errors, false);
+ testRun.rootSuite = rootSuite;
+ testRun.failureTracker.onRootSuite(rootSuite, topLevelProjects);
await collectProjectsAndTestFiles(testRun, false);
for (const [project, files] of testRun.projectFiles) {
const projectSuite = new Suite(project.project.name, 'project');
@@ -269,8 +271,9 @@ export function createLoadTask(mode: 'out-of-process' | 'in-process', options: {
testRun.config.preOnlyTestFilters.push(test => changedFiles.has(test.location.file));
}
- testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);
- testRun.failureTracker.onRootSuite(testRun.rootSuite);
+ const { rootSuite, topLevelProjects } = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);
+ testRun.rootSuite = rootSuite;
+ testRun.failureTracker.onRootSuite(rootSuite, topLevelProjects);
// Fail when no tests.
if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length && !testRun.config.cliPassWithNoTests && !testRun.config.config.shard && !testRun.config.cliOnlyChanged) {
if (testRun.config.cliArgs.length) {
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index b6d1b4fa6d..e5967c199b 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -73,6 +73,8 @@ export type RunTestsParams = {
connectWsEndpoint?: string;
pauseOnError?: boolean;
pauseAtEnd?: boolean;
+ doNotRunDepsOutsideProjectFilter?: boolean;
+ disableConfigReporters?: boolean;
};
type FullResultStatus = reporterTypes.FullResult['status'];
@@ -337,12 +339,12 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {
config.preOnlyTestFilters.push(test => testIdSet.has(test.id));
}
- const configReporters = await createReporters(config, 'test', true);
+ const configReporters = params.disableConfigReporters ? [] : await createReporters(config, 'test', true);
const reporter = new InternalReporter([...configReporters, userReporter]);
const stop = new ManualPromise();
const tasks = [
createApplyRebaselinesTask(),
- createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }),
+ createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: params.doNotRunDepsOutsideProjectFilter }),
...createRunTestsTasks(config),
];
const testRun = new TestRun(config, reporter, { pauseOnError: params.pauseOnError, pauseAtEnd: params.pauseAtEnd });
diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.ts
index 26ac3a0e52..08fc4415e7 100644
--- a/packages/playwright/src/runner/testServer.ts
+++ b/packages/playwright/src/runner/testServer.ts
@@ -200,7 +200,10 @@ export class TestServerDispatcher implements TestServerInterface {
async runTests(params: Parameters<TestServerInterface['runTests']>[0]): ReturnType<TestServerInterface['runTests']> {
const wireReporter = await this._wireReporter(e => this._dispatchEvent('report', e));
- const { status } = await this._testRunner.runTests(wireReporter, params);
+ const { status } = await this._testRunner.runTests(wireReporter, {
+ ...params,
+ doNotRunDepsOutsideProjectFilter: true,
+ });
return { status };
}
diff --git a/tests/mcp/test-tools.spec.ts b/tests/mcp/test-tools.spec.ts
index 8336833a94..9c64551ae2 100644
--- a/tests/mcp/test-tools.spec.ts
+++ b/tests/mcp/test-tools.spec.ts
@@ -17,6 +17,7 @@
import { test, expect, writeFiles, StartClient } from './fixtures';
import fs from 'fs';
+import path from 'path';
test.use({ mcpServerType: 'test-mcp' });
@@ -137,6 +138,51 @@ Running 2 tests using 1 worker
`);
});
+test('test_run should include dependencies', async ({ startClient }) => {
+ await writeFiles({
+ 'playwright.config.ts': `
+ module.exports = {
+ projects: [
+ { name: 'setup', testMatch: /.*setup\\.ts/ },
+ { name: 'chromium', dependencies: ['setup'] },
+ ],
+ };
+ `,
+ 'auth.setup.ts': `
+ import { test as setup, expect } from '@playwright/test';
+ setup('auth', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ `,
+ 'example.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('example1', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ test('example2', async ({}) => {
+ expect(1 + 1).toBe(2);
+ });
+ `
+ });
+
+ const { client } = await startClient();
+ expect(await client.callTool({
+ name: 'test_run',
+ arguments: {
+ locations: ['example.test.ts'],
+ projects: ['chromium'],
+ },
+ })).toHaveTextResponse(`
+Running 3 tests using 1 worker
+
+ ok 1 [id=<ID>] [project=setup] › auth.setup.ts:3:12 › auth (XXms)
+ ok 2 [id=<ID>] [project=chromium] › example.test.ts:3:11 › example1 (XXms)
+ ok 3 [id=<ID>] [project=chromium] › example.test.ts:6:11 › example2 (XXms)
+
+ 3 passed (XXms)
+`);
+});
+
test('test_debug (passed)', async ({ startClient }) => {
await writeFiles({
'a.test.ts': `
@@ -217,7 +263,7 @@ test('test_debug / browser_snapshot', async ({ startClient }) => {
});
});
-test('test_debug_test (pause/snapshot/resume)', async ({ startClient }) => {
+test('test_debug (pause/snapshot/resume)', async ({ startClient }) => {
const { client, id } = await prepareDebugTest(startClient);
expect(await client.callTool({
@@ -350,6 +396,49 @@ test('test_setup_page', async ({ startClient }) => {
});
});
+test('test_setup_page with dependencies', async ({ startClient }) => {
+ const baseDir = await writeFiles({
+ 'playwright.config.ts': `
+ module.exports = {
+ projects: [
+ { name: 'setup', testMatch: /.*setup\\.ts/ },
+ { name: 'chromium', dependencies: ['setup'] },
+ { name: 'ignored', dependencies: ['chromium'] },
+ ],
+ };
+ `,
+ 'auth.setup.ts': `
+ import { test as setup, expect } from '@playwright/test';
+ setup('auth', async ({ page }, testInfo) => {
+ require('fs').writeFileSync(testInfo.outputPath('auth.txt'), 'done');
+ });
+ `,
+ 'template.test.ts': `
+ import { test, expect } from '@playwright/test';
+ test('template', async ({ page }, testInfo) => {
+ require('fs').writeFileSync(testInfo.outputPath('template.txt'), 'done');
+ });
+ `,
+ });
+
+ const { client } = await startClient();
+ expect(await client.callTool({
+ name: 'test_setup_page',
+ arguments: {
+ testLocation: 'template.test.ts:3',
+ project: 'chromium',
+ },
+ })).toHaveTextResponse(`### Paused at end of test. ready for interaction
+
+### Current page snapshot:
+`);
+
+ // Should pause at the target test, not in a dependency or any other stray project.
+ expect(fs.existsSync(path.join(baseDir, 'test-results', 'auth.setup.ts-auth-setup', 'auth.txt'))).toBe(true);
+ expect(fs.existsSync(path.join(baseDir, 'test-results', 'template-template-chromium', 'template.txt'))).toBe(true);
+ expect(fs.existsSync(path.join(baseDir, 'test-results', 'template-template-ignored', 'template.txt'))).toBe(false);
+});
+
test('test_setup_page (no test location)', async ({ startClient }) => {
const { client } = await startClient();
expect(await client.callTool({
@@ -361,6 +450,31 @@ test('test_setup_page (no test location)', async ({ startClient }) => {
`);
});
+test('test_setup_page chooses top-level project', async ({ startClient }) => {
+ const baseDir = await writeFiles({
+ 'playwright.config.ts': `
+ module.exports = {
+ projects: [
+ { name: 'one', testDir: './one' },
+ { name: 'two', testDir: './two', dependencies: ['one'] },
+ ],
+ };
+ `,
+ });
+
+ const { client } = await startClient();
+ expect(await client.callTool({
+ name: 'test_setup_page',
+ arguments: {},
+ })).toHaveTextResponse(`### Paused at end of test. ready for interaction
+
+### Current page snapshot:
+`);
+
+ expect(fs.existsSync(path.join(baseDir, 'one', 'default.seed.spec.ts'))).toBe(false);
+ expect(fs.existsSync(path.join(baseDir, 'two', 'default.seed.spec.ts'))).toBe(true);
+});
+
test('test_setup_page without location respects testsDir', async ({ startClient }) => {
await writeFiles({
'playwright.config.ts': `