Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
fix(test-mcp): handle dependent projects in tools (#37459)
Dmitry Gozman8a6a4522e836
diff --git a/packages/playwright/src/mcp/test/testTools.ts b/packages/playwright/src/mcp/test/testTools.tsindex 894e64fe0f..b5ed411b0c 100644--- a/packages/playwright/src/mcp/test/testTools.ts+++ b/packages/playwright/src/mcp/test/testTools.ts@@ -23,6 +23,7 @@ import { z } from '../sdk/bundle';import { terminalScreen } from '../../reporters/base';import ListReporter from '../../reporters/list';import ListModeReporter from '../../reporters/listModeReporter';+import { findTopLevelProjects } from '../../runner/projectUtils';import { defineTestTool } from './testTool';import { StringWriteStream } from './streams';@@ -68,6 +69,7 @@ export const runTests = defineTestTool({const result = await testRunner.runTests(reporter, {locations: params.locations,projects: params.projects,+ disableConfigReporters: true,});const text = stream.content();@@ -106,6 +108,7 @@ export const debugTest = defineTestTool({timeout: 0,workers: 1,pauseOnError: true,+ disableConfigReporters: true,});const text = stream.content();@@ -140,10 +143,11 @@ export const setupPage = defineTestTool({if (!testLocation) {testLocation = 'default.seed.spec.ts';const config = await testRunner.loadConfig();- const project = params.project ? config.projects.find(p => p.project.name === params.project) : config.projects[0];+ const project = params.project ? config.projects.find(p => p.project.name === params.project) : findTopLevelProjects(config)[0];const testDir = project?.project.testDir || configDir;const seedFile = path.join(testDir, testLocation);if (!fs.existsSync(seedFile)) {+ await fs.promises.mkdir(path.dirname(seedFile), { recursive: true });await fs.promises.writeFile(seedFile, `import { test, expect } from '@playwright/test';test('seed', async ({ page }) => {});@@ -158,6 +162,7 @@ test('seed', async ({ page }) => {});timeout: 0,workers: 1,pauseAtEnd: true,+ disableConfigReporters: true,});const text = stream.content();diff --git a/packages/playwright/src/runner/dispatcher.ts b/packages/playwright/src/runner/dispatcher.tsindex 7d64c2f3a5..d9db512cd9 100644--- a/packages/playwright/src/runner/dispatcher.ts+++ b/packages/playwright/src/runner/dispatcher.ts@@ -224,7 +224,7 @@ export class Dispatcher {extraEnv: this._extraEnvByProjectId.get(testGroup.projectId) || {},outputDir,pauseOnError: this._failureTracker.pauseOnError(),- pauseAtEnd: this._failureTracker.pauseAtEnd(),+ pauseAtEnd: this._failureTracker.pauseAtEnd(projectConfig),});const handleOutput = (params: TestOutputPayload) => {const chunk = chunkFromParams(params);diff --git a/packages/playwright/src/runner/failureTracker.ts b/packages/playwright/src/runner/failureTracker.tsindex c2d709ec61..ef276fa509 100644--- a/packages/playwright/src/runner/failureTracker.ts+++ b/packages/playwright/src/runner/failureTracker.ts@@ -15,13 +15,14 @@*/import type { TestResult } from '../../types/testReporter';-import type { FullConfigInternal } from '../common/config';+import type { FullConfigInternal, FullProjectInternal } from '../common/config';import type { Suite, TestCase } from '../common/test';export class FailureTracker {private _failureCount = 0;private _hasWorkerErrors = false;private _rootSuite: Suite | undefined;+ private _topLevelProjects: FullProjectInternal[] = [];private _pauseOnError: boolean;private _pauseAtEnd: boolean;@@ -30,8 +31,9 @@ export class FailureTracker {this._pauseAtEnd = options?.pauseAtEnd ?? false;}- onRootSuite(rootSuite: Suite) {+ onRootSuite(rootSuite: Suite, topLevelProjects: FullProjectInternal[]) {this._rootSuite = rootSuite;+ this._topLevelProjects = topLevelProjects;}onTestEnd(test: TestCase, result: TestResult) {@@ -48,8 +50,8 @@ export class FailureTracker {return this._pauseOnError;}- pauseAtEnd(): boolean {- return this._pauseAtEnd;+ pauseAtEnd(inProject: FullProjectInternal): boolean {+ return this._pauseAtEnd && this._topLevelProjects.includes(inProject);}hasReachedMaxFailures() {diff --git a/packages/playwright/src/runner/loadUtils.ts b/packages/playwright/src/runner/loadUtils.tsindex 05c5bba718..065cd0cc4a 100644--- a/packages/playwright/src/runner/loadUtils.ts+++ b/packages/playwright/src/runner/loadUtils.ts@@ -120,7 +120,7 @@ export async function loadFileSuites(testRun: TestRun, mode: 'out-of-process' |}}-export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean): Promise<Suite> {+export async function createRootSuite(testRun: TestRun, errors: TestError[], shouldFilterOnly: boolean): Promise<{ rootSuite: Suite, topLevelProjects: FullProjectInternal[] }> {const config = testRun.config;// Create root suite, where each child will be a project suite with cloned file suites inside it.const rootSuite = new Suite('', 'root');@@ -200,6 +200,7 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], shoif (config.postShardTestFilters.length)filterTestsRemoveEmptySuites(rootSuite, test => config.postShardTestFilters.every(filter => filter(test)));+ const topLevelProjects = [];// Now prepend dependency projects without filtration.{// Filtering 'only' and sharding might have reduced the number of top-level projects.@@ -210,10 +211,12 @@ export async function createRootSuite(testRun: TestRun, errors: TestError[], shofor (const [project, level] of projectClosure.entries()) {if (level === 'dependency')rootSuite._prependSuite(buildProjectSuite(project, projectSuites.get(project)!));+ else+ topLevelProjects.push(project);}}- return rootSuite;+ return { rootSuite, topLevelProjects };}function createProjectSuite(project: FullProjectInternal, fileSuites: Suite[]): Suite {diff --git a/packages/playwright/src/runner/projectUtils.ts b/packages/playwright/src/runner/projectUtils.tsindex b52503c50c..3ad639a995 100644--- a/packages/playwright/src/runner/projectUtils.ts+++ b/packages/playwright/src/runner/projectUtils.ts@@ -23,7 +23,7 @@ import { minimatch } from 'playwright-core/lib/utilsBundle';import { createFileMatcher } from '../util';-import type { FullProjectInternal } from '../common/config';+import type { FullConfigInternal, FullProjectInternal } from '../common/config';const readFileAsync = promisify(fs.readFile);@@ -115,6 +115,11 @@ export function buildProjectsClosure(projects: FullProjectInternal[], hasTests?:return result;}+export function findTopLevelProjects(config: FullConfigInternal): FullProjectInternal[] {+ const closure = buildProjectsClosure(config.projects);+ return [...closure].filter(entry => entry[1] === 'top-level').map(entry => entry[0]);+}+export function buildDependentProjects(forProjects: FullProjectInternal[], projects: FullProjectInternal[]): Set<FullProjectInternal> {const reverseDeps = new Map<FullProjectInternal, FullProjectInternal[]>(projects.map(p => ([p, []])));for (const project of projects) {diff --git a/packages/playwright/src/runner/tasks.ts b/packages/playwright/src/runner/tasks.tsindex c383d71360..7d6245ad87 100644--- a/packages/playwright/src/runner/tasks.ts+++ b/packages/playwright/src/runner/tasks.ts@@ -63,6 +63,7 @@ export class TestRun {readonly phases: Phase[] = [];projectFiles: Map<FullProjectInternal, string[]> = new Map();projectSuites: Map<FullProjectInternal, Suite[]> = new Map();+ topLevelProjects: FullProjectInternal[] = [];constructor(config: FullConfigInternal, reporter: InternalReporter, options?: { pauseOnError?: boolean, pauseAtEnd?: boolean }) {this.config = config;@@ -232,8 +233,9 @@ export function createListFilesTask(): Task<TestRun> {return {title: 'load tests',setup: async (testRun, errors) => {- testRun.rootSuite = await createRootSuite(testRun, errors, false);- testRun.failureTracker.onRootSuite(testRun.rootSuite);+ const { rootSuite, topLevelProjects } = await createRootSuite(testRun, errors, false);+ testRun.rootSuite = rootSuite;+ testRun.failureTracker.onRootSuite(rootSuite, topLevelProjects);await collectProjectsAndTestFiles(testRun, false);for (const [project, files] of testRun.projectFiles) {const projectSuite = new Suite(project.project.name, 'project');@@ -269,8 +271,9 @@ export function createLoadTask(mode: 'out-of-process' | 'in-process', options: {testRun.config.preOnlyTestFilters.push(test => changedFiles.has(test.location.file));}- testRun.rootSuite = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);- testRun.failureTracker.onRootSuite(testRun.rootSuite);+ const { rootSuite, topLevelProjects } = await createRootSuite(testRun, options.failOnLoadErrors ? errors : softErrors, !!options.filterOnly);+ testRun.rootSuite = rootSuite;+ testRun.failureTracker.onRootSuite(rootSuite, topLevelProjects);// Fail when no tests.if (options.failOnLoadErrors && !testRun.rootSuite.allTests().length && !testRun.config.cliPassWithNoTests && !testRun.config.config.shard && !testRun.config.cliOnlyChanged) {if (testRun.config.cliArgs.length) {diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.tsindex b6d1b4fa6d..e5967c199b 100644--- a/packages/playwright/src/runner/testRunner.ts+++ b/packages/playwright/src/runner/testRunner.ts@@ -73,6 +73,8 @@ export type RunTestsParams = {connectWsEndpoint?: string;pauseOnError?: boolean;pauseAtEnd?: boolean;+ doNotRunDepsOutsideProjectFilter?: boolean;+ disableConfigReporters?: boolean;};type FullResultStatus = reporterTypes.FullResult['status'];@@ -337,12 +339,12 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {config.preOnlyTestFilters.push(test => testIdSet.has(test.id));}- const configReporters = await createReporters(config, 'test', true);+ const configReporters = params.disableConfigReporters ? [] : await createReporters(config, 'test', true);const reporter = new InternalReporter([...configReporters, userReporter]);const stop = new ManualPromise();const tasks = [createApplyRebaselinesTask(),- createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: true }),+ createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: params.doNotRunDepsOutsideProjectFilter }),...createRunTestsTasks(config),];const testRun = new TestRun(config, reporter, { pauseOnError: params.pauseOnError, pauseAtEnd: params.pauseAtEnd });diff --git a/packages/playwright/src/runner/testServer.ts b/packages/playwright/src/runner/testServer.tsindex 26ac3a0e52..08fc4415e7 100644--- a/packages/playwright/src/runner/testServer.ts+++ b/packages/playwright/src/runner/testServer.ts@@ -200,7 +200,10 @@ export class TestServerDispatcher implements TestServerInterface {async runTests(params: Parameters<TestServerInterface['runTests']>[0]): ReturnType<TestServerInterface['runTests']> {const wireReporter = await this._wireReporter(e => this._dispatchEvent('report', e));- const { status } = await this._testRunner.runTests(wireReporter, params);+ const { status } = await this._testRunner.runTests(wireReporter, {+ ...params,+ doNotRunDepsOutsideProjectFilter: true,+ });return { status };}diff --git a/tests/mcp/test-tools.spec.ts b/tests/mcp/test-tools.spec.tsindex 8336833a94..9c64551ae2 100644--- a/tests/mcp/test-tools.spec.ts+++ b/tests/mcp/test-tools.spec.ts@@ -17,6 +17,7 @@import { test, expect, writeFiles, StartClient } from './fixtures';import fs from 'fs';+import path from 'path';test.use({ mcpServerType: 'test-mcp' });@@ -137,6 +138,51 @@ Running 2 tests using 1 worker`);});+test('test_run should include dependencies', async ({ startClient }) => {+ await writeFiles({+ 'playwright.config.ts': `+ module.exports = {+ projects: [+ { name: 'setup', testMatch: /.*setup\\.ts/ },+ { name: 'chromium', dependencies: ['setup'] },+ ],+ };+ `,+ 'auth.setup.ts': `+ import { test as setup, expect } from '@playwright/test';+ setup('auth', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ `,+ 'example.test.ts': `+ import { test, expect } from '@playwright/test';+ test('example1', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ test('example2', async ({}) => {+ expect(1 + 1).toBe(2);+ });+ `+ });++ const { client } = await startClient();+ expect(await client.callTool({+ name: 'test_run',+ arguments: {+ locations: ['example.test.ts'],+ projects: ['chromium'],+ },+ })).toHaveTextResponse(`+Running 3 tests using 1 worker++ ok 1 [id=<ID>] [project=setup] › auth.setup.ts:3:12 › auth (XXms)+ ok 2 [id=<ID>] [project=chromium] › example.test.ts:3:11 › example1 (XXms)+ ok 3 [id=<ID>] [project=chromium] › example.test.ts:6:11 › example2 (XXms)++ 3 passed (XXms)+`);+});+test('test_debug (passed)', async ({ startClient }) => {await writeFiles({'a.test.ts': `@@ -217,7 +263,7 @@ test('test_debug / browser_snapshot', async ({ startClient }) => {});});-test('test_debug_test (pause/snapshot/resume)', async ({ startClient }) => {+test('test_debug (pause/snapshot/resume)', async ({ startClient }) => {const { client, id } = await prepareDebugTest(startClient);expect(await client.callTool({@@ -350,6 +396,49 @@ test('test_setup_page', async ({ startClient }) => {});});+test('test_setup_page with dependencies', async ({ startClient }) => {+ const baseDir = await writeFiles({+ 'playwright.config.ts': `+ module.exports = {+ projects: [+ { name: 'setup', testMatch: /.*setup\\.ts/ },+ { name: 'chromium', dependencies: ['setup'] },+ { name: 'ignored', dependencies: ['chromium'] },+ ],+ };+ `,+ 'auth.setup.ts': `+ import { test as setup, expect } from '@playwright/test';+ setup('auth', async ({ page }, testInfo) => {+ require('fs').writeFileSync(testInfo.outputPath('auth.txt'), 'done');+ });+ `,+ 'template.test.ts': `+ import { test, expect } from '@playwright/test';+ test('template', async ({ page }, testInfo) => {+ require('fs').writeFileSync(testInfo.outputPath('template.txt'), 'done');+ });+ `,+ });++ const { client } = await startClient();+ expect(await client.callTool({+ name: 'test_setup_page',+ arguments: {+ testLocation: 'template.test.ts:3',+ project: 'chromium',+ },+ })).toHaveTextResponse(`### Paused at end of test. ready for interaction++### Current page snapshot:+`);++ // Should pause at the target test, not in a dependency or any other stray project.+ expect(fs.existsSync(path.join(baseDir, 'test-results', 'auth.setup.ts-auth-setup', 'auth.txt'))).toBe(true);+ expect(fs.existsSync(path.join(baseDir, 'test-results', 'template-template-chromium', 'template.txt'))).toBe(true);+ expect(fs.existsSync(path.join(baseDir, 'test-results', 'template-template-ignored', 'template.txt'))).toBe(false);+});+test('test_setup_page (no test location)', async ({ startClient }) => {const { client } = await startClient();expect(await client.callTool({@@ -361,6 +450,31 @@ test('test_setup_page (no test location)', async ({ startClient }) => {`);});+test('test_setup_page chooses top-level project', async ({ startClient }) => {+ const baseDir = await writeFiles({+ 'playwright.config.ts': `+ module.exports = {+ projects: [+ { name: 'one', testDir: './one' },+ { name: 'two', testDir: './two', dependencies: ['one'] },+ ],+ };+ `,+ });++ const { client } = await startClient();+ expect(await client.callTool({+ name: 'test_setup_page',+ arguments: {},+ })).toHaveTextResponse(`### Paused at end of test. ready for interaction++### Current page snapshot:+`);++ expect(fs.existsSync(path.join(baseDir, 'one', 'default.seed.spec.ts'))).toBe(false);+ expect(fs.existsSync(path.join(baseDir, 'two', 'default.seed.spec.ts'))).toBe(true);+});+test('test_setup_page without location respects testsDir', async ({ startClient }) => {await writeFiles({'playwright.config.ts': `