Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

fix(test_setup_page): respond with error when failed to load the seed (#37576)

Dmitry Gozman6d41f6162851
diff --git a/packages/playwright/src/mcp/test/testTools.ts b/packages/playwright/src/mcp/test/testTools.ts
index ae37d68453..42e42b2821 100644
--- a/packages/playwright/src/mcp/test/testTools.ts
+++ b/packages/playwright/src/mcp/test/testTools.ts
@@ -127,7 +127,7 @@ export const setupPage = defineTestTool({
description: 'Setup the page for test',
inputSchema: z.object({
project: z.string().optional().describe('Project to use for setup. For example: "chromium", if no project is provided uses the first project in the config.'),
- testLocation: z.string().optional().describe('Location of the seed test to use for setup. For example: "test/seed/default.spec.ts:20".'),
+ testLocation: z.string().optional().describe('Location of the seed test to use for setup. For example: "tests/seed.spec.ts" or "tests/seed.spec.ts:20".'),
}),
type: 'readOnly',
},
@@ -162,8 +162,18 @@ test('seed', async ({ page }) => {});
workers: 1,
pauseAtEnd: true,
disableConfigReporters: true,
+ failOnLoadErrors: true,
});
+ // Ideally, we should check that page was indeed created and browser mcp has kicked in.
+ // However, that is handled in the upper layer, so hard to check here.
+ if (result.status === 'passed' && !reporter.suite?.allTests().length) {
+ return {
+ content: [{ type: 'text', text: 'Error: seed test not found.' }],
+ isError: true,
+ };
+ }
+
const text = stream.content();
return {
content: [{ type: 'text', text }],
diff --git a/packages/playwright/src/runner/testRunner.ts b/packages/playwright/src/runner/testRunner.ts
index 305247156c..59b4ce7400 100644
--- a/packages/playwright/src/runner/testRunner.ts
+++ b/packages/playwright/src/runner/testRunner.ts
@@ -75,6 +75,7 @@ export type RunTestsParams = {
pauseAtEnd?: boolean;
doNotRunDepsOutsideProjectFilter?: boolean;
disableConfigReporters?: boolean;
+ failOnLoadErrors?: boolean;
};
type FullResultStatus = reporterTypes.FullResult['status'];
@@ -344,7 +345,7 @@ export class TestRunner extends EventEmitter<TestRunnerEventMap> {
const stop = new ManualPromise();
const tasks = [
createApplyRebaselinesTask(),
- createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: false, doNotRunDepsOutsideProjectFilter: params.doNotRunDepsOutsideProjectFilter }),
+ createLoadTask('out-of-process', { filterOnly: true, failOnLoadErrors: !!params.failOnLoadErrors, doNotRunDepsOutsideProjectFilter: params.doNotRunDepsOutsideProjectFilter }),
...createRunTestsTasks(config),
];
const testRun = new TestRun(config, reporter, { pauseOnError: params.pauseOnError, pauseAtEnd: params.pauseAtEnd });
diff --git a/tests/mcp/test-tools.spec.ts b/tests/mcp/test-tools.spec.ts
index e7e9193fa2..1ec6ac2ecb 100644
--- a/tests/mcp/test-tools.spec.ts
+++ b/tests/mcp/test-tools.spec.ts
@@ -520,7 +520,7 @@ test('test_setup_page with dependencies', async ({ startClient }) => {
expect(await client.callTool({
name: 'test_setup_page',
arguments: {
- testLocation: 'template.test.ts:3',
+ testLocation: 'template.test.ts',
project: 'chromium',
},
})).toHaveTextResponse(`### Paused at end of test. ready for interaction
@@ -540,6 +540,31 @@ test('test_setup_page with dependencies', async ({ startClient }) => {
expect(fs.existsSync(path.join(baseDir, 'test-results', 'template-template-ignored', 'template.txt'))).toBe(false);
});
+test('test_setup_page (loading error)', async ({ startClient }) => {
+ await writeFiles({
+ 'template.test.ts': `
+ throw new Error('loading error');
+ `,
+ });
+ const { client } = await startClient();
+ expect(await client.callTool({
+ name: 'test_setup_page',
+ arguments: {
+ testLocation: 'template.test.ts',
+ },
+ })).toHaveTextResponse(expect.stringContaining('Error: loading error'));
+});
+
+test('test_setup_page (wrong test location)', async ({ startClient }) => {
+ const { client } = await startClient();
+ expect(await client.callTool({
+ name: 'test_setup_page',
+ arguments: {
+ testLocation: 'a.test.ts:6',
+ },
+ })).toHaveTextResponse(`Error: seed test not found.`);
+});
+
test('test_setup_page (no test location)', async ({ startClient }) => {
const { client } = await startClient();
expect(await client.callTool({