Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore: generate coverage.md with init-agents (#37876)

Pavel Feldman9f1fcc6d5ae3
diff --git a/examples/todomvc/.claude/agents/playwright-test-healer.md b/examples/todomvc/.claude/agents/playwright-test-healer.md
index 61efa5a1f5..0a31a92a52 100644
--- a/examples/todomvc/.claude/agents/playwright-test-healer.md
+++ b/examples/todomvc/.claude/agents/playwright-test-healer.md
@@ -11,8 +11,8 @@ resolving Playwright test failures. Your mission is to systematically identify,
broken Playwright tests using a methodical approach.
Your workflow:
-1. **Initial Execution**: Run all tests using playwright_test_run_test tool to identify failing tests
-2. **Debug failed tests**: For each failing test run playwright_test_debug_test.
+1. **Initial Execution**: Run all tests using `test_run` tool to identify failing tests
+2. **Debug failed tests**: For each failing test run `test_debug`.
3. **Error Investigation**: When the test pauses on errors, use available Playwright MCP tools to:
- Examine the error details
- Capture page snapshot to understand the context
diff --git a/examples/todomvc/.claude/agents/playwright-test-planner.md b/examples/todomvc/.claude/agents/playwright-test-planner.md
index 9e468a85d8..283a88d7fe 100644
--- a/examples/todomvc/.claude/agents/playwright-test-planner.md
+++ b/examples/todomvc/.claude/agents/playwright-test-planner.md
@@ -16,7 +16,7 @@ You will:
- Invoke the `planner_setup_page` tool once to set up page before using any other tools
- Explore the browser snapshot
- Do not take screenshots unless absolutely necessary
- - Use browser_* tools to navigate and discover interface
+ - Use `browser_*` tools to navigate and discover interface
- Thoroughly explore the interface, identifying all interactive elements, forms, navigation paths, and functionality
2. **Analyze User Flows**
diff --git "a/examples/todomvc/.github/chatmodes/ \360\237\216\255 planner.chatmode.md" "b/examples/todomvc/.github/chatmodes/ \360\237\216\255 planner.chatmode.md"
index 5eca16632b..ed3dc5e018 100644
--- "a/examples/todomvc/.github/chatmodes/ \360\237\216\255 planner.chatmode.md"
+++ "b/examples/todomvc/.github/chatmodes/ \360\237\216\255 planner.chatmode.md"
@@ -13,7 +13,7 @@ You will:
- Invoke the `planner_setup_page` tool once to set up page before using any other tools
- Explore the browser snapshot
- Do not take screenshots unless absolutely necessary
- - Use browser_* tools to navigate and discover interface
+ - Use `browser_*` tools to navigate and discover interface
- Thoroughly explore the interface, identifying all interactive elements, forms, navigation paths, and functionality
2. **Analyze User Flows**
diff --git "a/examples/todomvc/.github/chatmodes/\360\237\216\255 healer.chatmode.md" "b/examples/todomvc/.github/chatmodes/\360\237\216\255 healer.chatmode.md"
index 6604d74030..936786a7cc 100644
--- "a/examples/todomvc/.github/chatmodes/\360\237\216\255 healer.chatmode.md"
+++ "b/examples/todomvc/.github/chatmodes/\360\237\216\255 healer.chatmode.md"
@@ -8,8 +8,8 @@ resolving Playwright test failures. Your mission is to systematically identify,
broken Playwright tests using a methodical approach.
Your workflow:
-1. **Initial Execution**: Run all tests using playwright_test_run_test tool to identify failing tests
-2. **Debug failed tests**: For each failing test run playwright_test_debug_test.
+1. **Initial Execution**: Run all tests using `test_run` tool to identify failing tests
+2. **Debug failed tests**: For each failing test run `test_debug`.
3. **Error Investigation**: When the test pauses on errors, use available Playwright MCP tools to:
- Examine the error details
- Capture page snapshot to understand the context
diff --git a/examples/todomvc/prompts/README.md b/examples/todomvc/prompts/README.md
new file mode 100644
index 0000000000..2457872099
--- /dev/null
+++ b/examples/todomvc/prompts/README.md
@@ -0,0 +1,3 @@
+# Prompts
+
+This is a directory for useful prompts.
diff --git a/examples/todomvc/prompts/test-coverage.md b/examples/todomvc/prompts/test-coverage.md
new file mode 100644
index 0000000000..09d36ebaed
--- /dev/null
+++ b/examples/todomvc/prompts/test-coverage.md
@@ -0,0 +1,29 @@
+
+# Produce test coverage
+
+Parameters:
+- Task: the task to perform
+- Seed file (optional): the seed file to use, defaults to tests/seed.spec.ts
+- Test plan file (optional): the test plan file to write, under specs/ folder.
+
+1. Call #planner subagent with prompt:
+
+<plan>
+ <task><!-- the task --></task>
+ <seed-file><!-- seed file param --></seed-file>
+ <plan-file><!-- test plan file --></plan-file>
+</plan>
+
+2. For each test case from the test plan file (1.1, 1.2, ...), Call #generator subagent with prompt:
+
+<generate>
+ <test-file><!-- Name of the file to save the test into, should be unique for test --></test-file>
+ <test-suite><!-- Name of the top level test spec w/o ordinal--></test-suite>
+ <test-name><!--Name of the test case without the ordinal --></test-name>
+ <seed-file><!-- Seed file from test plan --></seed-file>
+ <body><!-- Test case content including steps and expectations --></body>
+</generate>
+
+3. Call #healer subagent with prompt:
+
+<heal>Run all tests and fix the failing ones one after another.</heal>
diff --git a/packages/playwright/src/agents/generateAgents.ts b/packages/playwright/src/agents/generateAgents.ts
index efa0ed0802..b349ad9234 100644
--- a/packages/playwright/src/agents/generateAgents.ts
+++ b/packages/playwright/src/agents/generateAgents.ts
@@ -16,7 +16,12 @@
import fs from 'fs';
import path from 'path';
+
import { colors, yaml } from 'playwright-core/lib/utilsBundle';
+import { mkdirIfNeeded } from 'playwright-core/lib/utils';
+
+import { FullConfigInternal } from '../common/config';
+import { defaultSeedFile, findSeedFile, seedFileContent, seedProject } from '../mcp/test/seed';
interface AgentHeader {
name: string;
@@ -33,6 +38,8 @@ interface Agent {
examples: string[];
}
+/* eslint-disable no-console */
+
class AgentParser {
static async parseFile(filePath: string): Promise<Agent> {
const rawMarkdown = await fs.promises.readFile(filePath, 'utf-8');
@@ -186,18 +193,55 @@ async function loadAgents(): Promise<Agent[]> {
return Promise.all(files.filter(file => file.endsWith('.md')).map(file => AgentParser.parseFile(path.join(__dirname, file))));
}
-async function writeFile(filePath: string, content: string) {
- // eslint-disable-next-line no-console
- console.log(`Writing file: ${filePath}`);
+async function writeFile(filePath: string, content: string, icon: string, description: string) {
+ console.log(`- ${icon} ${path.relative(process.cwd(), filePath)} ${colors.dim('- ' + description)}`);
+ await mkdirIfNeeded(filePath);
await fs.promises.writeFile(filePath, content, 'utf-8');
}
-export async function initClaudeCodeRepo() {
+async function initRepo(config: FullConfigInternal, projectName: string) {
+ const project = seedProject(config, projectName);
+ console.log(`🎭 Using project "${project.project.name}" as a primary project`);
+
+ if (!fs.existsSync('specs')) {
+ await fs.promises.mkdir('specs');
+ await writeFile(path.join('specs', 'README.md'), `# Specs
+
+This is a directory for test plans.
+`, 'πŸ“', 'directory for test plans');
+ }
+
+ if (!fs.existsSync('prompts')) {
+ await fs.promises.mkdir('prompts');
+ await writeFile(path.join('prompts', 'README.md'), `# Prompts
+
+This is a directory for useful prompts.
+`, 'πŸ“', 'useful prompts');
+ }
+
+ let seedFile = await findSeedFile(project);
+ if (!seedFile) {
+ seedFile = defaultSeedFile(project);
+ await writeFile(seedFile, seedFileContent, '🌱', 'default environment seed file');
+ }
+
+ const coveragePromptFile = path.join('prompts', 'test-coverage.md');
+ if (!fs.existsSync(coveragePromptFile))
+ await writeFile(coveragePromptFile, coveragePrompt(seedFile), 'πŸ“', 'test coverage prompt');
+}
+
+function initRepoDone() {
+ console.log('βœ… Done.');
+}
+
+export async function initClaudeCodeRepo(config: FullConfigInternal, projectName: string) {
+ await initRepo(config, projectName);
+
const agents = await loadAgents();
await fs.promises.mkdir('.claude/agents', { recursive: true });
for (const agent of agents)
- await writeFile(`.claude/agents/playwright-test-${agent.header.name}.md`, saveAsClaudeCode(agent));
+ await writeFile(`.claude/agents/playwright-test-${agent.header.name}.md`, saveAsClaudeCode(agent), 'πŸ€–', 'agent definition');
await writeFile('.mcp.json', JSON.stringify({
mcpServers: {
@@ -206,7 +250,9 @@ export async function initClaudeCodeRepo() {
args: commonMcpServers.playwrightTest.args,
}
}
- }, null, 2));
+ }, null, 2), 'πŸ”§', 'mcp configuration');
+
+ initRepoDone();
}
const vscodeToolMap = new Map<string, string[]>([
@@ -251,12 +297,13 @@ function saveAsVSCodeChatmode(agent: Agent): string {
return lines.join('\n');
}
-export async function initVSCodeRepo() {
+export async function initVSCodeRepo(config: FullConfigInternal, projectName: string) {
+ await initRepo(config, projectName);
const agents = await loadAgents();
await fs.promises.mkdir('.github/chatmodes', { recursive: true });
for (const agent of agents)
- await writeFile(`.github/chatmodes/${agent.header.name === 'planner' ? ' ' : ''}🎭 ${agent.header.name}.chatmode.md`, saveAsVSCodeChatmode(agent));
+ await writeFile(`.github/chatmodes/${agent.header.name === 'planner' ? ' ' : ''}🎭 ${agent.header.name}.chatmode.md`, saveAsVSCodeChatmode(agent), 'πŸ€–', 'chatmode definition');
await fs.promises.mkdir('.vscode', { recursive: true });
@@ -279,12 +326,14 @@ export async function initVSCodeRepo() {
args: commonMcpServers.playwrightTest.args,
cwd: '${workspaceFolder}',
};
- await writeFile(mcpJsonPath, JSON.stringify(mcpJson, null, 2));
- // eslint-disable-next-line no-console
- console.log(colors.yellow(`${colors.bold('Note:')} Playwright Test Agents require VSCode version 1.105+ or VSCode Insiders`));
+ await writeFile(mcpJsonPath, JSON.stringify(mcpJson, null, 2), 'πŸ”§', 'mcp configuration');
+
+ initRepoDone();
}
-export async function initOpencodeRepo() {
+export async function initOpencodeRepo(config: FullConfigInternal, projectName: string) {
+ await initRepo(config, projectName);
+
const agents = await loadAgents();
await fs.promises.mkdir('.opencode/prompts', { recursive: true });
@@ -292,7 +341,40 @@ export async function initOpencodeRepo() {
const prompt = [agent.instructions];
prompt.push('');
prompt.push(...agent.examples.map(example => `<example>${example}</example>`));
- await writeFile(`.opencode/prompts/playwright-test-${agent.header.name}.md`, prompt.join('\n'));
+ await writeFile(`.opencode/prompts/playwright-test-${agent.header.name}.md`, prompt.join('\n'), 'πŸ€–', 'agent definition');
}
- await writeFile('opencode.json', saveAsOpencodeJson(agents));
+ await writeFile('opencode.json', saveAsOpencodeJson(agents), 'πŸ”§', 'opencode configuration');
+
+ initRepoDone();
}
+
+const coveragePrompt = (seedFile: string) => `
+# Produce test coverage
+
+Parameters:
+- Task: the task to perform
+- Seed file (optional): the seed file to use, defaults to ${path.relative(process.cwd(), seedFile)}
+- Test plan file (optional): the test plan file to write, under specs/ folder.
+
+1. Call #planner subagent with prompt:
+
+<plan>
+ <task><!-- the task --></task>
+ <seed-file><!-- seed file param --></seed-file>
+ <plan-file><!-- test plan file --></plan-file>
+</plan>
+
+2. For each test case from the test plan file (1.1, 1.2, ...), Call #generator subagent with prompt:
+
+<generate>
+ <test-file><!-- Name of the file to save the test into, should be unique for test --></test-file>
+ <test-suite><!-- Name of the top level test spec w/o ordinal--></test-suite>
+ <test-name><!--Name of the test case without the ordinal --></test-name>
+ <seed-file><!-- Seed file from test plan --></seed-file>
+ <body><!-- Test case content including steps and expectations --></body>
+</generate>
+
+3. Call #healer subagent with prompt:
+
+<heal>Run all tests and fix the failing ones one after another.</heal>
+`;
diff --git a/packages/playwright/src/mcp/test/seed.ts b/packages/playwright/src/mcp/test/seed.ts
index 346ef1dfd2..04ebd74b8b 100644
--- a/packages/playwright/src/mcp/test/seed.ts
+++ b/packages/playwright/src/mcp/test/seed.ts
@@ -18,7 +18,6 @@ import fs from 'fs';
import path from 'path';
import { mkdirIfNeeded } from 'playwright-core/lib/utils';
-
import { collectFilesForProject, findTopLevelProjects } from '../../runner/projectUtils';
import type { FullConfigInternal, FullProjectInternal } from '../../common/config';
@@ -32,28 +31,31 @@ export function seedProject(config: FullConfigInternal, projectName?: string) {
return project;
}
-export async function ensureSeedTest(project: FullProjectInternal, logNew: boolean) {
+export async function findSeedFile(project: FullProjectInternal) {
const files = await collectFilesForProject(project);
- const seed = files.find(file => path.basename(file).includes('seed'));
- if (seed)
- return seed;
+ return files.find(file => path.basename(file).includes('seed'));
+}
+export function defaultSeedFile(project: FullProjectInternal) {
const testDir = project.project.testDir;
- const seedFile = path.resolve(testDir, 'seed.spec.ts');
+ return path.resolve(testDir, 'seed.spec.ts');
+}
- if (logNew) {
- // eslint-disable-next-line no-console
- console.log(`Writing file: ${path.relative(process.cwd(), seedFile)}`);
- }
+export async function ensureSeedFile(project: FullProjectInternal) {
+ const seedFile = await findSeedFile(project);
+ if (seedFile)
+ return seedFile;
+ const seedFilePath = defaultSeedFile(project);
+ await mkdirIfNeeded(seedFilePath);
+ await fs.promises.writeFile(seedFilePath, seedFileContent);
+ return seedFilePath;
+}
- await mkdirIfNeeded(seedFile);
- await fs.promises.writeFile(seedFile, `import { test, expect } from '@playwright/test';
+export const seedFileContent = `import { test, expect } from '@playwright/test';
test.describe('Test group', () => {
test('seed', async ({ page }) => {
// generate code here.
});
});
-`);
- return seedFile;
-}
+`;
diff --git a/packages/playwright/src/mcp/test/testContext.ts b/packages/playwright/src/mcp/test/testContext.ts
index a2204e24db..60a2d16b8f 100644
--- a/packages/playwright/src/mcp/test/testContext.ts
+++ b/packages/playwright/src/mcp/test/testContext.ts
@@ -24,7 +24,7 @@ import ListReporter from '../../reporters/list';
import { StringWriteStream } from './streams';
import { fileExistsAsync } from '../../util';
import { TestRunner, TestRunnerEvent } from '../../runner/testRunner';
-import { ensureSeedTest, seedProject } from './seed';
+import { ensureSeedFile, seedProject } from './seed';
import type { ProgressCallback } from '../sdk/server';
import type { ConfigLocation } from '../../common/config';
@@ -110,7 +110,7 @@ export class TestContext {
const project = seedProject(config, projectName);
if (!seedFile) {
- seedFile = await ensureSeedTest(project, false);
+ seedFile = await ensureSeedFile(project);
} else {
const candidateFiles: string[] = [];
const testDir = project.project.testDir;
diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.ts
index 7ea4813d20..cd9c8e801b 100644
--- a/packages/playwright/src/program.ts
+++ b/packages/playwright/src/program.ts
@@ -35,7 +35,6 @@ import { runAllTestsWithConfig, TestRunner } from './runner/testRunner';
import { createErrorCollectingReporter } from './runner/reporters';
import { ServerBackendFactory, runMainBackend } from './mcp/sdk/exports';
import { TestServerBackend } from './mcp/test/testBackend';
-import { ensureSeedTest, seedProject } from './mcp/test/seed';
import { decorateCommand } from './mcp/program';
import { setupExitWatchdog } from './mcp/browser/watchdog';
import { initClaudeCodeRepo, initOpencodeRepo, initVSCodeRepo } from './agents/generateAgents';
@@ -189,19 +188,17 @@ function addInitAgentsCommand(program: Command) {
command.option('-c, --config <file>', `Configuration file to find a project to use for seed test`);
command.option('--project <project>', 'Project to use for seed test');
command.action(async opts => {
+ const config = await loadConfigFromFile(opts.config);
if (opts.loop === 'opencode') {
- await initOpencodeRepo();
+ await initOpencodeRepo(config, opts.project);
} else if (opts.loop === 'vscode') {
- await initVSCodeRepo();
+ await initVSCodeRepo(config, opts.project);
} else if (opts.loop === 'claude') {
- await initClaudeCodeRepo();
+ await initClaudeCodeRepo(config, opts.project);
} else {
command.help();
return;
}
- const config = await loadConfigFromFile(opts.config);
- const project = seedProject(config, opts.project);
- await ensureSeedTest(project, true);
});
}