Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
chore: generate coverage.md with init-agents (#37876)
Pavel Feldman9f1fcc6d5ae3
diff --git a/examples/todomvc/.claude/agents/playwright-test-healer.md b/examples/todomvc/.claude/agents/playwright-test-healer.mdindex 61efa5a1f5..0a31a92a52 100644--- a/examples/todomvc/.claude/agents/playwright-test-healer.md+++ b/examples/todomvc/.claude/agents/playwright-test-healer.md@@ -11,8 +11,8 @@ resolving Playwright test failures. Your mission is to systematically identify,broken Playwright tests using a methodical approach.Your workflow:-1. **Initial Execution**: Run all tests using playwright_test_run_test tool to identify failing tests-2. **Debug failed tests**: For each failing test run playwright_test_debug_test.+1. **Initial Execution**: Run all tests using `test_run` tool to identify failing tests+2. **Debug failed tests**: For each failing test run `test_debug`.3. **Error Investigation**: When the test pauses on errors, use available Playwright MCP tools to:- Examine the error details- Capture page snapshot to understand the contextdiff --git a/examples/todomvc/.claude/agents/playwright-test-planner.md b/examples/todomvc/.claude/agents/playwright-test-planner.mdindex 9e468a85d8..283a88d7fe 100644--- a/examples/todomvc/.claude/agents/playwright-test-planner.md+++ b/examples/todomvc/.claude/agents/playwright-test-planner.md@@ -16,7 +16,7 @@ You will:- Invoke the `planner_setup_page` tool once to set up page before using any other tools- Explore the browser snapshot- Do not take screenshots unless absolutely necessary- - Use browser_* tools to navigate and discover interface+ - Use `browser_*` tools to navigate and discover interface- Thoroughly explore the interface, identifying all interactive elements, forms, navigation paths, and functionality2. **Analyze User Flows**diff --git "a/examples/todomvc/.github/chatmodes/ \360\237\216\255 planner.chatmode.md" "b/examples/todomvc/.github/chatmodes/ \360\237\216\255 planner.chatmode.md"index 5eca16632b..ed3dc5e018 100644--- "a/examples/todomvc/.github/chatmodes/ \360\237\216\255 planner.chatmode.md"+++ "b/examples/todomvc/.github/chatmodes/ \360\237\216\255 planner.chatmode.md"@@ -13,7 +13,7 @@ You will:- Invoke the `planner_setup_page` tool once to set up page before using any other tools- Explore the browser snapshot- Do not take screenshots unless absolutely necessary- - Use browser_* tools to navigate and discover interface+ - Use `browser_*` tools to navigate and discover interface- Thoroughly explore the interface, identifying all interactive elements, forms, navigation paths, and functionality2. **Analyze User Flows**diff --git "a/examples/todomvc/.github/chatmodes/\360\237\216\255 healer.chatmode.md" "b/examples/todomvc/.github/chatmodes/\360\237\216\255 healer.chatmode.md"index 6604d74030..936786a7cc 100644--- "a/examples/todomvc/.github/chatmodes/\360\237\216\255 healer.chatmode.md"+++ "b/examples/todomvc/.github/chatmodes/\360\237\216\255 healer.chatmode.md"@@ -8,8 +8,8 @@ resolving Playwright test failures. Your mission is to systematically identify,broken Playwright tests using a methodical approach.Your workflow:-1. **Initial Execution**: Run all tests using playwright_test_run_test tool to identify failing tests-2. **Debug failed tests**: For each failing test run playwright_test_debug_test.+1. **Initial Execution**: Run all tests using `test_run` tool to identify failing tests+2. **Debug failed tests**: For each failing test run `test_debug`.3. **Error Investigation**: When the test pauses on errors, use available Playwright MCP tools to:- Examine the error details- Capture page snapshot to understand the contextdiff --git a/examples/todomvc/prompts/README.md b/examples/todomvc/prompts/README.mdnew file mode 100644index 0000000000..2457872099--- /dev/null+++ b/examples/todomvc/prompts/README.md@@ -0,0 +1,3 @@+# Prompts++This is a directory for useful prompts.diff --git a/examples/todomvc/prompts/test-coverage.md b/examples/todomvc/prompts/test-coverage.mdnew file mode 100644index 0000000000..09d36ebaed--- /dev/null+++ b/examples/todomvc/prompts/test-coverage.md@@ -0,0 +1,29 @@++# Produce test coverage++Parameters:+- Task: the task to perform+- Seed file (optional): the seed file to use, defaults to tests/seed.spec.ts+- Test plan file (optional): the test plan file to write, under specs/ folder.++1. Call #planner subagent with prompt:++<plan>+ <task><!-- the task --></task>+ <seed-file><!-- seed file param --></seed-file>+ <plan-file><!-- test plan file --></plan-file>+</plan>++2. For each test case from the test plan file (1.1, 1.2, ...), Call #generator subagent with prompt:++<generate>+ <test-file><!-- Name of the file to save the test into, should be unique for test --></test-file>+ <test-suite><!-- Name of the top level test spec w/o ordinal--></test-suite>+ <test-name><!--Name of the test case without the ordinal --></test-name>+ <seed-file><!-- Seed file from test plan --></seed-file>+ <body><!-- Test case content including steps and expectations --></body>+</generate>++3. Call #healer subagent with prompt:++<heal>Run all tests and fix the failing ones one after another.</heal>diff --git a/packages/playwright/src/agents/generateAgents.ts b/packages/playwright/src/agents/generateAgents.tsindex efa0ed0802..b349ad9234 100644--- a/packages/playwright/src/agents/generateAgents.ts+++ b/packages/playwright/src/agents/generateAgents.ts@@ -16,7 +16,12 @@import fs from 'fs';import path from 'path';+import { colors, yaml } from 'playwright-core/lib/utilsBundle';+import { mkdirIfNeeded } from 'playwright-core/lib/utils';++import { FullConfigInternal } from '../common/config';+import { defaultSeedFile, findSeedFile, seedFileContent, seedProject } from '../mcp/test/seed';interface AgentHeader {name: string;@@ -33,6 +38,8 @@ interface Agent {examples: string[];}+/* eslint-disable no-console */+class AgentParser {static async parseFile(filePath: string): Promise<Agent> {const rawMarkdown = await fs.promises.readFile(filePath, 'utf-8');@@ -186,18 +193,55 @@ async function loadAgents(): Promise<Agent[]> {return Promise.all(files.filter(file => file.endsWith('.md')).map(file => AgentParser.parseFile(path.join(__dirname, file))));}-async function writeFile(filePath: string, content: string) {- // eslint-disable-next-line no-console- console.log(`Writing file: ${filePath}`);+async function writeFile(filePath: string, content: string, icon: string, description: string) {+ console.log(`- ${icon} ${path.relative(process.cwd(), filePath)} ${colors.dim('- ' + description)}`);+ await mkdirIfNeeded(filePath);await fs.promises.writeFile(filePath, content, 'utf-8');}-export async function initClaudeCodeRepo() {+async function initRepo(config: FullConfigInternal, projectName: string) {+ const project = seedProject(config, projectName);+ console.log(`π Using project "${project.project.name}" as a primary project`);++ if (!fs.existsSync('specs')) {+ await fs.promises.mkdir('specs');+ await writeFile(path.join('specs', 'README.md'), `# Specs++This is a directory for test plans.+`, 'π', 'directory for test plans');+ }++ if (!fs.existsSync('prompts')) {+ await fs.promises.mkdir('prompts');+ await writeFile(path.join('prompts', 'README.md'), `# Prompts++This is a directory for useful prompts.+`, 'π', 'useful prompts');+ }++ let seedFile = await findSeedFile(project);+ if (!seedFile) {+ seedFile = defaultSeedFile(project);+ await writeFile(seedFile, seedFileContent, 'π±', 'default environment seed file');+ }++ const coveragePromptFile = path.join('prompts', 'test-coverage.md');+ if (!fs.existsSync(coveragePromptFile))+ await writeFile(coveragePromptFile, coveragePrompt(seedFile), 'π', 'test coverage prompt');+}++function initRepoDone() {+ console.log('β Done.');+}++export async function initClaudeCodeRepo(config: FullConfigInternal, projectName: string) {+ await initRepo(config, projectName);+const agents = await loadAgents();await fs.promises.mkdir('.claude/agents', { recursive: true });for (const agent of agents)- await writeFile(`.claude/agents/playwright-test-${agent.header.name}.md`, saveAsClaudeCode(agent));+ await writeFile(`.claude/agents/playwright-test-${agent.header.name}.md`, saveAsClaudeCode(agent), 'π€', 'agent definition');await writeFile('.mcp.json', JSON.stringify({mcpServers: {@@ -206,7 +250,9 @@ export async function initClaudeCodeRepo() {args: commonMcpServers.playwrightTest.args,}}- }, null, 2));+ }, null, 2), 'π§', 'mcp configuration');++ initRepoDone();}const vscodeToolMap = new Map<string, string[]>([@@ -251,12 +297,13 @@ function saveAsVSCodeChatmode(agent: Agent): string {return lines.join('\n');}-export async function initVSCodeRepo() {+export async function initVSCodeRepo(config: FullConfigInternal, projectName: string) {+ await initRepo(config, projectName);const agents = await loadAgents();await fs.promises.mkdir('.github/chatmodes', { recursive: true });for (const agent of agents)- await writeFile(`.github/chatmodes/${agent.header.name === 'planner' ? ' ' : ''}π ${agent.header.name}.chatmode.md`, saveAsVSCodeChatmode(agent));+ await writeFile(`.github/chatmodes/${agent.header.name === 'planner' ? ' ' : ''}π ${agent.header.name}.chatmode.md`, saveAsVSCodeChatmode(agent), 'π€', 'chatmode definition');await fs.promises.mkdir('.vscode', { recursive: true });@@ -279,12 +326,14 @@ export async function initVSCodeRepo() {args: commonMcpServers.playwrightTest.args,cwd: '${workspaceFolder}',};- await writeFile(mcpJsonPath, JSON.stringify(mcpJson, null, 2));- // eslint-disable-next-line no-console- console.log(colors.yellow(`${colors.bold('Note:')} Playwright Test Agents require VSCode version 1.105+ or VSCode Insiders`));+ await writeFile(mcpJsonPath, JSON.stringify(mcpJson, null, 2), 'π§', 'mcp configuration');++ initRepoDone();}-export async function initOpencodeRepo() {+export async function initOpencodeRepo(config: FullConfigInternal, projectName: string) {+ await initRepo(config, projectName);+const agents = await loadAgents();await fs.promises.mkdir('.opencode/prompts', { recursive: true });@@ -292,7 +341,40 @@ export async function initOpencodeRepo() {const prompt = [agent.instructions];prompt.push('');prompt.push(...agent.examples.map(example => `<example>${example}</example>`));- await writeFile(`.opencode/prompts/playwright-test-${agent.header.name}.md`, prompt.join('\n'));+ await writeFile(`.opencode/prompts/playwright-test-${agent.header.name}.md`, prompt.join('\n'), 'π€', 'agent definition');}- await writeFile('opencode.json', saveAsOpencodeJson(agents));+ await writeFile('opencode.json', saveAsOpencodeJson(agents), 'π§', 'opencode configuration');++ initRepoDone();}++const coveragePrompt = (seedFile: string) => `+# Produce test coverage++Parameters:+- Task: the task to perform+- Seed file (optional): the seed file to use, defaults to ${path.relative(process.cwd(), seedFile)}+- Test plan file (optional): the test plan file to write, under specs/ folder.++1. Call #planner subagent with prompt:++<plan>+ <task><!-- the task --></task>+ <seed-file><!-- seed file param --></seed-file>+ <plan-file><!-- test plan file --></plan-file>+</plan>++2. For each test case from the test plan file (1.1, 1.2, ...), Call #generator subagent with prompt:++<generate>+ <test-file><!-- Name of the file to save the test into, should be unique for test --></test-file>+ <test-suite><!-- Name of the top level test spec w/o ordinal--></test-suite>+ <test-name><!--Name of the test case without the ordinal --></test-name>+ <seed-file><!-- Seed file from test plan --></seed-file>+ <body><!-- Test case content including steps and expectations --></body>+</generate>++3. Call #healer subagent with prompt:++<heal>Run all tests and fix the failing ones one after another.</heal>+`;diff --git a/packages/playwright/src/mcp/test/seed.ts b/packages/playwright/src/mcp/test/seed.tsindex 346ef1dfd2..04ebd74b8b 100644--- a/packages/playwright/src/mcp/test/seed.ts+++ b/packages/playwright/src/mcp/test/seed.ts@@ -18,7 +18,6 @@ import fs from 'fs';import path from 'path';import { mkdirIfNeeded } from 'playwright-core/lib/utils';-import { collectFilesForProject, findTopLevelProjects } from '../../runner/projectUtils';import type { FullConfigInternal, FullProjectInternal } from '../../common/config';@@ -32,28 +31,31 @@ export function seedProject(config: FullConfigInternal, projectName?: string) {return project;}-export async function ensureSeedTest(project: FullProjectInternal, logNew: boolean) {+export async function findSeedFile(project: FullProjectInternal) {const files = await collectFilesForProject(project);- const seed = files.find(file => path.basename(file).includes('seed'));- if (seed)- return seed;+ return files.find(file => path.basename(file).includes('seed'));+}+export function defaultSeedFile(project: FullProjectInternal) {const testDir = project.project.testDir;- const seedFile = path.resolve(testDir, 'seed.spec.ts');+ return path.resolve(testDir, 'seed.spec.ts');+}- if (logNew) {- // eslint-disable-next-line no-console- console.log(`Writing file: ${path.relative(process.cwd(), seedFile)}`);- }+export async function ensureSeedFile(project: FullProjectInternal) {+ const seedFile = await findSeedFile(project);+ if (seedFile)+ return seedFile;+ const seedFilePath = defaultSeedFile(project);+ await mkdirIfNeeded(seedFilePath);+ await fs.promises.writeFile(seedFilePath, seedFileContent);+ return seedFilePath;+}- await mkdirIfNeeded(seedFile);- await fs.promises.writeFile(seedFile, `import { test, expect } from '@playwright/test';+export const seedFileContent = `import { test, expect } from '@playwright/test';test.describe('Test group', () => {test('seed', async ({ page }) => {// generate code here.});});-`);- return seedFile;-}+`;diff --git a/packages/playwright/src/mcp/test/testContext.ts b/packages/playwright/src/mcp/test/testContext.tsindex a2204e24db..60a2d16b8f 100644--- a/packages/playwright/src/mcp/test/testContext.ts+++ b/packages/playwright/src/mcp/test/testContext.ts@@ -24,7 +24,7 @@ import ListReporter from '../../reporters/list';import { StringWriteStream } from './streams';import { fileExistsAsync } from '../../util';import { TestRunner, TestRunnerEvent } from '../../runner/testRunner';-import { ensureSeedTest, seedProject } from './seed';+import { ensureSeedFile, seedProject } from './seed';import type { ProgressCallback } from '../sdk/server';import type { ConfigLocation } from '../../common/config';@@ -110,7 +110,7 @@ export class TestContext {const project = seedProject(config, projectName);if (!seedFile) {- seedFile = await ensureSeedTest(project, false);+ seedFile = await ensureSeedFile(project);} else {const candidateFiles: string[] = [];const testDir = project.project.testDir;diff --git a/packages/playwright/src/program.ts b/packages/playwright/src/program.tsindex 7ea4813d20..cd9c8e801b 100644--- a/packages/playwright/src/program.ts+++ b/packages/playwright/src/program.ts@@ -35,7 +35,6 @@ import { runAllTestsWithConfig, TestRunner } from './runner/testRunner';import { createErrorCollectingReporter } from './runner/reporters';import { ServerBackendFactory, runMainBackend } from './mcp/sdk/exports';import { TestServerBackend } from './mcp/test/testBackend';-import { ensureSeedTest, seedProject } from './mcp/test/seed';import { decorateCommand } from './mcp/program';import { setupExitWatchdog } from './mcp/browser/watchdog';import { initClaudeCodeRepo, initOpencodeRepo, initVSCodeRepo } from './agents/generateAgents';@@ -189,19 +188,17 @@ function addInitAgentsCommand(program: Command) {command.option('-c, --config <file>', `Configuration file to find a project to use for seed test`);command.option('--project <project>', 'Project to use for seed test');command.action(async opts => {+ const config = await loadConfigFromFile(opts.config);if (opts.loop === 'opencode') {- await initOpencodeRepo();+ await initOpencodeRepo(config, opts.project);} else if (opts.loop === 'vscode') {- await initVSCodeRepo();+ await initVSCodeRepo(config, opts.project);} else if (opts.loop === 'claude') {- await initClaudeCodeRepo();+ await initClaudeCodeRepo(config, opts.project);} else {command.help();return;}- const config = await loadConfigFromFile(opts.config);- const project = seedProject(config, opts.project);- await ensureSeedTest(project, true);});}