Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(cli): add annotate command (#40520)

Simon Knott3a04dfd68834
diff --git a/packages/playwright-core/src/tools/cli-client/program.ts b/packages/playwright-core/src/tools/cli-client/program.ts
index 216221327b..eec45f46b6 100644
--- a/packages/playwright-core/src/tools/cli-client/program.ts
+++ b/packages/playwright-core/src/tools/cli-client/program.ts
@@ -216,13 +216,6 @@ export async function program(options?: { embedderVersion?: string}) {
await new Promise<void>(resolve => child.on('exit', () => resolve()));
return;
}
- if (args.annotate) {
- const dashboard = spawn(process.execPath, daemonArgs, { detached: true, stdio: 'ignore' });
- dashboard.unref();
- const annotate = spawn(process.execPath, [...daemonArgs, '--annotate'], { stdio: 'inherit' });
- await new Promise<void>(resolve => annotate.on('exit', () => resolve()));
- return;
- }
const foreground = args.port !== undefined;
const child = spawn(process.execPath, daemonArgs, {
detached: !foreground,
diff --git a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md
index f9bc753e06..4111868657 100644
--- a/packages/playwright-core/src/tools/cli-client/skill/SKILL.md
+++ b/packages/playwright-core/src/tools/cli-client/skill/SKILL.md
@@ -163,8 +163,8 @@ playwright-cli video-start video.webm
playwright-cli video-chapter "Chapter Title" --description="Details" --duration=2000
playwright-cli video-stop
-# launch the dashboard with annotation prompt to ask the user for input
-playwright-cli show --annotate
+# launch the dashboard for UI review / design feedback — user annotates the page, you receive the annotated screenshot, snapshot, and notes
+playwright-cli annotate
# generate a Playwright locator for an element from its ref or selector
playwright-cli generate-locator e5 --raw
@@ -367,11 +367,11 @@ playwright-cli close
## Example: Interactive session
-Ask the user to annotate the UI. User can provide contextual tasks or ask contextual questions using annotations:
+Ask the user for UI review or design feedback. The user draws boxes on the live page and types comments; you receive the annotated screenshot, the snapshot of the marked region, and the user's notes. Use this whenever the user asks for "UI review", "design feedback", or to "ask the user what they think / want / mean":
```bash
playwright-cli open https://example.com
-playwright-cli show --annotate
+playwright-cli annotate
```
## Specific tasks
diff --git a/packages/playwright-core/src/tools/cli-client/skill/references/spec-driven-testing.md b/packages/playwright-core/src/tools/cli-client/skill/references/spec-driven-testing.md
index f37bdb44af..5097a67cdf 100644
--- a/packages/playwright-core/src/tools/cli-client/skill/references/spec-driven-testing.md
+++ b/packages/playwright-core/src/tools/cli-client/skill/references/spec-driven-testing.md
@@ -88,7 +88,7 @@ playwright-cli resume # resume so that seed test runs fully
playwright-cli snapshot # inventory of interactive elements
playwright-cli click e5 # follow a flow
playwright-cli eval "location.href" # read URL / state
-playwright-cli show --annotate # ask the user to point at something
+playwright-cli annotate # ask the user to point at something
```
Map out:
@@ -262,7 +262,7 @@ The test is paused at the start. Step forward or run to until just before the fa
playwright-cli snapshot # did the element change / move / rename?
playwright-cli console # app-side errors?
playwright-cli network # failed request? wrong payload?
-playwright-cli show --annotate # ask the user to point somewhere
+playwright-cli annotate # ask the user to point somewhere
```
Common causes: selector drift, new wrapper element, label/ARIA rename, timing (transition, async load), assertion text updated in the app, test data leaking between runs.
diff --git a/packages/playwright-core/src/tools/cli-daemon/commands.ts b/packages/playwright-core/src/tools/cli-daemon/commands.ts
index 0a6596ad8a..b36cce71af 100644
--- a/packages/playwright-core/src/tools/cli-daemon/commands.ts
+++ b/packages/playwright-core/src/tools/cli-daemon/commands.ts
@@ -977,13 +977,22 @@ const dashboardShow = declareCommand({
options: z.object({
port: numberArg.optional().describe('Start as a blocking HTTP server on this port (use 0 for a random port)'),
host: z.string().optional().describe('Host to bind to when using --port (defaults to localhost)'),
- annotate: z.boolean().optional().describe('Switch the dashboard into annotation mode.'),
kill: z.boolean().optional().describe('Kill the dashboard daemon.'),
}),
toolName: '',
toolParams: () => ({}),
});
+const annotate = declareCommand({
+ name: 'annotate',
+ description: 'Ask the user to annotate the current page.',
+ category: 'devtools',
+ raw: true,
+ args: z.object({}),
+ toolName: 'browser_annotate',
+ toolParams: () => ({}),
+});
+
const resume = declareCommand({
name: 'resume',
description: 'Resume the test execution',
@@ -1198,6 +1207,7 @@ const commandsArray: AnyCommandSchema[] = [
videoStop,
videoChapter,
dashboardShow,
+ annotate,
pauseAt,
resume,
stepOver,
diff --git a/packages/playwright-core/src/tools/cli-daemon/daemon.ts b/packages/playwright-core/src/tools/cli-daemon/daemon.ts
index 84c6154610..61f576a02f 100644
--- a/packages/playwright-core/src/tools/cli-daemon/daemon.ts
+++ b/packages/playwright-core/src/tools/cli-daemon/daemon.ts
@@ -78,6 +78,8 @@ export async function startCliDaemonServer(
const server = net.createServer(socket => {
const connection = new SocketConnection(socket);
+ const abortController = new AbortController();
+ connection.onclose = () => abortController.abort();
connection.onmessage = async message => {
const { id, method, params } = message;
try {
@@ -91,7 +93,7 @@ export async function startCliDaemonServer(
} else if (method === 'run') {
const { toolName, toolParams } = parseCliCommand(params.args);
toolParams._meta = { cwd: params.cwd, raw: params.raw || params.json, json: !!params.json };
- const response = await backend.callTool(toolName, toolParams);
+ const response = await backend.callTool(toolName, toolParams, abortController.signal);
await connection.send({ id, result: formatResult(response) });
} else {
throw new Error(`Unknown method: ${method}`);
diff --git a/tests/mcp/dashboard.spec.ts b/tests/mcp/dashboard.spec.ts
index 7b11e9e96d..bcd72ca58a 100644
--- a/tests/mcp/dashboard.spec.ts
+++ b/tests/mcp/dashboard.spec.ts
@@ -153,16 +153,15 @@ async function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page
}
function verifyAnnotateOutput(output: string, expectedText: string, outputDir: string) {
- const lines = output.trim().split('\n');
- expect(lines[0]).toMatch(new RegExp(`^\\{ x: \\d+, y: \\d+, width: \\d+, height: \\d+ \\}: ${expectedText}$`));
- expect(lines[lines.length - 1]).toMatch(/^image: \.playwright-cli[\\/]annotations-.*\.png$/);
- const pngRel = lines[lines.length - 1].replace(/^image: /, '');
- const pngPath = path.resolve(outputDir, pngRel);
+ expect(output).toMatch(new RegExp(`\\{ x: \\d+, y: \\d+, width: \\d+, height: \\d+ \\}: ${expectedText}`));
+ const imageMatch = output.match(/- \[Annotation image\]\((\.playwright-cli[\\/]annotations-.*\.png)\)/);
+ expect(imageMatch).not.toBeNull();
+ const pngPath = path.resolve(outputDir, imageMatch![1]);
expect(fs.existsSync(pngPath)).toBe(true);
expect(fs.statSync(pngPath).size).toBeGreaterThan(0);
}
-test('should capture annotations via show --annotate', async ({ connectToDashboard, cli, server }) => {
+test('should capture annotations via annotate', async ({ connectToDashboard, cli, server }) => {
await cli('open', server.EMPTY_PAGE);
const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
await cli('show', { bindTitle });
@@ -171,7 +170,7 @@ test('should capture annotations via show --annotate', async ({ connectToDashboa
const dashboard = browser.contexts()[0].pages()[0];
await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();
- const annotatePromise = cli('show', '--annotate');
+ const annotatePromise = cli('annotate');
let done = false;
void annotatePromise.finally(() => { done = true; });
@@ -184,10 +183,10 @@ test('should capture annotations via show --annotate', async ({ connectToDashboa
});
test('should start dashboard and annotate when no dashboard is running', async ({ connectToDashboard, cli, server }) => {
- await cli('open', server.EMPTY_PAGE);
-
const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- const annotatePromise = cli('show', '--annotate', { bindTitle });
+ await cli('open', server.EMPTY_PAGE, { bindTitle });
+
+ const annotatePromise = cli('annotate');
let done = false;
void annotatePromise.finally(() => { done = true; });
@@ -205,12 +204,12 @@ test('should start dashboard and annotate when no dashboard is running', async (
verifyAnnotateOutput(output, 'hi', test.info().outputDir);
});
-test('should enter annotate mode on fresh dashboard.tsx mount with -s --annotate', async ({ connectToDashboard, cli, server }) => {
- await cli('-s=first', 'open', server.EMPTY_PAGE);
- await cli('-s=second', 'open', server.EMPTY_PAGE);
-
+test('should enter annotate mode on fresh dashboard.tsx mount with -s annotate', async ({ connectToDashboard, cli, server }) => {
const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- const annotatePromise = cli('-s=second', 'show', '--annotate', { bindTitle });
+ await cli('-s=first', 'open', server.EMPTY_PAGE, { bindTitle });
+ await cli('-s=second', 'open', server.EMPTY_PAGE, { bindTitle });
+
+ const annotatePromise = cli('-s=second', 'annotate');
let done = false;
void annotatePromise.finally(() => { done = true; });
@@ -322,7 +321,7 @@ test('should cancel browser_annotate when the MCP client disconnects', async ({
});
-test('should switch screencast to -s session on show --annotate', async ({ connectToDashboard, cli, server }) => {
+test('should switch screencast to -s session on annotate', async ({ connectToDashboard, cli, server }) => {
server.setContent('/red', '<html><head><style>html,body{margin:0;height:100vh;background:#ff0000}</style></head><body></body></html>', 'text/html');
server.setContent('/green', '<html><head><style>html,body{margin:0;height:100vh;background:#00ff00}</style></head><body></body></html>', 'text/html');
@@ -353,7 +352,7 @@ test('should switch screencast to -s session on show --annotate', async ({ conne
return !!(c && c.r > 200 && c.g < 50);
}, { timeout: 15000 }).toBe(true);
- const annotatePromise = cli('-s=second', 'show', '--annotate');
+ const annotatePromise = cli('-s=second', 'annotate');
let done = false;
void annotatePromise.finally(() => { done = true; });
@@ -371,7 +370,7 @@ test('should switch screencast to -s session on show --annotate', async ({ conne
expect(exitCode).toBe(0);
});
-test('should disengage annotate mode when --annotate client disconnects', async ({ connectToDashboard, cli, childProcess, cliEnv, mcpBrowser, mcpHeadless, server }) => {
+test('should disengage annotate mode when annotate client disconnects', async ({ connectToDashboard, cli, childProcess, cliEnv, mcpBrowser, mcpHeadless, server }) => {
await cli('open', server.EMPTY_PAGE);
const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
await cli('show', { bindTitle });
@@ -381,7 +380,7 @@ test('should disengage annotate mode when --annotate client disconnects', async
await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();
const annotateClient = childProcess({
- command: [process.execPath, require.resolve('../../packages/playwright-core/lib/tools/cli-client/cli.js'), 'show', '--annotate'],
+ command: [process.execPath, require.resolve('../../packages/playwright-core/lib/tools/cli-client/cli.js'), 'annotate'],
cwd: test.info().outputPath(),
env: inheritAndCleanEnv({
...cliEnv,