Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(mcp): introduce browser_annotate tool (#40491)

Simon Knottb19250e9c645
diff --git a/packages/playwright-core/src/tools/backend/devtools.ts b/packages/playwright-core/src/tools/backend/devtools.ts
index 2f7fe50a05..e60eb634a3 100644
--- a/packages/playwright-core/src/tools/backend/devtools.ts
+++ b/packages/playwright-core/src/tools/backend/devtools.ts
@@ -14,10 +14,16 @@
* limitations under the License.
*/
+import { spawn } from 'child_process';
+
import * as z from 'zod';
+
+import { libPath } from '../../package';
import { defineTabTool, defineTool } from './tool';
import { elementSchema, optionalElementSchema } from './snapshot';
+import type { AnnotationData } from '@dashboard/dashboardChannel';
+
const resume = defineTool({
capability: 'devtools',
@@ -108,4 +114,51 @@ const hideHighlight = defineTabTool({
},
});
-export default [resume, highlight, hideHighlight];
+const annotate = defineTabTool({
+ capability: 'devtools',
+ schema: {
+ name: 'browser_annotate',
+ title: 'Annotate the current page',
+ description: 'Open the Playwright Dashboard in annotation mode for the current page and wait for the user to draw annotations. Returns the annotated screenshot, ARIA snapshot, and the list of annotations.',
+ inputSchema: z.object({}),
+ type: 'readOnly',
+ },
+
+ handle: async (tab, params, response) => {
+ // eslint-disable-next-line no-restricted-syntax -- _guid is the cross-process page identifier shared with the dashboard daemon.
+ const pageId = (tab.page as any)._guid as string;
+ const daemonScript = libPath('entry', 'dashboardApp.js');
+ const daemonArgs = [daemonScript, `--pageId=${pageId}`];
+
+ // Spawn the dashboard daemon (idempotent — the singleton socket guards against duplicates).
+ const daemon = spawn(process.execPath, daemonArgs, { detached: true, stdio: 'ignore' });
+ daemon.unref();
+
+ // Spawn the annotate client in JSON mode to capture the raw payload over stdout.
+ const client = spawn(process.execPath, [...daemonArgs, '--annotate', '--json'], {
+ stdio: ['pipe', 'pipe', 'inherit'],
+ });
+ const stdoutChunks: Buffer[] = [];
+ client.stdout!.on('data', chunk => stdoutChunks.push(chunk));
+ const exitCode = await new Promise<number | null>(resolve => client.on('exit', code => resolve(code)));
+ if (exitCode !== 0) {
+ response.addError(`Annotation client exited with code ${exitCode}`);
+ return;
+ }
+ const text = Buffer.concat(stdoutChunks).toString('utf8').trim();
+ if (!text) {
+ response.addTextResult('No annotations were submitted.');
+ return;
+ }
+ const { png, ariaSnapshot, annotations } = JSON.parse(text) as { png?: string; ariaSnapshot?: string; annotations: AnnotationData[] };
+ for (const a of annotations)
+ response.addTextResult(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);
+ const date = new Date();
+ if (png)
+ await response.addResult('Annotation image', Buffer.from(png, 'base64'), { prefix: 'annotations', ext: 'png', date });
+ if (ariaSnapshot)
+ await response.addResult('Annotation snapshot', Buffer.from(ariaSnapshot, 'utf8'), { prefix: 'annotations', ext: 'yaml', date });
+ },
+});
+
+export default [resume, highlight, hideHighlight, annotate];
diff --git a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
index db3b9af3b6..20dfa9fe61 100644
--- a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
+++ b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
@@ -68,7 +68,9 @@ async function startDashboardServer(options: DashboardOptions): Promise<Dashboar
let connection: DashboardConnection;
// eslint-disable-next-line prefer-const
connection = new DashboardConnection(() => connections.delete(connection), () => {
- if (currentReveal.sessionName)
+ if (currentReveal.pageId)
+ connection.revealPage(currentReveal.pageId);
+ else if (currentReveal.sessionName)
connection.revealSession(currentReveal.sessionName, currentReveal.workspaceDir);
if (pendingAnnotate) {
pendingAnnotate = false;
@@ -91,6 +93,11 @@ async function startDashboardServer(options: DashboardOptions): Promise<Dashboar
const reveal = (next: DashboardOptions) => {
currentReveal = next;
+ if (next.pageId) {
+ for (const connection of connections)
+ connection.revealPage(next.pageId);
+ return;
+ }
if (!next.sessionName)
return;
for (const connection of connections)
@@ -228,23 +235,27 @@ function dashboardSocketPath() {
}
type DashboardOptions = {
- sessionName: string;
- workspaceDir: string;
+ sessionName?: string;
+ workspaceDir?: string;
+ pageId?: string;
kill?: boolean;
annotate?: boolean;
+ json?: boolean;
port?: number;
host?: string;
};
function parseOpenArgs(): DashboardOptions {
- const args = minimist(process.argv.slice(2), { string: ['sessionName', 'workspaceDir', 'host'], boolean: ['annotate', 'kill'] });
+ const args = minimist(process.argv.slice(2), { string: ['sessionName', 'workspaceDir', 'host', 'pageId'], boolean: ['annotate', 'kill', 'json'] });
const portStr = args.port as string | undefined;
return {
- sessionName: args.sessionName as string,
- workspaceDir: args.workspaceDir as string,
+ sessionName: args.sessionName as string | undefined,
+ workspaceDir: args.workspaceDir as string | undefined,
+ pageId: args.pageId as string | undefined,
port: portStr !== undefined ? Number(portStr) : undefined,
host: args.host as string | undefined,
annotate: !!args.annotate,
+ json: !!args.json,
kill: !!args.kill,
};
}
@@ -388,6 +399,11 @@ async function runAnnotateClient(options: DashboardOptions): Promise<void> {
const text = Buffer.concat(chunks).toString();
if (!text)
return;
+ if (options.json) {
+ // eslint-disable-next-line no-console
+ console.log(text);
+ return;
+ }
const { png, annotations, ariaSnapshot } = JSON.parse(text) as { png: string; annotations: AnnotationData[], ariaSnapshot: string };
for (const a of annotations) {
// eslint-disable-next-line no-console
diff --git a/packages/playwright-core/src/tools/dashboard/dashboardController.ts b/packages/playwright-core/src/tools/dashboard/dashboardController.ts
index a52db4d9f8..8b6305a166 100644
--- a/packages/playwright-core/src/tools/dashboard/dashboardController.ts
+++ b/packages/playwright-core/src/tools/dashboard/dashboardController.ts
@@ -115,7 +115,7 @@ export class DashboardConnection implements Transport {
private _pushSessionsScheduled = false;
private _pushTabsScheduled = false;
private _visible = true;
- private _pendingReveal: { sessionName: string; workspaceDir?: string } | undefined;
+ private _pendingReveal: { sessionName?: string; workspaceDir?: string; pageId?: string } | undefined;
private _annotateWaitingForAttach = false;
_recordingDir: string;
@@ -220,16 +220,24 @@ export class DashboardConnection implements Transport {
void this._tryRevealPending();
}
+ revealPage(pageId: string) {
+ this._pendingReveal = { pageId };
+ void this._tryRevealPending();
+ }
+
private async _tryRevealPending() {
const pending = this._pendingReveal;
if (!pending)
return;
- const slot = [...this._browsers.values()].find(s =>
- s.descriptor.title === pending.sessionName
- && (pending.workspaceDir === undefined || s.descriptor.workspaceDir === pending.workspaceDir));
- if (!slot)
- return;
- const page = slot.browser.contexts().flatMap(c => c.pages())[0];
+ const allPages = [...this._browsers.values()].flatMap(s => s.browser.contexts().flatMap(c => c.pages().map(p => ({ slot: s, page: p }))));
+ let page: api.Page | undefined;
+ if (pending.pageId !== undefined) {
+ page = allPages.find(({ page }) => pageId(page) === pending.pageId)?.page;
+ } else if (pending.sessionName !== undefined) {
+ page = allPages.find(({ slot }) =>
+ slot.descriptor.title === pending.sessionName
+ && (pending.workspaceDir === undefined || slot.descriptor.workspaceDir === pending.workspaceDir))?.page;
+ }
if (!page)
return;
this._pendingReveal = undefined;
diff --git a/tests/mcp/dashboard.spec.ts b/tests/mcp/dashboard.spec.ts
index dfdf05f5b6..74ba8fc2a6 100644
--- a/tests/mcp/dashboard.spec.ts
+++ b/tests/mcp/dashboard.spec.ts
@@ -137,7 +137,8 @@ test('daemon show: closing page exits the process', async ({ cli, connectToDashb
async function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page, text: string) {
await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
- const box = await dashboard.locator('img#display').boundingBox();
+ await expect(dashboard.locator('.annotate-modal-image')).toBeVisible();
+ const box = await dashboard.locator('.annotate-modal-image').boundingBox();
const x0 = box!.x + box!.width * 0.3;
const y0 = box!.y + box!.height * 0.3;
const x1 = box!.x + box!.width * 0.6;
@@ -228,6 +229,39 @@ test('should enter annotate mode on fresh dashboard.tsx mount with -s --annotate
expect(exitCode).toBe(0);
});
+test('should annotate via direct browser_annotate MCP call', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {
+ const page = await boundBrowser.newPage();
+ await page.goto(server.EMPTY_PAGE);
+
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ const { client } = await startClient({
+ args: ['--endpoint=default', '--caps=devtools'],
+ env: {
+ ...cliEnv,
+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,
+ },
+ });
+
+ const annotatePromise = client.callTool({ name: 'browser_annotate' });
+ let done = false;
+ void annotatePromise.then(() => { done = true; });
+
+ const browser = await connectToDashboard(bindTitle);
+ try {
+ const dashboard = browser.contexts()[0].pages()[0];
+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
+ await drawAndSubmitAnnotation(dashboard, 'direct-mcp');
+ } finally {
+ await browser.close().catch(() => {});
+ }
+
+ const result = await annotatePromise;
+ expect(done).toBe(true);
+ const text = (result.content as any).map(c => c.text ?? '').join('\n');
+ expect(text).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: direct-mcp/);
+ expect(text).toMatch(/- \[Annotation image\]\(.*\.png\)/);
+});
+
test('should switch screencast to -s session on show --annotate', async ({ connectToDashboard, cli, server }) => {
server.setContent('/red', '<html><head><style>html,body{margin:0;height:100vh;background:#ff0000}</style></head><body></body></html>', 'text/html');
server.setContent('/green', '<html><head><style>html,body{margin:0;height:100vh;background:#00ff00}</style></head><body></body></html>', 'text/html');