Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(mcp): introduce browser_annotate tool (#40491)
Simon Knottb19250e9c645
diff --git a/packages/playwright-core/src/tools/backend/devtools.ts b/packages/playwright-core/src/tools/backend/devtools.tsindex 2f7fe50a05..e60eb634a3 100644--- a/packages/playwright-core/src/tools/backend/devtools.ts+++ b/packages/playwright-core/src/tools/backend/devtools.ts@@ -14,10 +14,16 @@* limitations under the License.*/+import { spawn } from 'child_process';+import * as z from 'zod';++import { libPath } from '../../package';import { defineTabTool, defineTool } from './tool';import { elementSchema, optionalElementSchema } from './snapshot';+import type { AnnotationData } from '@dashboard/dashboardChannel';+const resume = defineTool({capability: 'devtools',@@ -108,4 +114,51 @@ const hideHighlight = defineTabTool({},});-export default [resume, highlight, hideHighlight];+const annotate = defineTabTool({+ capability: 'devtools',+ schema: {+ name: 'browser_annotate',+ title: 'Annotate the current page',+ description: 'Open the Playwright Dashboard in annotation mode for the current page and wait for the user to draw annotations. Returns the annotated screenshot, ARIA snapshot, and the list of annotations.',+ inputSchema: z.object({}),+ type: 'readOnly',+ },++ handle: async (tab, params, response) => {+ // eslint-disable-next-line no-restricted-syntax -- _guid is the cross-process page identifier shared with the dashboard daemon.+ const pageId = (tab.page as any)._guid as string;+ const daemonScript = libPath('entry', 'dashboardApp.js');+ const daemonArgs = [daemonScript, `--pageId=${pageId}`];++ // Spawn the dashboard daemon (idempotent — the singleton socket guards against duplicates).+ const daemon = spawn(process.execPath, daemonArgs, { detached: true, stdio: 'ignore' });+ daemon.unref();++ // Spawn the annotate client in JSON mode to capture the raw payload over stdout.+ const client = spawn(process.execPath, [...daemonArgs, '--annotate', '--json'], {+ stdio: ['pipe', 'pipe', 'inherit'],+ });+ const stdoutChunks: Buffer[] = [];+ client.stdout!.on('data', chunk => stdoutChunks.push(chunk));+ const exitCode = await new Promise<number | null>(resolve => client.on('exit', code => resolve(code)));+ if (exitCode !== 0) {+ response.addError(`Annotation client exited with code ${exitCode}`);+ return;+ }+ const text = Buffer.concat(stdoutChunks).toString('utf8').trim();+ if (!text) {+ response.addTextResult('No annotations were submitted.');+ return;+ }+ const { png, ariaSnapshot, annotations } = JSON.parse(text) as { png?: string; ariaSnapshot?: string; annotations: AnnotationData[] };+ for (const a of annotations)+ response.addTextResult(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);+ const date = new Date();+ if (png)+ await response.addResult('Annotation image', Buffer.from(png, 'base64'), { prefix: 'annotations', ext: 'png', date });+ if (ariaSnapshot)+ await response.addResult('Annotation snapshot', Buffer.from(ariaSnapshot, 'utf8'), { prefix: 'annotations', ext: 'yaml', date });+ },+});++export default [resume, highlight, hideHighlight, annotate];diff --git a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts b/packages/playwright-core/src/tools/dashboard/dashboardApp.tsindex db3b9af3b6..20dfa9fe61 100644--- a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts+++ b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts@@ -68,7 +68,9 @@ async function startDashboardServer(options: DashboardOptions): Promise<Dashboarlet connection: DashboardConnection;// eslint-disable-next-line prefer-constconnection = new DashboardConnection(() => connections.delete(connection), () => {- if (currentReveal.sessionName)+ if (currentReveal.pageId)+ connection.revealPage(currentReveal.pageId);+ else if (currentReveal.sessionName)connection.revealSession(currentReveal.sessionName, currentReveal.workspaceDir);if (pendingAnnotate) {pendingAnnotate = false;@@ -91,6 +93,11 @@ async function startDashboardServer(options: DashboardOptions): Promise<Dashboarconst reveal = (next: DashboardOptions) => {currentReveal = next;+ if (next.pageId) {+ for (const connection of connections)+ connection.revealPage(next.pageId);+ return;+ }if (!next.sessionName)return;for (const connection of connections)@@ -228,23 +235,27 @@ function dashboardSocketPath() {}type DashboardOptions = {- sessionName: string;- workspaceDir: string;+ sessionName?: string;+ workspaceDir?: string;+ pageId?: string;kill?: boolean;annotate?: boolean;+ json?: boolean;port?: number;host?: string;};function parseOpenArgs(): DashboardOptions {- const args = minimist(process.argv.slice(2), { string: ['sessionName', 'workspaceDir', 'host'], boolean: ['annotate', 'kill'] });+ const args = minimist(process.argv.slice(2), { string: ['sessionName', 'workspaceDir', 'host', 'pageId'], boolean: ['annotate', 'kill', 'json'] });const portStr = args.port as string | undefined;return {- sessionName: args.sessionName as string,- workspaceDir: args.workspaceDir as string,+ sessionName: args.sessionName as string | undefined,+ workspaceDir: args.workspaceDir as string | undefined,+ pageId: args.pageId as string | undefined,port: portStr !== undefined ? Number(portStr) : undefined,host: args.host as string | undefined,annotate: !!args.annotate,+ json: !!args.json,kill: !!args.kill,};}@@ -388,6 +399,11 @@ async function runAnnotateClient(options: DashboardOptions): Promise<void> {const text = Buffer.concat(chunks).toString();if (!text)return;+ if (options.json) {+ // eslint-disable-next-line no-console+ console.log(text);+ return;+ }const { png, annotations, ariaSnapshot } = JSON.parse(text) as { png: string; annotations: AnnotationData[], ariaSnapshot: string };for (const a of annotations) {// eslint-disable-next-line no-consolediff --git a/packages/playwright-core/src/tools/dashboard/dashboardController.ts b/packages/playwright-core/src/tools/dashboard/dashboardController.tsindex a52db4d9f8..8b6305a166 100644--- a/packages/playwright-core/src/tools/dashboard/dashboardController.ts+++ b/packages/playwright-core/src/tools/dashboard/dashboardController.ts@@ -115,7 +115,7 @@ export class DashboardConnection implements Transport {private _pushSessionsScheduled = false;private _pushTabsScheduled = false;private _visible = true;- private _pendingReveal: { sessionName: string; workspaceDir?: string } | undefined;+ private _pendingReveal: { sessionName?: string; workspaceDir?: string; pageId?: string } | undefined;private _annotateWaitingForAttach = false;_recordingDir: string;@@ -220,16 +220,24 @@ export class DashboardConnection implements Transport {void this._tryRevealPending();}+ revealPage(pageId: string) {+ this._pendingReveal = { pageId };+ void this._tryRevealPending();+ }+private async _tryRevealPending() {const pending = this._pendingReveal;if (!pending)return;- const slot = [...this._browsers.values()].find(s =>- s.descriptor.title === pending.sessionName- && (pending.workspaceDir === undefined || s.descriptor.workspaceDir === pending.workspaceDir));- if (!slot)- return;- const page = slot.browser.contexts().flatMap(c => c.pages())[0];+ const allPages = [...this._browsers.values()].flatMap(s => s.browser.contexts().flatMap(c => c.pages().map(p => ({ slot: s, page: p }))));+ let page: api.Page | undefined;+ if (pending.pageId !== undefined) {+ page = allPages.find(({ page }) => pageId(page) === pending.pageId)?.page;+ } else if (pending.sessionName !== undefined) {+ page = allPages.find(({ slot }) =>+ slot.descriptor.title === pending.sessionName+ && (pending.workspaceDir === undefined || slot.descriptor.workspaceDir === pending.workspaceDir))?.page;+ }if (!page)return;this._pendingReveal = undefined;diff --git a/tests/mcp/dashboard.spec.ts b/tests/mcp/dashboard.spec.tsindex dfdf05f5b6..74ba8fc2a6 100644--- a/tests/mcp/dashboard.spec.ts+++ b/tests/mcp/dashboard.spec.ts@@ -137,7 +137,8 @@ test('daemon show: closing page exits the process', async ({ cli, connectToDashbasync function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page, text: string) {await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();- const box = await dashboard.locator('img#display').boundingBox();+ await expect(dashboard.locator('.annotate-modal-image')).toBeVisible();+ const box = await dashboard.locator('.annotate-modal-image').boundingBox();const x0 = box!.x + box!.width * 0.3;const y0 = box!.y + box!.height * 0.3;const x1 = box!.x + box!.width * 0.6;@@ -228,6 +229,39 @@ test('should enter annotate mode on fresh dashboard.tsx mount with -s --annotateexpect(exitCode).toBe(0);});+test('should annotate via direct browser_annotate MCP call', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {+ const page = await boundBrowser.newPage();+ await page.goto(server.EMPTY_PAGE);++ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ const { client } = await startClient({+ args: ['--endpoint=default', '--caps=devtools'],+ env: {+ ...cliEnv,+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,+ },+ });++ const annotatePromise = client.callTool({ name: 'browser_annotate' });+ let done = false;+ void annotatePromise.then(() => { done = true; });++ const browser = await connectToDashboard(bindTitle);+ try {+ const dashboard = browser.contexts()[0].pages()[0];+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();+ await drawAndSubmitAnnotation(dashboard, 'direct-mcp');+ } finally {+ await browser.close().catch(() => {});+ }++ const result = await annotatePromise;+ expect(done).toBe(true);+ const text = (result.content as any).map(c => c.text ?? '').join('\n');+ expect(text).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: direct-mcp/);+ expect(text).toMatch(/- \[Annotation image\]\(.*\.png\)/);+});+test('should switch screencast to -s session on show --annotate', async ({ connectToDashboard, cli, server }) => {server.setContent('/red', '<html><head><style>html,body{margin:0;height:100vh;background:#ff0000}</style></head><body></body></html>', 'text/html');server.setContent('/green', '<html><head><style>html,body{margin:0;height:100vh;background:#00ff00}</style></head><body></body></html>', 'text/html');