Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(dashboard): add 'show --annotate' to capture annotations as PNG (#40262)

Pavel Feldmane575acda24bc
diff --git a/packages/dashboard/src/annotations.tsx b/packages/dashboard/src/annotations.tsx
index 628a1bc736..6b507049af 100644
--- a/packages/dashboard/src/annotations.tsx
+++ b/packages/dashboard/src/annotations.tsx
@@ -118,12 +118,12 @@ export const Annotations: React.FC<{
screenRef: React.RefObject<HTMLDivElement | null>;
viewportWidth: number;
viewportHeight: number;
-}> = ({ active, displayRef, screenRef, viewportWidth, viewportHeight }) => {
+ onSubmit?: (blob: Blob, annotations: Annotation[]) => Promise<void> | void;
+}> = ({ active, displayRef, screenRef, viewportWidth, viewportHeight, onSubmit }) => {
const [annotations, setAnnotations] = React.useState<Annotation[]>([]);
const [draft, setDraft] = React.useState<{ startX: number; startY: number; x: number; y: number } | null>(null);
const [selection, setSelection] = React.useState<Selection>(null);
const [drag, setDrag] = React.useState<DragState | null>(null);
- const [submittedJson, setSubmittedJson] = React.useState<string | null>(null);
const [, setTick] = React.useState(0);
const forceRender = React.useCallback(() => setTick(t => t + 1), []);
const layerRef = React.useRef<HTMLDivElement>(null);
@@ -283,10 +283,81 @@ export const Annotations: React.FC<{
}
}
- function submitAnnotations() {
- const json = JSON.stringify({ viewportWidth, viewportHeight, annotations }, null, 2);
- console.log('Annotations:', json);
- setSubmittedJson(json);
+ async function submitAnnotations() {
+ const img = displayRef.current;
+ if (!img || !img.naturalWidth || !img.naturalHeight || !viewportWidth || !viewportHeight)
+ return;
+ const canvas = document.createElement('canvas');
+ canvas.width = img.naturalWidth;
+ canvas.height = img.naturalHeight;
+ const ctx = canvas.getContext('2d');
+ if (!ctx)
+ return;
+ ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
+ const sx = canvas.width / viewportWidth;
+ const sy = canvas.height / viewportHeight;
+ const blue = 'rgb(54, 116, 209)';
+ const fontSize = Math.max(11, Math.round(14 * sy));
+ ctx.font = `500 ${fontSize}px -apple-system, system-ui, sans-serif`;
+ ctx.textBaseline = 'middle';
+ for (const a of annotations) {
+ const x = a.x * sx;
+ const y = a.y * sy;
+ const w = a.width * sx;
+ const h = a.height * sy;
+ ctx.fillStyle = 'rgba(54, 116, 209, 0.12)';
+ ctx.fillRect(x, y, w, h);
+ ctx.lineWidth = Math.max(2, Math.round(2 * sy));
+ ctx.strokeStyle = blue;
+ ctx.strokeRect(x, y, w, h);
+ if (a.text) {
+ const padX = Math.max(4, Math.round(6 * sy));
+ const padY = Math.max(2, Math.round(3 * sy));
+ const metrics = ctx.measureText(a.text);
+ const labelW = metrics.width + padX * 2;
+ const labelH = fontSize + padY * 2;
+ const labelX = x - ctx.lineWidth / 2;
+ const labelY = y - labelH;
+ ctx.fillStyle = blue;
+ ctx.fillRect(labelX, labelY, labelW, labelH);
+ ctx.fillStyle = '#fff';
+ ctx.fillText(a.text, labelX + padX, labelY + labelH / 2);
+ }
+ }
+ const blob = await new Promise<Blob | null>(resolve => canvas.toBlob(resolve, 'image/png'));
+ if (!blob)
+ return;
+ if (onSubmit) {
+ await onSubmit(blob, annotations);
+ return;
+ }
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const suggestedName = `annotations-${stamp}.png`;
+ const picker = (window as any).showSaveFilePicker as undefined | ((opts: any) => Promise<any>);
+ if (picker) {
+ try {
+ const handle = await picker({
+ suggestedName,
+ startIn: 'downloads',
+ types: [{ description: 'PNG image', accept: { 'image/png': ['.png'] } }],
+ });
+ const writable = await handle.createWritable();
+ await writable.write(blob);
+ await writable.close();
+ } catch (e: any) {
+ if (e?.name !== 'AbortError')
+ throw e;
+ }
+ return;
+ }
+ const url = URL.createObjectURL(blob);
+ const link = document.createElement('a');
+ link.href = url;
+ link.download = suggestedName;
+ document.body.appendChild(link);
+ link.click();
+ document.body.removeChild(link);
+ URL.revokeObjectURL(url);
}
if (!active)
@@ -325,7 +396,7 @@ export const Annotations: React.FC<{
className='annotate-action-btn primary'
onClick={submitAnnotations}
disabled={annotations.length === 0}
- title='Submit annotations'
+ title='Submit annotation'
>
Submit
</button>
@@ -434,25 +505,6 @@ export const Annotations: React.FC<{
);
})()}
- {submittedJson !== null && (
- <div className='annotation-modal-backdrop' onMouseDown={e => { e.stopPropagation(); setSubmittedJson(null); }}>
- <div className='annotation-modal' onMouseDown={e => e.stopPropagation()}>
- <div className='annotation-modal-header'>
- <span>Annotations JSON</span>
- <button className='annotate-action-btn' onClick={() => setSubmittedJson(null)}>Close</button>
- </div>
- <textarea className='annotation-modal-text' readOnly value={submittedJson} />
- <div className='annotation-modal-actions'>
- <button
- className='annotate-action-btn primary'
- onClick={() => { navigator.clipboard?.writeText(submittedJson).catch(() => {}); }}
- >
- Copy
- </button>
- </div>
- </div>
- </div>
- )}
</div>
);
};
diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsx
index 5910a41c33..2fa52d3eb5 100644
--- a/packages/dashboard/src/dashboard.tsx
+++ b/packages/dashboard/src/dashboard.tsx
@@ -20,6 +20,8 @@ import { DashboardClientContext } from './index';
import { asLocator } from '@isomorphic/locatorGenerators';
import { ChevronLeftIcon, ChevronRightIcon, ReloadIcon } from './icons';
import { Annotations, getImageLayout, clientToViewport } from './annotations';
+
+import type { Annotation } from './annotations';
import { ToolbarButton } from '@web/components/toolbarButton';
import { useMeasureForRef } from '@web/uiUtils';
@@ -70,6 +72,8 @@ export const Dashboard: React.FC = () => {
const [recording, setRecording] = React.useState(false);
const [screenshotIcon, setScreenshotIcon] = React.useState<'device-camera' | 'clippy'>('device-camera');
const [flashTick, setFlashTick] = React.useState(0);
+ const [pendingAnnotate, setPendingAnnotate] = React.useState(false);
+ const [cliAnnotate, setCliAnnotate] = React.useState(false);
const displayRef = React.useRef<HTMLImageElement>(null);
const screenRef = React.useRef<HTMLDivElement>(null);
@@ -129,6 +133,37 @@ export const Dashboard: React.FC = () => {
};
}, [flashTick, interactive]);
+ const hasFrame = !!frame;
+ React.useEffect(() => {
+ if (!pendingAnnotate || !hasFrame)
+ return;
+ setMode('annotate');
+ setCliAnnotate(true);
+ setPendingAnnotate(false);
+ }, [pendingAnnotate, hasFrame]);
+
+ React.useEffect(() => {
+ if (!annotating)
+ setCliAnnotate(false);
+ }, [annotating]);
+
+ const submitAnnotationToCli = React.useCallback(async (blob: Blob, annotations: Annotation[]) => {
+ if (!client)
+ return;
+ const dataUrl = await new Promise<string>((resolve, reject) => {
+ const reader = new FileReader();
+ reader.onload = () => resolve(reader.result as string);
+ reader.onerror = () => reject(reader.error);
+ reader.readAsDataURL(blob);
+ });
+ const data = dataUrl.slice(dataUrl.indexOf(',') + 1);
+ await client.submitAnnotation({
+ data,
+ annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),
+ });
+ setMode('readonly');
+ }, [client]);
+
function flashInteractiveHint() {
setFlashTick(tick => tick + 1);
}
@@ -173,15 +208,18 @@ export const Dashboard: React.FC = () => {
setMode('interactive');
setPicking(true);
};
+ const onAnnotate = () => setPendingAnnotate(true);
client.on('tabs', onTabs);
client.on('frame', onFrame);
client.on('elementPicked', onElementPicked);
client.on('pickLocator', onPickLocator);
+ client.on('annotate', onAnnotate);
return () => {
client.off('tabs', onTabs);
client.off('frame', onFrame);
client.off('elementPicked', onElementPicked);
client.off('pickLocator', onPickLocator);
+ client.off('annotate', onAnnotate);
};
}, [client]);
@@ -458,6 +496,7 @@ export const Dashboard: React.FC = () => {
screenRef={screenRef}
viewportWidth={frame?.viewportWidth ?? 0}
viewportHeight={frame?.viewportHeight ?? 0}
+ onSubmit={cliAnnotate ? submitAnnotationToCli : undefined}
/>
</div>
{overlayText && <div className={'screen-overlay' + (frame ? ' has-frame' : '')}><span>{overlayText}</span></div>}
diff --git a/packages/dashboard/src/dashboardChannel.ts b/packages/dashboard/src/dashboardChannel.ts
index bbde6fbb0e..cda3f96462 100644
--- a/packages/dashboard/src/dashboardChannel.ts
+++ b/packages/dashboard/src/dashboardChannel.ts
@@ -28,12 +28,15 @@ export type Tab = {
inspectorUrl?: string;
};
+export type AnnotationData = { x: number; y: number; width: number; height: number; text: string };
+
export type DashboardChannelEvents = {
sessions: { sessions: SessionStatus[]; clientInfo: ClientInfo };
tabs: { tabs: Tab[] };
frame: { data: string; viewportWidth: number; viewportHeight: number };
elementPicked: { selector: string; ariaSnapshot?: string };
pickLocator: {};
+ annotate: {};
};
export type MouseButton = 'left' | 'middle' | 'right';
@@ -63,6 +66,7 @@ export interface DashboardChannel {
stopRecording(): Promise<{ streamId: string }>;
readStream(params: { streamId: string }): Promise<{ data: string; eof: boolean }>;
screenshot(): Promise<string>;
+ submitAnnotation(params: { data: string; annotations: AnnotationData[] }): Promise<void>;
on<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;
off<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;
diff --git a/packages/playwright-core/src/tools/cli-client/program.ts b/packages/playwright-core/src/tools/cli-client/program.ts
index 5a7dabd03a..2e27bb5a32 100644
--- a/packages/playwright-core/src/tools/cli-client/program.ts
+++ b/packages/playwright-core/src/tools/cli-client/program.ts
@@ -187,6 +187,13 @@ export async function program(options?: { embedderVersion?: string}) {
daemonArgs.push(`--port=${args.port}`);
if (args.host !== undefined)
daemonArgs.push(`--host=${args.host as string}`);
+ if (args.annotate) {
+ const dashboard = spawn(process.execPath, daemonArgs, { detached: true, stdio: 'ignore' });
+ dashboard.unref();
+ const annotate = spawn(process.execPath, [...daemonArgs, '--annotate'], { stdio: 'inherit' });
+ await new Promise<void>(resolve => annotate.on('exit', () => resolve()));
+ return;
+ }
const foreground = args.port !== undefined;
const child = spawn(process.execPath, daemonArgs, {
detached: !foreground,
diff --git a/packages/playwright-core/src/tools/cli-daemon/commands.ts b/packages/playwright-core/src/tools/cli-daemon/commands.ts
index 00baa60e2b..3863c30dd6 100644
--- a/packages/playwright-core/src/tools/cli-daemon/commands.ts
+++ b/packages/playwright-core/src/tools/cli-daemon/commands.ts
@@ -857,6 +857,7 @@ const devtoolsShow = declareCommand({
options: z.object({
port: numberArg.optional().describe('Start as a blocking HTTP server on this port (use 0 for a random port)'),
host: z.string().optional().describe('Host to bind to when using --port (defaults to localhost)'),
+ annotate: z.boolean().optional().describe('Switch the dashboard into annotation mode.'),
}),
toolName: '',
toolParams: () => ({}),
diff --git a/packages/playwright-core/src/tools/dashboard/DEPS.list b/packages/playwright-core/src/tools/dashboard/DEPS.list
index b2feab4381..855fcdbbff 100644
--- a/packages/playwright-core/src/tools/dashboard/DEPS.list
+++ b/packages/playwright-core/src/tools/dashboard/DEPS.list
@@ -8,4 +8,5 @@
../cli-client/minimist.ts
../cli-client/registry.ts
../trace/traceParser.ts
+../trace/traceUtils.ts
../utils/**
diff --git a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
index fb1feba7e6..d8fe1c4e0d 100644
--- a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
+++ b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
@@ -26,25 +26,54 @@ import { libPath } from '../../package';
import { playwright } from '../../inprocess';
import { findChromiumChannelBestEffort, registryDirectory } from '../../server/registry/index';
import { minimist } from '../cli-client/minimist';
+import { saveOutputFile } from '../trace/traceUtils';
import { DashboardConnection } from './dashboardController';
import type * as api from '../../..';
+import type { AnnotationData } from '@dashboard/dashboardChannel';
type RevealOptions = { sessionName?: string; workspaceDir?: string };
-async function startDashboardServer(options: { port?: number; host?: string; reveal?: RevealOptions } = {}): Promise<{ url: string; reveal: (options: RevealOptions) => void }> {
+type DashboardCommand = RevealOptions & { command: 'bringToFront' | 'annotate' };
+
+type DashboardServer = {
+ url: string;
+ reveal: (options: RevealOptions) => void;
+ triggerAnnotate: () => void;
+ registerAnnotateWaiter: (socket: net.Socket) => void;
+};
+
+async function startDashboardServer(options: { port?: number; host?: string; reveal?: RevealOptions } = {}): Promise<DashboardServer> {
const httpServer = new HttpServer();
const dashboardDir = libPath('vite', 'dashboard');
const connections = new Set<DashboardConnection>();
let currentReveal: RevealOptions = options.reveal ?? {};
+ let pendingAnnotate = false;
+ const waitingSockets = new Set<net.Socket>();
+
+ const submitAnnotation = (base64Png: string, annotations: AnnotationData[]) => {
+ if (waitingSockets.size === 0)
+ return;
+ const payload = JSON.stringify({ png: base64Png, annotations });
+ for (const socket of waitingSockets) {
+ socket.write(payload);
+ socket.end();
+ }
+ waitingSockets.clear();
+ };
httpServer.createWebSocket(() => {
let connection: DashboardConnection;
// eslint-disable-next-line prefer-const
- connection = new DashboardConnection(() => connections.delete(connection));
- if (currentReveal.sessionName)
- connection.revealSession(currentReveal.sessionName, currentReveal.workspaceDir);
+ connection = new DashboardConnection(() => connections.delete(connection), () => {
+ if (currentReveal.sessionName)
+ connection.revealSession(currentReveal.sessionName, currentReveal.workspaceDir);
+ if (pendingAnnotate) {
+ pendingAnnotate = false;
+ connection.emitAnnotate();
+ }
+ }, submitAnnotation);
connections.add(connection);
return connection;
}, 'ws');
@@ -67,14 +96,30 @@ async function startDashboardServer(options: { port?: number; host?: string; rev
connection.revealSession(next.sessionName, next.workspaceDir);
};
- return { url: httpServer.urlPrefix('human-readable'), reveal };
+ const triggerAnnotate = () => {
+ if (connections.size === 0) {
+ pendingAnnotate = true;
+ return;
+ }
+ for (const connection of connections)
+ connection.emitAnnotate();
+ };
+
+ const registerAnnotateWaiter = (socket: net.Socket) => {
+ waitingSockets.add(socket);
+ const cleanup = () => waitingSockets.delete(socket);
+ socket.on('close', cleanup);
+ socket.on('error', cleanup);
+ };
+
+ return { url: httpServer.urlPrefix('human-readable'), reveal, triggerAnnotate, registerAnnotateWaiter };
}
-async function innerOpenDashboardApp(initialReveal: RevealOptions): Promise<{ page: api.Page; reveal: (options: RevealOptions) => void }> {
- const { url, reveal } = await startDashboardServer({ reveal: initialReveal });
+async function innerOpenDashboardApp(initialReveal: RevealOptions): Promise<{ page: api.Page; server: DashboardServer }> {
+ const server = await startDashboardServer({ reveal: initialReveal });
const { page } = await launchApp('dashboard');
- await page.goto(url);
- return { page, reveal };
+ await page.goto(server.url);
+ return { page, server };
}
async function launchApp(appName: string) {
@@ -148,10 +193,10 @@ function dashboardSocketPath() {
return makeSocketPath('dashboard', 'app');
}
-type OpenArgs = { reveal: RevealOptions; port?: number; host?: string };
+type OpenArgs = { reveal: RevealOptions; port?: number; host?: string; annotate: boolean };
function parseOpenArgs(): OpenArgs {
- const args = minimist(process.argv.slice(2), { string: ['session', 'workspace', 'host'] });
+ const args = minimist(process.argv.slice(2), { string: ['session', 'workspace', 'host'], boolean: ['annotate'] });
const portStr = args.port as string | undefined;
return {
reveal: {
@@ -160,6 +205,7 @@ function parseOpenArgs(): OpenArgs {
},
port: portStr !== undefined ? Number(portStr) : undefined,
host: (args.host as string) || undefined,
+ annotate: !!args.annotate,
};
}
@@ -169,13 +215,14 @@ async function acquireSingleton(reveal: RevealOptions): Promise<net.Server> {
await fs.promises.mkdir(path.dirname(socketPath), { recursive: true });
return await new Promise((resolve, reject) => {
- const server = net.createServer();
+ const server = net.createServer({ allowHalfOpen: true });
server.listen(socketPath, () => resolve(server));
server.on('error', (err: NodeJS.ErrnoException) => {
if (err.code !== 'EADDRINUSE')
return reject(err);
const client = net.connect(socketPath, () => {
- client.write(JSON.stringify({ command: 'bringToFront', ...reveal }));
+ const message: DashboardCommand = { command: 'bringToFront', ...reveal };
+ client.write(JSON.stringify(message));
client.end();
reject(new Error('already running'));
});
@@ -189,43 +236,108 @@ async function acquireSingleton(reveal: RevealOptions): Promise<net.Server> {
}
export async function openDashboardApp() {
- const { reveal: revealOptions, port, host } = parseOpenArgs();
+ const args = parseOpenArgs();
+ if (args.annotate) {
+ await runAnnotateClient(args);
+ return;
+ }
process.on('unhandledRejection', error => {
// eslint-disable-next-line no-console
console.error('Unhandled promise rejection:', error);
});
- if (port !== undefined) {
- const { url } = await startDashboardServer({ port, host, reveal: revealOptions });
+ if (args.port !== undefined) {
+ const { url } = await startDashboardServer({ port: args.port, host: args.host, reveal: args.reveal });
// eslint-disable-next-line no-console
console.log(`Listening on ${url}`);
+ selfDestructOnParentGone();
return;
}
let server: net.Server | undefined;
process.on('exit', () => server?.close());
- const underTest = !!process.env.PLAYWRIGHT_DASHBOARD_DEBUG_PORT;
- if (!underTest) {
- try {
- server = await acquireSingleton(revealOptions);
- } catch {
- return;
- }
+ try {
+ server = await acquireSingleton(args.reveal);
+ } catch {
+ return;
}
- const { page, reveal } = await innerOpenDashboardApp(revealOptions);
+ const statePromise = innerOpenDashboardApp(args.reveal);
server?.on('connection', socket => {
const chunks: Buffer[] = [];
socket.on('data', data => chunks.push(data));
- socket.on('end', () => {
+ socket.on('end', async () => {
const message = Buffer.concat(chunks).toString();
- let parsed: { command?: string; sessionName?: string; workspaceDir?: string } | undefined;
+ let parsed: DashboardCommand | undefined;
try {
parsed = JSON.parse(message);
} catch {
// no-op
}
- if (parsed?.command !== 'bringToFront')
+ if (!parsed?.command)
return;
- page?.bringToFront().catch(() => {});
- reveal({ sessionName: parsed.sessionName, workspaceDir: parsed.workspaceDir });
+ const { page, server: dashboard } = await statePromise;
+ const revealTo = { sessionName: parsed.sessionName, workspaceDir: parsed.workspaceDir };
+ if (parsed.command === 'bringToFront') {
+ page?.bringToFront().catch(() => {});
+ dashboard.reveal(revealTo);
+ } else if (parsed.command === 'annotate') {
+ page?.bringToFront().catch(() => {});
+ dashboard.reveal(revealTo);
+ dashboard.triggerAnnotate();
+ dashboard.registerAnnotateWaiter(socket);
+ }
});
});
+ await statePromise;
+}
+
+async function runAnnotateClient(args: OpenArgs): Promise<void> {
+ selfDestructOnParentGone();
+
+ const socketPath = dashboardSocketPath();
+ const tryConnect = () => new Promise<net.Socket | undefined>(resolve => {
+ const s = net.connect(socketPath);
+ const onError = () => { s.destroy(); resolve(undefined); };
+ s.once('connect', () => { s.off('error', onError); resolve(s); });
+ s.once('error', onError);
+ });
+ const deadline = Date.now() + 15000;
+ let socket: net.Socket | undefined;
+ while (Date.now() < deadline) {
+ socket = await tryConnect();
+ if (socket)
+ break;
+ await new Promise(r => setTimeout(r, 200));
+ }
+ if (!socket) {
+ // eslint-disable-next-line no-console
+ console.error('Dashboard did not start in time.');
+ gracefullyProcessExitDoNotHang(1);
+ return;
+ }
+ const message: DashboardCommand = { command: 'annotate', ...args.reveal };
+ socket.end(JSON.stringify(message));
+ const chunks: Buffer[] = [];
+ await new Promise<void>((resolve, reject) => {
+ socket!.on('data', chunk => chunks.push(chunk));
+ socket!.on('end', () => resolve());
+ socket!.on('error', reject);
+ });
+ socket.destroy();
+ const text = Buffer.concat(chunks).toString();
+ if (!text)
+ return;
+ const { png, annotations } = JSON.parse(text) as { png: string; annotations: AnnotationData[] };
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const filePath = await saveOutputFile(`annotations-${timestamp}.png`, Buffer.from(png, 'base64'));
+ for (const a of annotations) {
+ // eslint-disable-next-line no-console
+ console.log(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);
+ }
+ // eslint-disable-next-line no-console
+ console.log(`image available at: ${path.relative(process.cwd(), filePath)}`);
+}
+
+function selfDestructOnParentGone() {
+ process.stdin.on('close', () => {
+ gracefullyProcessExitDoNotHang(0);
+ });
}
diff --git a/packages/playwright-core/src/tools/dashboard/dashboardController.ts b/packages/playwright-core/src/tools/dashboard/dashboardController.ts
index 3f47aae614..efc1e58dca 100644
--- a/packages/playwright-core/src/tools/dashboard/dashboardController.ts
+++ b/packages/playwright-core/src/tools/dashboard/dashboardController.ts
@@ -26,7 +26,7 @@ import { createClientInfo } from '../cli-client/registry';
import type * as api from '../../..';
import type { Transport } from '@utils/httpServer';
-import type { Tab } from '@dashboard/dashboardChannel';
+import type { AnnotationData, Tab } from '@dashboard/dashboardChannel';
import type { BrowserDescriptor, BrowserStatus } from '../../serverRegistry';
type Disposable = { dispose: () => Promise<void> };
@@ -46,6 +46,8 @@ export class DashboardConnection implements Transport {
private _browsers = new Map<string, BrowserSlot>();
private _attachedBrowser: AttachedBrowser | undefined;
private _onclose: () => void;
+ private _onconnected?: () => void;
+ private _onAnnotationSubmit?: (base64Png: string, annotations: AnnotationData[]) => void;
private _serverRegistryDispose?: () => void;
private _pushSessionsScheduled = false;
private _pushTabsScheduled = false;
@@ -55,8 +57,10 @@ export class DashboardConnection implements Transport {
_recordingDir: string;
_streams = new Map<string, { handle: fs.promises.FileHandle; path: string }>();
- constructor(onclose: () => void) {
+ constructor(onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string, annotations: AnnotationData[]) => void) {
this._onclose = onclose;
+ this._onconnected = onconnected;
+ this._onAnnotationSubmit = onAnnotationSubmit;
this._recordingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'playwright-recordings-'));
}
@@ -66,6 +70,7 @@ export class DashboardConnection implements Transport {
serverRegistry.on('removed', this._pushSessions);
serverRegistry.on('changed', this._pushSessions);
this._pushSessions();
+ this._onconnected?.();
}
onclose() {
@@ -168,6 +173,10 @@ export class DashboardConnection implements Transport {
this._pushTabs();
}
+ async submitAnnotation(params: { data: string; annotations: AnnotationData[] }) {
+ this._onAnnotationSubmit?.(params.data, params.annotations);
+ }
+
async reveal(params: { path: string }) {
switch (os.platform()) {
case 'darwin':
@@ -221,6 +230,10 @@ export class DashboardConnection implements Transport {
this.sendEvent?.('pickLocator', {});
}
+ emitAnnotate() {
+ this.sendEvent?.('annotate', {});
+ }
+
_pushTabs() {
if (this._pushTabsScheduled)
return;
@@ -279,8 +292,13 @@ export class DashboardConnection implements Transport {
try {
const byWs = await serverRegistry.list();
const sessions: BrowserStatus[] = [];
- for (const list of byWs.values())
- sessions.push(...list);
+ for (const list of byWs.values()) {
+ for (const status of list) {
+ if (status.title.startsWith('--playwright-internal'))
+ continue;
+ sessions.push(status);
+ }
+ }
await this._reconcile(sessions);
await this._tryRevealPending();
this.emitSessions(sessions);
diff --git a/tests/mcp/cli-devtools.spec.ts b/tests/mcp/cli-devtools.spec.ts
index 4716a3dd92..d1df0e71b7 100644
--- a/tests/mcp/cli-devtools.spec.ts
+++ b/tests/mcp/cli-devtools.spec.ts
@@ -181,7 +181,7 @@ test('pick', async ({ cdpServer, cli, server }) => {
expect(output).toContain(`locator: getByRole('button', { name: 'Submit' })`);
});
-test('pick activates dashboard session', async ({ cdpServer, cli, server, openDashboard }) => {
+test('pick activates dashboard session', async ({ cdpServer, cli, server, startDashboardServer }) => {
server.setContent('/', `<button>Submit</button>`, 'text/html');
const browserContext = await cdpServer.start();
const [page] = browserContext.pages();
@@ -190,7 +190,7 @@ test('pick activates dashboard session', async ({ cdpServer, cli, server, openDa
await cli('attach', `--cdp=${cdpServer.endpoint}`);
await cli('snapshot');
- const dashboard = await openDashboard();
+ const dashboard = await startDashboardServer();
await expect(dashboard.locator('div.dashboard-view')).toBeVisible();
const scriptReady = page.waitForEvent('console', msg => msg.text() === 'Recorder script ready for test');
diff --git a/tests/mcp/cli-fixtures.ts b/tests/mcp/cli-fixtures.ts
index 673705307d..0758f4ab73 100644
--- a/tests/mcp/cli-fixtures.ts
+++ b/tests/mcp/cli-fixtures.ts
@@ -15,19 +15,21 @@
*/
import fs from 'fs';
+import os from 'os';
import path from 'path';
-import { test as baseTest } from './fixtures';
+import { test as baseTest, expect } from './fixtures';
import { killProcessGroup } from '../config/commonFixtures';
import { inheritAndCleanEnv } from '../config/utils';
-import type { Page } from 'playwright-core';
+import type { Page, Browser } from 'playwright-core';
import type { CommonFixtures } from '../config/commonFixtures';
export { expect } from './fixtures';
export const test = baseTest.extend<{
cliEnv: Record<string, string>,
- openDashboard: (options?: { cwd?: string, session?: string }) => Promise<Page>,
+ startDashboardServer: (options?: { cwd?: string, session?: string }) => Promise<Page>,
+ connectToDashboard: (bindTitle: string) => Promise<Browser>;
cli: (...args: any[]) => Promise<{
output: string,
error: string,
@@ -41,7 +43,7 @@ export const test = baseTest.extend<{
cliEnv: async ({}, use) => {
await use(cliEnv());
},
- openDashboard: async ({ childProcess, page }, use) => {
+ startDashboardServer: async ({ childProcess, page }, use) => {
await use(async (options?: { cwd?: string, session?: string }) => {
const testInfo = test.info();
const showArgs = options?.session ? [`-s=${options.session}`, 'show'] : ['show'];
@@ -55,23 +57,34 @@ export const test = baseTest.extend<{
return page;
});
},
+ connectToDashboard: async ({ cli, playwright }, use) => {
+ await use(async (bindTitle: string) => {
+ let endpoint = '';
+ await expect(async () => {
+ const { output } = await cli('list', '--all', '--json');
+ const { servers } = JSON.parse(output);
+ const server = servers.find(s => s.title === bindTitle);
+ endpoint = server.endpoint;
+ }).toPass();
+ return await playwright.chromium.connect(endpoint);
+ });
+ },
cli: async ({ mcpBrowser, mcpHeadless, childProcess }, use) => {
const sessions: { name: string, pid: number }[] = [];
await fs.promises.mkdir(test.info().outputPath('.playwright'), { recursive: true });
+ const pids: number[] = [];
await use(async (...args: string[]) => {
const cliArgs = args.filter(arg => typeof arg === 'string');
const cliOptions = args.findLast(arg => typeof arg === 'object') || {};
- return await runCli(childProcess, cliArgs, cliOptions, { mcpBrowser, mcpHeadless }, sessions);
+ const result = await runCli(childProcess, cliArgs, cliOptions, { mcpBrowser, mcpHeadless }, sessions);
+ if (result.pid !== undefined)
+ pids.push(result.pid);
+ return result;
});
- for (const session of sessions) {
- await runCli(childProcess, ['--session=' + session.name, 'close'], {}, { mcpBrowser, mcpHeadless }, []).catch(e => {
- if (!e.message.includes('is not running'))
- throw e;
- });
- killProcessGroup(session.pid);
- }
+ for (const pid of pids)
+ killProcessGroup(pid);
const daemonDir = path.join(test.info().outputDir, 'daemon');
const userDataDirs = await fs.promises.readdir(daemonDir).catch(() => []);
@@ -85,7 +98,7 @@ function cliEnv() {
PLAYWRIGHT_SERVER_REGISTRY: test.info().outputPath('registry'),
PLAYWRIGHT_DASHBOARD_SETTINGS_FILE_FOR_TEST: test.info().outputPath('dashboard.settings.json'),
PLAYWRIGHT_DAEMON_SESSION_DIR: test.info().outputPath('daemon'),
- PLAYWRIGHT_SOCKETS_DIR: path.join(test.info().project.outputDir, 'ds', String(test.info().parallelIndex)),
+ PLAYWRIGHT_SOCKETS_DIR: path.join(os.tmpdir(), 'ds' + String(test.info().workerIndex)),
PLAYWRIGHT_CLI_CHANNEL_SCAN_DISABLED_FOR_TEST: '1',
};
}
@@ -101,6 +114,7 @@ async function runCli(childProcess: CommonFixtures['childProcess'], args: string
...cliEnv(),
PLAYWRIGHT_MCP_BROWSER: options.mcpBrowser,
PLAYWRIGHT_MCP_HEADLESS: String(options.mcpHeadless),
+ PLAYWRIGHT_PRINT_DASHBOARD_PID_FOR_TEST: '1',
...cliOptions.env,
}),
});
diff --git a/tests/mcp/cli-json.spec.ts b/tests/mcp/cli-json.spec.ts
index d026b8ca9e..d223db8194 100644
--- a/tests/mcp/cli-json.spec.ts
+++ b/tests/mcp/cli-json.spec.ts
@@ -16,7 +16,7 @@
import { test, expect } from './cli-fixtures';
-const SNAPSHOT_FILE = expect.stringMatching(/^\.playwright-cli\/page-[\dTZ:.-]+\.yml$/);
+const SNAPSHOT_FILE = expect.stringMatching(/^\.playwright-cli[\\/]page-[\dTZ:.-]+\.yml$/);
test('output is pretty-printed', async ({ cli }) => {
const { output } = await cli('--json', 'list');
@@ -81,7 +81,7 @@ test('open returns envelope with session, pid and snapshot file', async ({ cli,
});
});
-test('list after open returns one browser entry', async ({ cli, server, mcpBrowser }) => {
+test('list after open returns one browser entry', async ({ cli, server, mcpBrowserNormalized }) => {
await cli('open', server.HELLO_WORLD);
const { output } = await cli('--json', 'list');
expect(JSON.parse(output)).toEqual({
@@ -89,7 +89,7 @@ test('list after open returns one browser entry', async ({ cli, server, mcpBrows
name: 'default',
workspace: expect.any(String),
status: 'open',
- browserType: mcpBrowser,
+ browserType: mcpBrowserNormalized,
userDataDir: null,
headed: false,
persistent: false,
diff --git a/tests/mcp/cli-killall.spec.ts b/tests/mcp/cli-killall.spec.ts
index 47221ae6a7..de05ec650f 100644
--- a/tests/mcp/cli-killall.spec.ts
+++ b/tests/mcp/cli-killall.spec.ts
@@ -44,16 +44,11 @@ test('kill-all kills filtered dashboard pid', async ({ cli }) => {
expect(pid).toBeDefined();
await expect.poll(() => isAlive(pid!)).toBe(true);
- try {
- const { output } = await cli('kill-all', {
- env: { PLAYWRIGHT_KILL_ALL_PID_FILTER_FOR_TEST: String(pid) },
- });
- expect(output).toContain(`Killed daemon process ${pid}`);
- expect(output).toContain('Killed 1 daemon process.');
-
- await expect.poll(() => isAlive(pid!)).toBe(false);
- } finally {
- if (isAlive(pid!))
- try { process.kill(pid!, 'SIGKILL'); } catch {}
- }
+ const { output } = await cli('kill-all', {
+ env: { PLAYWRIGHT_KILL_ALL_PID_FILTER_FOR_TEST: String(pid) },
+ });
+ expect(output).toContain(`Killed daemon process ${pid}`);
+ expect(output).toContain('Killed 1 daemon process.');
+
+ await expect.poll(() => isAlive(pid!)).toBe(false);
});
diff --git a/tests/mcp/dashboard.spec.ts b/tests/mcp/dashboard.spec.ts
index 34ce4d145a..04e5e10e41 100644
--- a/tests/mcp/dashboard.spec.ts
+++ b/tests/mcp/dashboard.spec.ts
@@ -33,15 +33,15 @@ test.beforeEach(({}, testInfo) => {
process.env.PLAYWRIGHT_SERVER_REGISTRY = testInfo.outputPath('registry');
});
-test('should show browser session chip', async ({ cli, server, openDashboard }) => {
+test('should show browser session chip', async ({ cli, server, startDashboardServer }) => {
await cli('open', server.EMPTY_PAGE);
- const dashboard = await openDashboard();
+ const dashboard = await startDashboardServer();
const chips = dashboard.locator('.session-chip');
await expect(chips).toHaveCount(1);
});
-test('should show current workspace sessions first', async ({ cli, server, openDashboard }) => {
+test('should show current workspace sessions first', async ({ cli, server, startDashboardServer }) => {
const wsA = test.info().outputPath('workspace-a');
const wsB = test.info().outputPath('workspace-b');
@@ -52,7 +52,7 @@ test('should show current workspace sessions first', async ({ cli, server, openD
await cli('open', server.EMPTY_PAGE, { cwd: wsB });
const checkOrder = async (first: string, second: string) => {
- const dashboard = await openDashboard({ cwd: first });
+ const dashboard = await startDashboardServer({ cwd: first });
const workspaceGroups = dashboard.locator('.workspace-group');
await expect(workspaceGroups).toHaveCount(2);
@@ -74,11 +74,11 @@ test('should show current workspace sessions first', async ({ cli, server, openD
});
});
-test('should activate session when show is called with -s', async ({ cli, server, openDashboard }) => {
+test('should activate session when show is called with -s', async ({ cli, server, startDashboardServer }) => {
await cli('-s=sessA', 'open', server.EMPTY_PAGE);
await cli('-s=sessB', 'open', server.EMPTY_PAGE);
- const dashboard = await openDashboard({ session: 'sessB' });
+ const dashboard = await startDashboardServer({ session: 'sessB' });
const activeSession = dashboard.locator('.sidebar-session:has(.sidebar-tab.active)');
await expect(activeSession.locator('.session-chip-name')).toHaveText('sessB');
});
@@ -92,34 +92,95 @@ function isAlive(pid: number): boolean {
}
}
-test('daemon show: closing page exits the process', async ({ playwright, cli }) => {
+test('daemon show: closing page exits the process', async ({ cli, connectToDashboard }) => {
const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- const { exitCode, pid } = await cli('show', { env: { PLAYWRIGHT_PRINT_DASHBOARD_PID_FOR_TEST: '1', PW_DASHBOARD_APP_BIND_TITLE: bindTitle } });
+ const { exitCode, pid } = await cli('show', { env: { PW_DASHBOARD_APP_BIND_TITLE: bindTitle } });
expect(exitCode).toBe(0);
expect(pid).toBeDefined();
expect(isAlive(pid!)).toBe(true);
- let endpoint = '';
- await expect(async () => {
- const { output } = await cli('list', '--all', '--json');
- const { servers } = JSON.parse(output);
- expect(servers[0].title).toBe(bindTitle);
- endpoint = servers[0].endpoint;
- }).toPass();
-
- const browser = await playwright.chromium.connect(endpoint);
+ const browser = await connectToDashboard(bindTitle);
const page = browser.contexts()[0].pages()[0];
await page.close();
await expect(() => expect(isAlive(pid!)).toBe(false)).toPass();
});
-test('should pick locator from browser', async ({ cli, server, openDashboard }) => {
+async function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page, text: string) {
+ await expect(dashboard.locator('div.dashboard-view.annotate')).toBeVisible();
+ const box = await dashboard.locator('img#display').boundingBox();
+ const x0 = box!.x + box!.width * 0.3;
+ const y0 = box!.y + box!.height * 0.3;
+ const x1 = box!.x + box!.width * 0.6;
+ const y1 = box!.y + box!.height * 0.6;
+ await dashboard.mouse.move(x0, y0);
+ await dashboard.mouse.down();
+ await dashboard.mouse.move(x1, y1);
+ await dashboard.mouse.up();
+ await dashboard.locator('.annotation-textarea').fill(text);
+ await dashboard.locator('.annotation-textarea').press('Enter');
+ await dashboard.locator('.annotate-action-btn.primary').click();
+}
+
+function verifyAnnotateOutput(output: string, expectedText: string, outputDir: string) {
+ const lines = output.trim().split('\n');
+ expect(lines[0]).toMatch(new RegExp(`^\\{ x: \\d+, y: \\d+, width: \\d+, height: \\d+ \\}: ${expectedText}$`));
+ expect(lines[lines.length - 1]).toMatch(/^image available at: \.playwright-cli[\\/]annotations-.*\.png$/);
+ const pngRel = lines[lines.length - 1].replace(/^image available at: /, '');
+ const pngPath = path.resolve(outputDir, pngRel);
+ expect(fs.existsSync(pngPath)).toBe(true);
+ expect(fs.statSync(pngPath).size).toBeGreaterThan(0);
+}
+
+test('should capture annotations via show --annotate', async ({ connectToDashboard, cli, server }) => {
+ await cli('open', server.EMPTY_PAGE);
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ await cli('show', { env: { PW_DASHBOARD_APP_BIND_TITLE: bindTitle } });
+ const browser = await connectToDashboard(bindTitle);
+
+ const dashboard = browser.contexts()[0].pages()[0];
+ await dashboard.locator('.sidebar-tab').first().click();
+
+ const annotatePromise = cli('show', '--annotate');
+ let done = false;
+ void annotatePromise.finally(() => { done = true; });
+
+ await drawAndSubmitAnnotation(dashboard, 'hello');
+
+ const { output, exitCode } = await annotatePromise;
+ expect(done).toBe(true);
+ expect(exitCode).toBe(0);
+ verifyAnnotateOutput(output, 'hello', test.info().outputDir);
+});
+
+test('should start dashboard and annotate when no dashboard is running', async ({ connectToDashboard, cli, server }) => {
+ await cli('open', server.EMPTY_PAGE);
+
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ const annotatePromise = cli('show', '--annotate', { env: { PW_DASHBOARD_APP_BIND_TITLE: bindTitle } });
+ let done = false;
+ void annotatePromise.finally(() => { done = true; });
+
+ const browser = await connectToDashboard(bindTitle);
+ try {
+ const dashboard = browser.contexts()[0].pages()[0];
+ await drawAndSubmitAnnotation(dashboard, 'hi');
+ } finally {
+ await browser.close().catch(() => {});
+ }
+
+ const { output, exitCode } = await annotatePromise;
+ expect(done).toBe(true);
+ expect(exitCode).toBe(0);
+ verifyAnnotateOutput(output, 'hi', test.info().outputDir);
+});
+
+test('should pick locator from browser', async ({ cli, server, startDashboardServer }) => {
server.setContent('/', '<button style="position:fixed;inset:0;width:100vw;height:100vh">Submit</button>', 'text/html');
await cli('open', server.PREFIX);
- const dashboard = await openDashboard();
+ const dashboard = await startDashboardServer();
await dashboard.locator('.sidebar-tab').first().click();
const pickPromise = cli('pick');
@@ -179,11 +240,11 @@ async function installSaveFilePickerMock(page: import('playwright-core').Page):
};
}
-test('screenshot writes PNG bytes to the chosen file', async ({ cli, server, page, openDashboard }) => {
+test('screenshot writes PNG bytes to the chosen file', async ({ cli, server, page, startDashboardServer }) => {
await cli('open', server.EMPTY_PAGE);
const awaitBytes = await installSaveFilePickerMock(page);
- const dashboard = await openDashboard();
+ const dashboard = await startDashboardServer();
await dashboard.locator('.sidebar-tab').first().click();
await expect(dashboard.locator('img#display')).toBeVisible();
await expect(dashboard.locator('.screenshot')).toBeEnabled();
@@ -194,11 +255,11 @@ test('screenshot writes PNG bytes to the chosen file', async ({ cli, server, pag
expect(bytes.subarray(0, 8)).toEqual(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]));
});
-test('stop recording streams WebM bytes to the chosen file', async ({ cli, server, page, openDashboard }) => {
+test('stop recording streams WebM bytes to the chosen file', async ({ cli, server, page, startDashboardServer }) => {
await cli('open', server.EMPTY_PAGE);
const awaitBytes = await installSaveFilePickerMock(page);
- const dashboard = await openDashboard();
+ const dashboard = await startDashboardServer();
await dashboard.locator('.sidebar-tab').first().click();
await expect(dashboard.locator('img#display')).toBeVisible();