Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(dashboard): annotate multiple screenshots (#40664)

Simon Knott2f1d9c062a9b
diff --git a/packages/dashboard/src/annotateView.tsx b/packages/dashboard/src/annotateView.tsx
new file mode 100644
index 0000000000..4eb339bf16
--- /dev/null
+++ b/packages/dashboard/src/annotateView.tsx
@@ -0,0 +1,221 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import { DownloadIcon } from './icons';
+import { Annotations } from './annotations';
+import { buildAnnotatedImage, saveAnnotationAsDownload } from './annotationImage';
+import { ToolbarButton } from '@web/components/toolbarButton';
+
+import type { Annotation, AnnotationsHandle } from './annotations';
+import type { DashboardModel, AnnotateFrame } from './dashboardModel';
+
+export type AnnotateSidebarProps = {
+ model: DashboardModel;
+ session: NonNullable<DashboardModel['state']['annotateSession']>;
+ onSubmit: () => Promise<void> | void;
+};
+
+export const AnnotateSidebar: React.FC<AnnotateSidebarProps> = ({ model, session, onSubmit }) => {
+ const [submitting, setSubmitting] = React.useState(false);
+ const [hoveredAnnotationId, setHoveredAnnotationId] = React.useState<string | null>(null);
+
+ return (
+ <aside className='annotate-sidebar' aria-label='Annotation screenshots'>
+ <div className='annotate-sidebar-header dashboard-shell-sidebar-header'>
+ <h2 className='dashboard-shell-sidebar-title'>UI Review</h2>
+ </div>
+ <div className='annotate-sidebar-list'>
+ {session.frames.map(frame => {
+ const selected = frame.id === session.selectedFrameId;
+ const comments = frame.annotations;
+ return (
+ <div
+ key={frame.id}
+ className={'annotate-sidebar-thumb' + (selected ? ' selected' : '')}
+ >
+ <button
+ className='annotate-sidebar-thumb-button'
+ onClick={() => model.selectAnnotateFrame(frame.id)}
+ title={`${frame.sessionTitle || 'session'} · ${frame.title || 'tab'}\n${frame.url}`}
+ aria-pressed={selected}
+ >
+ <span
+ className='annotate-sidebar-thumb-img-wrap'
+ style={{ aspectRatio: `${frame.viewportWidth} / ${frame.viewportHeight}` }}
+ >
+ <img
+ className='annotate-sidebar-thumb-img'
+ alt=''
+ src={'data:image/png;base64,' + frame.data}
+ />
+ {frame.annotations.map(a => (
+ <span
+ key={a.id}
+ className={'annotate-sidebar-thumb-rect' + (a.id === hoveredAnnotationId ? ' hovered' : '')}
+ style={{
+ left: `${(a.x / frame.viewportWidth) * 100}%`,
+ top: `${(a.y / frame.viewportHeight) * 100}%`,
+ width: `${(a.width / frame.viewportWidth) * 100}%`,
+ height: `${(a.height / frame.viewportHeight) * 100}%`,
+ ['--annotation-color' as any]: a.color,
+ }}
+ />
+ ))}
+ </span>
+ </button>
+ {comments.length > 0 && (
+ <ul className='annotate-sidebar-comments'>
+ {comments.map(a => (
+ <li key={a.id}>
+ <button
+ className={'annotate-sidebar-comment' + (a.text ? '' : ' empty')}
+ style={{ '--annotation-color': a.color } as React.CSSProperties}
+ onClick={() => model.selectAnnotateFrame(frame.id, a.id)}
+ onMouseEnter={() => setHoveredAnnotationId(a.id)}
+ onMouseLeave={() => setHoveredAnnotationId(null)}
+ title='Edit comment'
+ >
+ {a.text || 'Empty comment'}
+ </button>
+ </li>
+ ))}
+ </ul>
+ )}
+ <ToolbarButton
+ className='annotate-sidebar-thumb-remove'
+ icon='close'
+ title='Remove screenshot'
+ onClick={e => {
+ e.stopPropagation();
+ model.removeAnnotateFrame(frame.id);
+ }}
+ />
+ </div>
+ );
+ })}
+ </div>
+ <div
+ className='annotate-sidebar-feedback-wrap'
+ data-value={session.feedback}
+ >
+ <textarea
+ className='annotate-sidebar-feedback'
+ placeholder='Feedback…'
+ value={session.feedback}
+ onChange={e => model.updateFeedback(e.target.value)}
+ />
+ </div>
+ <button
+ className='annotate-sidebar-submit'
+ disabled={submitting}
+ onClick={async () => {
+ setSubmitting(true);
+ try {
+ await onSubmit();
+ } finally {
+ setSubmitting(false);
+ }
+ }}
+ >
+ {submitting ? 'Submitting…' : 'Submit'}
+ </button>
+ </aside>
+ );
+};
+
+export type AnnotateOverlayProps = {
+ model: DashboardModel;
+ frame: AnnotateFrame;
+ focusAnnotationId?: string | null;
+};
+
+export const AnnotateOverlay: React.FC<AnnotateOverlayProps> = ({ model, frame, focusAnnotationId }) => {
+ const annotationsRef = React.useRef<AnnotationsHandle>(null);
+ const displayRef = React.useRef<HTMLImageElement>(null);
+ const containerRef = React.useRef<HTMLDivElement>(null);
+
+ const onAnnotationsChange = React.useCallback((next: Annotation[]) => {
+ model.updateFrameAnnotations(frame.id, next);
+ }, [model, frame.id]);
+
+ const onSave = React.useCallback(async () => {
+ const img = displayRef.current;
+ if (!img)
+ return;
+ const blob = await buildAnnotatedImage(img, frame.viewportWidth, frame.viewportHeight, frame.annotations);
+ if (!blob)
+ return;
+ const safe = (frame.title || frame.url || 'screenshot').replace(/[^a-z0-9]+/gi, '-').slice(0, 40) || 'screenshot';
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+ await saveAnnotationAsDownload(blob, `annotations-${stamp}-${safe}.png`);
+ }, [frame]);
+
+ const onClear = React.useCallback(() => {
+ model.updateFrameAnnotations(frame.id, []);
+ annotationsRef.current?.clearSelection();
+ }, [model, frame.id]);
+
+ return (
+ <div className='annotate-overlay' role='dialog' aria-label={`Annotate screenshot from ${frame.title || frame.url || 'page'}`}>
+ <div className='annotate-overlay-window'>
+ <div className='annotate-overlay-chrome'>
+ <span className='annotate-overlay-titlebar'>
+ <span className='annotate-overlay-title-text'>{frame.title || 'untitled'}</span>
+ <span className='annotate-overlay-title-sep'>·</span>
+ <span className='annotate-overlay-title-url'>{frame.url}</span>
+ </span>
+ <ToolbarButton
+ title='Save screenshot'
+ onClick={onSave}
+ >
+ <DownloadIcon />
+ </ToolbarButton>
+ <ToolbarButton
+ title='Clear annotations'
+ icon='circle-slash'
+ disabled={frame.annotations.length === 0}
+ onClick={onClear}
+ />
+ <ToolbarButton
+ title='Done annotating'
+ icon='check'
+ onClick={() => model.deselectFrame()}
+ />
+ </div>
+ <div ref={containerRef} className='annotate-overlay-canvas'>
+ <img
+ ref={displayRef}
+ className='annotate-modal-image'
+ alt='annotation'
+ src={'data:image/png;base64,' + frame.data}
+ />
+ <Annotations
+ ref={annotationsRef}
+ active={true}
+ displayRef={displayRef}
+ screenRef={containerRef}
+ viewportWidth={frame.viewportWidth}
+ viewportHeight={frame.viewportHeight}
+ annotations={frame.annotations}
+ onAnnotationsChange={onAnnotationsChange}
+ focusAnnotationId={focusAnnotationId}
+ />
+ </div>
+ </div>
+ </div>
+ );
+};
diff --git a/packages/dashboard/src/annotationImage.ts b/packages/dashboard/src/annotationImage.ts
new file mode 100644
index 0000000000..2881aa2c49
--- /dev/null
+++ b/packages/dashboard/src/annotationImage.ts
@@ -0,0 +1,108 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+export type RectAnnotation = { x: number; y: number; width: number; height: number; text: string; color: string };
+
+export async function buildAnnotatedImage(
+ img: HTMLImageElement,
+ viewportWidth: number,
+ viewportHeight: number,
+ annotations: RectAnnotation[],
+): Promise<Blob | null> {
+ if (!img.naturalWidth || !img.naturalHeight || !viewportWidth || !viewportHeight)
+ return null;
+ const canvas = document.createElement('canvas');
+ canvas.width = img.naturalWidth;
+ canvas.height = img.naturalHeight;
+ const ctx = canvas.getContext('2d');
+ if (!ctx)
+ return null;
+ ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
+ const sx = canvas.width / viewportWidth;
+ const sy = canvas.height / viewportHeight;
+ const fontSize = Math.max(11, Math.round(14 * sy));
+ ctx.font = `500 ${fontSize}px -apple-system, system-ui, sans-serif`;
+ ctx.textBaseline = 'middle';
+ for (const a of annotations) {
+ const [r, g, b] = a.color.split(' ').map(Number);
+ const solid = `rgb(${r}, ${g}, ${b})`;
+ const wash = `rgba(${r}, ${g}, ${b}, 0.12)`;
+ const x = a.x * sx;
+ const y = a.y * sy;
+ const w = a.width * sx;
+ const h = a.height * sy;
+ ctx.fillStyle = wash;
+ ctx.fillRect(x, y, w, h);
+ ctx.lineWidth = Math.max(2, Math.round(2 * sy));
+ ctx.strokeStyle = solid;
+ ctx.strokeRect(x, y, w, h);
+ if (a.text) {
+ const padX = Math.max(4, Math.round(6 * sy));
+ const padY = Math.max(2, Math.round(3 * sy));
+ const metrics = ctx.measureText(a.text);
+ const labelW = metrics.width + padX * 2;
+ const labelH = fontSize + padY * 2;
+ const labelX = x - ctx.lineWidth / 2;
+ const labelY = y - labelH;
+ ctx.fillStyle = solid;
+ ctx.fillRect(labelX, labelY, labelW, labelH);
+ ctx.fillStyle = '#fff';
+ ctx.fillText(a.text, labelX + padX, labelY + labelH / 2);
+ }
+ }
+ return await new Promise<Blob | null>(resolve => canvas.toBlob(resolve, 'image/png'));
+}
+
+export async function saveAnnotationAsDownload(blob: Blob, suggestedName?: string): Promise<boolean> {
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const name = suggestedName ?? `annotations-${stamp}.png`;
+ const ext = (name.match(/\.[^./\\]+$/)?.[0] ?? '.bin').toLowerCase();
+ const mime = blob.type || ({
+ '.png': 'image/png',
+ '.zip': 'application/zip',
+ } as Record<string, string>)[ext] || 'application/octet-stream';
+ const description = ({
+ '.png': 'PNG image',
+ '.zip': 'Zip archive',
+ } as Record<string, string>)[ext] || 'File';
+ const picker = (window as any).showSaveFilePicker as undefined | ((opts: any) => Promise<any>);
+ if (picker) {
+ try {
+ const handle = await picker({
+ suggestedName: name,
+ startIn: 'downloads',
+ types: [{ description, accept: { [mime]: [ext] } }],
+ });
+ const writable = await handle.createWritable();
+ await writable.write(blob);
+ await writable.close();
+ } catch (e: any) {
+ if (e?.name === 'AbortError')
+ return false;
+ throw e;
+ }
+ return true;
+ }
+ const url = URL.createObjectURL(blob);
+ const link = document.createElement('a');
+ link.href = url;
+ link.download = name;
+ document.body.appendChild(link);
+ link.click();
+ document.body.removeChild(link);
+ URL.revokeObjectURL(url);
+ return true;
+}
diff --git a/packages/dashboard/src/annotationZip.ts b/packages/dashboard/src/annotationZip.ts
new file mode 100644
index 0000000000..bba3fe9e33
--- /dev/null
+++ b/packages/dashboard/src/annotationZip.ts
@@ -0,0 +1,62 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// @ts-ignore
+import * as zipImport from '@zip.js/zip.js/lib/zip-no-worker-deflate.js';
+
+import type * as zip from '@zip.js/zip.js';
+import type { SubmittedAnnotationFrame } from './dashboardChannel';
+
+const zipjs = zipImport as typeof zip;
+
+export async function buildAnnotationZip(frames: SubmittedAnnotationFrame[], feedback: string): Promise<Blob> {
+ zipjs.configure({ useWebWorkers: false } as any);
+ const writer = new zipjs.ZipWriter(new zipjs.BlobWriter('application/zip'));
+
+ const mdLines: string[] = [];
+ if (feedback)
+ mdLines.push(feedback, '');
+
+ for (let i = 0; i < frames.length; i++) {
+ const idx = i + 1;
+ const frame = frames[i];
+ const session = frame.sessionTitle || 'session';
+ const tab = frame.title || 'tab';
+ const header = `screenshot ${idx}: ${session} / ${tab} @ ${frame.url} (${frame.viewportWidth}x${frame.viewportHeight})`;
+
+ mdLines.push(`## ${header}`);
+ if (frame.data) {
+ const pngFile = `annotations-${idx}.png`;
+ await writer.add(pngFile, new zipjs.Uint8ArrayReader(Uint8Array.fromBase64(frame.data)));
+ mdLines.push(`- [Screenshot ${idx}](${pngFile})`);
+ }
+ if (frame.ariaSnapshot) {
+ const yamlFile = `annotations-${idx}.yaml`;
+ const annotationLines = frame.annotations.map(a => `# { x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);
+ const yamlContent = [...annotationLines, frame.ariaSnapshot].join('\n');
+ await writer.add(yamlFile, new zipjs.TextReader(yamlContent));
+ mdLines.push(`- [Aria snapshot ${idx}](${yamlFile})`);
+ }
+ for (const a of frame.annotations) {
+ if (a.text)
+ mdLines.push(`- { x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);
+ }
+ mdLines.push('');
+ }
+
+ await writer.add('feedback.md', new zipjs.TextReader(mdLines.join('\n')));
+ return writer.close();
+}
diff --git a/packages/dashboard/src/annotations.css b/packages/dashboard/src/annotations.css
index 41bdfd35f4..a6c9f75700 100644
--- a/packages/dashboard/src/annotations.css
+++ b/packages/dashboard/src/annotations.css
@@ -232,3 +232,317 @@
-webkit-user-drag: none;
box-shadow: 0 0 0 1px var(--color-border-muted), 0 0 8px var(--color-border-muted);
}
+
+
+/* -- Annotate sidebar -- */
+.annotate-sidebar {
+ width: 260px;
+ flex-shrink: 0;
+ display: flex;
+ flex-direction: column;
+ background: var(--color-canvas-subtle);
+ border-left: 1px solid var(--color-border-muted);
+ min-height: 0;
+}
+
+.annotate-sidebar-header {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ gap: 6px;
+ height: 40px;
+ min-height: 40px;
+ padding: 0 8px;
+ border-bottom: 1px solid var(--color-border-muted);
+ background: var(--color-canvas-overlay);
+}
+
+:root.light-mode .annotate-sidebar-header {
+ background: var(--color-canvas-subtle);
+}
+
+.annotate-sidebar-list {
+ flex: 1;
+ min-height: 0;
+ overflow-y: auto;
+ padding: 8px;
+ display: flex;
+ flex-direction: column;
+ gap: 6px;
+}
+
+.annotate-sidebar-thumb {
+ position: relative;
+ flex-shrink: 0;
+ border: 1px solid var(--color-border-muted);
+ border-radius: 6px;
+ background: var(--color-canvas-default);
+ overflow: hidden;
+}
+
+.annotate-sidebar-thumb:hover {
+ border-color: var(--color-border-default);
+}
+
+.annotate-sidebar-thumb.selected {
+ border-color: var(--color-accent-fg, #2f81f7);
+ box-shadow: 0 0 0 1px var(--color-accent-fg, #2f81f7);
+}
+
+.annotate-sidebar-thumb-button {
+ display: block;
+ width: 100%;
+ padding: 0;
+ background: transparent;
+ border: 0;
+ text-align: left;
+ color: inherit;
+ font: inherit;
+ cursor: pointer;
+}
+
+.annotate-sidebar-thumb-img-wrap {
+ position: relative;
+ display: block;
+ width: 100%;
+ background: var(--color-canvas-overlay);
+ overflow: hidden;
+}
+
+.annotate-sidebar-thumb-img {
+ display: block;
+ width: 100%;
+ height: 100%;
+ object-fit: cover;
+}
+
+.annotate-sidebar-thumb-overlay {
+ position: absolute;
+ inset: 0;
+ width: 100%;
+ height: 100%;
+ pointer-events: none;
+}
+
+.annotate-sidebar-thumb-rect {
+ position: absolute;
+ border: 1.5px solid rgb(var(--annotation-color));
+ background: rgb(var(--annotation-color) / 0.12);
+ border-radius: 2px;
+ pointer-events: none;
+ box-sizing: border-box;
+}
+
+.annotate-sidebar-thumb-rect.hovered {
+ background: rgb(var(--annotation-color) / 0.32);
+}
+
+.annotate-sidebar-thumb-remove.toolbar-button {
+ position: absolute;
+ top: 4px;
+ right: 4px;
+ padding: 2px;
+ background: rgba(0, 0, 0, 0.55);
+ color: #fff;
+ border-radius: 50%;
+ opacity: 0;
+ transition: opacity 120ms ease;
+ z-index: 1;
+}
+
+.annotate-sidebar-thumb-remove.toolbar-button:not(:disabled):hover {
+ background: rgba(0, 0, 0, 0.75);
+ color: #fff;
+}
+
+.annotate-sidebar-thumb:hover .annotate-sidebar-thumb-remove,
+.annotate-sidebar-thumb-remove:focus-visible {
+ opacity: 1;
+}
+
+.annotate-sidebar-comments {
+ list-style: none;
+ margin: 0;
+ padding: 4px 8px 6px;
+ display: flex;
+ flex-direction: column;
+ gap: 3px;
+}
+
+.annotate-sidebar-comment {
+ display: block;
+ width: 100%;
+ text-align: left;
+ background: rgb(var(--annotation-color) / 0.12);
+ border: 1px solid rgb(var(--annotation-color) / 0.35);
+ border-radius: 4px;
+ color: var(--color-fg-default);
+ font-size: 11px;
+ line-height: 1.4;
+ padding: 3px 6px;
+ cursor: pointer;
+ white-space: pre-wrap;
+ word-break: break-word;
+}
+
+.annotate-sidebar-comment:hover {
+ background: rgb(var(--annotation-color) / 0.2);
+}
+
+.annotate-sidebar-comment.empty {
+ color: var(--color-fg-muted);
+ font-style: italic;
+}
+
+/* auto-grow textarea: the wrapper mirrors content in ::after so the grid row
+ expands to fit, and the textarea fills the same cell */
+.annotate-sidebar-feedback-wrap {
+ display: grid;
+ margin: 8px 8px 4px;
+ flex-shrink: 0;
+}
+
+.annotate-sidebar-feedback-wrap::after {
+ content: attr(data-value) ' ';
+ white-space: pre-wrap;
+ word-break: break-word;
+ visibility: hidden;
+ /* same box-model as the textarea */
+ padding: 6px 8px;
+ font-size: 12px;
+ font-family: inherit;
+ border: 1px solid transparent;
+ border-radius: 4px;
+ min-height: 56px;
+ grid-area: 1 / 1 / 2 / 2;
+}
+
+.annotate-sidebar-feedback {
+ grid-area: 1 / 1 / 2 / 2;
+ padding: 6px 8px;
+ background: var(--vscode-input-background, #1e1e1e);
+ color: var(--vscode-input-foreground, #cccccc);
+ border: 1px solid var(--color-border-muted);
+ border-radius: 4px;
+ font-size: 12px;
+ font-family: inherit;
+ resize: none;
+ min-height: 56px;
+ overflow: hidden;
+ outline: none;
+}
+
+.annotate-sidebar-feedback:focus {
+ border-color: var(--vscode-focusBorder, #007fd4);
+}
+
+.annotate-sidebar-submit {
+ margin: 4px 8px 8px;
+ padding: 8px 12px;
+ background: var(--color-accent-fg, #2f81f7);
+ color: white;
+ border: 0;
+ border-radius: 6px;
+ cursor: pointer;
+ font-weight: 500;
+ font-size: 13px;
+ flex-shrink: 0;
+}
+
+.annotate-sidebar-submit:hover:not(:disabled) {
+ filter: brightness(1.05);
+}
+
+.annotate-sidebar-submit:disabled {
+ opacity: 0.55;
+ cursor: not-allowed;
+}
+
+/* -- Annotate overlay (drawn on top of the live screen) -- */
+.annotate-overlay {
+ position: fixed;
+ inset: 0;
+ display: flex;
+ align-items: stretch;
+ justify-content: center;
+ padding: 16px;
+ background: rgba(0, 0, 0, 0.55);
+ z-index: 100;
+ min-height: 0;
+}
+
+.annotate-overlay-window {
+ flex: 1;
+ display: flex;
+ flex-direction: column;
+ border: 1px solid var(--color-border-default);
+ border-radius: 10px;
+ overflow: hidden;
+ box-shadow: 0 4px 20px rgba(0, 0, 0, 0.35);
+ background: var(--color-canvas-default);
+ min-height: 0;
+}
+
+.annotate-overlay-chrome {
+ display: flex;
+ align-items: center;
+ gap: 4px;
+ height: 40px;
+ min-height: 40px;
+ padding: 0 8px;
+ background: var(--color-canvas-overlay);
+ border-bottom: 1px solid var(--color-border-muted);
+}
+
+:root.light-mode .annotate-overlay-chrome {
+ background: var(--color-canvas-subtle);
+}
+
+.annotate-overlay-titlebar {
+ flex: 1;
+ display: flex;
+ align-items: center;
+ gap: 6px;
+ height: 30px;
+ padding: 0 4px;
+ font-size: 13px;
+ color: var(--color-fg-default);
+ min-width: 0;
+ overflow: hidden;
+ white-space: nowrap;
+}
+
+.annotate-overlay-title-text {
+ font-weight: 500;
+ color: var(--color-fg-default);
+ overflow: hidden;
+ text-overflow: ellipsis;
+ flex-shrink: 0;
+ max-width: 40%;
+}
+
+.annotate-overlay-title-sep {
+ color: var(--color-fg-subtle);
+}
+
+.annotate-overlay-title-url {
+ flex: 1;
+ font-family: var(--monospace-font, monospace);
+ font-size: 12px;
+ color: var(--color-fg-muted);
+ overflow: hidden;
+ text-overflow: ellipsis;
+}
+
+.annotate-overlay-canvas {
+ flex: 1;
+ position: relative;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ min-height: 0;
+ padding: 12px;
+ background: var(--color-canvas-default);
+}
+
+
diff --git a/packages/dashboard/src/annotations.tsx b/packages/dashboard/src/annotations.tsx
index 5430e82cef..6d2a41ca44 100644
--- a/packages/dashboard/src/annotations.tsx
+++ b/packages/dashboard/src/annotations.tsx
@@ -17,15 +17,14 @@
import React from 'react';
import './modal.css';
import './annotations.css';
-import { DownloadIcon } from './icons';
-import { ToolbarButton } from '@web/components/toolbarButton';
import { clientToViewport, getImageLayout } from './imageLayout';
import type { ImageLayout } from './imageLayout';
-import type { AnnotateFrame } from './dashboardModel';
export type Annotation = { id: string; x: number; y: number; width: number; height: number; text: string; color: string };
+export { buildAnnotatedImage, saveAnnotationAsDownload } from './annotationImage';
+
// Palette is 6 rgb triples — matches the `--annotations-blue` pattern so they
// compose nicely via `rgb(var(--annotation-color))` / `rgb(var(...) / 0.12)`.
const ANNOTATION_COLORS = [
@@ -103,90 +102,8 @@ function viewportRectToScreenStyle(layout: ImageLayout, screenRect: DOMRect, vw:
};
}
-export async function saveAnnotationAsDownload(blob: Blob): Promise<void> {
- const stamp = new Date().toISOString().replace(/[:.]/g, '-');
- const suggestedName = `annotations-${stamp}.png`;
- const picker = (window as any).showSaveFilePicker as undefined | ((opts: any) => Promise<any>);
- if (picker) {
- try {
- const handle = await picker({
- suggestedName,
- startIn: 'downloads',
- types: [{ description: 'PNG image', accept: { 'image/png': ['.png'] } }],
- });
- const writable = await handle.createWritable();
- await writable.write(blob);
- await writable.close();
- } catch (e: any) {
- if (e?.name !== 'AbortError')
- throw e;
- }
- return;
- }
- const url = URL.createObjectURL(blob);
- const link = document.createElement('a');
- link.href = url;
- link.download = suggestedName;
- document.body.appendChild(link);
- link.click();
- document.body.removeChild(link);
- URL.revokeObjectURL(url);
-}
-
-export async function buildAnnotatedImage(
- img: HTMLImageElement,
- viewportWidth: number,
- viewportHeight: number,
- annotations: Annotation[],
-): Promise<Blob | null> {
- if (!img.naturalWidth || !img.naturalHeight || !viewportWidth || !viewportHeight)
- return null;
- const canvas = document.createElement('canvas');
- canvas.width = img.naturalWidth;
- canvas.height = img.naturalHeight;
- const ctx = canvas.getContext('2d');
- if (!ctx)
- return null;
- ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
- const sx = canvas.width / viewportWidth;
- const sy = canvas.height / viewportHeight;
- const fontSize = Math.max(11, Math.round(14 * sy));
- ctx.font = `500 ${fontSize}px -apple-system, system-ui, sans-serif`;
- ctx.textBaseline = 'middle';
- for (const a of annotations) {
- const [r, g, b] = a.color.split(' ').map(Number);
- const solid = `rgb(${r}, ${g}, ${b})`;
- const wash = `rgba(${r}, ${g}, ${b}, 0.12)`;
- const x = a.x * sx;
- const y = a.y * sy;
- const w = a.width * sx;
- const h = a.height * sy;
- ctx.fillStyle = wash;
- ctx.fillRect(x, y, w, h);
- ctx.lineWidth = Math.max(2, Math.round(2 * sy));
- ctx.strokeStyle = solid;
- ctx.strokeRect(x, y, w, h);
- if (a.text) {
- const padX = Math.max(4, Math.round(6 * sy));
- const padY = Math.max(2, Math.round(3 * sy));
- const metrics = ctx.measureText(a.text);
- const labelW = metrics.width + padX * 2;
- const labelH = fontSize + padY * 2;
- const labelX = x - ctx.lineWidth / 2;
- const labelY = y - labelH;
- ctx.fillStyle = solid;
- ctx.fillRect(labelX, labelY, labelW, labelH);
- ctx.fillStyle = '#fff';
- ctx.fillText(a.text, labelX + padX, labelY + labelH / 2);
- }
- }
- return await new Promise<Blob | null>(resolve => canvas.toBlob(resolve, 'image/png'));
-}
-
export type AnnotationsHandle = {
- clear(): void;
- submit(): Promise<void>;
- save(): Promise<void>;
+ clearSelection(): void;
};
type AnnotationsProps = {
@@ -195,8 +112,9 @@ type AnnotationsProps = {
screenRef: React.RefObject<HTMLDivElement | null>;
viewportWidth: number;
viewportHeight: number;
- onSubmit: (blob: Blob, annotations: Annotation[]) => Promise<void> | void;
- onAnnotationsChange?: (count: number) => void;
+ annotations: Annotation[];
+ onAnnotationsChange: (annotations: Annotation[]) => void;
+ focusAnnotationId?: string | null;
};
const ColorPicker: React.FC<{ color: string; onChange: (color: string) => void }> = ({ color, onChange }) => (
@@ -216,8 +134,7 @@ const ColorPicker: React.FC<{ color: string; onChange: (color: string) => void }
</div>
);
-export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>(({ active, displayRef, screenRef, viewportWidth, viewportHeight, onSubmit, onAnnotationsChange }, ref) => {
- const [annotations, setAnnotations] = React.useState<Annotation[]>([]);
+export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>(({ active, displayRef, screenRef, viewportWidth, viewportHeight, annotations, onAnnotationsChange, focusAnnotationId }, ref) => {
const [draft, setDraft] = React.useState<{ startX: number; startY: number; x: number; y: number } | null>(null);
const [selection, setSelection] = React.useState<Selection>(null);
const [drag, setDrag] = React.useState<DragState | null>(null);
@@ -229,6 +146,14 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
const [, setTick] = React.useState(0);
const forceRender = React.useCallback(() => setTick(t => t + 1), []);
const layerRef = React.useRef<HTMLDivElement>(null);
+ // Refs kept in sync each render so effects can read current values without
+ // listing them as dependencies (avoids spurious effect re-runs).
+ const annotationsRef = React.useRef(annotations);
+ const focusAnnotationIdRef = React.useRef(focusAnnotationId);
+ React.useLayoutEffect(() => {
+ annotationsRef.current = annotations;
+ focusAnnotationIdRef.current = focusAnnotationId;
+ });
const selectedId = selection?.id ?? null;
const editingId = selection?.editing ? selection.id : null;
@@ -248,10 +173,20 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
}, [active, displayRef, forceRender]);
React.useEffect(() => {
- if (active)
+ if (active) {
+ const fid = focusAnnotationIdRef.current;
+ if (fid) {
+ const annotation = annotationsRef.current.find(a => a.id === fid);
+ if (annotation) {
+ setEditSnapshot({ ...annotation });
+ setSelection({ id: fid, editing: true });
+ return;
+ }
+ }
layerRef.current?.focus();
- else
+ } else {
setSelection(null);
+ }
}, [active]);
React.useEffect(() => {
@@ -266,7 +201,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
const dvy = vp.y - drag.startVy;
if (dvx === 0 && dvy === 0)
return;
- setAnnotations(prev => prev.map(a => a.id === drag.id ? { ...a, ...applyDrag(drag.orig, drag.kind, dvx, dvy) } : a));
+ onAnnotationsChange(annotationsRef.current.map(a => a.id === drag.id ? { ...a, ...applyDrag(drag.orig, drag.kind, dvx, dvy) } : a));
};
const onUp = () => setDrag(null);
window.addEventListener('mousemove', onMove);
@@ -275,7 +210,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
window.removeEventListener('mousemove', onMove);
window.removeEventListener('mouseup', onUp);
};
- }, [drag, displayRef, viewportWidth, viewportHeight]);
+ }, [drag, displayRef, viewportWidth, viewportHeight, onAnnotationsChange]);
function imgCoords(e: React.MouseEvent): { x: number; y: number } | null {
if (!viewportWidth || !viewportHeight)
@@ -333,7 +268,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
if (rect.width < MIN_ANNOTATION_SIZE || rect.height < MIN_ANNOTATION_SIZE)
return;
const id = newAnnotationId();
- setAnnotations(prev => [...prev, { id, ...rect, text: '', color: activeColor }]);
+ onAnnotationsChange([...annotations, { id, ...rect, text: '', color: activeColor }]);
setSelection({ id, editing: true });
}
@@ -351,7 +286,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
function nudgeSelected(dx: number, dy: number) {
if (!selectedId)
return;
- setAnnotations(prev => prev.map(a => a.id === selectedId ? { ...a, x: a.x + dx, y: a.y + dy } : a));
+ onAnnotationsChange(annotations.map(a => a.id === selectedId ? { ...a, x: a.x + dx, y: a.y + dy } : a));
}
function closeEditor() {
@@ -372,7 +307,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
}
if ((e.key === 'Delete' || e.key === 'Backspace') && selectedId && !editingId) {
e.preventDefault();
- setAnnotations(prev => prev.filter(a => a.id !== selectedId));
+ onAnnotationsChange(annotations.filter(a => a.id !== selectedId));
setSelection(null);
return;
}
@@ -385,10 +320,6 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
}
}
- React.useEffect(() => {
- onAnnotationsChange?.(annotations.length);
- }, [annotations.length, onAnnotationsChange]);
-
// Snapshot is only meaningful while an annotation is actively being edited.
React.useEffect(() => {
if (!editingId)
@@ -396,33 +327,11 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
}, [editingId]);
React.useImperativeHandle(ref, () => ({
- clear: () => {
- setAnnotations([]);
+ clearSelection: () => {
setDraft(null);
setSelection(null);
},
- submit: async () => {
- const img = displayRef.current;
- if (!img)
- return;
- const blob = await buildAnnotatedImage(img, viewportWidth, viewportHeight, annotations);
- if (!blob)
- return;
- await onSubmit(blob, annotations);
- setAnnotations([]);
- setSelection(null);
- setDraft(null);
- },
- save: async () => {
- const img = displayRef.current;
- if (!img)
- return;
- const blob = await buildAnnotatedImage(img, viewportWidth, viewportHeight, annotations);
- if (!blob)
- return;
- await saveAnnotationAsDownload(blob);
- },
- }), [displayRef, viewportWidth, viewportHeight, annotations, onSubmit]);
+ }), []);
if (!active)
return null;
@@ -515,11 +424,12 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
<textarea
className='annotations-textarea'
autoFocus
+ onFocus={e => { const len = e.target.value.length; e.target.setSelectionRange(len, len); }}
value={editingAnnotation.text}
placeholder='Task or comment…'
onChange={e => {
const text = e.target.value;
- setAnnotations(prev => prev.map(a => a.id === editingAnnotation.id ? { ...a, text } : a));
+ onAnnotationsChange(annotations.map(a => a.id === editingAnnotation.id ? { ...a, text } : a));
}}
onKeyDown={e => {
if (e.key === 'Enter' && !e.shiftKey && !e.ctrlKey && !e.metaKey) {
@@ -537,7 +447,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
<ColorPicker
color={editingAnnotation.color}
onChange={color => {
- setAnnotations(prev => prev.map(a => a.id === editingAnnotation.id ? { ...a, color } : a));
+ onAnnotationsChange(annotations.map(a => a.id === editingAnnotation.id ? { ...a, color } : a));
setActiveColor(color);
}}
/>
@@ -548,7 +458,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
className='annotations-action-btn'
onClick={() => {
const snap = editSnapshot;
- setAnnotations(prev => prev.map(a => a.id === snap.id ? snap : a));
+ onAnnotationsChange(annotations.map(a => a.id === snap.id ? snap : a));
setSelection(null);
layerRef.current?.focus();
}}
@@ -567,7 +477,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
<button
className='annotations-action-btn danger'
onClick={() => {
- setAnnotations(prev => prev.filter(a => a.id !== editingAnnotation.id));
+ onAnnotationsChange(annotations.filter(a => a.id !== editingAnnotation.id));
setSelection(null);
layerRef.current?.focus();
}}
@@ -592,70 +502,3 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
);
});
Annotations.displayName = 'Annotations';
-
-type AnnotateModalProps = {
- frame: AnnotateFrame;
- showSubmit: boolean;
- onSubmit: (blob: Blob, annotations: Annotation[]) => Promise<void>;
- onClose: () => void;
-};
-
-export const AnnotateModal: React.FC<AnnotateModalProps> = ({ frame, showSubmit, onSubmit, onClose }) => {
- const [annotationCount, setAnnotationCount] = React.useState(0);
- const annotationsRef = React.useRef<AnnotationsHandle>(null);
- const displayRef = React.useRef<HTMLImageElement>(null);
- const viewRef = React.useRef<HTMLDivElement>(null);
-
- return (
- <div className='modal-overlay' role='dialog' aria-modal='true' aria-label='Annotate screenshot'>
- <div className='modal'>
- <div className='modal-toolbar'>
- <div className='modal-title'>Annotate screenshot</div>
- {showSubmit && (
- <ToolbarButton
- title='Submit annotation'
- icon='check'
- disabled={annotationCount === 0}
- onClick={() => annotationsRef.current?.submit()}
- />
- )}
- <ToolbarButton
- title='Save annotated image'
- onClick={() => annotationsRef.current?.save()}
- >
- <DownloadIcon />
- </ToolbarButton>
- <ToolbarButton
- title='Clear annotations'
- icon='circle-slash'
- disabled={annotationCount === 0}
- onClick={() => annotationsRef.current?.clear()}
- />
- <ToolbarButton
- title='Discard'
- icon='close'
- onClick={onClose}
- />
- </div>
- <div ref={viewRef} className='modal-body'>
- <img
- ref={displayRef}
- className='annotate-modal-image'
- alt='annotation'
- src={'data:image/png;base64,' + frame.data}
- />
- <Annotations
- ref={annotationsRef}
- active={true}
- displayRef={displayRef}
- screenRef={viewRef}
- viewportWidth={frame.viewportWidth}
- viewportHeight={frame.viewportHeight}
- onSubmit={onSubmit}
- onAnnotationsChange={setAnnotationCount}
- />
- </div>
- </div>
- </div>
- );
-};
diff --git a/packages/dashboard/src/dashboard.css b/packages/dashboard/src/dashboard.css
index 4444e70521..288a4fbdce 100644
--- a/packages/dashboard/src/dashboard.css
+++ b/packages/dashboard/src/dashboard.css
@@ -25,6 +25,18 @@
position: relative;
}
+.dashboard-view.has-annotate-sidebar {
+ flex-direction: row;
+}
+
+.dashboard-main {
+ display: flex;
+ flex-direction: column;
+ flex: 1;
+ min-width: 0;
+ min-height: 0;
+}
+
/* -- Mode toggle buttons (interactive / record) -- */
.mode-toggle.toolbar-button.toggled,
.mode-toggle.toolbar-button.toggled > .codicon {
diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsx
index 184e536a20..c2f1e643ea 100644
--- a/packages/dashboard/src/dashboard.tsx
+++ b/packages/dashboard/src/dashboard.tsx
@@ -17,11 +17,10 @@
import React from 'react';
import './dashboard.css';
import { ChevronLeftIcon, ChevronRightIcon, LockIcon, LockOpenIcon, ReloadIcon, ScreenshotRegionIcon } from './icons';
-import { AnnotateModal } from './annotations';
import { clientToViewport, getImageLayout } from './imageLayout';
import { Recording } from './recording';
+import { AnnotateSidebar, AnnotateOverlay } from './annotateView';
-import type { Annotation } from './annotations';
import { ToolbarButton } from '@web/components/toolbarButton';
import { useMeasureForRef } from '@web/uiUtils';
@@ -67,8 +66,10 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
const [, setRevision] = React.useState(0);
React.useEffect(() => model.subscribe(() => setRevision(r => r + 1)), [model]);
- const { tabs, mode, recording, liveFrame, annotateFrame, annotateInitiator, pendingAnnotate } = model.state;
+ const { tabs, mode, recording, liveFrame, annotateSession, pendingCapture } = model.state;
const interactive = mode === 'interactive';
+ const annotateActive = !!annotateSession;
+ const selectedFrame = annotateSession?.frames.find(f => f.id === annotateSession.selectedFrameId) ?? null;
const [flashTick, setFlashTick] = React.useState(0);
@@ -131,17 +132,6 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
interactiveBtnRef.current?.classList.remove('flash');
}, [interactive]);
- const onSubmitAnnotations = React.useCallback(async (blob: Blob, annotations: Annotation[]) => {
- const dataUrl = await new Promise<string>((resolve, reject) => {
- const reader = new FileReader();
- reader.onload = () => resolve(reader.result as string);
- reader.onerror = () => reject(reader.error);
- reader.readAsDataURL(blob);
- });
- const data = dataUrl.slice(dataUrl.indexOf(',') + 1);
- await model.submitAnnotation(data, annotations);
- }, [model]);
-
function flashInteractiveHint() {
setFlashTick(tick => tick + 1);
}
@@ -155,13 +145,6 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
model.discardRecording();
}, [model]);
- const onCloseAnnotate = React.useCallback(() => {
- if (annotateInitiator === 'cli')
- model.completeAnnotation();
- else
- model.cancelAnnotate();
- }, [model, annotateInitiator]);
-
const selectedTab = tabs?.find(t => t.selected);
const ready = !!selectedTab;
@@ -247,148 +230,161 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
const overlayText = selectedTab ? undefined : 'Select a session';
const isRecording = recording?.phase === 'recording';
- const showAnnotateModal = !!annotateFrame;
const showRecording = recording?.phase === 'stopped';
- const modeLabel = showAnnotateModal ? 'Dashboard: annotate' : isRecording ? 'Dashboard: record' : 'Dashboard';
+ const modeLabel = annotateActive ? 'Dashboard: annotate' : isRecording ? 'Dashboard: record' : 'Dashboard';
+ const overlayOpen = !!selectedFrame;
return (
- <main className={'dashboard-view' + (interactive ? ' interactive' : '')} aria-label={modeLabel}>
- {/* Toolbar */}
- <div ref={toolbarRef} className='toolbar'>
- <ToolbarButton
- ref={interactiveBtnRef}
- className='mode-toggle mode-interactive'
- title={interactive ? 'Disable interactive mode' : 'Enable interactive mode'}
- toggled={interactive}
- disabled={!ready}
- onClick={() => {
- if (interactive)
- model.toggleInteractive();
- else
- model.enterInteractive();
- }}
- >
- {interactive ? <LockOpenIcon /> : <LockIcon />}
- </ToolbarButton>
- <ToolbarButton
- className='mode-annotate'
- title='Annotate screenshot'
- disabled={!ready || !!pendingAnnotate || showAnnotateModal}
- onClick={() => model.enterAnnotate('user')}
- >
- <ScreenshotRegionIcon />
- </ToolbarButton>
- <ToolbarButton
- className='mode-toggle mode-record'
- title={isRecording ? 'Stop recording' : 'Record video'}
- icon='record'
- toggled={isRecording}
- disabled={!ready || showRecording}
- onClick={() => {
- if (isRecording)
- model.stopRecording();
- else
- model.startRecording();
- }}
- >
- {isRecording && <span className='mode-record-label'>Recording...</span>}
- </ToolbarButton>
- </div>
+ <main className={'dashboard-view' + (interactive ? ' interactive' : '') + (annotateActive ? ' has-annotate-sidebar' : '')} aria-label={modeLabel}>
+ <div className='dashboard-main'>
+ {/* Toolbar */}
+ <div ref={toolbarRef} className='toolbar' hidden={overlayOpen}>
+ <ToolbarButton
+ ref={interactiveBtnRef}
+ className='mode-toggle mode-interactive'
+ title={interactive ? 'Disable interactive mode' : 'Enable interactive mode'}
+ toggled={interactive}
+ disabled={!ready}
+ onClick={() => {
+ if (interactive)
+ model.toggleInteractive();
+ else
+ model.enterInteractive();
+ }}
+ >
+ {interactive ? <LockOpenIcon /> : <LockIcon />}
+ </ToolbarButton>
+ <ToolbarButton
+ className='mode-annotate'
+ title={annotateActive ? 'Add screenshot' : 'Take screenshot'}
+ disabled={!ready || pendingCapture}
+ onClick={() => {
+ if (annotateActive)
+ model.addAnnotateFrame();
+ else
+ model.enterAnnotate('user');
+ }}
+ >
+ <ScreenshotRegionIcon />
+ </ToolbarButton>
+ <ToolbarButton
+ className='mode-toggle mode-record'
+ title={isRecording ? 'Stop recording' : 'Record video'}
+ icon='record'
+ toggled={isRecording}
+ disabled={!ready || showRecording}
+ onClick={() => {
+ if (isRecording)
+ model.stopRecording();
+ else
+ model.startRecording();
+ }}
+ >
+ {isRecording && <span className='mode-record-label'>Recording...</span>}
+ </ToolbarButton>
+ </div>
- {/* Viewport */}
- <div className='viewport-wrapper'>
- <div ref={viewportMainRef} className='viewport-main'>
- <div className='browser-window' style={windowStyle}>
- {showBrowserChrome && (
- <div ref={browserChromeRef} className='browser-chrome'>
- <button className='nav-btn' title='Back' aria-disabled={!interactive || undefined} onClick={() => {
- if (!interactive) {
- flashInteractiveHint();
- return;
- }
- model.back();
- }}>
- <ChevronLeftIcon />
- </button>
- <button className='nav-btn' title='Forward' aria-disabled={!interactive || undefined} onClick={() => {
- if (!interactive) {
- flashInteractiveHint();
- return;
- }
- model.forward();
- }}>
- <ChevronRightIcon />
- </button>
- <button className='nav-btn' title='Reload' aria-disabled={!interactive || undefined} onClick={() => {
- if (!interactive) {
- flashInteractiveHint();
- return;
- }
- model.reload();
- }}>
- <ReloadIcon />
- </button>
- <div className='omnibox-wrap'>
- <input
- id='omnibox'
- className='omnibox'
- type='text'
- placeholder='Search or enter URL'
- spellCheck={false}
- autoComplete='off'
- value={omniboxValue}
- onChange={e => setOmniboxValue(e.target.value)}
- onKeyDown={e => {
- if (!interactive)
- return;
- onOmniboxKeyDown(e);
- }}
- onFocus={e => {
- if (!interactive) {
- flashInteractiveHint();
- e.target.blur();
- return;
- }
- e.target.select();
- }}
- aria-disabled={!interactive || undefined}
- readOnly={!interactive}
- />
+ {/* Viewport */}
+ <div className='viewport-wrapper'>
+ <div ref={viewportMainRef} className='viewport-main'>
+ <div className='browser-window' style={windowStyle}>
+ {showBrowserChrome && (
+ <div ref={browserChromeRef} className='browser-chrome'>
+ <button className='nav-btn' title='Back' aria-disabled={!interactive || undefined} onClick={() => {
+ if (!interactive) {
+ flashInteractiveHint();
+ return;
+ }
+ model.back();
+ }}>
+ <ChevronLeftIcon />
+ </button>
+ <button className='nav-btn' title='Forward' aria-disabled={!interactive || undefined} onClick={() => {
+ if (!interactive) {
+ flashInteractiveHint();
+ return;
+ }
+ model.forward();
+ }}>
+ <ChevronRightIcon />
+ </button>
+ <button className='nav-btn' title='Reload' aria-disabled={!interactive || undefined} onClick={() => {
+ if (!interactive) {
+ flashInteractiveHint();
+ return;
+ }
+ model.reload();
+ }}>
+ <ReloadIcon />
+ </button>
+ <div className='omnibox-wrap'>
+ <input
+ id='omnibox'
+ className='omnibox'
+ type='text'
+ placeholder='Search or enter URL'
+ spellCheck={false}
+ autoComplete='off'
+ value={omniboxValue}
+ onChange={e => setOmniboxValue(e.target.value)}
+ onKeyDown={e => {
+ if (!interactive)
+ return;
+ onOmniboxKeyDown(e);
+ }}
+ onFocus={e => {
+ if (!interactive) {
+ flashInteractiveHint();
+ e.target.blur();
+ return;
+ }
+ e.target.select();
+ }}
+ aria-disabled={!interactive || undefined}
+ readOnly={!interactive}
+ />
+ </div>
</div>
+ )}
+ <div
+ ref={screenRef}
+ className='screen'
+ tabIndex={0}
+ style={{ display: liveFrame ? '' : 'none' }}
+ onMouseDown={onScreenMouseDown}
+ onMouseUp={onScreenMouseUp}
+ onMouseMove={onScreenMouseMove}
+ onWheel={onScreenWheel}
+ onKeyDown={onScreenKeyDown}
+ onKeyUp={onScreenKeyUp}
+ onContextMenu={e => e.preventDefault()}
+ >
+ <img
+ ref={displayRef}
+ id='display'
+ className='display'
+ alt='screencast'
+ src={liveFrame ? 'data:image/jpeg;base64,' + liveFrame.data : undefined}
+ />
</div>
- )}
- <div
- ref={screenRef}
- className='screen'
- tabIndex={0}
- style={{ display: liveFrame ? '' : 'none' }}
- onMouseDown={onScreenMouseDown}
- onMouseUp={onScreenMouseUp}
- onMouseMove={onScreenMouseMove}
- onWheel={onScreenWheel}
- onKeyDown={onScreenKeyDown}
- onKeyUp={onScreenKeyUp}
- onContextMenu={e => e.preventDefault()}
- >
- <img
- ref={displayRef}
- id='display'
- className='display'
- alt='screencast'
- src={liveFrame ? 'data:image/jpeg;base64,' + liveFrame.data : undefined}
- />
+ {overlayText && <div className={'screen-overlay' + (liveFrame ? ' has-frame' : '')}><span>{overlayText}</span></div>}
</div>
- {overlayText && <div className={'screen-overlay' + (liveFrame ? ' has-frame' : '')}><span>{overlayText}</span></div>}
</div>
</div>
</div>
-
- {showAnnotateModal && annotateFrame && (
- <AnnotateModal
- frame={annotateFrame}
- showSubmit={annotateInitiator === 'cli'}
- onSubmit={onSubmitAnnotations}
- onClose={onCloseAnnotate}
+ {selectedFrame && (
+ <AnnotateOverlay
+ key={selectedFrame.id + (annotateSession?.focusAnnotationId ?? '')}
+ model={model}
+ frame={selectedFrame}
+ focusAnnotationId={annotateSession?.focusAnnotationId}
+ />
+ )}
+ {annotateSession && (
+ <AnnotateSidebar
+ model={model}
+ session={annotateSession}
+ onSubmit={() => model.submitAnnotateSession()}
/>
)}
diff --git a/packages/dashboard/src/dashboardChannel.ts b/packages/dashboard/src/dashboardChannel.ts
index 2fe3b8799e..b09fc47e66 100644
--- a/packages/dashboard/src/dashboardChannel.ts
+++ b/packages/dashboard/src/dashboardChannel.ts
@@ -32,6 +32,17 @@ export type Tab = {
export type AnnotationData = { x: number; y: number; width: number; height: number; text: string };
+export type SubmittedAnnotationFrame = {
+ data: string;
+ ariaSnapshot: string;
+ annotations: AnnotationData[];
+ sessionTitle: string;
+ title: string;
+ url: string;
+ viewportWidth: number;
+ viewportHeight: number;
+};
+
export type DashboardChannelEvents = {
sessions: { sessions: SessionStatus[]; clientInfo: ClientInfo };
tabs: { tabs: Tab[] };
@@ -62,8 +73,9 @@ export interface DashboardChannel {
startRecording(): Promise<void>;
stopRecording(): Promise<{ streamId: string }>;
readStream(params: { streamId: string }): Promise<{ data: string; eof: boolean }>;
- screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number, ariaSnapshot: string }>;
- submitAnnotation(params: { data: string | undefined; annotations: AnnotationData[], ariaSnapshot: string }): Promise<void>;
+ screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number; ariaSnapshot: string }>;
+ submitAnnotation(params: { frames: SubmittedAnnotationFrame[]; feedback: string }): Promise<void>;
+ cancelAnnotation(): Promise<void>;
on<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;
off<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;
diff --git a/packages/dashboard/src/dashboardModel.ts b/packages/dashboard/src/dashboardModel.ts
index 3896ab71d1..30f4c38bf0 100644
--- a/packages/dashboard/src/dashboardModel.ts
+++ b/packages/dashboard/src/dashboardModel.ts
@@ -14,8 +14,11 @@
* limitations under the License.
*/
+import { buildAnnotatedImage, saveAnnotationAsDownload } from './annotationImage';
+import { buildAnnotationZip } from './annotationZip';
+
import type { Annotation } from './annotations';
-import type { DashboardChannel, DashboardChannelEvents, MouseButton, SessionStatus, Tab } from './dashboardChannel';
+import type { DashboardChannel, DashboardChannelEvents, MouseButton, SessionStatus, SubmittedAnnotationFrame, Tab } from './dashboardChannel';
import type { ClientInfo } from '../../playwright-core/src/tools/cli-client/registry';
import type { BrowserDescriptor } from '../../playwright-core/src/serverRegistry';
@@ -27,7 +30,25 @@ export type RecordingState =
| { phase: 'recording' }
| { phase: 'stopped'; blob: Blob; blobUrl: string };
-export type AnnotateFrame = { data: string; viewportWidth: number; viewportHeight: number; ariaSnapshot: string };
+export type AnnotateFrame = {
+ id: string;
+ data: string;
+ viewportWidth: number;
+ viewportHeight: number;
+ ariaSnapshot: string;
+ sessionTitle: string;
+ title: string;
+ url: string;
+ annotations: Annotation[];
+};
+
+export type AnnotateSession = {
+ initiator: 'cli' | 'user';
+ frames: AnnotateFrame[];
+ selectedFrameId: string | null;
+ focusAnnotationId: string | null;
+ feedback: string;
+};
export type DashboardState = {
// Session model state.
@@ -37,9 +58,8 @@ export type DashboardState = {
// Dashboard / page state.
tabs: Tab[] | null;
liveFrame: DashboardChannelEvents['frame'] | undefined;
- annotateFrame: AnnotateFrame | undefined;
- pendingAnnotate: { initiator: 'cli' | 'user' } | null;
- annotateInitiator: 'cli' | 'user' | null;
+ annotateSession: AnnotateSession | null;
+ pendingCapture: boolean;
mode: Mode;
recording: RecordingState | null;
};
@@ -52,9 +72,8 @@ const initialState: DashboardState = {
loadingSessions: true,
tabs: null,
liveFrame: undefined,
- annotateFrame: undefined,
- pendingAnnotate: null,
- annotateInitiator: null,
+ annotateSession: null,
+ pendingCapture: false,
mode: 'readonly',
recording: null,
};
@@ -74,7 +93,7 @@ export class DashboardModel {
client.on('tabs', params => this._emit({ tabs: params.tabs }));
client.on('frame', params => this._emit({ liveFrame: params }));
client.on('annotate', () => this.enterAnnotate('cli'));
- client.on('cancelAnnotate', () => this.cancelAnnotate());
+ client.on('cancelAnnotate', () => this.cancelAnnotate(false));
}
subscribe(listener: Listener): () => void {
@@ -158,7 +177,10 @@ export class DashboardModel {
toggleInteractive() {
const next: Mode = this.state.mode === 'interactive' ? 'readonly' : 'interactive';
- this._emit({ mode: next });
+ if (next === 'interactive')
+ void this._enterInteractive();
+ else
+ this._emit({ mode: next });
}
// Public action methods are sync fire-and-forget wrappers around the
@@ -174,6 +196,62 @@ export class DashboardModel {
void this._enterAnnotate(initiator);
}
+ addAnnotateFrame() {
+ void this._addAnnotateFrame();
+ }
+
+ selectAnnotateFrame(id: string, focusAnnotationId?: string) {
+ const session = this.state.annotateSession;
+ if (!session || !session.frames.find(f => f.id === id))
+ return;
+ this._emit({ annotateSession: { ...session, selectedFrameId: id, focusAnnotationId: focusAnnotationId ?? null }, mode: 'annotate' });
+ }
+
+ toggleSelectFrame(id: string) {
+ const session = this.state.annotateSession;
+ if (!session)
+ return;
+ if (session.selectedFrameId === id)
+ this._emit({ annotateSession: { ...session, selectedFrameId: null, focusAnnotationId: null }, mode: 'readonly' });
+ else
+ this.selectAnnotateFrame(id);
+ }
+
+ deselectFrame() {
+ const session = this.state.annotateSession;
+ if (!session || session.selectedFrameId === null)
+ return;
+ this._emit({ annotateSession: { ...session, selectedFrameId: null, focusAnnotationId: null }, mode: 'readonly' });
+ }
+
+ removeAnnotateFrame(id: string) {
+ const session = this.state.annotateSession;
+ if (!session)
+ return;
+ const frames = session.frames.filter(f => f.id !== id);
+ if (frames.length === 0) {
+ this.cancelAnnotate();
+ return;
+ }
+ const selectedFrameId = session.selectedFrameId === id ? null : session.selectedFrameId;
+ this._emit({ annotateSession: { ...session, frames, selectedFrameId } });
+ }
+
+ updateFrameAnnotations(frameId: string, annotations: Annotation[]) {
+ const session = this.state.annotateSession;
+ if (!session)
+ return;
+ const frames = session.frames.map(f => f.id === frameId ? { ...f, annotations } : f);
+ this._emit({ annotateSession: { ...session, frames } });
+ }
+
+ updateFeedback(feedback: string) {
+ const session = this.state.annotateSession;
+ if (!session)
+ return;
+ this._emit({ annotateSession: { ...session, feedback } });
+ }
+
completeAnnotation() {
void this._completeAnnotation();
}
@@ -190,47 +268,122 @@ export class DashboardModel {
void this._discardRecording();
}
- cancelAnnotate() {
+ cancelAnnotate(notifyServer = true) {
this._requestId++;
const s = this.state;
+ if (notifyServer && s.annotateSession?.initiator === 'cli')
+ void this._client.cancelAnnotation();
this._emit({
mode: s.mode === 'annotate' ? 'readonly' : s.mode,
- annotateFrame: undefined,
- pendingAnnotate: null,
- annotateInitiator: null,
+ annotateSession: null,
+ pendingCapture: false,
});
}
- async submitAnnotation(data: string | undefined, annotations: Annotation[]) {
- await this._client.submitAnnotation({
- data,
- ariaSnapshot: this.state.annotateFrame?.ariaSnapshot ?? '',
- annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),
- });
+ async submitAnnotateSession() {
+ const session = this.state.annotateSession;
+ if (!session)
+ return;
+ const frames: SubmittedAnnotationFrame[] = [];
+ for (const frame of session.frames) {
+ const data = await renderFrameToBase64Png(frame);
+ frames.push({
+ data: data ?? frame.data,
+ ariaSnapshot: frame.ariaSnapshot,
+ annotations: frame.annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),
+ sessionTitle: frame.sessionTitle,
+ title: frame.title,
+ url: frame.url,
+ viewportWidth: frame.viewportWidth,
+ viewportHeight: frame.viewportHeight,
+ });
+ }
+ if (session.initiator === 'cli') {
+ await this._client.submitAnnotation({ frames, feedback: session.feedback });
+ } else {
+ const blob = await buildAnnotationZip(frames, session.feedback);
+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const saved = await saveAnnotationAsDownload(blob, `annotations-${stamp}.zip`);
+ if (!saved)
+ return; // user cancelled the file picker — keep the session open
+ }
this.cancelAnnotate();
}
private async _enterInteractive() {
- // Interactive mode coexists with recording so the user can drive the
- // page while it is being captured. Only in-flight annotation is cleared.
- await this._completeAnnotation();
- this._emit({ mode: 'interactive' });
+ const s = this.state;
+ // Interactive coexists with the annotate sidebar; only the overlay is
+ // dismissed so the live page is reachable.
+ this._emit({
+ mode: 'interactive',
+ annotateSession: s.annotateSession?.selectedFrameId
+ ? { ...s.annotateSession, selectedFrameId: null, focusAnnotationId: null }
+ : s.annotateSession,
+ });
}
private async _enterAnnotate(initiator: 'cli' | 'user') {
- await this._cleanupOnModeSwitch();
- this._requestAnnotate(initiator);
+ const s = this.state;
+ if (s.annotateSession) {
+ // Session already open: capture another frame and select it.
+ if (initiator === 'cli' && s.annotateSession.initiator !== 'cli')
+ this._emit({ annotateSession: { ...s.annotateSession, initiator: 'cli' } });
+ await this._addAnnotateFrame();
+ return;
+ }
+ await this._discardRecording();
+ await this._addAnnotateFrame(initiator);
+ }
+
+ private async _addAnnotateFrame(initiator?: 'cli' | 'user') {
+ if (this.state.pendingCapture)
+ return;
+ const myId = ++this._requestId;
+ this._emit({ pendingCapture: true });
+ let frameData: Awaited<ReturnType<DashboardChannel['screenshot']>> | undefined;
+ try {
+ frameData = await this._client.screenshot();
+ } catch {
+ // ignore
+ }
+ if (myId !== this._requestId)
+ return;
+ if (!frameData) {
+ this._emit({ pendingCapture: false });
+ return;
+ }
+ const selectedTab = this.state.tabs?.find(t => t.selected);
+ const sessionTitle = this.state.sessions.find(s => s.browser.guid === selectedTab?.browser)?.title ?? '';
+ const frame: AnnotateFrame = {
+ id: 'frm-' + Math.random().toString(36).slice(2, 10),
+ data: frameData.data,
+ viewportWidth: frameData.viewportWidth,
+ viewportHeight: frameData.viewportHeight,
+ ariaSnapshot: frameData.ariaSnapshot,
+ sessionTitle,
+ title: selectedTab?.title ?? '',
+ url: selectedTab?.url ?? '',
+ annotations: [],
+ };
+ const existing = this.state.annotateSession;
+ const session: AnnotateSession = existing
+ ? { ...existing, frames: [...existing.frames, frame], selectedFrameId: frame.id, focusAnnotationId: null }
+ : { initiator: initiator ?? 'user', frames: [frame], selectedFrameId: frame.id, focusAnnotationId: null, feedback: '' };
+ this._emit({ annotateSession: session, pendingCapture: false, mode: 'annotate' });
}
private async _completeAnnotation() {
const s = this.state;
- if (s.mode === 'annotate' && s.annotateInitiator === 'cli' && s.annotateFrame)
- await this.submitAnnotation(undefined, []).catch(() => {});
- this.cancelAnnotate();
+ if (s.annotateSession?.initiator === 'cli')
+ await this.submitAnnotateSession().catch(() => {});
+ else
+ this.cancelAnnotate();
}
private async _startRecording() {
- await this._cleanupOnModeSwitch();
+ // Recording closes any open annotate session.
+ if (this.state.annotateSession)
+ this.cancelAnnotate();
await this._client.startRecording();
this._emit({ recording: { phase: 'recording' } });
}
@@ -269,52 +422,6 @@ export class DashboardModel {
this._emit({ recording: null });
}
- private async _cleanupOnModeSwitch() {
- await this._discardRecording();
- await this._completeAnnotation();
- }
-
- private _requestAnnotate(initiator: 'cli' | 'user') {
- const s = this.state;
- if (s.mode === 'annotate') {
- if (initiator === 'cli' && s.annotateInitiator !== 'cli')
- this._emit({ annotateInitiator: 'cli' });
- return;
- }
- if (s.pendingAnnotate) {
- if (initiator === 'cli' && s.pendingAnnotate.initiator !== 'cli')
- this._emit({ pendingAnnotate: { initiator: 'cli' } });
- return;
- }
- const myId = ++this._requestId;
- this._emit({ pendingAnnotate: { initiator } });
- void this._fetchScreenshot(myId);
- }
-
- private async _fetchScreenshot(id: number) {
- let frame: AnnotateFrame | undefined;
- try {
- frame = await this._client.screenshot();
- } catch {
- // frame stays undefined
- }
- if (id !== this._requestId)
- return;
- const s = this.state;
- if (!s.pendingAnnotate)
- return;
- if (!frame) {
- this._emit({ pendingAnnotate: null });
- return;
- }
- this._emit({
- mode: 'annotate',
- annotateFrame: frame,
- annotateInitiator: s.pendingAnnotate.initiator,
- pendingAnnotate: null,
- });
- }
-
private _emit(partial: Partial<DashboardState>) {
this.state = { ...this.state, ...partial };
for (const listener of this._listeners)
@@ -322,6 +429,21 @@ export class DashboardModel {
}
}
+async function renderFrameToBase64Png(frame: AnnotateFrame): Promise<string | undefined> {
+ const img = new Image();
+ img.src = 'data:image/png;base64,' + frame.data;
+ try {
+ await img.decode();
+ } catch {
+ return undefined;
+ }
+ const blob = await buildAnnotatedImage(img, frame.viewportWidth, frame.viewportHeight, frame.annotations);
+ if (!blob)
+ return undefined;
+ const buf = await blob.arrayBuffer();
+ return new Uint8Array(buf).toBase64();
+}
+
function base64ToBlob(base64: string, mime: string): Blob {
- return new Blob([(Uint8Array as any).fromBase64(base64)], { type: mime });
+ return new Blob([Uint8Array.fromBase64(base64)], { type: mime });
}
diff --git a/packages/playwright-core/src/tools/backend/devtools.ts b/packages/playwright-core/src/tools/backend/devtools.ts
index 4fc8cf7794..79e0fb9afc 100644
--- a/packages/playwright-core/src/tools/backend/devtools.ts
+++ b/packages/playwright-core/src/tools/backend/devtools.ts
@@ -22,7 +22,7 @@ import { libPath } from '../../package';
import { defineTabTool, defineTool } from './tool';
import { elementSchema, optionalElementSchema } from './snapshot';
-import type { AnnotationData } from '@dashboard/dashboardChannel';
+import type { SubmittedAnnotationFrame } from '@dashboard/dashboardChannel';
const resume = defineTool({
capability: 'devtools',
@@ -135,7 +135,7 @@ const annotate = defineTabTool({
daemon.unref();
// Spawn the annotate client in JSON mode to capture the raw payload over stdout.
- const client = spawn(process.execPath, [...daemonArgs, '--annotate', '--json'], {
+ const client = spawn(process.execPath, [...daemonArgs, '--annotate'], {
stdio: ['pipe', 'pipe', 'inherit'],
});
const onAbort = () => client.kill();
@@ -157,14 +157,28 @@ const annotate = defineTabTool({
response.addTextResult('No annotations were submitted.');
return;
}
- const { png, ariaSnapshot, annotations } = JSON.parse(text) as { png?: string; ariaSnapshot?: string; annotations: AnnotationData[] };
- for (const a of annotations)
- response.addTextResult(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);
+ const { frames, feedback } = JSON.parse(text) as { frames: SubmittedAnnotationFrame[]; feedback: string };
+ if (!frames || frames.length === 0) {
+ response.addTextResult('No annotations were submitted.');
+ return;
+ }
const date = new Date();
- if (png)
- await response.addResult('Annotation image', Buffer.from(png, 'base64'), { prefix: 'annotations', ext: 'png', date });
- if (ariaSnapshot)
- await response.addResult('Annotation snapshot', Buffer.from(ariaSnapshot, 'utf8'), { prefix: 'annotations', ext: 'yaml', date });
+ if (feedback)
+ response.addTextResult(feedback);
+ const multi = frames.length > 1;
+ for (let i = 0; i < frames.length; i++) {
+ const frame = frames[i];
+ const idx = i + 1;
+ const session = frame.sessionTitle || 'session';
+ const tab = frame.title || 'tab';
+ response.addTextResult(`${multi ? `## Screenshot ${idx}\n` : ''}${session} / ${tab} @ ${frame.url} (${frame.viewportWidth}x${frame.viewportHeight})`);
+ for (const a of frame.annotations)
+ response.addTextResult(` { x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);
+ if (frame.data)
+ await response.addResult(`Annotation image${multi ? ' ' + idx : ''}`, Buffer.from(frame.data, 'base64'), { prefix: `annotations${multi ? '-' + idx : ''}`, ext: 'png', date });
+ if (frame.ariaSnapshot)
+ await response.addResult(`Annotation snapshot${multi ? ' ' + idx : ''}`, Buffer.from(frame.ariaSnapshot, 'utf8'), { prefix: `annotations${multi ? '-' + idx : ''}`, ext: 'yaml', date });
+ }
},
});
diff --git a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
index 0673df147a..b4a8a1a962 100644
--- a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
+++ b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
@@ -26,13 +26,12 @@ import { libPath } from '../../package';
import { playwright } from '../../inprocess';
import { findChromiumChannelBestEffort, registryDirectory } from '../../server/registry/index';
import { minimist } from '../cli-client/minimist';
-import { saveOutputFile } from '../trace/traceUtils';
import { DashboardConnection } from './dashboardController';
import { RegistrySessionProvider } from './registrySessionProvider';
import { IdentitySessionProvider } from './identitySessionProvider';
import type * as api from '../../..';
-import type { AnnotationData } from '@dashboard/dashboardChannel';
+import type { SubmittedAnnotationFrame } from '@dashboard/dashboardChannel';
import type { SessionProvider } from './sessionProvider';
// HMR: build-time flag — `true` in watch builds, `false` in release. esbuild
@@ -57,10 +56,10 @@ async function startDashboardServer(provider: SessionProvider, options: Dashboar
let pendingAnnotate = false;
const waitingSockets = new Set<net.Socket>();
- const submitAnnotation = (base64Png: string | undefined, ariaSnapshot: string, annotations: AnnotationData[]) => {
+ const submitAnnotation = (frames: SubmittedAnnotationFrame[], feedback: string) => {
if (waitingSockets.size === 0)
return;
- const payload = JSON.stringify({ png: base64Png, ariaSnapshot, annotations });
+ const payload = JSON.stringify({ frames, feedback });
for (const socket of waitingSockets) {
socket.write(payload);
socket.end();
@@ -251,13 +250,12 @@ type DashboardOptions = {
pageId?: string;
kill?: boolean;
annotate?: boolean;
- json?: boolean;
port?: number;
host?: string;
};
function parseOpenArgs(): DashboardOptions {
- const args = minimist(process.argv.slice(2), { string: ['sessionName', 'workspaceDir', 'host', 'pageId'], boolean: ['annotate', 'kill', 'json'] });
+ const args = minimist(process.argv.slice(2), { string: ['sessionName', 'workspaceDir', 'host', 'pageId'], boolean: ['annotate', 'kill'] });
const portStr = args.port as string | undefined;
return {
sessionName: args.sessionName as string | undefined,
@@ -266,7 +264,6 @@ function parseOpenArgs(): DashboardOptions {
port: portStr !== undefined ? Number(portStr) : undefined,
host: args.host as string | undefined,
annotate: !!args.annotate,
- json: !!args.json,
kill: !!args.kill,
};
}
@@ -429,27 +426,8 @@ async function runAnnotateClient(options: DashboardOptions): Promise<void> {
const text = Buffer.concat(chunks).toString();
if (!text)
return;
- if (options.json) {
- // eslint-disable-next-line no-console
- console.log(text);
- return;
- }
- const { png, annotations, ariaSnapshot } = JSON.parse(text) as { png: string; annotations: AnnotationData[], ariaSnapshot: string };
- for (const a of annotations) {
- // eslint-disable-next-line no-console
- console.log(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);
- }
- const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
- if (png) {
- const filePath = await saveOutputFile(`annotations-${timestamp}.png`, Buffer.from(png, 'base64'));
- // eslint-disable-next-line no-console
- console.log(`image: ${path.relative(process.cwd(), filePath)}`);
- }
- if (ariaSnapshot) {
- const filePath = await saveOutputFile(`annotations-${timestamp}.yaml`, ariaSnapshot);
- // eslint-disable-next-line no-console
- console.log(`snapshot: ${path.relative(process.cwd(), filePath)}`);
- }
+ // eslint-disable-next-line no-console
+ console.log(text);
}
function selfDestructOnParentGone() {
diff --git a/packages/playwright-core/src/tools/dashboard/dashboardController.ts b/packages/playwright-core/src/tools/dashboard/dashboardController.ts
index 79e0c503c5..f27ecb3e0a 100644
--- a/packages/playwright-core/src/tools/dashboard/dashboardController.ts
+++ b/packages/playwright-core/src/tools/dashboard/dashboardController.ts
@@ -27,7 +27,7 @@ import { SessionProviderEvent } from './sessionProvider';
import type * as api from '../../..';
import type { Transport } from '@utils/httpServer';
-import type { AnnotationData, Tab } from '@dashboard/dashboardChannel';
+import type { SubmittedAnnotationFrame, Tab } from '@dashboard/dashboardChannel';
import type { BrowserDescriptor } from '../../serverRegistry';
import type { SessionProvider } from './sessionProvider';
@@ -39,7 +39,7 @@ export class DashboardConnection implements Transport {
private _attachedPage: AttachedPage | undefined;
private _onclose: () => void;
private _onconnected?: () => void;
- private _onAnnotationSubmit?: (base64Png: string | undefined, ariaSnapshot: string, annotations: AnnotationData[]) => void;
+ private _onAnnotationSubmit?: (frames: SubmittedAnnotationFrame[], feedback: string) => void;
private _pushTabsScheduled = false;
private _visible = true;
private _pendingReveal: { sessionName?: string; workspaceDir?: string; pageId?: string } | undefined;
@@ -48,7 +48,7 @@ export class DashboardConnection implements Transport {
_recordingDir: string;
_streams = new Map<string, { handle: fs.promises.FileHandle; path: string }>();
- constructor(provider: SessionProvider, onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string | undefined, ariaSnapshot: string, annotations: AnnotationData[]) => void) {
+ constructor(provider: SessionProvider, onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (frames: SubmittedAnnotationFrame[], feedback: string) => void) {
this._provider = provider;
this._onclose = onclose;
this._onconnected = onconnected;
@@ -163,8 +163,12 @@ export class DashboardConnection implements Transport {
this._pushTabs();
}
- async submitAnnotation(params: { data: string | undefined; ariaSnapshot: string; annotations: AnnotationData[] }) {
- this._onAnnotationSubmit?.(params.data, params.ariaSnapshot, params.annotations);
+ async submitAnnotation(params: { frames: SubmittedAnnotationFrame[]; feedback: string }) {
+ this._onAnnotationSubmit?.(params.frames, params.feedback);
+ }
+
+ async cancelAnnotation() {
+ this._onAnnotationSubmit?.([], '');
}
async reveal(params: { path: string }) {
@@ -425,10 +429,10 @@ class AttachedPage {
return { streamId };
}
- async screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number, ariaSnapshot: string }> {
+ async screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number; ariaSnapshot: string }> {
const buffer = await this._page.screenshot({ type: 'png' });
- const vp = await this._viewportSize();
const ariaSnapshot = await this._page.ariaSnapshot({ boxes: true, mode: 'ai' });
+ const vp = await this._viewportSize();
return {
data: buffer.toString('base64'),
viewportWidth: vp.width,
diff --git a/tests/mcp/annotate.spec.ts b/tests/mcp/annotate.spec.ts
new file mode 100644
index 0000000000..eb91e63be3
--- /dev/null
+++ b/tests/mcp/annotate.spec.ts
@@ -0,0 +1,500 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+import * as zipjs from '@zip.js/zip.js';
+
+import { test, expect, installSaveFilePickerMock, mockAbortingFilePicker } from './cli-fixtures';
+import { inheritAndCleanEnv } from '../config/utils';
+
+function activeSession(dashboard: import('playwright-core').Page) {
+ return dashboard.getByRole('region', { name: /^Session / }).filter({ has: dashboard.getByRole('option', { selected: true }) });
+}
+
+async function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page, text: string) {
+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
+ await expect(dashboard.locator('.annotate-modal-image')).toBeVisible();
+ const box = await dashboard.locator('.annotate-modal-image').boundingBox();
+ const x0 = box!.x + box!.width * 0.3;
+ const y0 = box!.y + box!.height * 0.3;
+ const x1 = box!.x + box!.width * 0.6;
+ const y1 = box!.y + box!.height * 0.6;
+ await dashboard.mouse.move(x0, y0);
+ await dashboard.mouse.down();
+ await dashboard.mouse.move(x1, y1);
+ await dashboard.mouse.up();
+ await dashboard.locator('.annotations-textarea').fill(text);
+ await dashboard.locator('.annotations-textarea').press('Enter');
+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();
+ await dashboard.getByRole('button', { name: 'Submit' }).click();
+}
+
+function verifyAnnotateOutput(output: string, expectedText: string, outputDir: string) {
+ expect(output).toMatch(/.* @ .* \(\d+x\d+\)/);
+ expect(output).toMatch(new RegExp(`\\{ x: \\d+, y: \\d+, width: \\d+, height: \\d+ \\}: ${expectedText}`));
+ const imageMatch = output.match(/- \[Annotation image\]\((\.playwright-cli[\\/]annotations-.*\.png)\)/);
+ expect(imageMatch).not.toBeNull();
+ const pngPath = path.resolve(outputDir, imageMatch![1]);
+ expect(fs.existsSync(pngPath)).toBe(true);
+ expect(fs.statSync(pngPath).size).toBeGreaterThan(0);
+}
+
+test('should capture multiple screenshots in one annotation', async ({ connectToDashboard, cli, server }) => {
+ await cli('open', server.PREFIX + '/title.html');
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ await cli('show', { bindTitle });
+ const browser = await connectToDashboard(bindTitle);
+
+ const dashboard = browser.contexts()[0].pages()[0];
+ await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();
+
+ const annotatePromise = cli('show', '--annotate');
+ let done = false;
+ void annotatePromise.finally(() => { done = true; });
+
+ await expect(dashboard.locator('.annotate-modal-image')).toBeVisible();
+ // First annotation on initial frame.
+ let box = await dashboard.locator('.annotate-modal-image').boundingBox();
+ await dashboard.mouse.move(box!.x + box!.width * 0.2, box!.y + box!.height * 0.2);
+ await dashboard.mouse.down();
+ await dashboard.mouse.move(box!.x + box!.width * 0.4, box!.y + box!.height * 0.4);
+ await dashboard.mouse.up();
+ await dashboard.locator('.annotations-textarea').fill('first');
+ await dashboard.locator('.annotations-textarea').press('Enter');
+
+ // Deselect overlay (sidebar stays), then capture a second frame via the toolbar button.
+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();
+ await expect(dashboard.locator('.annotate-overlay')).toHaveCount(0);
+ await dashboard.getByRole('button', { name: /^(Take|Add) screenshot$/ }).click();
+
+ // New frame auto-selected; draw + label.
+ await expect(dashboard.locator('.annotate-sidebar-thumb')).toHaveCount(2);
+ await expect(dashboard.locator('.annotate-overlay')).toBeVisible();
+ box = await dashboard.locator('.annotate-modal-image').boundingBox();
+ await dashboard.mouse.move(box!.x + box!.width * 0.5, box!.y + box!.height * 0.5);
+ await dashboard.mouse.down();
+ await dashboard.mouse.move(box!.x + box!.width * 0.7, box!.y + box!.height * 0.7);
+ await dashboard.mouse.up();
+ await dashboard.locator('.annotations-textarea').fill('second');
+ await dashboard.locator('.annotations-textarea').press('Enter');
+
+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();
+ await dashboard.getByRole('button', { name: 'Submit' }).click();
+
+ const { output, exitCode } = await annotatePromise;
+ expect(done).toBe(true);
+ expect(exitCode).toBe(0);
+ expect(output).toMatch(/## Screenshot 1/);
+ expect(output).toMatch(/## Screenshot 2/);
+ expect(output).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: first/);
+ expect(output).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: second/);
+ expect(output).toMatch(/- \[Annotation image 1\]\(.*annotations-1-.*\.png\)/);
+ expect(output).toMatch(/- \[Annotation image 2\]\(.*annotations-2-.*\.png\)/);
+});
+
+test('should abort annotation when last screenshot is removed', async ({ connectToDashboard, cli, server }) => {
+ await cli('open', server.EMPTY_PAGE);
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ await cli('show', { bindTitle });
+ const browser = await connectToDashboard(bindTitle);
+
+ const dashboard = browser.contexts()[0].pages()[0];
+ await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();
+
+ const annotatePromise = cli('show', '--annotate');
+ let done = false;
+ void annotatePromise.finally(() => { done = true; });
+
+ await expect(dashboard.locator('.annotate-sidebar-thumb')).toHaveCount(1);
+
+ // Close the fullscreen overlay first so the sidebar remove button is accessible.
+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();
+
+ // Remove the only screenshot — should abort the annotation
+ await dashboard.locator('.annotate-sidebar-thumb-remove').click();
+
+ const { output, exitCode } = await annotatePromise;
+ expect(done).toBe(true);
+ expect(exitCode).toBe(0);
+ expect(output).toContain('No annotations were submitted');
+});
+
+test('should abort MCP annotation when last screenshot is removed', async ({ connectToDashboard, boundBrowser, startClient, cliEnv }) => {
+ await boundBrowser.newPage();
+
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ const { client } = await startClient({
+ args: ['--endpoint=default', '--caps=devtools'],
+ env: {
+ ...cliEnv,
+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,
+ },
+ });
+
+ const annotatePromise = client.callTool({ name: 'browser_annotate' });
+ let done = false;
+ void annotatePromise.then(() => { done = true; });
+
+ const browser = await connectToDashboard(bindTitle);
+ try {
+ const dashboard = browser.contexts()[0].pages()[0];
+ await expect(dashboard.locator('.annotate-sidebar-thumb')).toHaveCount(1);
+
+ // Close the fullscreen overlay first so the sidebar remove button is accessible.
+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();
+
+ // Remove the only screenshot — should abort
+ await dashboard.locator('.annotate-sidebar-thumb-remove').click();
+ } finally {
+ await browser.close().catch(() => {});
+ }
+
+ const result = await annotatePromise;
+ expect(done).toBe(true);
+ const text = (result.content as any).map((c: any) => c.text ?? '').join('\n');
+ expect(text).toContain('No annotations were submitted');
+});
+
+test('user-initiated annotate downloads zip with feedback.md', async ({ connectToDashboard, cli, server }) => {
+ await cli('open', server.EMPTY_PAGE);
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ await cli('show', { bindTitle });
+ const browser = await connectToDashboard(bindTitle);
+
+ const dashboard = browser.contexts()[0].pages()[0];
+ await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();
+
+ // Start with an aborting picker to verify cancellation keeps the session intact.
+ await mockAbortingFilePicker(dashboard);
+
+ // Enter annotate via toolbar (user-initiated).
+ await dashboard.getByRole('button', { name: /^(Take|Add) screenshot$/ }).click();
+ await expect(dashboard.locator('.annotate-modal-image')).toBeVisible();
+
+ // Draw an annotation.
+ const box = await dashboard.locator('.annotate-modal-image').boundingBox();
+ await dashboard.mouse.move(box!.x + box!.width * 0.2, box!.y + box!.height * 0.2);
+ await dashboard.mouse.down();
+ await dashboard.mouse.move(box!.x + box!.width * 0.5, box!.y + box!.height * 0.5);
+ await dashboard.mouse.up();
+ await dashboard.locator('.annotations-textarea').fill('zip-test');
+ await dashboard.locator('.annotations-textarea').press('Enter');
+
+ // Close overlay, type feedback, submit → picker aborts.
+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();
+ await dashboard.locator('.annotate-sidebar-feedback').fill('My feedback');
+ await dashboard.getByRole('button', { name: 'Submit', exact: true }).click();
+
+ // Session must still be open with data intact.
+ await expect(dashboard.locator('.annotate-sidebar')).toBeVisible();
+ await expect(dashboard.locator('.annotate-sidebar-thumb')).toHaveCount(1);
+ // Wait until the in-flight (aborted) submit fully resolves and the button is re-enabled,
+ // otherwise installing the next picker mid-flight would race.
+ await expect(dashboard.getByRole('button', { name: 'Submit', exact: true })).toBeEnabled();
+
+ // Now install a capturing picker and submit for real.
+ const awaitZipBytes = await installSaveFilePickerMock(dashboard);
+ await dashboard.getByRole('button', { name: 'Submit', exact: true }).click();
+
+ const zipBytes = await awaitZipBytes();
+
+ zipjs.configure({ useWebWorkers: false });
+ const entries = await new zipjs.ZipReader(new zipjs.Uint8ArrayReader(zipBytes)).getEntries();
+ const names = entries.map(e => e.filename);
+ expect(names).toContain('feedback.md');
+ expect(names).toContain('annotations-1.png');
+
+ const mdEntry = entries.find(e => e.filename === 'feedback.md')!;
+ const mdText = await mdEntry.getData!(new zipjs.TextWriter());
+ expect(mdText).toContain('My feedback');
+ expect(mdText).toContain('Screenshot 1');
+ expect(mdText).toContain('[Screenshot 1](annotations-1.png)');
+ expect(mdText).toMatch(/\d+x\d+/);
+});
+
+test('should capture annotations via show --annotate', async ({ connectToDashboard, cli, server }) => {
+ await cli('open', server.EMPTY_PAGE);
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ await cli('show', { bindTitle });
+ const browser = await connectToDashboard(bindTitle);
+
+ const dashboard = browser.contexts()[0].pages()[0];
+ await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();
+
+ const annotatePromise = cli('show', '--annotate');
+ let done = false;
+ void annotatePromise.finally(() => { done = true; });
+
+ await drawAndSubmitAnnotation(dashboard, 'hello');
+
+ const { output, exitCode } = await annotatePromise;
+ expect(done).toBe(true);
+ expect(exitCode).toBe(0);
+ verifyAnnotateOutput(output, 'hello', test.info().outputDir);
+});
+
+test('should start dashboard and annotate when no dashboard is running', async ({ connectToDashboard, cli, server }) => {
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ await cli('open', server.EMPTY_PAGE, { bindTitle });
+
+ const annotatePromise = cli('show', '--annotate');
+ let done = false;
+ void annotatePromise.finally(() => { done = true; });
+
+ const browser = await connectToDashboard(bindTitle);
+ try {
+ const dashboard = browser.contexts()[0].pages()[0];
+ await drawAndSubmitAnnotation(dashboard, 'hi');
+ } finally {
+ await browser.close().catch(() => {});
+ }
+
+ const { output, exitCode } = await annotatePromise;
+ expect(done).toBe(true);
+ expect(exitCode).toBe(0);
+ verifyAnnotateOutput(output, 'hi', test.info().outputDir);
+});
+
+test('should enter annotate mode on fresh dashboard.tsx mount with -s --annotate', async ({ connectToDashboard, cli, server }) => {
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ await cli('-s=first', 'open', server.EMPTY_PAGE, { bindTitle });
+ await cli('-s=second', 'open', server.EMPTY_PAGE, { bindTitle });
+
+ const annotatePromise = cli('-s=second', 'show', '--annotate');
+ let done = false;
+ void annotatePromise.finally(() => { done = true; });
+
+ const browser = await connectToDashboard(bindTitle);
+ try {
+ const dashboard = browser.contexts()[0].pages()[0];
+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
+ await expect(activeSession(dashboard)).toHaveAccessibleName('Session second');
+ await drawAndSubmitAnnotation(dashboard, 'fresh');
+ } finally {
+ await browser.close().catch(() => {});
+ }
+
+ const { exitCode } = await annotatePromise;
+ expect(done).toBe(true);
+ expect(exitCode).toBe(0);
+});
+
+test('should annotate via direct browser_annotate MCP call', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {
+ const page = await boundBrowser.newPage();
+ await page.goto(server.EMPTY_PAGE);
+
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ const { client } = await startClient({
+ args: ['--endpoint=default', '--caps=devtools'],
+ env: {
+ ...cliEnv,
+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,
+ },
+ });
+
+ const annotatePromise = client.callTool({ name: 'browser_annotate' });
+ let done = false;
+ void annotatePromise.then(() => { done = true; });
+
+ const browser = await connectToDashboard(bindTitle);
+ try {
+ const dashboard = browser.contexts()[0].pages()[0];
+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
+ await drawAndSubmitAnnotation(dashboard, 'direct-mcp');
+ } finally {
+ await browser.close().catch(() => {});
+ }
+
+ const result = await annotatePromise;
+ expect(done).toBe(true);
+ const text = (result.content as any).map(c => c.text ?? '').join('\n');
+ expect(text).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: direct-mcp/);
+ expect(text).toMatch(/- \[Annotation image\]\(.*\.png\)/);
+});
+
+test('should annotate when context has no fixed viewport', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {
+ // Simulates headed `playwright-cli open --headed`, which launches with viewport: null
+ // so that the browser window controls the page size. https://github.com/microsoft/playwright/issues/40565
+ const context = await boundBrowser.newContext({ viewport: null });
+ const page = await context.newPage();
+ expect(page.viewportSize()).toBe(null);
+ await page.goto(server.EMPTY_PAGE);
+
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ const { client } = await startClient({
+ args: ['--endpoint=default', '--caps=devtools'],
+ env: {
+ ...cliEnv,
+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,
+ },
+ });
+
+ const annotatePromise = client.callTool({ name: 'browser_annotate' });
+ let done = false;
+ void annotatePromise.then(() => { done = true; });
+
+ const browser = await connectToDashboard(bindTitle);
+ try {
+ const dashboard = browser.contexts()[0].pages()[0];
+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
+ await drawAndSubmitAnnotation(dashboard, 'no-viewport');
+ } finally {
+ await browser.close().catch(() => {});
+ }
+
+ const result = await annotatePromise;
+ expect(done).toBe(true);
+ const text = (result.content as any).map(c => c.text ?? '').join('\n');
+ expect(text).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: no-viewport/);
+ expect(text).toMatch(/- \[Annotation image\]\(.*\.png\)/);
+});
+
+test('should cancel browser_annotate when the MCP request is aborted', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {
+ const page = await boundBrowser.newPage();
+ await page.goto(server.EMPTY_PAGE);
+
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ const { client } = await startClient({
+ args: ['--endpoint=default', '--caps=devtools'],
+ env: {
+ ...cliEnv,
+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,
+ },
+ });
+
+ const controller = new AbortController();
+ const annotatePromise = client.callTool({ name: 'browser_annotate' }, undefined, { signal: controller.signal }).catch(() => {});
+
+ const browser = await connectToDashboard(bindTitle);
+ try {
+ const dashboard = browser.contexts()[0].pages()[0];
+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
+
+ controller.abort();
+
+ await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();
+ } finally {
+ await browser.close().catch(() => {});
+ }
+
+ await annotatePromise;
+});
+
+test('should cancel browser_annotate when the MCP client disconnects', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {
+ const page = await boundBrowser.newPage();
+ await page.goto(server.EMPTY_PAGE);
+
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ const { client } = await startClient({
+ args: ['--endpoint=default', '--caps=devtools'],
+ env: {
+ ...cliEnv,
+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,
+ },
+ });
+
+ void client.callTool({ name: 'browser_annotate' }).catch(() => {});
+
+ const browser = await connectToDashboard(bindTitle);
+ try {
+ const dashboard = browser.contexts()[0].pages()[0];
+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
+
+ await client.close();
+
+ await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();
+ } finally {
+ await browser.close().catch(() => {});
+ }
+});
+
+
+test('should switch screencast to -s session on show --annotate', async ({ connectToDashboard, cli, server }) => {
+ server.setContent('/red', '<html><head><style>html,body{margin:0;height:100vh;background:#ff0000}</style></head><body></body></html>', 'text/html');
+ server.setContent('/green', '<html><head><style>html,body{margin:0;height:100vh;background:#00ff00}</style></head><body></body></html>', 'text/html');
+
+ await cli('-s=first', 'open', server.PREFIX + '/red');
+ await cli('-s=second', 'open', server.PREFIX + '/green');
+
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ await cli('-s=first', 'show', { bindTitle });
+ const browser = await connectToDashboard(bindTitle);
+ const dashboard = browser.contexts()[0].pages()[0];
+ await expect(dashboard.locator('#display')).toBeVisible();
+
+ const sampleCenter = () => dashboard.evaluate(() => {
+ const img = document.querySelector('#display') as HTMLImageElement | null;
+ if (!img || !img.naturalWidth)
+ return null;
+ const canvas = document.createElement('canvas');
+ canvas.width = 1;
+ canvas.height = 1;
+ const ctx = canvas.getContext('2d')!;
+ ctx.drawImage(img, img.naturalWidth / 2, img.naturalHeight / 2, 1, 1, 0, 0, 1, 1);
+ const [r, g] = ctx.getImageData(0, 0, 1, 1).data;
+ return { r, g };
+ });
+
+ await expect.poll(async () => {
+ const c = await sampleCenter();
+ return !!(c && c.r > 200 && c.g < 50);
+ }, { timeout: 15000 }).toBe(true);
+
+ const annotatePromise = cli('-s=second', 'show', '--annotate');
+ let done = false;
+ void annotatePromise.finally(() => { done = true; });
+
+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
+ await expect(activeSession(dashboard)).toHaveAccessibleName('Session second');
+
+ await expect.poll(async () => {
+ const c = await sampleCenter();
+ return !!(c && c.g > 200 && c.r < 50);
+ }, { timeout: 15000 }).toBe(true);
+
+ await drawAndSubmitAnnotation(dashboard, 'session switch');
+ const { exitCode } = await annotatePromise;
+ expect(done).toBe(true);
+ expect(exitCode).toBe(0);
+});
+
+test('should disengage annotate mode when --annotate client disconnects', async ({ connectToDashboard, cli, childProcess, cliEnv, mcpBrowser, mcpHeadless, server }) => {
+ await cli('open', server.EMPTY_PAGE);
+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
+ await cli('show', { bindTitle });
+ const browser = await connectToDashboard(bindTitle);
+
+ const dashboard = browser.contexts()[0].pages()[0];
+ await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();
+
+ const annotateClient = childProcess({
+ command: [process.execPath, require.resolve('../../packages/playwright-core/lib/tools/cli-client/cli.js'), 'show', '--annotate'],
+ cwd: test.info().outputPath(),
+ env: inheritAndCleanEnv({
+ ...cliEnv,
+ PLAYWRIGHT_MCP_BROWSER: mcpBrowser,
+ PLAYWRIGHT_MCP_HEADLESS: String(mcpHeadless),
+ }),
+ });
+
+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
+
+ await annotateClient.kill();
+
+ await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();
+});
diff --git a/tests/mcp/cli-fixtures.ts b/tests/mcp/cli-fixtures.ts
index dea738a7f3..4c7413f228 100644
--- a/tests/mcp/cli-fixtures.ts
+++ b/tests/mcp/cli-fixtures.ts
@@ -261,3 +261,45 @@ export async function daemonFolder() {
}
return null;
}
+
+export async function installSaveFilePickerMock(page: import('playwright-core').Page): Promise<() => Promise<Buffer>> {
+ await page.evaluate(() => {
+ (window as any).__testCaptureBytes = undefined as string | undefined;
+ (window as any).showSaveFilePicker = async () => ({
+ createWritable: async () => {
+ const chunks: Uint8Array[] = [];
+ return {
+ write: async (chunk: Blob | BufferSource) => {
+ const buf = chunk instanceof Blob
+ ? new Uint8Array(await chunk.arrayBuffer())
+ : new Uint8Array(chunk instanceof ArrayBuffer ? chunk : (chunk as ArrayBufferView).buffer);
+ chunks.push(buf);
+ },
+ close: async () => {
+ const total = chunks.reduce((n, c) => n + c.byteLength, 0);
+ const merged = new Uint8Array(total);
+ let offset = 0;
+ for (const c of chunks) {
+ merged.set(c, offset);
+ offset += c.byteLength;
+ }
+ (window as any).__testCaptureBytes = (merged as any).toBase64();
+ },
+ };
+ },
+ });
+ });
+ return async () => {
+ await expect.poll(() => page.evaluate(() => !!(window as any).__testCaptureBytes), { timeout: 10000 }).toBe(true);
+ const b64: string = await page.evaluate(() => (window as any).__testCaptureBytes);
+ return Buffer.from(b64, 'base64');
+ };
+}
+
+export async function mockAbortingFilePicker(page: import('playwright-core').Page): Promise<void> {
+ await page.evaluate(() => {
+ (window as any).showSaveFilePicker = async () => {
+ throw new DOMException('The user aborted a request.', 'AbortError');
+ };
+ });
+}
diff --git a/tests/mcp/dashboard.spec.ts b/tests/mcp/dashboard.spec.ts
index dd25d767e0..f9a52b515f 100644
--- a/tests/mcp/dashboard.spec.ts
+++ b/tests/mcp/dashboard.spec.ts
@@ -18,8 +18,7 @@ import fs from 'fs';
import os from 'os';
import path from 'path';
-import { test, expect } from './cli-fixtures';
-import { inheritAndCleanEnv } from '../config/utils';
+import { test, expect, installSaveFilePickerMock } from './cli-fixtures';
function displayPath(p: string): string {
const home = os.homedir();
@@ -135,345 +134,6 @@ test('daemon show: closing page exits the process', async ({ cli, connectToDashb
await expect(() => expect(isAlive(dashboardPid)).toBe(false)).toPass();
});
-async function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page, text: string) {
- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
- await expect(dashboard.locator('.annotate-modal-image')).toBeVisible();
- const box = await dashboard.locator('.annotate-modal-image').boundingBox();
- const x0 = box!.x + box!.width * 0.3;
- const y0 = box!.y + box!.height * 0.3;
- const x1 = box!.x + box!.width * 0.6;
- const y1 = box!.y + box!.height * 0.6;
- await dashboard.mouse.move(x0, y0);
- await dashboard.mouse.down();
- await dashboard.mouse.move(x1, y1);
- await dashboard.mouse.up();
- await dashboard.locator('.annotations-textarea').fill(text);
- await dashboard.locator('.annotations-textarea').press('Enter');
- await dashboard.getByRole('button', { name: 'Submit annotation' }).click();
-}
-
-function verifyAnnotateOutput(output: string, expectedText: string, outputDir: string) {
- expect(output).toMatch(new RegExp(`\\{ x: \\d+, y: \\d+, width: \\d+, height: \\d+ \\}: ${expectedText}`));
- const imageMatch = output.match(/- \[Annotation image\]\((\.playwright-cli[\\/]annotations-.*\.png)\)/);
- expect(imageMatch).not.toBeNull();
- const pngPath = path.resolve(outputDir, imageMatch![1]);
- expect(fs.existsSync(pngPath)).toBe(true);
- expect(fs.statSync(pngPath).size).toBeGreaterThan(0);
-}
-
-test('should capture annotations via show --annotate', async ({ connectToDashboard, cli, server }) => {
- await cli('open', server.EMPTY_PAGE);
- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- await cli('show', { bindTitle });
- const browser = await connectToDashboard(bindTitle);
-
- const dashboard = browser.contexts()[0].pages()[0];
- await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();
-
- const annotatePromise = cli('show', '--annotate');
- let done = false;
- void annotatePromise.finally(() => { done = true; });
-
- await drawAndSubmitAnnotation(dashboard, 'hello');
-
- const { output, exitCode } = await annotatePromise;
- expect(done).toBe(true);
- expect(exitCode).toBe(0);
- verifyAnnotateOutput(output, 'hello', test.info().outputDir);
-});
-
-test('should start dashboard and annotate when no dashboard is running', async ({ connectToDashboard, cli, server }) => {
- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- await cli('open', server.EMPTY_PAGE, { bindTitle });
-
- const annotatePromise = cli('show', '--annotate');
- let done = false;
- void annotatePromise.finally(() => { done = true; });
-
- const browser = await connectToDashboard(bindTitle);
- try {
- const dashboard = browser.contexts()[0].pages()[0];
- await drawAndSubmitAnnotation(dashboard, 'hi');
- } finally {
- await browser.close().catch(() => {});
- }
-
- const { output, exitCode } = await annotatePromise;
- expect(done).toBe(true);
- expect(exitCode).toBe(0);
- verifyAnnotateOutput(output, 'hi', test.info().outputDir);
-});
-
-test('should enter annotate mode on fresh dashboard.tsx mount with -s --annotate', async ({ connectToDashboard, cli, server }) => {
- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- await cli('-s=first', 'open', server.EMPTY_PAGE, { bindTitle });
- await cli('-s=second', 'open', server.EMPTY_PAGE, { bindTitle });
-
- const annotatePromise = cli('-s=second', 'show', '--annotate');
- let done = false;
- void annotatePromise.finally(() => { done = true; });
-
- const browser = await connectToDashboard(bindTitle);
- try {
- const dashboard = browser.contexts()[0].pages()[0];
- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
- await expect(activeSession(dashboard)).toHaveAccessibleName('Session second');
- await drawAndSubmitAnnotation(dashboard, 'fresh');
- } finally {
- await browser.close().catch(() => {});
- }
-
- const { exitCode } = await annotatePromise;
- expect(done).toBe(true);
- expect(exitCode).toBe(0);
-});
-
-test('should annotate via direct browser_annotate MCP call', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {
- const page = await boundBrowser.newPage();
- await page.goto(server.EMPTY_PAGE);
-
- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- const { client } = await startClient({
- args: ['--endpoint=default', '--caps=devtools'],
- env: {
- ...cliEnv,
- PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,
- },
- });
-
- const annotatePromise = client.callTool({ name: 'browser_annotate' });
- let done = false;
- void annotatePromise.then(() => { done = true; });
-
- const browser = await connectToDashboard(bindTitle);
- try {
- const dashboard = browser.contexts()[0].pages()[0];
- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
- await drawAndSubmitAnnotation(dashboard, 'direct-mcp');
- } finally {
- await browser.close().catch(() => {});
- }
-
- const result = await annotatePromise;
- expect(done).toBe(true);
- const text = (result.content as any).map(c => c.text ?? '').join('\n');
- expect(text).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: direct-mcp/);
- expect(text).toMatch(/- \[Annotation image\]\(.*\.png\)/);
-});
-
-test('should annotate when context has no fixed viewport', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {
- // Simulates headed `playwright-cli open --headed`, which launches with viewport: null
- // so that the browser window controls the page size. https://github.com/microsoft/playwright/issues/40565
- const context = await boundBrowser.newContext({ viewport: null });
- const page = await context.newPage();
- expect(page.viewportSize()).toBe(null);
- await page.goto(server.EMPTY_PAGE);
-
- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- const { client } = await startClient({
- args: ['--endpoint=default', '--caps=devtools'],
- env: {
- ...cliEnv,
- PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,
- },
- });
-
- const annotatePromise = client.callTool({ name: 'browser_annotate' });
- let done = false;
- void annotatePromise.then(() => { done = true; });
-
- const browser = await connectToDashboard(bindTitle);
- try {
- const dashboard = browser.contexts()[0].pages()[0];
- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
- await drawAndSubmitAnnotation(dashboard, 'no-viewport');
- } finally {
- await browser.close().catch(() => {});
- }
-
- const result = await annotatePromise;
- expect(done).toBe(true);
- const text = (result.content as any).map(c => c.text ?? '').join('\n');
- expect(text).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: no-viewport/);
- expect(text).toMatch(/- \[Annotation image\]\(.*\.png\)/);
-});
-
-test('should cancel browser_annotate when the MCP request is aborted', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {
- const page = await boundBrowser.newPage();
- await page.goto(server.EMPTY_PAGE);
-
- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- const { client } = await startClient({
- args: ['--endpoint=default', '--caps=devtools'],
- env: {
- ...cliEnv,
- PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,
- },
- });
-
- const controller = new AbortController();
- const annotatePromise = client.callTool({ name: 'browser_annotate' }, undefined, { signal: controller.signal }).catch(() => {});
-
- const browser = await connectToDashboard(bindTitle);
- try {
- const dashboard = browser.contexts()[0].pages()[0];
- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
-
- controller.abort();
-
- await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();
- } finally {
- await browser.close().catch(() => {});
- }
-
- await annotatePromise;
-});
-
-test('should cancel browser_annotate when the MCP client disconnects', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {
- const page = await boundBrowser.newPage();
- await page.goto(server.EMPTY_PAGE);
-
- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- const { client } = await startClient({
- args: ['--endpoint=default', '--caps=devtools'],
- env: {
- ...cliEnv,
- PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,
- },
- });
-
- void client.callTool({ name: 'browser_annotate' }).catch(() => {});
-
- const browser = await connectToDashboard(bindTitle);
- try {
- const dashboard = browser.contexts()[0].pages()[0];
- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
-
- await client.close();
-
- await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();
- } finally {
- await browser.close().catch(() => {});
- }
-});
-
-
-test('should switch screencast to -s session on show --annotate', async ({ connectToDashboard, cli, server }) => {
- server.setContent('/red', '<html><head><style>html,body{margin:0;height:100vh;background:#ff0000}</style></head><body></body></html>', 'text/html');
- server.setContent('/green', '<html><head><style>html,body{margin:0;height:100vh;background:#00ff00}</style></head><body></body></html>', 'text/html');
-
- await cli('-s=first', 'open', server.PREFIX + '/red');
- await cli('-s=second', 'open', server.PREFIX + '/green');
-
- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- await cli('-s=first', 'show', { bindTitle });
- const browser = await connectToDashboard(bindTitle);
- const dashboard = browser.contexts()[0].pages()[0];
- await expect(dashboard.locator('#display')).toBeVisible();
-
- const sampleCenter = () => dashboard.evaluate(() => {
- const img = document.querySelector('#display') as HTMLImageElement | null;
- if (!img || !img.naturalWidth)
- return null;
- const canvas = document.createElement('canvas');
- canvas.width = 1;
- canvas.height = 1;
- const ctx = canvas.getContext('2d')!;
- ctx.drawImage(img, img.naturalWidth / 2, img.naturalHeight / 2, 1, 1, 0, 0, 1, 1);
- const [r, g] = ctx.getImageData(0, 0, 1, 1).data;
- return { r, g };
- });
-
- await expect.poll(async () => {
- const c = await sampleCenter();
- return !!(c && c.r > 200 && c.g < 50);
- }, { timeout: 15000 }).toBe(true);
-
- const annotatePromise = cli('-s=second', 'show', '--annotate');
- let done = false;
- void annotatePromise.finally(() => { done = true; });
-
- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
- await expect(activeSession(dashboard)).toHaveAccessibleName('Session second');
-
- await expect.poll(async () => {
- const c = await sampleCenter();
- return !!(c && c.g > 200 && c.r < 50);
- }, { timeout: 15000 }).toBe(true);
-
- await drawAndSubmitAnnotation(dashboard, 'session switch');
- const { exitCode } = await annotatePromise;
- expect(done).toBe(true);
- expect(exitCode).toBe(0);
-});
-
-test('should disengage annotate mode when --annotate client disconnects', async ({ connectToDashboard, cli, childProcess, cliEnv, mcpBrowser, mcpHeadless, server }) => {
- await cli('open', server.EMPTY_PAGE);
- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;
- await cli('show', { bindTitle });
- const browser = await connectToDashboard(bindTitle);
-
- const dashboard = browser.contexts()[0].pages()[0];
- await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();
-
- const annotateClient = childProcess({
- command: [process.execPath, require.resolve('../../packages/playwright-core/lib/tools/cli-client/cli.js'), 'show', '--annotate'],
- cwd: test.info().outputPath(),
- env: inheritAndCleanEnv({
- ...cliEnv,
- PLAYWRIGHT_MCP_BROWSER: mcpBrowser,
- PLAYWRIGHT_MCP_HEADLESS: String(mcpHeadless),
- }),
- });
-
- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();
-
- await annotateClient.kill();
-
- await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();
-});
-
-async function installSaveFilePickerMock(page: import('playwright-core').Page): Promise<() => Promise<Buffer>> {
- let captured: string | undefined;
- let resolveCaptured: ((b64: string) => void) | undefined;
- const waitForCapture = new Promise<string>(resolve => {
- resolveCaptured = resolve;
- });
- await page.exposeBinding('__testCaptureBytes', (_, b64: string) => {
- captured = b64;
- resolveCaptured!(b64);
- });
- await page.addInitScript(() => {
- (window as any).showSaveFilePicker = async () => ({
- createWritable: async () => {
- const chunks: Uint8Array[] = [];
- return {
- write: async (chunk: Blob | BufferSource) => {
- const buf = chunk instanceof Blob
- ? new Uint8Array(await chunk.arrayBuffer())
- : new Uint8Array(chunk instanceof ArrayBuffer ? chunk : (chunk as ArrayBufferView).buffer);
- chunks.push(buf);
- },
- close: async () => {
- const total = chunks.reduce((n, c) => n + c.byteLength, 0);
- const merged = new Uint8Array(total);
- let offset = 0;
- for (const c of chunks) {
- merged.set(c, offset);
- offset += c.byteLength;
- }
- await (window as any).__testCaptureBytes((merged as any).toBase64());
- },
- };
- },
- });
- });
- return async () => {
- const b64 = captured ?? await waitForCapture;
- return Buffer.from(b64, 'base64');
- };
-}
-
test('should allow typing in omnibox in interactive mode', async ({ cli, server, startDashboardServer }) => {
server.setContent('/page1', '<html><body>Page 1</body></html>', 'text/html');
server.setContent('/page2', '<html><body>Page 2</body></html>', 'text/html');
@@ -496,11 +156,11 @@ test('should allow typing in omnibox in interactive mode', async ({ cli, server,
await expect(dashboard.locator('#omnibox')).toHaveValue(server.PREFIX + '/page2', { timeout: 10000 });
});
-test('save recording streams WebM bytes to the chosen file', async ({ cli, server, page, startDashboardServer }) => {
+test('save recording streams WebM bytes to the chosen file', async ({ cli, server, startDashboardServer }) => {
await cli('open', server.EMPTY_PAGE);
- const awaitBytes = await installSaveFilePickerMock(page);
const dashboard = await startDashboardServer();
+ const awaitBytes = await installSaveFilePickerMock(dashboard);
await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();
await expect(dashboard.locator('img#display')).toBeVisible();