Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(dashboard): annotate multiple screenshots (#40664)
Simon Knott2f1d9c062a9b
diff --git a/packages/dashboard/src/annotateView.tsx b/packages/dashboard/src/annotateView.tsxnew file mode 100644index 0000000000..4eb339bf16--- /dev/null+++ b/packages/dashboard/src/annotateView.tsx@@ -0,0 +1,221 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import React from 'react';+import { DownloadIcon } from './icons';+import { Annotations } from './annotations';+import { buildAnnotatedImage, saveAnnotationAsDownload } from './annotationImage';+import { ToolbarButton } from '@web/components/toolbarButton';++import type { Annotation, AnnotationsHandle } from './annotations';+import type { DashboardModel, AnnotateFrame } from './dashboardModel';++export type AnnotateSidebarProps = {+ model: DashboardModel;+ session: NonNullable<DashboardModel['state']['annotateSession']>;+ onSubmit: () => Promise<void> | void;+};++export const AnnotateSidebar: React.FC<AnnotateSidebarProps> = ({ model, session, onSubmit }) => {+ const [submitting, setSubmitting] = React.useState(false);+ const [hoveredAnnotationId, setHoveredAnnotationId] = React.useState<string | null>(null);++ return (+ <aside className='annotate-sidebar' aria-label='Annotation screenshots'>+ <div className='annotate-sidebar-header dashboard-shell-sidebar-header'>+ <h2 className='dashboard-shell-sidebar-title'>UI Review</h2>+ </div>+ <div className='annotate-sidebar-list'>+ {session.frames.map(frame => {+ const selected = frame.id === session.selectedFrameId;+ const comments = frame.annotations;+ return (+ <div+ key={frame.id}+ className={'annotate-sidebar-thumb' + (selected ? ' selected' : '')}+ >+ <button+ className='annotate-sidebar-thumb-button'+ onClick={() => model.selectAnnotateFrame(frame.id)}+ title={`${frame.sessionTitle || 'session'} · ${frame.title || 'tab'}\n${frame.url}`}+ aria-pressed={selected}+ >+ <span+ className='annotate-sidebar-thumb-img-wrap'+ style={{ aspectRatio: `${frame.viewportWidth} / ${frame.viewportHeight}` }}+ >+ <img+ className='annotate-sidebar-thumb-img'+ alt=''+ src={'data:image/png;base64,' + frame.data}+ />+ {frame.annotations.map(a => (+ <span+ key={a.id}+ className={'annotate-sidebar-thumb-rect' + (a.id === hoveredAnnotationId ? ' hovered' : '')}+ style={{+ left: `${(a.x / frame.viewportWidth) * 100}%`,+ top: `${(a.y / frame.viewportHeight) * 100}%`,+ width: `${(a.width / frame.viewportWidth) * 100}%`,+ height: `${(a.height / frame.viewportHeight) * 100}%`,+ ['--annotation-color' as any]: a.color,+ }}+ />+ ))}+ </span>+ </button>+ {comments.length > 0 && (+ <ul className='annotate-sidebar-comments'>+ {comments.map(a => (+ <li key={a.id}>+ <button+ className={'annotate-sidebar-comment' + (a.text ? '' : ' empty')}+ style={{ '--annotation-color': a.color } as React.CSSProperties}+ onClick={() => model.selectAnnotateFrame(frame.id, a.id)}+ onMouseEnter={() => setHoveredAnnotationId(a.id)}+ onMouseLeave={() => setHoveredAnnotationId(null)}+ title='Edit comment'+ >+ {a.text || 'Empty comment'}+ </button>+ </li>+ ))}+ </ul>+ )}+ <ToolbarButton+ className='annotate-sidebar-thumb-remove'+ icon='close'+ title='Remove screenshot'+ onClick={e => {+ e.stopPropagation();+ model.removeAnnotateFrame(frame.id);+ }}+ />+ </div>+ );+ })}+ </div>+ <div+ className='annotate-sidebar-feedback-wrap'+ data-value={session.feedback}+ >+ <textarea+ className='annotate-sidebar-feedback'+ placeholder='Feedback…'+ value={session.feedback}+ onChange={e => model.updateFeedback(e.target.value)}+ />+ </div>+ <button+ className='annotate-sidebar-submit'+ disabled={submitting}+ onClick={async () => {+ setSubmitting(true);+ try {+ await onSubmit();+ } finally {+ setSubmitting(false);+ }+ }}+ >+ {submitting ? 'Submitting…' : 'Submit'}+ </button>+ </aside>+ );+};++export type AnnotateOverlayProps = {+ model: DashboardModel;+ frame: AnnotateFrame;+ focusAnnotationId?: string | null;+};++export const AnnotateOverlay: React.FC<AnnotateOverlayProps> = ({ model, frame, focusAnnotationId }) => {+ const annotationsRef = React.useRef<AnnotationsHandle>(null);+ const displayRef = React.useRef<HTMLImageElement>(null);+ const containerRef = React.useRef<HTMLDivElement>(null);++ const onAnnotationsChange = React.useCallback((next: Annotation[]) => {+ model.updateFrameAnnotations(frame.id, next);+ }, [model, frame.id]);++ const onSave = React.useCallback(async () => {+ const img = displayRef.current;+ if (!img)+ return;+ const blob = await buildAnnotatedImage(img, frame.viewportWidth, frame.viewportHeight, frame.annotations);+ if (!blob)+ return;+ const safe = (frame.title || frame.url || 'screenshot').replace(/[^a-z0-9]+/gi, '-').slice(0, 40) || 'screenshot';+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');+ await saveAnnotationAsDownload(blob, `annotations-${stamp}-${safe}.png`);+ }, [frame]);++ const onClear = React.useCallback(() => {+ model.updateFrameAnnotations(frame.id, []);+ annotationsRef.current?.clearSelection();+ }, [model, frame.id]);++ return (+ <div className='annotate-overlay' role='dialog' aria-label={`Annotate screenshot from ${frame.title || frame.url || 'page'}`}>+ <div className='annotate-overlay-window'>+ <div className='annotate-overlay-chrome'>+ <span className='annotate-overlay-titlebar'>+ <span className='annotate-overlay-title-text'>{frame.title || 'untitled'}</span>+ <span className='annotate-overlay-title-sep'>·</span>+ <span className='annotate-overlay-title-url'>{frame.url}</span>+ </span>+ <ToolbarButton+ title='Save screenshot'+ onClick={onSave}+ >+ <DownloadIcon />+ </ToolbarButton>+ <ToolbarButton+ title='Clear annotations'+ icon='circle-slash'+ disabled={frame.annotations.length === 0}+ onClick={onClear}+ />+ <ToolbarButton+ title='Done annotating'+ icon='check'+ onClick={() => model.deselectFrame()}+ />+ </div>+ <div ref={containerRef} className='annotate-overlay-canvas'>+ <img+ ref={displayRef}+ className='annotate-modal-image'+ alt='annotation'+ src={'data:image/png;base64,' + frame.data}+ />+ <Annotations+ ref={annotationsRef}+ active={true}+ displayRef={displayRef}+ screenRef={containerRef}+ viewportWidth={frame.viewportWidth}+ viewportHeight={frame.viewportHeight}+ annotations={frame.annotations}+ onAnnotationsChange={onAnnotationsChange}+ focusAnnotationId={focusAnnotationId}+ />+ </div>+ </div>+ </div>+ );+};diff --git a/packages/dashboard/src/annotationImage.ts b/packages/dashboard/src/annotationImage.tsnew file mode 100644index 0000000000..2881aa2c49--- /dev/null+++ b/packages/dashboard/src/annotationImage.ts@@ -0,0 +1,108 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++export type RectAnnotation = { x: number; y: number; width: number; height: number; text: string; color: string };++export async function buildAnnotatedImage(+ img: HTMLImageElement,+ viewportWidth: number,+ viewportHeight: number,+ annotations: RectAnnotation[],+): Promise<Blob | null> {+ if (!img.naturalWidth || !img.naturalHeight || !viewportWidth || !viewportHeight)+ return null;+ const canvas = document.createElement('canvas');+ canvas.width = img.naturalWidth;+ canvas.height = img.naturalHeight;+ const ctx = canvas.getContext('2d');+ if (!ctx)+ return null;+ ctx.drawImage(img, 0, 0, canvas.width, canvas.height);+ const sx = canvas.width / viewportWidth;+ const sy = canvas.height / viewportHeight;+ const fontSize = Math.max(11, Math.round(14 * sy));+ ctx.font = `500 ${fontSize}px -apple-system, system-ui, sans-serif`;+ ctx.textBaseline = 'middle';+ for (const a of annotations) {+ const [r, g, b] = a.color.split(' ').map(Number);+ const solid = `rgb(${r}, ${g}, ${b})`;+ const wash = `rgba(${r}, ${g}, ${b}, 0.12)`;+ const x = a.x * sx;+ const y = a.y * sy;+ const w = a.width * sx;+ const h = a.height * sy;+ ctx.fillStyle = wash;+ ctx.fillRect(x, y, w, h);+ ctx.lineWidth = Math.max(2, Math.round(2 * sy));+ ctx.strokeStyle = solid;+ ctx.strokeRect(x, y, w, h);+ if (a.text) {+ const padX = Math.max(4, Math.round(6 * sy));+ const padY = Math.max(2, Math.round(3 * sy));+ const metrics = ctx.measureText(a.text);+ const labelW = metrics.width + padX * 2;+ const labelH = fontSize + padY * 2;+ const labelX = x - ctx.lineWidth / 2;+ const labelY = y - labelH;+ ctx.fillStyle = solid;+ ctx.fillRect(labelX, labelY, labelW, labelH);+ ctx.fillStyle = '#fff';+ ctx.fillText(a.text, labelX + padX, labelY + labelH / 2);+ }+ }+ return await new Promise<Blob | null>(resolve => canvas.toBlob(resolve, 'image/png'));+}++export async function saveAnnotationAsDownload(blob: Blob, suggestedName?: string): Promise<boolean> {+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');+ const name = suggestedName ?? `annotations-${stamp}.png`;+ const ext = (name.match(/\.[^./\\]+$/)?.[0] ?? '.bin').toLowerCase();+ const mime = blob.type || ({+ '.png': 'image/png',+ '.zip': 'application/zip',+ } as Record<string, string>)[ext] || 'application/octet-stream';+ const description = ({+ '.png': 'PNG image',+ '.zip': 'Zip archive',+ } as Record<string, string>)[ext] || 'File';+ const picker = (window as any).showSaveFilePicker as undefined | ((opts: any) => Promise<any>);+ if (picker) {+ try {+ const handle = await picker({+ suggestedName: name,+ startIn: 'downloads',+ types: [{ description, accept: { [mime]: [ext] } }],+ });+ const writable = await handle.createWritable();+ await writable.write(blob);+ await writable.close();+ } catch (e: any) {+ if (e?.name === 'AbortError')+ return false;+ throw e;+ }+ return true;+ }+ const url = URL.createObjectURL(blob);+ const link = document.createElement('a');+ link.href = url;+ link.download = name;+ document.body.appendChild(link);+ link.click();+ document.body.removeChild(link);+ URL.revokeObjectURL(url);+ return true;+}diff --git a/packages/dashboard/src/annotationZip.ts b/packages/dashboard/src/annotationZip.tsnew file mode 100644index 0000000000..bba3fe9e33--- /dev/null+++ b/packages/dashboard/src/annotationZip.ts@@ -0,0 +1,62 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++// @ts-ignore+import * as zipImport from '@zip.js/zip.js/lib/zip-no-worker-deflate.js';++import type * as zip from '@zip.js/zip.js';+import type { SubmittedAnnotationFrame } from './dashboardChannel';++const zipjs = zipImport as typeof zip;++export async function buildAnnotationZip(frames: SubmittedAnnotationFrame[], feedback: string): Promise<Blob> {+ zipjs.configure({ useWebWorkers: false } as any);+ const writer = new zipjs.ZipWriter(new zipjs.BlobWriter('application/zip'));++ const mdLines: string[] = [];+ if (feedback)+ mdLines.push(feedback, '');++ for (let i = 0; i < frames.length; i++) {+ const idx = i + 1;+ const frame = frames[i];+ const session = frame.sessionTitle || 'session';+ const tab = frame.title || 'tab';+ const header = `screenshot ${idx}: ${session} / ${tab} @ ${frame.url} (${frame.viewportWidth}x${frame.viewportHeight})`;++ mdLines.push(`## ${header}`);+ if (frame.data) {+ const pngFile = `annotations-${idx}.png`;+ await writer.add(pngFile, new zipjs.Uint8ArrayReader(Uint8Array.fromBase64(frame.data)));+ mdLines.push(`- [Screenshot ${idx}](${pngFile})`);+ }+ if (frame.ariaSnapshot) {+ const yamlFile = `annotations-${idx}.yaml`;+ const annotationLines = frame.annotations.map(a => `# { x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);+ const yamlContent = [...annotationLines, frame.ariaSnapshot].join('\n');+ await writer.add(yamlFile, new zipjs.TextReader(yamlContent));+ mdLines.push(`- [Aria snapshot ${idx}](${yamlFile})`);+ }+ for (const a of frame.annotations) {+ if (a.text)+ mdLines.push(`- { x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);+ }+ mdLines.push('');+ }++ await writer.add('feedback.md', new zipjs.TextReader(mdLines.join('\n')));+ return writer.close();+}diff --git a/packages/dashboard/src/annotations.css b/packages/dashboard/src/annotations.cssindex 41bdfd35f4..a6c9f75700 100644--- a/packages/dashboard/src/annotations.css+++ b/packages/dashboard/src/annotations.css@@ -232,3 +232,317 @@-webkit-user-drag: none;box-shadow: 0 0 0 1px var(--color-border-muted), 0 0 8px var(--color-border-muted);}+++/* -- Annotate sidebar -- */+.annotate-sidebar {+ width: 260px;+ flex-shrink: 0;+ display: flex;+ flex-direction: column;+ background: var(--color-canvas-subtle);+ border-left: 1px solid var(--color-border-muted);+ min-height: 0;+}++.annotate-sidebar-header {+ display: flex;+ align-items: center;+ justify-content: space-between;+ gap: 6px;+ height: 40px;+ min-height: 40px;+ padding: 0 8px;+ border-bottom: 1px solid var(--color-border-muted);+ background: var(--color-canvas-overlay);+}++:root.light-mode .annotate-sidebar-header {+ background: var(--color-canvas-subtle);+}++.annotate-sidebar-list {+ flex: 1;+ min-height: 0;+ overflow-y: auto;+ padding: 8px;+ display: flex;+ flex-direction: column;+ gap: 6px;+}++.annotate-sidebar-thumb {+ position: relative;+ flex-shrink: 0;+ border: 1px solid var(--color-border-muted);+ border-radius: 6px;+ background: var(--color-canvas-default);+ overflow: hidden;+}++.annotate-sidebar-thumb:hover {+ border-color: var(--color-border-default);+}++.annotate-sidebar-thumb.selected {+ border-color: var(--color-accent-fg, #2f81f7);+ box-shadow: 0 0 0 1px var(--color-accent-fg, #2f81f7);+}++.annotate-sidebar-thumb-button {+ display: block;+ width: 100%;+ padding: 0;+ background: transparent;+ border: 0;+ text-align: left;+ color: inherit;+ font: inherit;+ cursor: pointer;+}++.annotate-sidebar-thumb-img-wrap {+ position: relative;+ display: block;+ width: 100%;+ background: var(--color-canvas-overlay);+ overflow: hidden;+}++.annotate-sidebar-thumb-img {+ display: block;+ width: 100%;+ height: 100%;+ object-fit: cover;+}++.annotate-sidebar-thumb-overlay {+ position: absolute;+ inset: 0;+ width: 100%;+ height: 100%;+ pointer-events: none;+}++.annotate-sidebar-thumb-rect {+ position: absolute;+ border: 1.5px solid rgb(var(--annotation-color));+ background: rgb(var(--annotation-color) / 0.12);+ border-radius: 2px;+ pointer-events: none;+ box-sizing: border-box;+}++.annotate-sidebar-thumb-rect.hovered {+ background: rgb(var(--annotation-color) / 0.32);+}++.annotate-sidebar-thumb-remove.toolbar-button {+ position: absolute;+ top: 4px;+ right: 4px;+ padding: 2px;+ background: rgba(0, 0, 0, 0.55);+ color: #fff;+ border-radius: 50%;+ opacity: 0;+ transition: opacity 120ms ease;+ z-index: 1;+}++.annotate-sidebar-thumb-remove.toolbar-button:not(:disabled):hover {+ background: rgba(0, 0, 0, 0.75);+ color: #fff;+}++.annotate-sidebar-thumb:hover .annotate-sidebar-thumb-remove,+.annotate-sidebar-thumb-remove:focus-visible {+ opacity: 1;+}++.annotate-sidebar-comments {+ list-style: none;+ margin: 0;+ padding: 4px 8px 6px;+ display: flex;+ flex-direction: column;+ gap: 3px;+}++.annotate-sidebar-comment {+ display: block;+ width: 100%;+ text-align: left;+ background: rgb(var(--annotation-color) / 0.12);+ border: 1px solid rgb(var(--annotation-color) / 0.35);+ border-radius: 4px;+ color: var(--color-fg-default);+ font-size: 11px;+ line-height: 1.4;+ padding: 3px 6px;+ cursor: pointer;+ white-space: pre-wrap;+ word-break: break-word;+}++.annotate-sidebar-comment:hover {+ background: rgb(var(--annotation-color) / 0.2);+}++.annotate-sidebar-comment.empty {+ color: var(--color-fg-muted);+ font-style: italic;+}++/* auto-grow textarea: the wrapper mirrors content in ::after so the grid row+ expands to fit, and the textarea fills the same cell */+.annotate-sidebar-feedback-wrap {+ display: grid;+ margin: 8px 8px 4px;+ flex-shrink: 0;+}++.annotate-sidebar-feedback-wrap::after {+ content: attr(data-value) ' ';+ white-space: pre-wrap;+ word-break: break-word;+ visibility: hidden;+ /* same box-model as the textarea */+ padding: 6px 8px;+ font-size: 12px;+ font-family: inherit;+ border: 1px solid transparent;+ border-radius: 4px;+ min-height: 56px;+ grid-area: 1 / 1 / 2 / 2;+}++.annotate-sidebar-feedback {+ grid-area: 1 / 1 / 2 / 2;+ padding: 6px 8px;+ background: var(--vscode-input-background, #1e1e1e);+ color: var(--vscode-input-foreground, #cccccc);+ border: 1px solid var(--color-border-muted);+ border-radius: 4px;+ font-size: 12px;+ font-family: inherit;+ resize: none;+ min-height: 56px;+ overflow: hidden;+ outline: none;+}++.annotate-sidebar-feedback:focus {+ border-color: var(--vscode-focusBorder, #007fd4);+}++.annotate-sidebar-submit {+ margin: 4px 8px 8px;+ padding: 8px 12px;+ background: var(--color-accent-fg, #2f81f7);+ color: white;+ border: 0;+ border-radius: 6px;+ cursor: pointer;+ font-weight: 500;+ font-size: 13px;+ flex-shrink: 0;+}++.annotate-sidebar-submit:hover:not(:disabled) {+ filter: brightness(1.05);+}++.annotate-sidebar-submit:disabled {+ opacity: 0.55;+ cursor: not-allowed;+}++/* -- Annotate overlay (drawn on top of the live screen) -- */+.annotate-overlay {+ position: fixed;+ inset: 0;+ display: flex;+ align-items: stretch;+ justify-content: center;+ padding: 16px;+ background: rgba(0, 0, 0, 0.55);+ z-index: 100;+ min-height: 0;+}++.annotate-overlay-window {+ flex: 1;+ display: flex;+ flex-direction: column;+ border: 1px solid var(--color-border-default);+ border-radius: 10px;+ overflow: hidden;+ box-shadow: 0 4px 20px rgba(0, 0, 0, 0.35);+ background: var(--color-canvas-default);+ min-height: 0;+}++.annotate-overlay-chrome {+ display: flex;+ align-items: center;+ gap: 4px;+ height: 40px;+ min-height: 40px;+ padding: 0 8px;+ background: var(--color-canvas-overlay);+ border-bottom: 1px solid var(--color-border-muted);+}++:root.light-mode .annotate-overlay-chrome {+ background: var(--color-canvas-subtle);+}++.annotate-overlay-titlebar {+ flex: 1;+ display: flex;+ align-items: center;+ gap: 6px;+ height: 30px;+ padding: 0 4px;+ font-size: 13px;+ color: var(--color-fg-default);+ min-width: 0;+ overflow: hidden;+ white-space: nowrap;+}++.annotate-overlay-title-text {+ font-weight: 500;+ color: var(--color-fg-default);+ overflow: hidden;+ text-overflow: ellipsis;+ flex-shrink: 0;+ max-width: 40%;+}++.annotate-overlay-title-sep {+ color: var(--color-fg-subtle);+}++.annotate-overlay-title-url {+ flex: 1;+ font-family: var(--monospace-font, monospace);+ font-size: 12px;+ color: var(--color-fg-muted);+ overflow: hidden;+ text-overflow: ellipsis;+}++.annotate-overlay-canvas {+ flex: 1;+ position: relative;+ display: flex;+ align-items: center;+ justify-content: center;+ min-height: 0;+ padding: 12px;+ background: var(--color-canvas-default);+}++diff --git a/packages/dashboard/src/annotations.tsx b/packages/dashboard/src/annotations.tsxindex 5430e82cef..6d2a41ca44 100644--- a/packages/dashboard/src/annotations.tsx+++ b/packages/dashboard/src/annotations.tsx@@ -17,15 +17,14 @@import React from 'react';import './modal.css';import './annotations.css';-import { DownloadIcon } from './icons';-import { ToolbarButton } from '@web/components/toolbarButton';import { clientToViewport, getImageLayout } from './imageLayout';import type { ImageLayout } from './imageLayout';-import type { AnnotateFrame } from './dashboardModel';export type Annotation = { id: string; x: number; y: number; width: number; height: number; text: string; color: string };+export { buildAnnotatedImage, saveAnnotationAsDownload } from './annotationImage';+// Palette is 6 rgb triples — matches the `--annotations-blue` pattern so they// compose nicely via `rgb(var(--annotation-color))` / `rgb(var(...) / 0.12)`.const ANNOTATION_COLORS = [@@ -103,90 +102,8 @@ function viewportRectToScreenStyle(layout: ImageLayout, screenRect: DOMRect, vw:};}-export async function saveAnnotationAsDownload(blob: Blob): Promise<void> {- const stamp = new Date().toISOString().replace(/[:.]/g, '-');- const suggestedName = `annotations-${stamp}.png`;- const picker = (window as any).showSaveFilePicker as undefined | ((opts: any) => Promise<any>);- if (picker) {- try {- const handle = await picker({- suggestedName,- startIn: 'downloads',- types: [{ description: 'PNG image', accept: { 'image/png': ['.png'] } }],- });- const writable = await handle.createWritable();- await writable.write(blob);- await writable.close();- } catch (e: any) {- if (e?.name !== 'AbortError')- throw e;- }- return;- }- const url = URL.createObjectURL(blob);- const link = document.createElement('a');- link.href = url;- link.download = suggestedName;- document.body.appendChild(link);- link.click();- document.body.removeChild(link);- URL.revokeObjectURL(url);-}--export async function buildAnnotatedImage(- img: HTMLImageElement,- viewportWidth: number,- viewportHeight: number,- annotations: Annotation[],-): Promise<Blob | null> {- if (!img.naturalWidth || !img.naturalHeight || !viewportWidth || !viewportHeight)- return null;- const canvas = document.createElement('canvas');- canvas.width = img.naturalWidth;- canvas.height = img.naturalHeight;- const ctx = canvas.getContext('2d');- if (!ctx)- return null;- ctx.drawImage(img, 0, 0, canvas.width, canvas.height);- const sx = canvas.width / viewportWidth;- const sy = canvas.height / viewportHeight;- const fontSize = Math.max(11, Math.round(14 * sy));- ctx.font = `500 ${fontSize}px -apple-system, system-ui, sans-serif`;- ctx.textBaseline = 'middle';- for (const a of annotations) {- const [r, g, b] = a.color.split(' ').map(Number);- const solid = `rgb(${r}, ${g}, ${b})`;- const wash = `rgba(${r}, ${g}, ${b}, 0.12)`;- const x = a.x * sx;- const y = a.y * sy;- const w = a.width * sx;- const h = a.height * sy;- ctx.fillStyle = wash;- ctx.fillRect(x, y, w, h);- ctx.lineWidth = Math.max(2, Math.round(2 * sy));- ctx.strokeStyle = solid;- ctx.strokeRect(x, y, w, h);- if (a.text) {- const padX = Math.max(4, Math.round(6 * sy));- const padY = Math.max(2, Math.round(3 * sy));- const metrics = ctx.measureText(a.text);- const labelW = metrics.width + padX * 2;- const labelH = fontSize + padY * 2;- const labelX = x - ctx.lineWidth / 2;- const labelY = y - labelH;- ctx.fillStyle = solid;- ctx.fillRect(labelX, labelY, labelW, labelH);- ctx.fillStyle = '#fff';- ctx.fillText(a.text, labelX + padX, labelY + labelH / 2);- }- }- return await new Promise<Blob | null>(resolve => canvas.toBlob(resolve, 'image/png'));-}-export type AnnotationsHandle = {- clear(): void;- submit(): Promise<void>;- save(): Promise<void>;+ clearSelection(): void;};type AnnotationsProps = {@@ -195,8 +112,9 @@ type AnnotationsProps = {screenRef: React.RefObject<HTMLDivElement | null>;viewportWidth: number;viewportHeight: number;- onSubmit: (blob: Blob, annotations: Annotation[]) => Promise<void> | void;- onAnnotationsChange?: (count: number) => void;+ annotations: Annotation[];+ onAnnotationsChange: (annotations: Annotation[]) => void;+ focusAnnotationId?: string | null;};const ColorPicker: React.FC<{ color: string; onChange: (color: string) => void }> = ({ color, onChange }) => (@@ -216,8 +134,7 @@ const ColorPicker: React.FC<{ color: string; onChange: (color: string) => void }</div>);-export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>(({ active, displayRef, screenRef, viewportWidth, viewportHeight, onSubmit, onAnnotationsChange }, ref) => {- const [annotations, setAnnotations] = React.useState<Annotation[]>([]);+export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>(({ active, displayRef, screenRef, viewportWidth, viewportHeight, annotations, onAnnotationsChange, focusAnnotationId }, ref) => {const [draft, setDraft] = React.useState<{ startX: number; startY: number; x: number; y: number } | null>(null);const [selection, setSelection] = React.useState<Selection>(null);const [drag, setDrag] = React.useState<DragState | null>(null);@@ -229,6 +146,14 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>const [, setTick] = React.useState(0);const forceRender = React.useCallback(() => setTick(t => t + 1), []);const layerRef = React.useRef<HTMLDivElement>(null);+ // Refs kept in sync each render so effects can read current values without+ // listing them as dependencies (avoids spurious effect re-runs).+ const annotationsRef = React.useRef(annotations);+ const focusAnnotationIdRef = React.useRef(focusAnnotationId);+ React.useLayoutEffect(() => {+ annotationsRef.current = annotations;+ focusAnnotationIdRef.current = focusAnnotationId;+ });const selectedId = selection?.id ?? null;const editingId = selection?.editing ? selection.id : null;@@ -248,10 +173,20 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>}, [active, displayRef, forceRender]);React.useEffect(() => {- if (active)+ if (active) {+ const fid = focusAnnotationIdRef.current;+ if (fid) {+ const annotation = annotationsRef.current.find(a => a.id === fid);+ if (annotation) {+ setEditSnapshot({ ...annotation });+ setSelection({ id: fid, editing: true });+ return;+ }+ }layerRef.current?.focus();- else+ } else {setSelection(null);+ }}, [active]);React.useEffect(() => {@@ -266,7 +201,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>const dvy = vp.y - drag.startVy;if (dvx === 0 && dvy === 0)return;- setAnnotations(prev => prev.map(a => a.id === drag.id ? { ...a, ...applyDrag(drag.orig, drag.kind, dvx, dvy) } : a));+ onAnnotationsChange(annotationsRef.current.map(a => a.id === drag.id ? { ...a, ...applyDrag(drag.orig, drag.kind, dvx, dvy) } : a));};const onUp = () => setDrag(null);window.addEventListener('mousemove', onMove);@@ -275,7 +210,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>window.removeEventListener('mousemove', onMove);window.removeEventListener('mouseup', onUp);};- }, [drag, displayRef, viewportWidth, viewportHeight]);+ }, [drag, displayRef, viewportWidth, viewportHeight, onAnnotationsChange]);function imgCoords(e: React.MouseEvent): { x: number; y: number } | null {if (!viewportWidth || !viewportHeight)@@ -333,7 +268,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>if (rect.width < MIN_ANNOTATION_SIZE || rect.height < MIN_ANNOTATION_SIZE)return;const id = newAnnotationId();- setAnnotations(prev => [...prev, { id, ...rect, text: '', color: activeColor }]);+ onAnnotationsChange([...annotations, { id, ...rect, text: '', color: activeColor }]);setSelection({ id, editing: true });}@@ -351,7 +286,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>function nudgeSelected(dx: number, dy: number) {if (!selectedId)return;- setAnnotations(prev => prev.map(a => a.id === selectedId ? { ...a, x: a.x + dx, y: a.y + dy } : a));+ onAnnotationsChange(annotations.map(a => a.id === selectedId ? { ...a, x: a.x + dx, y: a.y + dy } : a));}function closeEditor() {@@ -372,7 +307,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>}if ((e.key === 'Delete' || e.key === 'Backspace') && selectedId && !editingId) {e.preventDefault();- setAnnotations(prev => prev.filter(a => a.id !== selectedId));+ onAnnotationsChange(annotations.filter(a => a.id !== selectedId));setSelection(null);return;}@@ -385,10 +320,6 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>}}- React.useEffect(() => {- onAnnotationsChange?.(annotations.length);- }, [annotations.length, onAnnotationsChange]);-// Snapshot is only meaningful while an annotation is actively being edited.React.useEffect(() => {if (!editingId)@@ -396,33 +327,11 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>}, [editingId]);React.useImperativeHandle(ref, () => ({- clear: () => {- setAnnotations([]);+ clearSelection: () => {setDraft(null);setSelection(null);},- submit: async () => {- const img = displayRef.current;- if (!img)- return;- const blob = await buildAnnotatedImage(img, viewportWidth, viewportHeight, annotations);- if (!blob)- return;- await onSubmit(blob, annotations);- setAnnotations([]);- setSelection(null);- setDraft(null);- },- save: async () => {- const img = displayRef.current;- if (!img)- return;- const blob = await buildAnnotatedImage(img, viewportWidth, viewportHeight, annotations);- if (!blob)- return;- await saveAnnotationAsDownload(blob);- },- }), [displayRef, viewportWidth, viewportHeight, annotations, onSubmit]);+ }), []);if (!active)return null;@@ -515,11 +424,12 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps><textareaclassName='annotations-textarea'autoFocus+ onFocus={e => { const len = e.target.value.length; e.target.setSelectionRange(len, len); }}value={editingAnnotation.text}placeholder='Task or comment…'onChange={e => {const text = e.target.value;- setAnnotations(prev => prev.map(a => a.id === editingAnnotation.id ? { ...a, text } : a));+ onAnnotationsChange(annotations.map(a => a.id === editingAnnotation.id ? { ...a, text } : a));}}onKeyDown={e => {if (e.key === 'Enter' && !e.shiftKey && !e.ctrlKey && !e.metaKey) {@@ -537,7 +447,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps><ColorPickercolor={editingAnnotation.color}onChange={color => {- setAnnotations(prev => prev.map(a => a.id === editingAnnotation.id ? { ...a, color } : a));+ onAnnotationsChange(annotations.map(a => a.id === editingAnnotation.id ? { ...a, color } : a));setActiveColor(color);}}/>@@ -548,7 +458,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>className='annotations-action-btn'onClick={() => {const snap = editSnapshot;- setAnnotations(prev => prev.map(a => a.id === snap.id ? snap : a));+ onAnnotationsChange(annotations.map(a => a.id === snap.id ? snap : a));setSelection(null);layerRef.current?.focus();}}@@ -567,7 +477,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps><buttonclassName='annotations-action-btn danger'onClick={() => {- setAnnotations(prev => prev.filter(a => a.id !== editingAnnotation.id));+ onAnnotationsChange(annotations.filter(a => a.id !== editingAnnotation.id));setSelection(null);layerRef.current?.focus();}}@@ -592,70 +502,3 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>);});Annotations.displayName = 'Annotations';--type AnnotateModalProps = {- frame: AnnotateFrame;- showSubmit: boolean;- onSubmit: (blob: Blob, annotations: Annotation[]) => Promise<void>;- onClose: () => void;-};--export const AnnotateModal: React.FC<AnnotateModalProps> = ({ frame, showSubmit, onSubmit, onClose }) => {- const [annotationCount, setAnnotationCount] = React.useState(0);- const annotationsRef = React.useRef<AnnotationsHandle>(null);- const displayRef = React.useRef<HTMLImageElement>(null);- const viewRef = React.useRef<HTMLDivElement>(null);-- return (- <div className='modal-overlay' role='dialog' aria-modal='true' aria-label='Annotate screenshot'>- <div className='modal'>- <div className='modal-toolbar'>- <div className='modal-title'>Annotate screenshot</div>- {showSubmit && (- <ToolbarButton- title='Submit annotation'- icon='check'- disabled={annotationCount === 0}- onClick={() => annotationsRef.current?.submit()}- />- )}- <ToolbarButton- title='Save annotated image'- onClick={() => annotationsRef.current?.save()}- >- <DownloadIcon />- </ToolbarButton>- <ToolbarButton- title='Clear annotations'- icon='circle-slash'- disabled={annotationCount === 0}- onClick={() => annotationsRef.current?.clear()}- />- <ToolbarButton- title='Discard'- icon='close'- onClick={onClose}- />- </div>- <div ref={viewRef} className='modal-body'>- <img- ref={displayRef}- className='annotate-modal-image'- alt='annotation'- src={'data:image/png;base64,' + frame.data}- />- <Annotations- ref={annotationsRef}- active={true}- displayRef={displayRef}- screenRef={viewRef}- viewportWidth={frame.viewportWidth}- viewportHeight={frame.viewportHeight}- onSubmit={onSubmit}- onAnnotationsChange={setAnnotationCount}- />- </div>- </div>- </div>- );-};diff --git a/packages/dashboard/src/dashboard.css b/packages/dashboard/src/dashboard.cssindex 4444e70521..288a4fbdce 100644--- a/packages/dashboard/src/dashboard.css+++ b/packages/dashboard/src/dashboard.css@@ -25,6 +25,18 @@position: relative;}+.dashboard-view.has-annotate-sidebar {+ flex-direction: row;+}++.dashboard-main {+ display: flex;+ flex-direction: column;+ flex: 1;+ min-width: 0;+ min-height: 0;+}+/* -- Mode toggle buttons (interactive / record) -- */.mode-toggle.toolbar-button.toggled,.mode-toggle.toolbar-button.toggled > .codicon {diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsxindex 184e536a20..c2f1e643ea 100644--- a/packages/dashboard/src/dashboard.tsx+++ b/packages/dashboard/src/dashboard.tsx@@ -17,11 +17,10 @@import React from 'react';import './dashboard.css';import { ChevronLeftIcon, ChevronRightIcon, LockIcon, LockOpenIcon, ReloadIcon, ScreenshotRegionIcon } from './icons';-import { AnnotateModal } from './annotations';import { clientToViewport, getImageLayout } from './imageLayout';import { Recording } from './recording';+import { AnnotateSidebar, AnnotateOverlay } from './annotateView';-import type { Annotation } from './annotations';import { ToolbarButton } from '@web/components/toolbarButton';import { useMeasureForRef } from '@web/uiUtils';@@ -67,8 +66,10 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {const [, setRevision] = React.useState(0);React.useEffect(() => model.subscribe(() => setRevision(r => r + 1)), [model]);- const { tabs, mode, recording, liveFrame, annotateFrame, annotateInitiator, pendingAnnotate } = model.state;+ const { tabs, mode, recording, liveFrame, annotateSession, pendingCapture } = model.state;const interactive = mode === 'interactive';+ const annotateActive = !!annotateSession;+ const selectedFrame = annotateSession?.frames.find(f => f.id === annotateSession.selectedFrameId) ?? null;const [flashTick, setFlashTick] = React.useState(0);@@ -131,17 +132,6 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {interactiveBtnRef.current?.classList.remove('flash');}, [interactive]);- const onSubmitAnnotations = React.useCallback(async (blob: Blob, annotations: Annotation[]) => {- const dataUrl = await new Promise<string>((resolve, reject) => {- const reader = new FileReader();- reader.onload = () => resolve(reader.result as string);- reader.onerror = () => reject(reader.error);- reader.readAsDataURL(blob);- });- const data = dataUrl.slice(dataUrl.indexOf(',') + 1);- await model.submitAnnotation(data, annotations);- }, [model]);-function flashInteractiveHint() {setFlashTick(tick => tick + 1);}@@ -155,13 +145,6 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {model.discardRecording();}, [model]);- const onCloseAnnotate = React.useCallback(() => {- if (annotateInitiator === 'cli')- model.completeAnnotation();- else- model.cancelAnnotate();- }, [model, annotateInitiator]);-const selectedTab = tabs?.find(t => t.selected);const ready = !!selectedTab;@@ -247,148 +230,161 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {const overlayText = selectedTab ? undefined : 'Select a session';const isRecording = recording?.phase === 'recording';- const showAnnotateModal = !!annotateFrame;const showRecording = recording?.phase === 'stopped';- const modeLabel = showAnnotateModal ? 'Dashboard: annotate' : isRecording ? 'Dashboard: record' : 'Dashboard';+ const modeLabel = annotateActive ? 'Dashboard: annotate' : isRecording ? 'Dashboard: record' : 'Dashboard';+ const overlayOpen = !!selectedFrame;return (- <main className={'dashboard-view' + (interactive ? ' interactive' : '')} aria-label={modeLabel}>- {/* Toolbar */}- <div ref={toolbarRef} className='toolbar'>- <ToolbarButton- ref={interactiveBtnRef}- className='mode-toggle mode-interactive'- title={interactive ? 'Disable interactive mode' : 'Enable interactive mode'}- toggled={interactive}- disabled={!ready}- onClick={() => {- if (interactive)- model.toggleInteractive();- else- model.enterInteractive();- }}- >- {interactive ? <LockOpenIcon /> : <LockIcon />}- </ToolbarButton>- <ToolbarButton- className='mode-annotate'- title='Annotate screenshot'- disabled={!ready || !!pendingAnnotate || showAnnotateModal}- onClick={() => model.enterAnnotate('user')}- >- <ScreenshotRegionIcon />- </ToolbarButton>- <ToolbarButton- className='mode-toggle mode-record'- title={isRecording ? 'Stop recording' : 'Record video'}- icon='record'- toggled={isRecording}- disabled={!ready || showRecording}- onClick={() => {- if (isRecording)- model.stopRecording();- else- model.startRecording();- }}- >- {isRecording && <span className='mode-record-label'>Recording...</span>}- </ToolbarButton>- </div>+ <main className={'dashboard-view' + (interactive ? ' interactive' : '') + (annotateActive ? ' has-annotate-sidebar' : '')} aria-label={modeLabel}>+ <div className='dashboard-main'>+ {/* Toolbar */}+ <div ref={toolbarRef} className='toolbar' hidden={overlayOpen}>+ <ToolbarButton+ ref={interactiveBtnRef}+ className='mode-toggle mode-interactive'+ title={interactive ? 'Disable interactive mode' : 'Enable interactive mode'}+ toggled={interactive}+ disabled={!ready}+ onClick={() => {+ if (interactive)+ model.toggleInteractive();+ else+ model.enterInteractive();+ }}+ >+ {interactive ? <LockOpenIcon /> : <LockIcon />}+ </ToolbarButton>+ <ToolbarButton+ className='mode-annotate'+ title={annotateActive ? 'Add screenshot' : 'Take screenshot'}+ disabled={!ready || pendingCapture}+ onClick={() => {+ if (annotateActive)+ model.addAnnotateFrame();+ else+ model.enterAnnotate('user');+ }}+ >+ <ScreenshotRegionIcon />+ </ToolbarButton>+ <ToolbarButton+ className='mode-toggle mode-record'+ title={isRecording ? 'Stop recording' : 'Record video'}+ icon='record'+ toggled={isRecording}+ disabled={!ready || showRecording}+ onClick={() => {+ if (isRecording)+ model.stopRecording();+ else+ model.startRecording();+ }}+ >+ {isRecording && <span className='mode-record-label'>Recording...</span>}+ </ToolbarButton>+ </div>- {/* Viewport */}- <div className='viewport-wrapper'>- <div ref={viewportMainRef} className='viewport-main'>- <div className='browser-window' style={windowStyle}>- {showBrowserChrome && (- <div ref={browserChromeRef} className='browser-chrome'>- <button className='nav-btn' title='Back' aria-disabled={!interactive || undefined} onClick={() => {- if (!interactive) {- flashInteractiveHint();- return;- }- model.back();- }}>- <ChevronLeftIcon />- </button>- <button className='nav-btn' title='Forward' aria-disabled={!interactive || undefined} onClick={() => {- if (!interactive) {- flashInteractiveHint();- return;- }- model.forward();- }}>- <ChevronRightIcon />- </button>- <button className='nav-btn' title='Reload' aria-disabled={!interactive || undefined} onClick={() => {- if (!interactive) {- flashInteractiveHint();- return;- }- model.reload();- }}>- <ReloadIcon />- </button>- <div className='omnibox-wrap'>- <input- id='omnibox'- className='omnibox'- type='text'- placeholder='Search or enter URL'- spellCheck={false}- autoComplete='off'- value={omniboxValue}- onChange={e => setOmniboxValue(e.target.value)}- onKeyDown={e => {- if (!interactive)- return;- onOmniboxKeyDown(e);- }}- onFocus={e => {- if (!interactive) {- flashInteractiveHint();- e.target.blur();- return;- }- e.target.select();- }}- aria-disabled={!interactive || undefined}- readOnly={!interactive}- />+ {/* Viewport */}+ <div className='viewport-wrapper'>+ <div ref={viewportMainRef} className='viewport-main'>+ <div className='browser-window' style={windowStyle}>+ {showBrowserChrome && (+ <div ref={browserChromeRef} className='browser-chrome'>+ <button className='nav-btn' title='Back' aria-disabled={!interactive || undefined} onClick={() => {+ if (!interactive) {+ flashInteractiveHint();+ return;+ }+ model.back();+ }}>+ <ChevronLeftIcon />+ </button>+ <button className='nav-btn' title='Forward' aria-disabled={!interactive || undefined} onClick={() => {+ if (!interactive) {+ flashInteractiveHint();+ return;+ }+ model.forward();+ }}>+ <ChevronRightIcon />+ </button>+ <button className='nav-btn' title='Reload' aria-disabled={!interactive || undefined} onClick={() => {+ if (!interactive) {+ flashInteractiveHint();+ return;+ }+ model.reload();+ }}>+ <ReloadIcon />+ </button>+ <div className='omnibox-wrap'>+ <input+ id='omnibox'+ className='omnibox'+ type='text'+ placeholder='Search or enter URL'+ spellCheck={false}+ autoComplete='off'+ value={omniboxValue}+ onChange={e => setOmniboxValue(e.target.value)}+ onKeyDown={e => {+ if (!interactive)+ return;+ onOmniboxKeyDown(e);+ }}+ onFocus={e => {+ if (!interactive) {+ flashInteractiveHint();+ e.target.blur();+ return;+ }+ e.target.select();+ }}+ aria-disabled={!interactive || undefined}+ readOnly={!interactive}+ />+ </div></div>+ )}+ <div+ ref={screenRef}+ className='screen'+ tabIndex={0}+ style={{ display: liveFrame ? '' : 'none' }}+ onMouseDown={onScreenMouseDown}+ onMouseUp={onScreenMouseUp}+ onMouseMove={onScreenMouseMove}+ onWheel={onScreenWheel}+ onKeyDown={onScreenKeyDown}+ onKeyUp={onScreenKeyUp}+ onContextMenu={e => e.preventDefault()}+ >+ <img+ ref={displayRef}+ id='display'+ className='display'+ alt='screencast'+ src={liveFrame ? 'data:image/jpeg;base64,' + liveFrame.data : undefined}+ /></div>- )}- <div- ref={screenRef}- className='screen'- tabIndex={0}- style={{ display: liveFrame ? '' : 'none' }}- onMouseDown={onScreenMouseDown}- onMouseUp={onScreenMouseUp}- onMouseMove={onScreenMouseMove}- onWheel={onScreenWheel}- onKeyDown={onScreenKeyDown}- onKeyUp={onScreenKeyUp}- onContextMenu={e => e.preventDefault()}- >- <img- ref={displayRef}- id='display'- className='display'- alt='screencast'- src={liveFrame ? 'data:image/jpeg;base64,' + liveFrame.data : undefined}- />+ {overlayText && <div className={'screen-overlay' + (liveFrame ? ' has-frame' : '')}><span>{overlayText}</span></div>}</div>- {overlayText && <div className={'screen-overlay' + (liveFrame ? ' has-frame' : '')}><span>{overlayText}</span></div>}</div></div></div>-- {showAnnotateModal && annotateFrame && (- <AnnotateModal- frame={annotateFrame}- showSubmit={annotateInitiator === 'cli'}- onSubmit={onSubmitAnnotations}- onClose={onCloseAnnotate}+ {selectedFrame && (+ <AnnotateOverlay+ key={selectedFrame.id + (annotateSession?.focusAnnotationId ?? '')}+ model={model}+ frame={selectedFrame}+ focusAnnotationId={annotateSession?.focusAnnotationId}+ />+ )}+ {annotateSession && (+ <AnnotateSidebar+ model={model}+ session={annotateSession}+ onSubmit={() => model.submitAnnotateSession()}/>)}diff --git a/packages/dashboard/src/dashboardChannel.ts b/packages/dashboard/src/dashboardChannel.tsindex 2fe3b8799e..b09fc47e66 100644--- a/packages/dashboard/src/dashboardChannel.ts+++ b/packages/dashboard/src/dashboardChannel.ts@@ -32,6 +32,17 @@ export type Tab = {export type AnnotationData = { x: number; y: number; width: number; height: number; text: string };+export type SubmittedAnnotationFrame = {+ data: string;+ ariaSnapshot: string;+ annotations: AnnotationData[];+ sessionTitle: string;+ title: string;+ url: string;+ viewportWidth: number;+ viewportHeight: number;+};+export type DashboardChannelEvents = {sessions: { sessions: SessionStatus[]; clientInfo: ClientInfo };tabs: { tabs: Tab[] };@@ -62,8 +73,9 @@ export interface DashboardChannel {startRecording(): Promise<void>;stopRecording(): Promise<{ streamId: string }>;readStream(params: { streamId: string }): Promise<{ data: string; eof: boolean }>;- screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number, ariaSnapshot: string }>;- submitAnnotation(params: { data: string | undefined; annotations: AnnotationData[], ariaSnapshot: string }): Promise<void>;+ screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number; ariaSnapshot: string }>;+ submitAnnotation(params: { frames: SubmittedAnnotationFrame[]; feedback: string }): Promise<void>;+ cancelAnnotation(): Promise<void>;on<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;off<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;diff --git a/packages/dashboard/src/dashboardModel.ts b/packages/dashboard/src/dashboardModel.tsindex 3896ab71d1..30f4c38bf0 100644--- a/packages/dashboard/src/dashboardModel.ts+++ b/packages/dashboard/src/dashboardModel.ts@@ -14,8 +14,11 @@* limitations under the License.*/+import { buildAnnotatedImage, saveAnnotationAsDownload } from './annotationImage';+import { buildAnnotationZip } from './annotationZip';+import type { Annotation } from './annotations';-import type { DashboardChannel, DashboardChannelEvents, MouseButton, SessionStatus, Tab } from './dashboardChannel';+import type { DashboardChannel, DashboardChannelEvents, MouseButton, SessionStatus, SubmittedAnnotationFrame, Tab } from './dashboardChannel';import type { ClientInfo } from '../../playwright-core/src/tools/cli-client/registry';import type { BrowserDescriptor } from '../../playwright-core/src/serverRegistry';@@ -27,7 +30,25 @@ export type RecordingState =| { phase: 'recording' }| { phase: 'stopped'; blob: Blob; blobUrl: string };-export type AnnotateFrame = { data: string; viewportWidth: number; viewportHeight: number; ariaSnapshot: string };+export type AnnotateFrame = {+ id: string;+ data: string;+ viewportWidth: number;+ viewportHeight: number;+ ariaSnapshot: string;+ sessionTitle: string;+ title: string;+ url: string;+ annotations: Annotation[];+};++export type AnnotateSession = {+ initiator: 'cli' | 'user';+ frames: AnnotateFrame[];+ selectedFrameId: string | null;+ focusAnnotationId: string | null;+ feedback: string;+};export type DashboardState = {// Session model state.@@ -37,9 +58,8 @@ export type DashboardState = {// Dashboard / page state.tabs: Tab[] | null;liveFrame: DashboardChannelEvents['frame'] | undefined;- annotateFrame: AnnotateFrame | undefined;- pendingAnnotate: { initiator: 'cli' | 'user' } | null;- annotateInitiator: 'cli' | 'user' | null;+ annotateSession: AnnotateSession | null;+ pendingCapture: boolean;mode: Mode;recording: RecordingState | null;};@@ -52,9 +72,8 @@ const initialState: DashboardState = {loadingSessions: true,tabs: null,liveFrame: undefined,- annotateFrame: undefined,- pendingAnnotate: null,- annotateInitiator: null,+ annotateSession: null,+ pendingCapture: false,mode: 'readonly',recording: null,};@@ -74,7 +93,7 @@ export class DashboardModel {client.on('tabs', params => this._emit({ tabs: params.tabs }));client.on('frame', params => this._emit({ liveFrame: params }));client.on('annotate', () => this.enterAnnotate('cli'));- client.on('cancelAnnotate', () => this.cancelAnnotate());+ client.on('cancelAnnotate', () => this.cancelAnnotate(false));}subscribe(listener: Listener): () => void {@@ -158,7 +177,10 @@ export class DashboardModel {toggleInteractive() {const next: Mode = this.state.mode === 'interactive' ? 'readonly' : 'interactive';- this._emit({ mode: next });+ if (next === 'interactive')+ void this._enterInteractive();+ else+ this._emit({ mode: next });}// Public action methods are sync fire-and-forget wrappers around the@@ -174,6 +196,62 @@ export class DashboardModel {void this._enterAnnotate(initiator);}+ addAnnotateFrame() {+ void this._addAnnotateFrame();+ }++ selectAnnotateFrame(id: string, focusAnnotationId?: string) {+ const session = this.state.annotateSession;+ if (!session || !session.frames.find(f => f.id === id))+ return;+ this._emit({ annotateSession: { ...session, selectedFrameId: id, focusAnnotationId: focusAnnotationId ?? null }, mode: 'annotate' });+ }++ toggleSelectFrame(id: string) {+ const session = this.state.annotateSession;+ if (!session)+ return;+ if (session.selectedFrameId === id)+ this._emit({ annotateSession: { ...session, selectedFrameId: null, focusAnnotationId: null }, mode: 'readonly' });+ else+ this.selectAnnotateFrame(id);+ }++ deselectFrame() {+ const session = this.state.annotateSession;+ if (!session || session.selectedFrameId === null)+ return;+ this._emit({ annotateSession: { ...session, selectedFrameId: null, focusAnnotationId: null }, mode: 'readonly' });+ }++ removeAnnotateFrame(id: string) {+ const session = this.state.annotateSession;+ if (!session)+ return;+ const frames = session.frames.filter(f => f.id !== id);+ if (frames.length === 0) {+ this.cancelAnnotate();+ return;+ }+ const selectedFrameId = session.selectedFrameId === id ? null : session.selectedFrameId;+ this._emit({ annotateSession: { ...session, frames, selectedFrameId } });+ }++ updateFrameAnnotations(frameId: string, annotations: Annotation[]) {+ const session = this.state.annotateSession;+ if (!session)+ return;+ const frames = session.frames.map(f => f.id === frameId ? { ...f, annotations } : f);+ this._emit({ annotateSession: { ...session, frames } });+ }++ updateFeedback(feedback: string) {+ const session = this.state.annotateSession;+ if (!session)+ return;+ this._emit({ annotateSession: { ...session, feedback } });+ }+completeAnnotation() {void this._completeAnnotation();}@@ -190,47 +268,122 @@ export class DashboardModel {void this._discardRecording();}- cancelAnnotate() {+ cancelAnnotate(notifyServer = true) {this._requestId++;const s = this.state;+ if (notifyServer && s.annotateSession?.initiator === 'cli')+ void this._client.cancelAnnotation();this._emit({mode: s.mode === 'annotate' ? 'readonly' : s.mode,- annotateFrame: undefined,- pendingAnnotate: null,- annotateInitiator: null,+ annotateSession: null,+ pendingCapture: false,});}- async submitAnnotation(data: string | undefined, annotations: Annotation[]) {- await this._client.submitAnnotation({- data,- ariaSnapshot: this.state.annotateFrame?.ariaSnapshot ?? '',- annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),- });+ async submitAnnotateSession() {+ const session = this.state.annotateSession;+ if (!session)+ return;+ const frames: SubmittedAnnotationFrame[] = [];+ for (const frame of session.frames) {+ const data = await renderFrameToBase64Png(frame);+ frames.push({+ data: data ?? frame.data,+ ariaSnapshot: frame.ariaSnapshot,+ annotations: frame.annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),+ sessionTitle: frame.sessionTitle,+ title: frame.title,+ url: frame.url,+ viewportWidth: frame.viewportWidth,+ viewportHeight: frame.viewportHeight,+ });+ }+ if (session.initiator === 'cli') {+ await this._client.submitAnnotation({ frames, feedback: session.feedback });+ } else {+ const blob = await buildAnnotationZip(frames, session.feedback);+ const stamp = new Date().toISOString().replace(/[:.]/g, '-');+ const saved = await saveAnnotationAsDownload(blob, `annotations-${stamp}.zip`);+ if (!saved)+ return; // user cancelled the file picker — keep the session open+ }this.cancelAnnotate();}private async _enterInteractive() {- // Interactive mode coexists with recording so the user can drive the- // page while it is being captured. Only in-flight annotation is cleared.- await this._completeAnnotation();- this._emit({ mode: 'interactive' });+ const s = this.state;+ // Interactive coexists with the annotate sidebar; only the overlay is+ // dismissed so the live page is reachable.+ this._emit({+ mode: 'interactive',+ annotateSession: s.annotateSession?.selectedFrameId+ ? { ...s.annotateSession, selectedFrameId: null, focusAnnotationId: null }+ : s.annotateSession,+ });}private async _enterAnnotate(initiator: 'cli' | 'user') {- await this._cleanupOnModeSwitch();- this._requestAnnotate(initiator);+ const s = this.state;+ if (s.annotateSession) {+ // Session already open: capture another frame and select it.+ if (initiator === 'cli' && s.annotateSession.initiator !== 'cli')+ this._emit({ annotateSession: { ...s.annotateSession, initiator: 'cli' } });+ await this._addAnnotateFrame();+ return;+ }+ await this._discardRecording();+ await this._addAnnotateFrame(initiator);+ }++ private async _addAnnotateFrame(initiator?: 'cli' | 'user') {+ if (this.state.pendingCapture)+ return;+ const myId = ++this._requestId;+ this._emit({ pendingCapture: true });+ let frameData: Awaited<ReturnType<DashboardChannel['screenshot']>> | undefined;+ try {+ frameData = await this._client.screenshot();+ } catch {+ // ignore+ }+ if (myId !== this._requestId)+ return;+ if (!frameData) {+ this._emit({ pendingCapture: false });+ return;+ }+ const selectedTab = this.state.tabs?.find(t => t.selected);+ const sessionTitle = this.state.sessions.find(s => s.browser.guid === selectedTab?.browser)?.title ?? '';+ const frame: AnnotateFrame = {+ id: 'frm-' + Math.random().toString(36).slice(2, 10),+ data: frameData.data,+ viewportWidth: frameData.viewportWidth,+ viewportHeight: frameData.viewportHeight,+ ariaSnapshot: frameData.ariaSnapshot,+ sessionTitle,+ title: selectedTab?.title ?? '',+ url: selectedTab?.url ?? '',+ annotations: [],+ };+ const existing = this.state.annotateSession;+ const session: AnnotateSession = existing+ ? { ...existing, frames: [...existing.frames, frame], selectedFrameId: frame.id, focusAnnotationId: null }+ : { initiator: initiator ?? 'user', frames: [frame], selectedFrameId: frame.id, focusAnnotationId: null, feedback: '' };+ this._emit({ annotateSession: session, pendingCapture: false, mode: 'annotate' });}private async _completeAnnotation() {const s = this.state;- if (s.mode === 'annotate' && s.annotateInitiator === 'cli' && s.annotateFrame)- await this.submitAnnotation(undefined, []).catch(() => {});- this.cancelAnnotate();+ if (s.annotateSession?.initiator === 'cli')+ await this.submitAnnotateSession().catch(() => {});+ else+ this.cancelAnnotate();}private async _startRecording() {- await this._cleanupOnModeSwitch();+ // Recording closes any open annotate session.+ if (this.state.annotateSession)+ this.cancelAnnotate();await this._client.startRecording();this._emit({ recording: { phase: 'recording' } });}@@ -269,52 +422,6 @@ export class DashboardModel {this._emit({ recording: null });}- private async _cleanupOnModeSwitch() {- await this._discardRecording();- await this._completeAnnotation();- }-- private _requestAnnotate(initiator: 'cli' | 'user') {- const s = this.state;- if (s.mode === 'annotate') {- if (initiator === 'cli' && s.annotateInitiator !== 'cli')- this._emit({ annotateInitiator: 'cli' });- return;- }- if (s.pendingAnnotate) {- if (initiator === 'cli' && s.pendingAnnotate.initiator !== 'cli')- this._emit({ pendingAnnotate: { initiator: 'cli' } });- return;- }- const myId = ++this._requestId;- this._emit({ pendingAnnotate: { initiator } });- void this._fetchScreenshot(myId);- }-- private async _fetchScreenshot(id: number) {- let frame: AnnotateFrame | undefined;- try {- frame = await this._client.screenshot();- } catch {- // frame stays undefined- }- if (id !== this._requestId)- return;- const s = this.state;- if (!s.pendingAnnotate)- return;- if (!frame) {- this._emit({ pendingAnnotate: null });- return;- }- this._emit({- mode: 'annotate',- annotateFrame: frame,- annotateInitiator: s.pendingAnnotate.initiator,- pendingAnnotate: null,- });- }-private _emit(partial: Partial<DashboardState>) {this.state = { ...this.state, ...partial };for (const listener of this._listeners)@@ -322,6 +429,21 @@ export class DashboardModel {}}+async function renderFrameToBase64Png(frame: AnnotateFrame): Promise<string | undefined> {+ const img = new Image();+ img.src = 'data:image/png;base64,' + frame.data;+ try {+ await img.decode();+ } catch {+ return undefined;+ }+ const blob = await buildAnnotatedImage(img, frame.viewportWidth, frame.viewportHeight, frame.annotations);+ if (!blob)+ return undefined;+ const buf = await blob.arrayBuffer();+ return new Uint8Array(buf).toBase64();+}+function base64ToBlob(base64: string, mime: string): Blob {- return new Blob([(Uint8Array as any).fromBase64(base64)], { type: mime });+ return new Blob([Uint8Array.fromBase64(base64)], { type: mime });}diff --git a/packages/playwright-core/src/tools/backend/devtools.ts b/packages/playwright-core/src/tools/backend/devtools.tsindex 4fc8cf7794..79e0fb9afc 100644--- a/packages/playwright-core/src/tools/backend/devtools.ts+++ b/packages/playwright-core/src/tools/backend/devtools.ts@@ -22,7 +22,7 @@ import { libPath } from '../../package';import { defineTabTool, defineTool } from './tool';import { elementSchema, optionalElementSchema } from './snapshot';-import type { AnnotationData } from '@dashboard/dashboardChannel';+import type { SubmittedAnnotationFrame } from '@dashboard/dashboardChannel';const resume = defineTool({capability: 'devtools',@@ -135,7 +135,7 @@ const annotate = defineTabTool({daemon.unref();// Spawn the annotate client in JSON mode to capture the raw payload over stdout.- const client = spawn(process.execPath, [...daemonArgs, '--annotate', '--json'], {+ const client = spawn(process.execPath, [...daemonArgs, '--annotate'], {stdio: ['pipe', 'pipe', 'inherit'],});const onAbort = () => client.kill();@@ -157,14 +157,28 @@ const annotate = defineTabTool({response.addTextResult('No annotations were submitted.');return;}- const { png, ariaSnapshot, annotations } = JSON.parse(text) as { png?: string; ariaSnapshot?: string; annotations: AnnotationData[] };- for (const a of annotations)- response.addTextResult(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);+ const { frames, feedback } = JSON.parse(text) as { frames: SubmittedAnnotationFrame[]; feedback: string };+ if (!frames || frames.length === 0) {+ response.addTextResult('No annotations were submitted.');+ return;+ }const date = new Date();- if (png)- await response.addResult('Annotation image', Buffer.from(png, 'base64'), { prefix: 'annotations', ext: 'png', date });- if (ariaSnapshot)- await response.addResult('Annotation snapshot', Buffer.from(ariaSnapshot, 'utf8'), { prefix: 'annotations', ext: 'yaml', date });+ if (feedback)+ response.addTextResult(feedback);+ const multi = frames.length > 1;+ for (let i = 0; i < frames.length; i++) {+ const frame = frames[i];+ const idx = i + 1;+ const session = frame.sessionTitle || 'session';+ const tab = frame.title || 'tab';+ response.addTextResult(`${multi ? `## Screenshot ${idx}\n` : ''}${session} / ${tab} @ ${frame.url} (${frame.viewportWidth}x${frame.viewportHeight})`);+ for (const a of frame.annotations)+ response.addTextResult(` { x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);+ if (frame.data)+ await response.addResult(`Annotation image${multi ? ' ' + idx : ''}`, Buffer.from(frame.data, 'base64'), { prefix: `annotations${multi ? '-' + idx : ''}`, ext: 'png', date });+ if (frame.ariaSnapshot)+ await response.addResult(`Annotation snapshot${multi ? ' ' + idx : ''}`, Buffer.from(frame.ariaSnapshot, 'utf8'), { prefix: `annotations${multi ? '-' + idx : ''}`, ext: 'yaml', date });+ }},});diff --git a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts b/packages/playwright-core/src/tools/dashboard/dashboardApp.tsindex 0673df147a..b4a8a1a962 100644--- a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts+++ b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts@@ -26,13 +26,12 @@ import { libPath } from '../../package';import { playwright } from '../../inprocess';import { findChromiumChannelBestEffort, registryDirectory } from '../../server/registry/index';import { minimist } from '../cli-client/minimist';-import { saveOutputFile } from '../trace/traceUtils';import { DashboardConnection } from './dashboardController';import { RegistrySessionProvider } from './registrySessionProvider';import { IdentitySessionProvider } from './identitySessionProvider';import type * as api from '../../..';-import type { AnnotationData } from '@dashboard/dashboardChannel';+import type { SubmittedAnnotationFrame } from '@dashboard/dashboardChannel';import type { SessionProvider } from './sessionProvider';// HMR: build-time flag — `true` in watch builds, `false` in release. esbuild@@ -57,10 +56,10 @@ async function startDashboardServer(provider: SessionProvider, options: Dashboarlet pendingAnnotate = false;const waitingSockets = new Set<net.Socket>();- const submitAnnotation = (base64Png: string | undefined, ariaSnapshot: string, annotations: AnnotationData[]) => {+ const submitAnnotation = (frames: SubmittedAnnotationFrame[], feedback: string) => {if (waitingSockets.size === 0)return;- const payload = JSON.stringify({ png: base64Png, ariaSnapshot, annotations });+ const payload = JSON.stringify({ frames, feedback });for (const socket of waitingSockets) {socket.write(payload);socket.end();@@ -251,13 +250,12 @@ type DashboardOptions = {pageId?: string;kill?: boolean;annotate?: boolean;- json?: boolean;port?: number;host?: string;};function parseOpenArgs(): DashboardOptions {- const args = minimist(process.argv.slice(2), { string: ['sessionName', 'workspaceDir', 'host', 'pageId'], boolean: ['annotate', 'kill', 'json'] });+ const args = minimist(process.argv.slice(2), { string: ['sessionName', 'workspaceDir', 'host', 'pageId'], boolean: ['annotate', 'kill'] });const portStr = args.port as string | undefined;return {sessionName: args.sessionName as string | undefined,@@ -266,7 +264,6 @@ function parseOpenArgs(): DashboardOptions {port: portStr !== undefined ? Number(portStr) : undefined,host: args.host as string | undefined,annotate: !!args.annotate,- json: !!args.json,kill: !!args.kill,};}@@ -429,27 +426,8 @@ async function runAnnotateClient(options: DashboardOptions): Promise<void> {const text = Buffer.concat(chunks).toString();if (!text)return;- if (options.json) {- // eslint-disable-next-line no-console- console.log(text);- return;- }- const { png, annotations, ariaSnapshot } = JSON.parse(text) as { png: string; annotations: AnnotationData[], ariaSnapshot: string };- for (const a of annotations) {- // eslint-disable-next-line no-console- console.log(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);- }- const timestamp = new Date().toISOString().replace(/[:.]/g, '-');- if (png) {- const filePath = await saveOutputFile(`annotations-${timestamp}.png`, Buffer.from(png, 'base64'));- // eslint-disable-next-line no-console- console.log(`image: ${path.relative(process.cwd(), filePath)}`);- }- if (ariaSnapshot) {- const filePath = await saveOutputFile(`annotations-${timestamp}.yaml`, ariaSnapshot);- // eslint-disable-next-line no-console- console.log(`snapshot: ${path.relative(process.cwd(), filePath)}`);- }+ // eslint-disable-next-line no-console+ console.log(text);}function selfDestructOnParentGone() {diff --git a/packages/playwright-core/src/tools/dashboard/dashboardController.ts b/packages/playwright-core/src/tools/dashboard/dashboardController.tsindex 79e0c503c5..f27ecb3e0a 100644--- a/packages/playwright-core/src/tools/dashboard/dashboardController.ts+++ b/packages/playwright-core/src/tools/dashboard/dashboardController.ts@@ -27,7 +27,7 @@ import { SessionProviderEvent } from './sessionProvider';import type * as api from '../../..';import type { Transport } from '@utils/httpServer';-import type { AnnotationData, Tab } from '@dashboard/dashboardChannel';+import type { SubmittedAnnotationFrame, Tab } from '@dashboard/dashboardChannel';import type { BrowserDescriptor } from '../../serverRegistry';import type { SessionProvider } from './sessionProvider';@@ -39,7 +39,7 @@ export class DashboardConnection implements Transport {private _attachedPage: AttachedPage | undefined;private _onclose: () => void;private _onconnected?: () => void;- private _onAnnotationSubmit?: (base64Png: string | undefined, ariaSnapshot: string, annotations: AnnotationData[]) => void;+ private _onAnnotationSubmit?: (frames: SubmittedAnnotationFrame[], feedback: string) => void;private _pushTabsScheduled = false;private _visible = true;private _pendingReveal: { sessionName?: string; workspaceDir?: string; pageId?: string } | undefined;@@ -48,7 +48,7 @@ export class DashboardConnection implements Transport {_recordingDir: string;_streams = new Map<string, { handle: fs.promises.FileHandle; path: string }>();- constructor(provider: SessionProvider, onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string | undefined, ariaSnapshot: string, annotations: AnnotationData[]) => void) {+ constructor(provider: SessionProvider, onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (frames: SubmittedAnnotationFrame[], feedback: string) => void) {this._provider = provider;this._onclose = onclose;this._onconnected = onconnected;@@ -163,8 +163,12 @@ export class DashboardConnection implements Transport {this._pushTabs();}- async submitAnnotation(params: { data: string | undefined; ariaSnapshot: string; annotations: AnnotationData[] }) {- this._onAnnotationSubmit?.(params.data, params.ariaSnapshot, params.annotations);+ async submitAnnotation(params: { frames: SubmittedAnnotationFrame[]; feedback: string }) {+ this._onAnnotationSubmit?.(params.frames, params.feedback);+ }++ async cancelAnnotation() {+ this._onAnnotationSubmit?.([], '');}async reveal(params: { path: string }) {@@ -425,10 +429,10 @@ class AttachedPage {return { streamId };}- async screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number, ariaSnapshot: string }> {+ async screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number; ariaSnapshot: string }> {const buffer = await this._page.screenshot({ type: 'png' });- const vp = await this._viewportSize();const ariaSnapshot = await this._page.ariaSnapshot({ boxes: true, mode: 'ai' });+ const vp = await this._viewportSize();return {data: buffer.toString('base64'),viewportWidth: vp.width,diff --git a/tests/mcp/annotate.spec.ts b/tests/mcp/annotate.spec.tsnew file mode 100644index 0000000000..eb91e63be3--- /dev/null+++ b/tests/mcp/annotate.spec.ts@@ -0,0 +1,500 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import fs from 'fs';+import path from 'path';++import * as zipjs from '@zip.js/zip.js';++import { test, expect, installSaveFilePickerMock, mockAbortingFilePicker } from './cli-fixtures';+import { inheritAndCleanEnv } from '../config/utils';++function activeSession(dashboard: import('playwright-core').Page) {+ return dashboard.getByRole('region', { name: /^Session / }).filter({ has: dashboard.getByRole('option', { selected: true }) });+}++async function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page, text: string) {+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();+ await expect(dashboard.locator('.annotate-modal-image')).toBeVisible();+ const box = await dashboard.locator('.annotate-modal-image').boundingBox();+ const x0 = box!.x + box!.width * 0.3;+ const y0 = box!.y + box!.height * 0.3;+ const x1 = box!.x + box!.width * 0.6;+ const y1 = box!.y + box!.height * 0.6;+ await dashboard.mouse.move(x0, y0);+ await dashboard.mouse.down();+ await dashboard.mouse.move(x1, y1);+ await dashboard.mouse.up();+ await dashboard.locator('.annotations-textarea').fill(text);+ await dashboard.locator('.annotations-textarea').press('Enter');+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();+ await dashboard.getByRole('button', { name: 'Submit' }).click();+}++function verifyAnnotateOutput(output: string, expectedText: string, outputDir: string) {+ expect(output).toMatch(/.* @ .* \(\d+x\d+\)/);+ expect(output).toMatch(new RegExp(`\\{ x: \\d+, y: \\d+, width: \\d+, height: \\d+ \\}: ${expectedText}`));+ const imageMatch = output.match(/- \[Annotation image\]\((\.playwright-cli[\\/]annotations-.*\.png)\)/);+ expect(imageMatch).not.toBeNull();+ const pngPath = path.resolve(outputDir, imageMatch![1]);+ expect(fs.existsSync(pngPath)).toBe(true);+ expect(fs.statSync(pngPath).size).toBeGreaterThan(0);+}++test('should capture multiple screenshots in one annotation', async ({ connectToDashboard, cli, server }) => {+ await cli('open', server.PREFIX + '/title.html');+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ await cli('show', { bindTitle });+ const browser = await connectToDashboard(bindTitle);++ const dashboard = browser.contexts()[0].pages()[0];+ await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();++ const annotatePromise = cli('show', '--annotate');+ let done = false;+ void annotatePromise.finally(() => { done = true; });++ await expect(dashboard.locator('.annotate-modal-image')).toBeVisible();+ // First annotation on initial frame.+ let box = await dashboard.locator('.annotate-modal-image').boundingBox();+ await dashboard.mouse.move(box!.x + box!.width * 0.2, box!.y + box!.height * 0.2);+ await dashboard.mouse.down();+ await dashboard.mouse.move(box!.x + box!.width * 0.4, box!.y + box!.height * 0.4);+ await dashboard.mouse.up();+ await dashboard.locator('.annotations-textarea').fill('first');+ await dashboard.locator('.annotations-textarea').press('Enter');++ // Deselect overlay (sidebar stays), then capture a second frame via the toolbar button.+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();+ await expect(dashboard.locator('.annotate-overlay')).toHaveCount(0);+ await dashboard.getByRole('button', { name: /^(Take|Add) screenshot$/ }).click();++ // New frame auto-selected; draw + label.+ await expect(dashboard.locator('.annotate-sidebar-thumb')).toHaveCount(2);+ await expect(dashboard.locator('.annotate-overlay')).toBeVisible();+ box = await dashboard.locator('.annotate-modal-image').boundingBox();+ await dashboard.mouse.move(box!.x + box!.width * 0.5, box!.y + box!.height * 0.5);+ await dashboard.mouse.down();+ await dashboard.mouse.move(box!.x + box!.width * 0.7, box!.y + box!.height * 0.7);+ await dashboard.mouse.up();+ await dashboard.locator('.annotations-textarea').fill('second');+ await dashboard.locator('.annotations-textarea').press('Enter');++ await dashboard.getByRole('button', { name: 'Done annotating' }).click();+ await dashboard.getByRole('button', { name: 'Submit' }).click();++ const { output, exitCode } = await annotatePromise;+ expect(done).toBe(true);+ expect(exitCode).toBe(0);+ expect(output).toMatch(/## Screenshot 1/);+ expect(output).toMatch(/## Screenshot 2/);+ expect(output).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: first/);+ expect(output).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: second/);+ expect(output).toMatch(/- \[Annotation image 1\]\(.*annotations-1-.*\.png\)/);+ expect(output).toMatch(/- \[Annotation image 2\]\(.*annotations-2-.*\.png\)/);+});++test('should abort annotation when last screenshot is removed', async ({ connectToDashboard, cli, server }) => {+ await cli('open', server.EMPTY_PAGE);+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ await cli('show', { bindTitle });+ const browser = await connectToDashboard(bindTitle);++ const dashboard = browser.contexts()[0].pages()[0];+ await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();++ const annotatePromise = cli('show', '--annotate');+ let done = false;+ void annotatePromise.finally(() => { done = true; });++ await expect(dashboard.locator('.annotate-sidebar-thumb')).toHaveCount(1);++ // Close the fullscreen overlay first so the sidebar remove button is accessible.+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();++ // Remove the only screenshot — should abort the annotation+ await dashboard.locator('.annotate-sidebar-thumb-remove').click();++ const { output, exitCode } = await annotatePromise;+ expect(done).toBe(true);+ expect(exitCode).toBe(0);+ expect(output).toContain('No annotations were submitted');+});++test('should abort MCP annotation when last screenshot is removed', async ({ connectToDashboard, boundBrowser, startClient, cliEnv }) => {+ await boundBrowser.newPage();++ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ const { client } = await startClient({+ args: ['--endpoint=default', '--caps=devtools'],+ env: {+ ...cliEnv,+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,+ },+ });++ const annotatePromise = client.callTool({ name: 'browser_annotate' });+ let done = false;+ void annotatePromise.then(() => { done = true; });++ const browser = await connectToDashboard(bindTitle);+ try {+ const dashboard = browser.contexts()[0].pages()[0];+ await expect(dashboard.locator('.annotate-sidebar-thumb')).toHaveCount(1);++ // Close the fullscreen overlay first so the sidebar remove button is accessible.+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();++ // Remove the only screenshot — should abort+ await dashboard.locator('.annotate-sidebar-thumb-remove').click();+ } finally {+ await browser.close().catch(() => {});+ }++ const result = await annotatePromise;+ expect(done).toBe(true);+ const text = (result.content as any).map((c: any) => c.text ?? '').join('\n');+ expect(text).toContain('No annotations were submitted');+});++test('user-initiated annotate downloads zip with feedback.md', async ({ connectToDashboard, cli, server }) => {+ await cli('open', server.EMPTY_PAGE);+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ await cli('show', { bindTitle });+ const browser = await connectToDashboard(bindTitle);++ const dashboard = browser.contexts()[0].pages()[0];+ await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();++ // Start with an aborting picker to verify cancellation keeps the session intact.+ await mockAbortingFilePicker(dashboard);++ // Enter annotate via toolbar (user-initiated).+ await dashboard.getByRole('button', { name: /^(Take|Add) screenshot$/ }).click();+ await expect(dashboard.locator('.annotate-modal-image')).toBeVisible();++ // Draw an annotation.+ const box = await dashboard.locator('.annotate-modal-image').boundingBox();+ await dashboard.mouse.move(box!.x + box!.width * 0.2, box!.y + box!.height * 0.2);+ await dashboard.mouse.down();+ await dashboard.mouse.move(box!.x + box!.width * 0.5, box!.y + box!.height * 0.5);+ await dashboard.mouse.up();+ await dashboard.locator('.annotations-textarea').fill('zip-test');+ await dashboard.locator('.annotations-textarea').press('Enter');++ // Close overlay, type feedback, submit → picker aborts.+ await dashboard.getByRole('button', { name: 'Done annotating' }).click();+ await dashboard.locator('.annotate-sidebar-feedback').fill('My feedback');+ await dashboard.getByRole('button', { name: 'Submit', exact: true }).click();++ // Session must still be open with data intact.+ await expect(dashboard.locator('.annotate-sidebar')).toBeVisible();+ await expect(dashboard.locator('.annotate-sidebar-thumb')).toHaveCount(1);+ // Wait until the in-flight (aborted) submit fully resolves and the button is re-enabled,+ // otherwise installing the next picker mid-flight would race.+ await expect(dashboard.getByRole('button', { name: 'Submit', exact: true })).toBeEnabled();++ // Now install a capturing picker and submit for real.+ const awaitZipBytes = await installSaveFilePickerMock(dashboard);+ await dashboard.getByRole('button', { name: 'Submit', exact: true }).click();++ const zipBytes = await awaitZipBytes();++ zipjs.configure({ useWebWorkers: false });+ const entries = await new zipjs.ZipReader(new zipjs.Uint8ArrayReader(zipBytes)).getEntries();+ const names = entries.map(e => e.filename);+ expect(names).toContain('feedback.md');+ expect(names).toContain('annotations-1.png');++ const mdEntry = entries.find(e => e.filename === 'feedback.md')!;+ const mdText = await mdEntry.getData!(new zipjs.TextWriter());+ expect(mdText).toContain('My feedback');+ expect(mdText).toContain('Screenshot 1');+ expect(mdText).toContain('[Screenshot 1](annotations-1.png)');+ expect(mdText).toMatch(/\d+x\d+/);+});++test('should capture annotations via show --annotate', async ({ connectToDashboard, cli, server }) => {+ await cli('open', server.EMPTY_PAGE);+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ await cli('show', { bindTitle });+ const browser = await connectToDashboard(bindTitle);++ const dashboard = browser.contexts()[0].pages()[0];+ await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();++ const annotatePromise = cli('show', '--annotate');+ let done = false;+ void annotatePromise.finally(() => { done = true; });++ await drawAndSubmitAnnotation(dashboard, 'hello');++ const { output, exitCode } = await annotatePromise;+ expect(done).toBe(true);+ expect(exitCode).toBe(0);+ verifyAnnotateOutput(output, 'hello', test.info().outputDir);+});++test('should start dashboard and annotate when no dashboard is running', async ({ connectToDashboard, cli, server }) => {+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ await cli('open', server.EMPTY_PAGE, { bindTitle });++ const annotatePromise = cli('show', '--annotate');+ let done = false;+ void annotatePromise.finally(() => { done = true; });++ const browser = await connectToDashboard(bindTitle);+ try {+ const dashboard = browser.contexts()[0].pages()[0];+ await drawAndSubmitAnnotation(dashboard, 'hi');+ } finally {+ await browser.close().catch(() => {});+ }++ const { output, exitCode } = await annotatePromise;+ expect(done).toBe(true);+ expect(exitCode).toBe(0);+ verifyAnnotateOutput(output, 'hi', test.info().outputDir);+});++test('should enter annotate mode on fresh dashboard.tsx mount with -s --annotate', async ({ connectToDashboard, cli, server }) => {+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ await cli('-s=first', 'open', server.EMPTY_PAGE, { bindTitle });+ await cli('-s=second', 'open', server.EMPTY_PAGE, { bindTitle });++ const annotatePromise = cli('-s=second', 'show', '--annotate');+ let done = false;+ void annotatePromise.finally(() => { done = true; });++ const browser = await connectToDashboard(bindTitle);+ try {+ const dashboard = browser.contexts()[0].pages()[0];+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();+ await expect(activeSession(dashboard)).toHaveAccessibleName('Session second');+ await drawAndSubmitAnnotation(dashboard, 'fresh');+ } finally {+ await browser.close().catch(() => {});+ }++ const { exitCode } = await annotatePromise;+ expect(done).toBe(true);+ expect(exitCode).toBe(0);+});++test('should annotate via direct browser_annotate MCP call', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {+ const page = await boundBrowser.newPage();+ await page.goto(server.EMPTY_PAGE);++ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ const { client } = await startClient({+ args: ['--endpoint=default', '--caps=devtools'],+ env: {+ ...cliEnv,+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,+ },+ });++ const annotatePromise = client.callTool({ name: 'browser_annotate' });+ let done = false;+ void annotatePromise.then(() => { done = true; });++ const browser = await connectToDashboard(bindTitle);+ try {+ const dashboard = browser.contexts()[0].pages()[0];+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();+ await drawAndSubmitAnnotation(dashboard, 'direct-mcp');+ } finally {+ await browser.close().catch(() => {});+ }++ const result = await annotatePromise;+ expect(done).toBe(true);+ const text = (result.content as any).map(c => c.text ?? '').join('\n');+ expect(text).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: direct-mcp/);+ expect(text).toMatch(/- \[Annotation image\]\(.*\.png\)/);+});++test('should annotate when context has no fixed viewport', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {+ // Simulates headed `playwright-cli open --headed`, which launches with viewport: null+ // so that the browser window controls the page size. https://github.com/microsoft/playwright/issues/40565+ const context = await boundBrowser.newContext({ viewport: null });+ const page = await context.newPage();+ expect(page.viewportSize()).toBe(null);+ await page.goto(server.EMPTY_PAGE);++ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ const { client } = await startClient({+ args: ['--endpoint=default', '--caps=devtools'],+ env: {+ ...cliEnv,+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,+ },+ });++ const annotatePromise = client.callTool({ name: 'browser_annotate' });+ let done = false;+ void annotatePromise.then(() => { done = true; });++ const browser = await connectToDashboard(bindTitle);+ try {+ const dashboard = browser.contexts()[0].pages()[0];+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();+ await drawAndSubmitAnnotation(dashboard, 'no-viewport');+ } finally {+ await browser.close().catch(() => {});+ }++ const result = await annotatePromise;+ expect(done).toBe(true);+ const text = (result.content as any).map(c => c.text ?? '').join('\n');+ expect(text).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: no-viewport/);+ expect(text).toMatch(/- \[Annotation image\]\(.*\.png\)/);+});++test('should cancel browser_annotate when the MCP request is aborted', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {+ const page = await boundBrowser.newPage();+ await page.goto(server.EMPTY_PAGE);++ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ const { client } = await startClient({+ args: ['--endpoint=default', '--caps=devtools'],+ env: {+ ...cliEnv,+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,+ },+ });++ const controller = new AbortController();+ const annotatePromise = client.callTool({ name: 'browser_annotate' }, undefined, { signal: controller.signal }).catch(() => {});++ const browser = await connectToDashboard(bindTitle);+ try {+ const dashboard = browser.contexts()[0].pages()[0];+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();++ controller.abort();++ await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();+ } finally {+ await browser.close().catch(() => {});+ }++ await annotatePromise;+});++test('should cancel browser_annotate when the MCP client disconnects', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {+ const page = await boundBrowser.newPage();+ await page.goto(server.EMPTY_PAGE);++ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ const { client } = await startClient({+ args: ['--endpoint=default', '--caps=devtools'],+ env: {+ ...cliEnv,+ PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,+ },+ });++ void client.callTool({ name: 'browser_annotate' }).catch(() => {});++ const browser = await connectToDashboard(bindTitle);+ try {+ const dashboard = browser.contexts()[0].pages()[0];+ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();++ await client.close();++ await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();+ } finally {+ await browser.close().catch(() => {});+ }+});+++test('should switch screencast to -s session on show --annotate', async ({ connectToDashboard, cli, server }) => {+ server.setContent('/red', '<html><head><style>html,body{margin:0;height:100vh;background:#ff0000}</style></head><body></body></html>', 'text/html');+ server.setContent('/green', '<html><head><style>html,body{margin:0;height:100vh;background:#00ff00}</style></head><body></body></html>', 'text/html');++ await cli('-s=first', 'open', server.PREFIX + '/red');+ await cli('-s=second', 'open', server.PREFIX + '/green');++ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ await cli('-s=first', 'show', { bindTitle });+ const browser = await connectToDashboard(bindTitle);+ const dashboard = browser.contexts()[0].pages()[0];+ await expect(dashboard.locator('#display')).toBeVisible();++ const sampleCenter = () => dashboard.evaluate(() => {+ const img = document.querySelector('#display') as HTMLImageElement | null;+ if (!img || !img.naturalWidth)+ return null;+ const canvas = document.createElement('canvas');+ canvas.width = 1;+ canvas.height = 1;+ const ctx = canvas.getContext('2d')!;+ ctx.drawImage(img, img.naturalWidth / 2, img.naturalHeight / 2, 1, 1, 0, 0, 1, 1);+ const [r, g] = ctx.getImageData(0, 0, 1, 1).data;+ return { r, g };+ });++ await expect.poll(async () => {+ const c = await sampleCenter();+ return !!(c && c.r > 200 && c.g < 50);+ }, { timeout: 15000 }).toBe(true);++ const annotatePromise = cli('-s=second', 'show', '--annotate');+ let done = false;+ void annotatePromise.finally(() => { done = true; });++ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();+ await expect(activeSession(dashboard)).toHaveAccessibleName('Session second');++ await expect.poll(async () => {+ const c = await sampleCenter();+ return !!(c && c.g > 200 && c.r < 50);+ }, { timeout: 15000 }).toBe(true);++ await drawAndSubmitAnnotation(dashboard, 'session switch');+ const { exitCode } = await annotatePromise;+ expect(done).toBe(true);+ expect(exitCode).toBe(0);+});++test('should disengage annotate mode when --annotate client disconnects', async ({ connectToDashboard, cli, childProcess, cliEnv, mcpBrowser, mcpHeadless, server }) => {+ await cli('open', server.EMPTY_PAGE);+ const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;+ await cli('show', { bindTitle });+ const browser = await connectToDashboard(bindTitle);++ const dashboard = browser.contexts()[0].pages()[0];+ await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();++ const annotateClient = childProcess({+ command: [process.execPath, require.resolve('../../packages/playwright-core/lib/tools/cli-client/cli.js'), 'show', '--annotate'],+ cwd: test.info().outputPath(),+ env: inheritAndCleanEnv({+ ...cliEnv,+ PLAYWRIGHT_MCP_BROWSER: mcpBrowser,+ PLAYWRIGHT_MCP_HEADLESS: String(mcpHeadless),+ }),+ });++ await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();++ await annotateClient.kill();++ await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();+});diff --git a/tests/mcp/cli-fixtures.ts b/tests/mcp/cli-fixtures.tsindex dea738a7f3..4c7413f228 100644--- a/tests/mcp/cli-fixtures.ts+++ b/tests/mcp/cli-fixtures.ts@@ -261,3 +261,45 @@ export async function daemonFolder() {}return null;}++export async function installSaveFilePickerMock(page: import('playwright-core').Page): Promise<() => Promise<Buffer>> {+ await page.evaluate(() => {+ (window as any).__testCaptureBytes = undefined as string | undefined;+ (window as any).showSaveFilePicker = async () => ({+ createWritable: async () => {+ const chunks: Uint8Array[] = [];+ return {+ write: async (chunk: Blob | BufferSource) => {+ const buf = chunk instanceof Blob+ ? new Uint8Array(await chunk.arrayBuffer())+ : new Uint8Array(chunk instanceof ArrayBuffer ? chunk : (chunk as ArrayBufferView).buffer);+ chunks.push(buf);+ },+ close: async () => {+ const total = chunks.reduce((n, c) => n + c.byteLength, 0);+ const merged = new Uint8Array(total);+ let offset = 0;+ for (const c of chunks) {+ merged.set(c, offset);+ offset += c.byteLength;+ }+ (window as any).__testCaptureBytes = (merged as any).toBase64();+ },+ };+ },+ });+ });+ return async () => {+ await expect.poll(() => page.evaluate(() => !!(window as any).__testCaptureBytes), { timeout: 10000 }).toBe(true);+ const b64: string = await page.evaluate(() => (window as any).__testCaptureBytes);+ return Buffer.from(b64, 'base64');+ };+}++export async function mockAbortingFilePicker(page: import('playwright-core').Page): Promise<void> {+ await page.evaluate(() => {+ (window as any).showSaveFilePicker = async () => {+ throw new DOMException('The user aborted a request.', 'AbortError');+ };+ });+}diff --git a/tests/mcp/dashboard.spec.ts b/tests/mcp/dashboard.spec.tsindex dd25d767e0..f9a52b515f 100644--- a/tests/mcp/dashboard.spec.ts+++ b/tests/mcp/dashboard.spec.ts@@ -18,8 +18,7 @@ import fs from 'fs';import os from 'os';import path from 'path';-import { test, expect } from './cli-fixtures';-import { inheritAndCleanEnv } from '../config/utils';+import { test, expect, installSaveFilePickerMock } from './cli-fixtures';function displayPath(p: string): string {const home = os.homedir();@@ -135,345 +134,6 @@ test('daemon show: closing page exits the process', async ({ cli, connectToDashbawait expect(() => expect(isAlive(dashboardPid)).toBe(false)).toPass();});-async function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page, text: string) {- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();- await expect(dashboard.locator('.annotate-modal-image')).toBeVisible();- const box = await dashboard.locator('.annotate-modal-image').boundingBox();- const x0 = box!.x + box!.width * 0.3;- const y0 = box!.y + box!.height * 0.3;- const x1 = box!.x + box!.width * 0.6;- const y1 = box!.y + box!.height * 0.6;- await dashboard.mouse.move(x0, y0);- await dashboard.mouse.down();- await dashboard.mouse.move(x1, y1);- await dashboard.mouse.up();- await dashboard.locator('.annotations-textarea').fill(text);- await dashboard.locator('.annotations-textarea').press('Enter');- await dashboard.getByRole('button', { name: 'Submit annotation' }).click();-}--function verifyAnnotateOutput(output: string, expectedText: string, outputDir: string) {- expect(output).toMatch(new RegExp(`\\{ x: \\d+, y: \\d+, width: \\d+, height: \\d+ \\}: ${expectedText}`));- const imageMatch = output.match(/- \[Annotation image\]\((\.playwright-cli[\\/]annotations-.*\.png)\)/);- expect(imageMatch).not.toBeNull();- const pngPath = path.resolve(outputDir, imageMatch![1]);- expect(fs.existsSync(pngPath)).toBe(true);- expect(fs.statSync(pngPath).size).toBeGreaterThan(0);-}--test('should capture annotations via show --annotate', async ({ connectToDashboard, cli, server }) => {- await cli('open', server.EMPTY_PAGE);- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;- await cli('show', { bindTitle });- const browser = await connectToDashboard(bindTitle);-- const dashboard = browser.contexts()[0].pages()[0];- await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();-- const annotatePromise = cli('show', '--annotate');- let done = false;- void annotatePromise.finally(() => { done = true; });-- await drawAndSubmitAnnotation(dashboard, 'hello');-- const { output, exitCode } = await annotatePromise;- expect(done).toBe(true);- expect(exitCode).toBe(0);- verifyAnnotateOutput(output, 'hello', test.info().outputDir);-});--test('should start dashboard and annotate when no dashboard is running', async ({ connectToDashboard, cli, server }) => {- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;- await cli('open', server.EMPTY_PAGE, { bindTitle });-- const annotatePromise = cli('show', '--annotate');- let done = false;- void annotatePromise.finally(() => { done = true; });-- const browser = await connectToDashboard(bindTitle);- try {- const dashboard = browser.contexts()[0].pages()[0];- await drawAndSubmitAnnotation(dashboard, 'hi');- } finally {- await browser.close().catch(() => {});- }-- const { output, exitCode } = await annotatePromise;- expect(done).toBe(true);- expect(exitCode).toBe(0);- verifyAnnotateOutput(output, 'hi', test.info().outputDir);-});--test('should enter annotate mode on fresh dashboard.tsx mount with -s --annotate', async ({ connectToDashboard, cli, server }) => {- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;- await cli('-s=first', 'open', server.EMPTY_PAGE, { bindTitle });- await cli('-s=second', 'open', server.EMPTY_PAGE, { bindTitle });-- const annotatePromise = cli('-s=second', 'show', '--annotate');- let done = false;- void annotatePromise.finally(() => { done = true; });-- const browser = await connectToDashboard(bindTitle);- try {- const dashboard = browser.contexts()[0].pages()[0];- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();- await expect(activeSession(dashboard)).toHaveAccessibleName('Session second');- await drawAndSubmitAnnotation(dashboard, 'fresh');- } finally {- await browser.close().catch(() => {});- }-- const { exitCode } = await annotatePromise;- expect(done).toBe(true);- expect(exitCode).toBe(0);-});--test('should annotate via direct browser_annotate MCP call', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {- const page = await boundBrowser.newPage();- await page.goto(server.EMPTY_PAGE);-- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;- const { client } = await startClient({- args: ['--endpoint=default', '--caps=devtools'],- env: {- ...cliEnv,- PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,- },- });-- const annotatePromise = client.callTool({ name: 'browser_annotate' });- let done = false;- void annotatePromise.then(() => { done = true; });-- const browser = await connectToDashboard(bindTitle);- try {- const dashboard = browser.contexts()[0].pages()[0];- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();- await drawAndSubmitAnnotation(dashboard, 'direct-mcp');- } finally {- await browser.close().catch(() => {});- }-- const result = await annotatePromise;- expect(done).toBe(true);- const text = (result.content as any).map(c => c.text ?? '').join('\n');- expect(text).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: direct-mcp/);- expect(text).toMatch(/- \[Annotation image\]\(.*\.png\)/);-});--test('should annotate when context has no fixed viewport', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {- // Simulates headed `playwright-cli open --headed`, which launches with viewport: null- // so that the browser window controls the page size. https://github.com/microsoft/playwright/issues/40565- const context = await boundBrowser.newContext({ viewport: null });- const page = await context.newPage();- expect(page.viewportSize()).toBe(null);- await page.goto(server.EMPTY_PAGE);-- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;- const { client } = await startClient({- args: ['--endpoint=default', '--caps=devtools'],- env: {- ...cliEnv,- PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,- },- });-- const annotatePromise = client.callTool({ name: 'browser_annotate' });- let done = false;- void annotatePromise.then(() => { done = true; });-- const browser = await connectToDashboard(bindTitle);- try {- const dashboard = browser.contexts()[0].pages()[0];- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();- await drawAndSubmitAnnotation(dashboard, 'no-viewport');- } finally {- await browser.close().catch(() => {});- }-- const result = await annotatePromise;- expect(done).toBe(true);- const text = (result.content as any).map(c => c.text ?? '').join('\n');- expect(text).toMatch(/\{ x: \d+, y: \d+, width: \d+, height: \d+ \}: no-viewport/);- expect(text).toMatch(/- \[Annotation image\]\(.*\.png\)/);-});--test('should cancel browser_annotate when the MCP request is aborted', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {- const page = await boundBrowser.newPage();- await page.goto(server.EMPTY_PAGE);-- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;- const { client } = await startClient({- args: ['--endpoint=default', '--caps=devtools'],- env: {- ...cliEnv,- PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,- },- });-- const controller = new AbortController();- const annotatePromise = client.callTool({ name: 'browser_annotate' }, undefined, { signal: controller.signal }).catch(() => {});-- const browser = await connectToDashboard(bindTitle);- try {- const dashboard = browser.contexts()[0].pages()[0];- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();-- controller.abort();-- await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();- } finally {- await browser.close().catch(() => {});- }-- await annotatePromise;-});--test('should cancel browser_annotate when the MCP client disconnects', async ({ connectToDashboard, boundBrowser, startClient, cliEnv, server }) => {- const page = await boundBrowser.newPage();- await page.goto(server.EMPTY_PAGE);-- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;- const { client } = await startClient({- args: ['--endpoint=default', '--caps=devtools'],- env: {- ...cliEnv,- PWTEST_DASHBOARD_APP_BIND_TITLE: bindTitle,- },- });-- void client.callTool({ name: 'browser_annotate' }).catch(() => {});-- const browser = await connectToDashboard(bindTitle);- try {- const dashboard = browser.contexts()[0].pages()[0];- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();-- await client.close();-- await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();- } finally {- await browser.close().catch(() => {});- }-});---test('should switch screencast to -s session on show --annotate', async ({ connectToDashboard, cli, server }) => {- server.setContent('/red', '<html><head><style>html,body{margin:0;height:100vh;background:#ff0000}</style></head><body></body></html>', 'text/html');- server.setContent('/green', '<html><head><style>html,body{margin:0;height:100vh;background:#00ff00}</style></head><body></body></html>', 'text/html');-- await cli('-s=first', 'open', server.PREFIX + '/red');- await cli('-s=second', 'open', server.PREFIX + '/green');-- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;- await cli('-s=first', 'show', { bindTitle });- const browser = await connectToDashboard(bindTitle);- const dashboard = browser.contexts()[0].pages()[0];- await expect(dashboard.locator('#display')).toBeVisible();-- const sampleCenter = () => dashboard.evaluate(() => {- const img = document.querySelector('#display') as HTMLImageElement | null;- if (!img || !img.naturalWidth)- return null;- const canvas = document.createElement('canvas');- canvas.width = 1;- canvas.height = 1;- const ctx = canvas.getContext('2d')!;- ctx.drawImage(img, img.naturalWidth / 2, img.naturalHeight / 2, 1, 1, 0, 0, 1, 1);- const [r, g] = ctx.getImageData(0, 0, 1, 1).data;- return { r, g };- });-- await expect.poll(async () => {- const c = await sampleCenter();- return !!(c && c.r > 200 && c.g < 50);- }, { timeout: 15000 }).toBe(true);-- const annotatePromise = cli('-s=second', 'show', '--annotate');- let done = false;- void annotatePromise.finally(() => { done = true; });-- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();- await expect(activeSession(dashboard)).toHaveAccessibleName('Session second');-- await expect.poll(async () => {- const c = await sampleCenter();- return !!(c && c.g > 200 && c.r < 50);- }, { timeout: 15000 }).toBe(true);-- await drawAndSubmitAnnotation(dashboard, 'session switch');- const { exitCode } = await annotatePromise;- expect(done).toBe(true);- expect(exitCode).toBe(0);-});--test('should disengage annotate mode when --annotate client disconnects', async ({ connectToDashboard, cli, childProcess, cliEnv, mcpBrowser, mcpHeadless, server }) => {- await cli('open', server.EMPTY_PAGE);- const bindTitle = `--playwright-internal--${crypto.randomUUID()}`;- await cli('show', { bindTitle });- const browser = await connectToDashboard(bindTitle);-- const dashboard = browser.contexts()[0].pages()[0];- await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();-- const annotateClient = childProcess({- command: [process.execPath, require.resolve('../../packages/playwright-core/lib/tools/cli-client/cli.js'), 'show', '--annotate'],- cwd: test.info().outputPath(),- env: inheritAndCleanEnv({- ...cliEnv,- PLAYWRIGHT_MCP_BROWSER: mcpBrowser,- PLAYWRIGHT_MCP_HEADLESS: String(mcpHeadless),- }),- });-- await expect(dashboard.getByRole('main', { name: 'Dashboard: annotate' })).toBeVisible();-- await annotateClient.kill();-- await expect(dashboard.getByRole('main', { name: 'Dashboard', exact: true })).toBeVisible();-});--async function installSaveFilePickerMock(page: import('playwright-core').Page): Promise<() => Promise<Buffer>> {- let captured: string | undefined;- let resolveCaptured: ((b64: string) => void) | undefined;- const waitForCapture = new Promise<string>(resolve => {- resolveCaptured = resolve;- });- await page.exposeBinding('__testCaptureBytes', (_, b64: string) => {- captured = b64;- resolveCaptured!(b64);- });- await page.addInitScript(() => {- (window as any).showSaveFilePicker = async () => ({- createWritable: async () => {- const chunks: Uint8Array[] = [];- return {- write: async (chunk: Blob | BufferSource) => {- const buf = chunk instanceof Blob- ? new Uint8Array(await chunk.arrayBuffer())- : new Uint8Array(chunk instanceof ArrayBuffer ? chunk : (chunk as ArrayBufferView).buffer);- chunks.push(buf);- },- close: async () => {- const total = chunks.reduce((n, c) => n + c.byteLength, 0);- const merged = new Uint8Array(total);- let offset = 0;- for (const c of chunks) {- merged.set(c, offset);- offset += c.byteLength;- }- await (window as any).__testCaptureBytes((merged as any).toBase64());- },- };- },- });- });- return async () => {- const b64 = captured ?? await waitForCapture;- return Buffer.from(b64, 'base64');- };-}-test('should allow typing in omnibox in interactive mode', async ({ cli, server, startDashboardServer }) => {server.setContent('/page1', '<html><body>Page 1</body></html>', 'text/html');server.setContent('/page2', '<html><body>Page 2</body></html>', 'text/html');@@ -496,11 +156,11 @@ test('should allow typing in omnibox in interactive mode', async ({ cli, server,await expect(dashboard.locator('#omnibox')).toHaveValue(server.PREFIX + '/page2', { timeout: 10000 });});-test('save recording streams WebM bytes to the chosen file', async ({ cli, server, page, startDashboardServer }) => {+test('save recording streams WebM bytes to the chosen file', async ({ cli, server, startDashboardServer }) => {await cli('open', server.EMPTY_PAGE);- const awaitBytes = await installSaveFilePickerMock(page);const dashboard = await startDashboardServer();+ const awaitBytes = await installSaveFilePickerMock(dashboard);await dashboard.getByRole('navigation', { name: 'Sessions' }).getByRole('option').first().click();await expect(dashboard.locator('img#display')).toBeVisible();