Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(dashboard): add annotation mode (#40238)
Pavel Feldman1898141d5d17
diff --git a/packages/dashboard/src/annotations.css b/packages/dashboard/src/annotations.cssnew file mode 100644index 0000000000..0090d8e3d9--- /dev/null+++ b/packages/dashboard/src/annotations.css@@ -0,0 +1,255 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++.annotation-layer {+ position: absolute;+ inset: 0;+ z-index: 5;+ outline: none;+ cursor: crosshair;+}++.annotation-layer::after {+ content: '';+ position: absolute;+ inset: 0;+ pointer-events: none;+ box-shadow: inset 0 0 0 1px rgb(var(--annotate-blue) / 0.72), inset 0 0 26px rgb(var(--annotate-blue) / 0.28);+}++.annotation-toolbar {+ position: absolute;+ top: 8px;+ right: 8px;+ display: inline-flex;+ gap: 6px;+ padding: 6px;+ background: var(--color-canvas-overlay);+ border: 1px solid var(--color-border-muted);+ border-radius: 999px;+ box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2);+ z-index: 20;+ cursor: default;+}++.annotate-action-btn {+ height: 26px;+ padding: 0 12px;+ font-size: 11px;+ font-weight: 600;+ letter-spacing: 0.03em;+ text-transform: uppercase;+ background: var(--color-neutral-subtle);+ color: var(--color-fg-default);+ border: 1px solid var(--color-border-muted);+ border-radius: 999px;+ cursor: pointer;+ line-height: 1;+}++.annotate-action-btn:hover:not(:disabled) {+ background: var(--color-neutral-muted);+}++.annotate-action-btn:disabled {+ opacity: 0.5;+ cursor: default;+}++.annotate-action-btn.primary {+ background: rgb(var(--annotate-blue));+ color: #fff;+ border-color: transparent;+}++.annotate-action-btn.primary:hover:not(:disabled) {+ background: rgb(var(--annotate-blue) / 0.85);+}++.annotate-action-btn.danger {+ background: transparent;+ color: var(--color-danger-fg);+ border-color: var(--color-border-muted);+}++.annotate-action-btn.danger:hover:not(:disabled) {+ background: var(--color-danger-subtle);+}++.annotation-rect {+ position: absolute;+ box-sizing: border-box;+ border: 2px solid rgb(var(--annotate-blue));+ background: rgb(var(--annotate-blue) / 0.12);+ cursor: pointer;+ transition: box-shadow 120ms ease;+}++.annotation-rect:hover {+ box-shadow: 0 0 0 3px rgb(var(--annotate-blue) / 0.25);+}++.annotation-rect.selected {+ box-shadow: 0 0 0 3px rgb(var(--annotate-blue) / 0.4);+ border-style: solid;+ border-color: rgb(var(--annotate-blue));+ cursor: move;+}++.annotation-rect.draft {+ border-style: dashed;+ background: rgb(var(--annotate-blue) / 0.18);+ pointer-events: none;+}++.annotation-handle {+ position: absolute;+ width: 10px;+ height: 10px;+ background: #fff;+ border: 1.5px solid rgb(var(--annotate-blue));+ border-radius: 2px;+ box-sizing: border-box;+ z-index: 1;+}++.annotation-handle-nw { left: -6px; top: -6px; cursor: nwse-resize; }+.annotation-handle-n { left: 50%; top: -6px; margin-left: -5px; cursor: ns-resize; }+.annotation-handle-ne { right: -6px; top: -6px; cursor: nesw-resize; }+.annotation-handle-e { right: -6px; top: 50%; margin-top: -5px; cursor: ew-resize; }+.annotation-handle-se { right: -6px; bottom: -6px; cursor: nwse-resize; }+.annotation-handle-s { left: 50%; bottom: -6px; margin-left: -5px; cursor: ns-resize; }+.annotation-handle-sw { left: -6px; bottom: -6px; cursor: nesw-resize; }+.annotation-handle-w { left: -6px; top: 50%; margin-top: -5px; cursor: ew-resize; }++.annotation-label {+ position: absolute;+ top: -2px;+ left: -2px;+ max-width: calc(100% + 4px);+ padding: 2px 6px;+ background: rgb(var(--annotate-blue));+ color: #fff;+ font-size: 11px;+ font-weight: 500;+ border-radius: 2px 2px 2px 0;+ transform: translateY(-100%);+ white-space: nowrap;+ overflow: hidden;+ text-overflow: ellipsis;+ cursor: text;+}++.annotation-label:hover {+ background: rgb(var(--annotate-blue) / 0.85);+}++.annotation-popover {+ position: absolute;+ min-width: 220px;+ max-width: 320px;+ background: var(--color-canvas-overlay);+ border: 1px solid var(--color-border-default);+ border-radius: 8px;+ padding: 8px;+ box-shadow: 0 8px 24px rgba(0, 0, 0, 0.3);+ display: flex;+ flex-direction: column;+ gap: 6px;+ z-index: 10;+ cursor: default;+}++.annotation-textarea {+ width: 100%;+ min-height: 64px;+ resize: vertical;+ padding: 6px 8px;+ font-family: inherit;+ font-size: 12px;+ color: var(--color-fg-default);+ background: var(--color-canvas-default);+ border: 1px solid var(--color-border-muted);+ border-radius: 6px;+ outline: none;+ box-sizing: border-box;+}++.annotation-textarea:focus {+ border-color: rgb(var(--annotate-blue));+}++.annotation-popover-actions {+ display: flex;+ gap: 6px;+ justify-content: flex-end;+}++.annotation-modal-backdrop {+ position: absolute;+ inset: 0;+ background: rgba(0, 0, 0, 0.5);+ display: flex;+ align-items: center;+ justify-content: center;+ z-index: 100;+ cursor: default;+}++.annotation-modal {+ width: min(520px, 92%);+ max-height: 80%;+ background: var(--color-canvas-default);+ border: 1px solid var(--color-border-default);+ border-radius: 10px;+ box-shadow: 0 12px 40px rgba(0, 0, 0, 0.45);+ display: flex;+ flex-direction: column;+ overflow: hidden;+}++.annotation-modal-header {+ display: flex;+ align-items: center;+ justify-content: space-between;+ padding: 12px 16px;+ border-bottom: 1px solid var(--color-border-muted);+ font-size: 13px;+ font-weight: 600;+ color: var(--color-fg-default);+}++.annotation-modal-text {+ flex: 1;+ min-height: 200px;+ padding: 12px 16px;+ font-family: 'SF Mono', 'Cascadia Code', 'Fira Code', 'Consolas', monospace;+ font-size: 12px;+ color: var(--color-fg-default);+ background: var(--color-canvas-subtle);+ border: none;+ outline: none;+ resize: none;+ white-space: pre;+}++.annotation-modal-actions {+ display: flex;+ justify-content: flex-end;+ gap: 8px;+ padding: 10px 16px;+ border-top: 1px solid var(--color-border-muted);+}diff --git a/packages/dashboard/src/annotations.tsx b/packages/dashboard/src/annotations.tsxnew file mode 100644index 0000000000..628a1bc736--- /dev/null+++ b/packages/dashboard/src/annotations.tsx@@ -0,0 +1,458 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import React from 'react';+import './annotations.css';++export type Annotation = { id: string; x: number; y: number; width: number; height: number; text: string };++type Rect = { x: number; y: number; width: number; height: number };+type DragKind = 'move' | 'nw' | 'n' | 'ne' | 'e' | 'se' | 's' | 'sw' | 'w';+const HANDLES: Exclude<DragKind, 'move'>[] = ['nw', 'n', 'ne', 'e', 'se', 's', 'sw', 'w'];++type DragState = {+ kind: DragKind;+ id: string;+ orig: Rect;+ startVx: number;+ startVy: number;+};++type Selection = { id: string; editing: boolean } | null;++const MIN_ANNOTATION_SIZE = 4;++function newAnnotationId() {+ return 'ann-' + Math.random().toString(36).slice(2, 10);+}++function normalizeRect(a: { startX: number; startY: number; x: number; y: number }): Rect {+ return {+ x: Math.min(a.startX, a.x),+ y: Math.min(a.startY, a.y),+ width: Math.abs(a.x - a.startX),+ height: Math.abs(a.y - a.startY),+ };+}++function applyDrag(orig: Rect, kind: DragKind, dvx: number, dvy: number): Rect {+ let left = orig.x;+ let top = orig.y;+ let right = orig.x + orig.width;+ let bottom = orig.y + orig.height;+ switch (kind) {+ case 'move':+ left += dvx; top += dvy; right += dvx; bottom += dvy; break;+ case 'nw': left += dvx; top += dvy; break;+ case 'n': top += dvy; break;+ case 'ne': right += dvx; top += dvy; break;+ case 'e': right += dvx; break;+ case 'se': right += dvx; bottom += dvy; break;+ case 's': bottom += dvy; break;+ case 'sw': left += dvx; bottom += dvy; break;+ case 'w': left += dvx; break;+ }+ return {+ x: Math.min(left, right),+ y: Math.min(top, bottom),+ width: Math.abs(right - left),+ height: Math.abs(bottom - top),+ };+}++export type ImageLayout = {+ rect: DOMRect;+ renderW: number;+ renderH: number;+ offsetX: number;+ offsetY: number;+};++export function getImageLayout(display: HTMLImageElement | null): ImageLayout | null {+ if (!display || !display.naturalWidth || !display.naturalHeight)+ return null;+ const rect = display.getBoundingClientRect();+ const imgAspect = display.naturalWidth / display.naturalHeight;+ const elemAspect = rect.width / rect.height;+ if (imgAspect > elemAspect) {+ const renderH = rect.width / imgAspect;+ return { rect, renderW: rect.width, renderH, offsetX: 0, offsetY: (rect.height - renderH) / 2 };+ }+ const renderW = rect.height * imgAspect;+ return { rect, renderW, renderH: rect.height, offsetX: (rect.width - renderW) / 2, offsetY: 0 };+}++export function clientToViewport(layout: ImageLayout, vw: number, vh: number, clientX: number, clientY: number): { x: number; y: number } {+ const fracX = (clientX - layout.rect.left - layout.offsetX) / layout.renderW;+ const fracY = (clientY - layout.rect.top - layout.offsetY) / layout.renderH;+ return { x: Math.round(fracX * vw), y: Math.round(fracY * vh) };+}++function viewportRectToScreenStyle(layout: ImageLayout, screenRect: DOMRect, vw: number, vh: number, r: Rect): React.CSSProperties {+ const baseLeft = layout.rect.left - screenRect.left + layout.offsetX;+ const baseTop = layout.rect.top - screenRect.top + layout.offsetY;+ return {+ left: baseLeft + (r.x / vw) * layout.renderW,+ top: baseTop + (r.y / vh) * layout.renderH,+ width: (r.width / vw) * layout.renderW,+ height: (r.height / vh) * layout.renderH,+ };+}++export const Annotations: React.FC<{+ active: boolean;+ displayRef: React.RefObject<HTMLImageElement | null>;+ screenRef: React.RefObject<HTMLDivElement | null>;+ viewportWidth: number;+ viewportHeight: number;+}> = ({ active, displayRef, screenRef, viewportWidth, viewportHeight }) => {+ const [annotations, setAnnotations] = React.useState<Annotation[]>([]);+ const [draft, setDraft] = React.useState<{ startX: number; startY: number; x: number; y: number } | null>(null);+ const [selection, setSelection] = React.useState<Selection>(null);+ const [drag, setDrag] = React.useState<DragState | null>(null);+ const [submittedJson, setSubmittedJson] = React.useState<string | null>(null);+ const [, setTick] = React.useState(0);+ const forceRender = React.useCallback(() => setTick(t => t + 1), []);+ const layerRef = React.useRef<HTMLDivElement>(null);++ const selectedId = selection?.id ?? null;+ const editingId = selection?.editing ? selection.id : null;++ React.useEffect(() => {+ if (!active)+ return;+ const onResize = () => forceRender();+ const img = displayRef.current;+ const onLoad = () => forceRender();+ window.addEventListener('resize', onResize);+ img?.addEventListener('load', onLoad);+ return () => {+ window.removeEventListener('resize', onResize);+ img?.removeEventListener('load', onLoad);+ };+ }, [active, displayRef, forceRender]);++ React.useEffect(() => {+ if (active)+ layerRef.current?.focus();+ else+ setSelection(null);+ }, [active]);++ React.useEffect(() => {+ if (!drag)+ return;+ const onMove = (e: MouseEvent) => {+ const layout = getImageLayout(displayRef.current);+ if (!layout || !viewportWidth || !viewportHeight)+ return;+ const vp = clientToViewport(layout, viewportWidth, viewportHeight, e.clientX, e.clientY);+ const dvx = vp.x - drag.startVx;+ const dvy = vp.y - drag.startVy;+ if (dvx === 0 && dvy === 0)+ return;+ setAnnotations(prev => prev.map(a => a.id === drag.id ? { ...a, ...applyDrag(drag.orig, drag.kind, dvx, dvy) } : a));+ };+ const onUp = () => setDrag(null);+ window.addEventListener('mousemove', onMove);+ window.addEventListener('mouseup', onUp);+ return () => {+ window.removeEventListener('mousemove', onMove);+ window.removeEventListener('mouseup', onUp);+ };+ }, [drag, displayRef, viewportWidth, viewportHeight]);++ function imgCoords(e: React.MouseEvent): { x: number; y: number } | null {+ if (!viewportWidth || !viewportHeight)+ return null;+ const layout = getImageLayout(displayRef.current);+ if (!layout)+ return null;+ return clientToViewport(layout, viewportWidth, viewportHeight, e.clientX, e.clientY);+ }++ function hitTestAnnotation(x: number, y: number): Annotation | undefined {+ for (let i = annotations.length - 1; i >= 0; i--) {+ const a = annotations[i];+ if (x >= a.x && x <= a.x + a.width && y >= a.y && y <= a.y + a.height)+ return a;+ }+ return undefined;+ }++ function onLayerMouseDown(e: React.MouseEvent) {+ if (e.button !== 0)+ return;+ layerRef.current?.focus();+ const vp = imgCoords(e);+ if (!vp)+ return;+ const hit = hitTestAnnotation(vp.x, vp.y);+ e.preventDefault();+ if (hit) {+ setSelection({ id: hit.id, editing: false });+ setDrag({ kind: 'move', id: hit.id, orig: { x: hit.x, y: hit.y, width: hit.width, height: hit.height }, startVx: vp.x, startVy: vp.y });+ return;+ }+ setDraft({ startX: vp.x, startY: vp.y, x: vp.x, y: vp.y });+ setSelection(null);+ }++ function onLayerMouseMove(e: React.MouseEvent) {+ if (drag || !draft)+ return;+ const vp = imgCoords(e);+ if (!vp)+ return;+ if (draft.x === vp.x && draft.y === vp.y)+ return;+ setDraft({ ...draft, x: vp.x, y: vp.y });+ }++ function onLayerMouseUp(e: React.MouseEvent) {+ if (!draft)+ return;+ e.preventDefault();+ const rect = normalizeRect(draft);+ setDraft(null);+ if (rect.width < MIN_ANNOTATION_SIZE || rect.height < MIN_ANNOTATION_SIZE)+ return;+ const id = newAnnotationId();+ setAnnotations(prev => [...prev, { id, ...rect, text: '' }]);+ setSelection({ id, editing: true });+ }++ function startResize(kind: Exclude<DragKind, 'move'>, a: Annotation, e: React.MouseEvent) {+ e.preventDefault();+ e.stopPropagation();+ layerRef.current?.focus();+ const vp = imgCoords(e);+ if (!vp)+ return;+ setSelection({ id: a.id, editing: false });+ setDrag({ kind, id: a.id, orig: { x: a.x, y: a.y, width: a.width, height: a.height }, startVx: vp.x, startVy: vp.y });+ }++ function nudgeSelected(dx: number, dy: number) {+ if (!selectedId)+ return;+ setAnnotations(prev => prev.map(a => a.id === selectedId ? { ...a, x: a.x + dx, y: a.y + dy } : a));+ }++ function closeEditor() {+ setSelection(sel => sel?.editing ? { ...sel, editing: false } : sel);+ layerRef.current?.focus();+ }++ function onLayerKeyDown(e: React.KeyboardEvent) {+ if (e.key === 'Escape') {+ e.preventDefault();+ if (draft)+ setDraft(null);+ else if (editingId)+ closeEditor();+ else if (selectedId)+ setSelection(null);+ return;+ }+ if ((e.key === 'Delete' || e.key === 'Backspace') && selectedId && !editingId) {+ e.preventDefault();+ setAnnotations(prev => prev.filter(a => a.id !== selectedId));+ setSelection(null);+ return;+ }+ if (selectedId && !editingId) {+ const step = e.shiftKey ? 1 : 5;+ if (e.key === 'ArrowLeft') { e.preventDefault(); nudgeSelected(-step, 0); return; }+ if (e.key === 'ArrowRight') { e.preventDefault(); nudgeSelected(step, 0); return; }+ if (e.key === 'ArrowUp') { e.preventDefault(); nudgeSelected(0, -step); return; }+ if (e.key === 'ArrowDown') { e.preventDefault(); nudgeSelected(0, step); return; }+ }+ }++ function submitAnnotations() {+ const json = JSON.stringify({ viewportWidth, viewportHeight, annotations }, null, 2);+ console.log('Annotations:', json);+ setSubmittedJson(json);+ }++ if (!active)+ return null;++ const layout = getImageLayout(displayRef.current);+ const screenRect = screenRef.current?.getBoundingClientRect() ?? null;+ const mapRect = (r: Rect): React.CSSProperties | null =>+ (layout && screenRect && viewportWidth && viewportHeight)+ ? viewportRectToScreenStyle(layout, screenRect, viewportWidth, viewportHeight, r)+ : null;++ const editingAnnotation = editingId ? annotations.find(a => a.id === editingId) : undefined;++ return (+ <div+ ref={layerRef}+ className='annotation-layer'+ tabIndex={0}+ onMouseDown={onLayerMouseDown}+ onMouseMove={onLayerMouseMove}+ onMouseUp={onLayerMouseUp}+ onKeyDown={onLayerKeyDown}+ onContextMenu={e => e.preventDefault()}+ >+ <div className='annotation-toolbar' onMouseDown={e => e.stopPropagation()}>+ <button+ className='annotate-action-btn'+ onClick={() => { setAnnotations([]); setDraft(null); setSelection(null); }}+ disabled={annotations.length === 0}+ title='Remove all annotations'+ >+ Clear+ </button>+ <button+ className='annotate-action-btn primary'+ onClick={submitAnnotations}+ disabled={annotations.length === 0}+ title='Submit annotations'+ >+ Submit+ </button>+ </div>++ {annotations.map(a => {+ const style = mapRect(a);+ if (!style)+ return null;+ const isSelected = a.id === selectedId;+ const isEditing = a.id === editingId;+ return (+ <div+ key={a.id}+ className={'annotation-rect' + (isSelected ? ' selected' : '') + (isEditing ? ' editing' : '') + (a.text ? '' : ' empty')}+ style={style}+ onDoubleClick={e => {+ e.preventDefault();+ e.stopPropagation();+ setSelection({ id: a.id, editing: true });+ }}+ >+ {a.text && (+ <div+ className='annotation-label'+ onMouseDown={e => {+ e.preventDefault();+ e.stopPropagation();+ setSelection({ id: a.id, editing: true });+ }}+ >+ {a.text}+ </div>+ )}+ {isSelected && !isEditing && HANDLES.map(h => (+ <div+ key={h}+ className={'annotation-handle annotation-handle-' + h}+ onMouseDown={e => startResize(h, a, e)}+ />+ ))}+ </div>+ );+ })}++ {draft && (() => {+ const style = mapRect(normalizeRect(draft));+ return style ? <div className='annotation-rect draft' style={style} /> : null;+ })()}++ {editingAnnotation && (() => {+ const style = mapRect(editingAnnotation);+ if (!style)+ return null;+ const popoverStyle: React.CSSProperties = {+ left: style.left as number,+ top: (style.top as number) + (style.height as number) + 6,+ };+ return (+ <div+ className='annotation-popover'+ style={popoverStyle}+ onMouseDown={e => e.stopPropagation()}+ onClick={e => e.stopPropagation()}+ >+ <textarea+ className='annotation-textarea'+ autoFocus+ value={editingAnnotation.text}+ placeholder='Task or comment…'+ onChange={e => {+ const text = e.target.value;+ setAnnotations(prev => prev.map(a => a.id === editingAnnotation.id ? { ...a, text } : a));+ }}+ onKeyDown={e => {+ if (e.key === 'Enter' && !e.shiftKey && !e.ctrlKey && !e.metaKey) {+ e.preventDefault();+ closeEditor();+ } else if (e.key === 'Escape') {+ e.preventDefault();+ closeEditor();+ }+ e.stopPropagation();+ }}+ onKeyUp={e => e.stopPropagation()}+ />+ <div className='annotation-popover-actions'>+ <button+ className='annotate-action-btn danger'+ onClick={() => {+ setAnnotations(prev => prev.filter(a => a.id !== editingAnnotation.id));+ setSelection(null);+ layerRef.current?.focus();+ }}+ >+ Delete+ </button>+ <button+ className='annotate-action-btn'+ onClick={closeEditor}+ >+ Done+ </button>+ </div>+ </div>+ );+ })()}++ {submittedJson !== null && (+ <div className='annotation-modal-backdrop' onMouseDown={e => { e.stopPropagation(); setSubmittedJson(null); }}>+ <div className='annotation-modal' onMouseDown={e => e.stopPropagation()}>+ <div className='annotation-modal-header'>+ <span>Annotations JSON</span>+ <button className='annotate-action-btn' onClick={() => setSubmittedJson(null)}>Close</button>+ </div>+ <textarea className='annotation-modal-text' readOnly value={submittedJson} />+ <div className='annotation-modal-actions'>+ <button+ className='annotate-action-btn primary'+ onClick={() => { navigator.clipboard?.writeText(submittedJson).catch(() => {}); }}+ >+ Copy+ </button>+ </div>+ </div>+ </div>+ )}+ </div>+ );+};diff --git a/packages/dashboard/src/dashboard.css b/packages/dashboard/src/dashboard.cssindex f3b2269870..d6b411353d 100644--- a/packages/dashboard/src/dashboard.css+++ b/packages/dashboard/src/dashboard.css@@ -568,3 +568,41 @@color: var(--color-accent-fg);background: var(--color-accent-subtle);}++/* -- Annotate mode (dashboard chrome) -- */+.dashboard-view {+ --annotate-blue: 54 116 209;+}++.segmented-control-3 {+ padding: 2px;+}++.segmented-control-3::before {+ width: calc(33.333% - 2px);+ transform: translateX(0);+}++.segmented-control-3.interactive::before {+ transform: translateX(100%);+ background: rgb(var(--interactive-orange) / 0.95);+}++.segmented-control-3.annotate::before {+ transform: translateX(200%);+ background: rgb(var(--annotate-blue) / 0.95);+}++.segmented-control-3 .segmented-control-option {+ width: 88px;+}++.dashboard-view.annotate .toolbar {+ background: rgb(var(--annotate-blue));+ color: var(--color-fg-on-emphasis);+}++.dashboard-view.annotate .toolbar .nav-btn,+.dashboard-view.annotate .toolbar .omnibox {+ color: var(--color-fg-on-emphasis);+}diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsxindex 618410f784..e685950da0 100644--- a/packages/dashboard/src/dashboard.tsx+++ b/packages/dashboard/src/dashboard.tsx@@ -21,6 +21,7 @@ import { asLocator } from '@isomorphic/locatorGenerators';import { SplitView } from '@web/components/splitView';import { ChevronLeftIcon, ChevronRightIcon, CloseIcon, PlusIcon, ReloadIcon, PickLocatorIcon, InspectorPanelIcon } from './icons';import { SettingsButton } from './settingsView';+import { Annotations, getImageLayout, clientToViewport } from './annotations';import type { Tab, DashboardChannelEvents } from './dashboardChannel';@@ -36,18 +37,20 @@ function tabFavicon(url: string): string {const BUTTONS = ['left', 'middle', 'right'] as const;+type Mode = 'readonly' | 'interactive' | 'annotate';+export const Dashboard: React.FC<{browser: string;autoInteractive?: boolean;onAutoInteractiveConsumed?: () => void;}> = ({ browser, autoInteractive, onAutoInteractiveConsumed }) => {const client = React.useContext(DashboardClientContext);- const [interactive, setInteractive] = React.useState(false);+ const [mode, setMode] = React.useState<Mode>('readonly');React.useEffect(() => {if (!autoInteractive)return;- setInteractive(true);+ setMode('interactive');onAutoInteractiveConsumed?.();}, [autoInteractive, onAutoInteractiveConsumed]);const [tabs, setTabs] = React.useState<Tab[] | null>(null);@@ -63,6 +66,12 @@ export const Dashboard: React.FC<{const tabbarRef = React.useRef<HTMLDivElement>(null);const toolbarRef = React.useRef<HTMLDivElement>(null);const moveThrottleRef = React.useRef(0);+ const modeRef = React.useRef<Mode>('readonly');++ React.useEffect(() => { modeRef.current = mode; }, [mode]);++ const interactive = mode === 'interactive';+ const annotating = mode === 'annotate';React.useEffect(() => {if (!client)@@ -81,6 +90,8 @@ export const Dashboard: React.FC<{const onFrame = (params: DashboardChannelEvents['frame']) => {if (params.target.browser !== browser)return;+ if (modeRef.current === 'annotate')+ return;setFrame(params);const tabbar = tabbarRef.current;const toolbar = toolbarRef.current;@@ -124,7 +135,7 @@ export const Dashboard: React.FC<{setContext(undefined);setTabs(null);setFrame(undefined);- setInteractive(false);+ setMode('readonly');setPickingPage(null);setShowInspector(false);};@@ -139,32 +150,10 @@ export const Dashboard: React.FC<{function imgCoords(e: React.MouseEvent): { x: number; y: number } {const vw = frame?.viewportWidth ?? 0;const vh = frame?.viewportHeight ?? 0;- if (!vw || !vh)- return { x: 0, y: 0 };- const display = displayRef.current;- if (!display)+ const layout = getImageLayout(displayRef.current);+ if (!vw || !vh || !layout)return { x: 0, y: 0 };- const rect = display.getBoundingClientRect();- const imgAspect = display.naturalWidth / display.naturalHeight;- const elemAspect = rect.width / rect.height;- let renderW: number, renderH: number, offsetX: number, offsetY: number;- if (imgAspect > elemAspect) {- renderW = rect.width;- renderH = rect.width / imgAspect;- offsetX = 0;- offsetY = (rect.height - renderH) / 2;- } else {- renderH = rect.height;- renderW = rect.height * imgAspect;- offsetX = (rect.width - renderW) / 2;- offsetY = 0;- }- const fracX = (e.clientX - rect.left - offsetX) / renderW;- const fracY = (e.clientY - rect.top - offsetY) / renderH;- return {- x: Math.round(fracX * vw),- y: Math.round(fracY * vh),- };+ return clientToViewport(layout, vw, vh, e.clientX, e.clientY);}function sendMouseEvent(method: 'mousedown' | 'mouseup', e: React.MouseEvent) {@@ -175,26 +164,28 @@ export const Dashboard: React.FC<{}function onScreenMouseDown(e: React.MouseEvent) {+ if (annotating)+ return;e.preventDefault();screenRef.current?.focus();if (!ready)return;if (!interactive) {- setInteractive(true);+ setMode('interactive');return;}sendMouseEvent('mousedown', e);}function onScreenMouseUp(e: React.MouseEvent) {- if (!interactive)+ if (annotating || !interactive)return;e.preventDefault();sendMouseEvent('mouseup', e);}function onScreenMouseMove(e: React.MouseEvent) {- if (!interactive || !pageTarget)+ if (annotating || !interactive || !pageTarget)return;const now = Date.now();if (now - moveThrottleRef.current < 32)@@ -205,13 +196,15 @@ export const Dashboard: React.FC<{}function onScreenWheel(e: React.WheelEvent) {- if (!interactive || !pageTarget)+ if (annotating || !interactive || !pageTarget)return;e.preventDefault();client?.wheel({ ...pageTarget, deltaX: e.deltaX, deltaY: e.deltaY });}function onScreenKeyDown(e: React.KeyboardEvent) {+ if (annotating)+ return;if (pickingPage !== null && e.key === 'Escape') {e.preventDefault();if (pageTarget)@@ -226,7 +219,7 @@ export const Dashboard: React.FC<{}function onScreenKeyUp(e: React.KeyboardEvent) {- if (!interactive || !pageTarget)+ if (annotating || !interactive || !pageTarget)return;e.preventDefault();client?.keyup({ ...pageTarget, key: e.key });@@ -254,7 +247,7 @@ export const Dashboard: React.FC<{else if (tabs.length === 0)overlayText = 'No tabs open';- return (<div className={'dashboard-view' + (interactive ? ' interactive' : '')}+ return (<div className={'dashboard-view' + (interactive ? ' interactive' : '') + (annotating ? ' annotate' : '')}>{/* Tab bar */}<div ref={tabbarRef} className='tabbar'>@@ -294,31 +287,51 @@ export const Dashboard: React.FC<{<PlusIcon /></button><div className='interactive-controls'>- <div className={'segmented-control' + (interactive ? ' interactive' : '')} role='group' aria-label='Interaction mode' title={interactive ? 'Interactive mode: page input is forwarded' : 'Read-only mode: page input is blocked'}>+ <div+ className={'segmented-control segmented-control-3' + (interactive ? ' interactive' : '') + (annotating ? ' annotate' : '')}+ role='group'+ aria-label='Interaction mode'+ title={annotating ? 'Annotate mode: draw rectangular regions and add comments' : interactive ? 'Interactive mode: page input is forwarded' : 'Read-only mode: page input is blocked'}+ ><button- className={'segmented-control-option' + (!interactive ? ' active' : '')}+ className={'segmented-control-option' + (mode === 'readonly' ? ' active' : '')}disabled={!ready}- aria-pressed={!interactive}+ aria-pressed={mode === 'readonly'}title='Read-only mode'onClick={() => {if (pageTarget)client?.cancelPickLocator(pageTarget);setPickingPage(null);setShowInspector(false);- setInteractive(false);+ setMode('readonly');}}>Read-only</button><button- className={'segmented-control-option' + (interactive ? ' active' : '')}+ className={'segmented-control-option' + (mode === 'interactive' ? ' active' : '')}disabled={!ready}- aria-pressed={interactive}+ aria-pressed={mode === 'interactive'}title='Interactive mode'- onClick={() => setInteractive(true)}+ onClick={() => setMode('interactive')}>Interactive</button>+ <button+ className={'segmented-control-option' + (mode === 'annotate' ? ' active' : '')}+ disabled={!ready || !frame}+ aria-pressed={mode === 'annotate'}+ title='Annotate mode'+ onClick={() => {+ if (pageTarget)+ client?.cancelPickLocator(pageTarget);+ setPickingPage(null);+ setShowInspector(false);+ setMode('annotate');+ }}+ >+ Annotate+ </button></div><SettingsButton /></div>@@ -359,7 +372,7 @@ export const Dashboard: React.FC<{client?.cancelPickLocator(pageTarget);setPickingPage(null);} else {- setInteractive(true);+ setMode('interactive');setPickingPage(selectedTab?.page ?? null);screenRef.current?.focus();client?.pickLocator(pageTarget);@@ -375,7 +388,7 @@ export const Dashboard: React.FC<{aria-pressed={showInspector}disabled={!ready}onClick={() => {- setInteractive(true);+ setMode('interactive');setShowInspector(!showInspector);}}>@@ -413,6 +426,13 @@ export const Dashboard: React.FC<{alt='screencast'src={frame ? 'data:image/jpeg;base64,' + frame.data : undefined}/>+ <Annotations+ active={annotating}+ displayRef={displayRef}+ screenRef={screenRef}+ viewportWidth={frame?.viewportWidth ?? 0}+ viewportHeight={frame?.viewportHeight ?? 0}+ />{locatorToast? <div className='screen-toast visible'>Copied: <code>{locatorToast.text}</code></div>: picking