Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(dashboard): add annotation mode (#40238)

Pavel Feldman1898141d5d17
diff --git a/packages/dashboard/src/annotations.css b/packages/dashboard/src/annotations.css
new file mode 100644
index 0000000000..0090d8e3d9
--- /dev/null
+++ b/packages/dashboard/src/annotations.css
@@ -0,0 +1,255 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.annotation-layer {
+ position: absolute;
+ inset: 0;
+ z-index: 5;
+ outline: none;
+ cursor: crosshair;
+}
+
+.annotation-layer::after {
+ content: '';
+ position: absolute;
+ inset: 0;
+ pointer-events: none;
+ box-shadow: inset 0 0 0 1px rgb(var(--annotate-blue) / 0.72), inset 0 0 26px rgb(var(--annotate-blue) / 0.28);
+}
+
+.annotation-toolbar {
+ position: absolute;
+ top: 8px;
+ right: 8px;
+ display: inline-flex;
+ gap: 6px;
+ padding: 6px;
+ background: var(--color-canvas-overlay);
+ border: 1px solid var(--color-border-muted);
+ border-radius: 999px;
+ box-shadow: 0 4px 12px rgba(0, 0, 0, 0.2);
+ z-index: 20;
+ cursor: default;
+}
+
+.annotate-action-btn {
+ height: 26px;
+ padding: 0 12px;
+ font-size: 11px;
+ font-weight: 600;
+ letter-spacing: 0.03em;
+ text-transform: uppercase;
+ background: var(--color-neutral-subtle);
+ color: var(--color-fg-default);
+ border: 1px solid var(--color-border-muted);
+ border-radius: 999px;
+ cursor: pointer;
+ line-height: 1;
+}
+
+.annotate-action-btn:hover:not(:disabled) {
+ background: var(--color-neutral-muted);
+}
+
+.annotate-action-btn:disabled {
+ opacity: 0.5;
+ cursor: default;
+}
+
+.annotate-action-btn.primary {
+ background: rgb(var(--annotate-blue));
+ color: #fff;
+ border-color: transparent;
+}
+
+.annotate-action-btn.primary:hover:not(:disabled) {
+ background: rgb(var(--annotate-blue) / 0.85);
+}
+
+.annotate-action-btn.danger {
+ background: transparent;
+ color: var(--color-danger-fg);
+ border-color: var(--color-border-muted);
+}
+
+.annotate-action-btn.danger:hover:not(:disabled) {
+ background: var(--color-danger-subtle);
+}
+
+.annotation-rect {
+ position: absolute;
+ box-sizing: border-box;
+ border: 2px solid rgb(var(--annotate-blue));
+ background: rgb(var(--annotate-blue) / 0.12);
+ cursor: pointer;
+ transition: box-shadow 120ms ease;
+}
+
+.annotation-rect:hover {
+ box-shadow: 0 0 0 3px rgb(var(--annotate-blue) / 0.25);
+}
+
+.annotation-rect.selected {
+ box-shadow: 0 0 0 3px rgb(var(--annotate-blue) / 0.4);
+ border-style: solid;
+ border-color: rgb(var(--annotate-blue));
+ cursor: move;
+}
+
+.annotation-rect.draft {
+ border-style: dashed;
+ background: rgb(var(--annotate-blue) / 0.18);
+ pointer-events: none;
+}
+
+.annotation-handle {
+ position: absolute;
+ width: 10px;
+ height: 10px;
+ background: #fff;
+ border: 1.5px solid rgb(var(--annotate-blue));
+ border-radius: 2px;
+ box-sizing: border-box;
+ z-index: 1;
+}
+
+.annotation-handle-nw { left: -6px; top: -6px; cursor: nwse-resize; }
+.annotation-handle-n { left: 50%; top: -6px; margin-left: -5px; cursor: ns-resize; }
+.annotation-handle-ne { right: -6px; top: -6px; cursor: nesw-resize; }
+.annotation-handle-e { right: -6px; top: 50%; margin-top: -5px; cursor: ew-resize; }
+.annotation-handle-se { right: -6px; bottom: -6px; cursor: nwse-resize; }
+.annotation-handle-s { left: 50%; bottom: -6px; margin-left: -5px; cursor: ns-resize; }
+.annotation-handle-sw { left: -6px; bottom: -6px; cursor: nesw-resize; }
+.annotation-handle-w { left: -6px; top: 50%; margin-top: -5px; cursor: ew-resize; }
+
+.annotation-label {
+ position: absolute;
+ top: -2px;
+ left: -2px;
+ max-width: calc(100% + 4px);
+ padding: 2px 6px;
+ background: rgb(var(--annotate-blue));
+ color: #fff;
+ font-size: 11px;
+ font-weight: 500;
+ border-radius: 2px 2px 2px 0;
+ transform: translateY(-100%);
+ white-space: nowrap;
+ overflow: hidden;
+ text-overflow: ellipsis;
+ cursor: text;
+}
+
+.annotation-label:hover {
+ background: rgb(var(--annotate-blue) / 0.85);
+}
+
+.annotation-popover {
+ position: absolute;
+ min-width: 220px;
+ max-width: 320px;
+ background: var(--color-canvas-overlay);
+ border: 1px solid var(--color-border-default);
+ border-radius: 8px;
+ padding: 8px;
+ box-shadow: 0 8px 24px rgba(0, 0, 0, 0.3);
+ display: flex;
+ flex-direction: column;
+ gap: 6px;
+ z-index: 10;
+ cursor: default;
+}
+
+.annotation-textarea {
+ width: 100%;
+ min-height: 64px;
+ resize: vertical;
+ padding: 6px 8px;
+ font-family: inherit;
+ font-size: 12px;
+ color: var(--color-fg-default);
+ background: var(--color-canvas-default);
+ border: 1px solid var(--color-border-muted);
+ border-radius: 6px;
+ outline: none;
+ box-sizing: border-box;
+}
+
+.annotation-textarea:focus {
+ border-color: rgb(var(--annotate-blue));
+}
+
+.annotation-popover-actions {
+ display: flex;
+ gap: 6px;
+ justify-content: flex-end;
+}
+
+.annotation-modal-backdrop {
+ position: absolute;
+ inset: 0;
+ background: rgba(0, 0, 0, 0.5);
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ z-index: 100;
+ cursor: default;
+}
+
+.annotation-modal {
+ width: min(520px, 92%);
+ max-height: 80%;
+ background: var(--color-canvas-default);
+ border: 1px solid var(--color-border-default);
+ border-radius: 10px;
+ box-shadow: 0 12px 40px rgba(0, 0, 0, 0.45);
+ display: flex;
+ flex-direction: column;
+ overflow: hidden;
+}
+
+.annotation-modal-header {
+ display: flex;
+ align-items: center;
+ justify-content: space-between;
+ padding: 12px 16px;
+ border-bottom: 1px solid var(--color-border-muted);
+ font-size: 13px;
+ font-weight: 600;
+ color: var(--color-fg-default);
+}
+
+.annotation-modal-text {
+ flex: 1;
+ min-height: 200px;
+ padding: 12px 16px;
+ font-family: 'SF Mono', 'Cascadia Code', 'Fira Code', 'Consolas', monospace;
+ font-size: 12px;
+ color: var(--color-fg-default);
+ background: var(--color-canvas-subtle);
+ border: none;
+ outline: none;
+ resize: none;
+ white-space: pre;
+}
+
+.annotation-modal-actions {
+ display: flex;
+ justify-content: flex-end;
+ gap: 8px;
+ padding: 10px 16px;
+ border-top: 1px solid var(--color-border-muted);
+}
diff --git a/packages/dashboard/src/annotations.tsx b/packages/dashboard/src/annotations.tsx
new file mode 100644
index 0000000000..628a1bc736
--- /dev/null
+++ b/packages/dashboard/src/annotations.tsx
@@ -0,0 +1,458 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import './annotations.css';
+
+export type Annotation = { id: string; x: number; y: number; width: number; height: number; text: string };
+
+type Rect = { x: number; y: number; width: number; height: number };
+type DragKind = 'move' | 'nw' | 'n' | 'ne' | 'e' | 'se' | 's' | 'sw' | 'w';
+const HANDLES: Exclude<DragKind, 'move'>[] = ['nw', 'n', 'ne', 'e', 'se', 's', 'sw', 'w'];
+
+type DragState = {
+ kind: DragKind;
+ id: string;
+ orig: Rect;
+ startVx: number;
+ startVy: number;
+};
+
+type Selection = { id: string; editing: boolean } | null;
+
+const MIN_ANNOTATION_SIZE = 4;
+
+function newAnnotationId() {
+ return 'ann-' + Math.random().toString(36).slice(2, 10);
+}
+
+function normalizeRect(a: { startX: number; startY: number; x: number; y: number }): Rect {
+ return {
+ x: Math.min(a.startX, a.x),
+ y: Math.min(a.startY, a.y),
+ width: Math.abs(a.x - a.startX),
+ height: Math.abs(a.y - a.startY),
+ };
+}
+
+function applyDrag(orig: Rect, kind: DragKind, dvx: number, dvy: number): Rect {
+ let left = orig.x;
+ let top = orig.y;
+ let right = orig.x + orig.width;
+ let bottom = orig.y + orig.height;
+ switch (kind) {
+ case 'move':
+ left += dvx; top += dvy; right += dvx; bottom += dvy; break;
+ case 'nw': left += dvx; top += dvy; break;
+ case 'n': top += dvy; break;
+ case 'ne': right += dvx; top += dvy; break;
+ case 'e': right += dvx; break;
+ case 'se': right += dvx; bottom += dvy; break;
+ case 's': bottom += dvy; break;
+ case 'sw': left += dvx; bottom += dvy; break;
+ case 'w': left += dvx; break;
+ }
+ return {
+ x: Math.min(left, right),
+ y: Math.min(top, bottom),
+ width: Math.abs(right - left),
+ height: Math.abs(bottom - top),
+ };
+}
+
+export type ImageLayout = {
+ rect: DOMRect;
+ renderW: number;
+ renderH: number;
+ offsetX: number;
+ offsetY: number;
+};
+
+export function getImageLayout(display: HTMLImageElement | null): ImageLayout | null {
+ if (!display || !display.naturalWidth || !display.naturalHeight)
+ return null;
+ const rect = display.getBoundingClientRect();
+ const imgAspect = display.naturalWidth / display.naturalHeight;
+ const elemAspect = rect.width / rect.height;
+ if (imgAspect > elemAspect) {
+ const renderH = rect.width / imgAspect;
+ return { rect, renderW: rect.width, renderH, offsetX: 0, offsetY: (rect.height - renderH) / 2 };
+ }
+ const renderW = rect.height * imgAspect;
+ return { rect, renderW, renderH: rect.height, offsetX: (rect.width - renderW) / 2, offsetY: 0 };
+}
+
+export function clientToViewport(layout: ImageLayout, vw: number, vh: number, clientX: number, clientY: number): { x: number; y: number } {
+ const fracX = (clientX - layout.rect.left - layout.offsetX) / layout.renderW;
+ const fracY = (clientY - layout.rect.top - layout.offsetY) / layout.renderH;
+ return { x: Math.round(fracX * vw), y: Math.round(fracY * vh) };
+}
+
+function viewportRectToScreenStyle(layout: ImageLayout, screenRect: DOMRect, vw: number, vh: number, r: Rect): React.CSSProperties {
+ const baseLeft = layout.rect.left - screenRect.left + layout.offsetX;
+ const baseTop = layout.rect.top - screenRect.top + layout.offsetY;
+ return {
+ left: baseLeft + (r.x / vw) * layout.renderW,
+ top: baseTop + (r.y / vh) * layout.renderH,
+ width: (r.width / vw) * layout.renderW,
+ height: (r.height / vh) * layout.renderH,
+ };
+}
+
+export const Annotations: React.FC<{
+ active: boolean;
+ displayRef: React.RefObject<HTMLImageElement | null>;
+ screenRef: React.RefObject<HTMLDivElement | null>;
+ viewportWidth: number;
+ viewportHeight: number;
+}> = ({ active, displayRef, screenRef, viewportWidth, viewportHeight }) => {
+ const [annotations, setAnnotations] = React.useState<Annotation[]>([]);
+ const [draft, setDraft] = React.useState<{ startX: number; startY: number; x: number; y: number } | null>(null);
+ const [selection, setSelection] = React.useState<Selection>(null);
+ const [drag, setDrag] = React.useState<DragState | null>(null);
+ const [submittedJson, setSubmittedJson] = React.useState<string | null>(null);
+ const [, setTick] = React.useState(0);
+ const forceRender = React.useCallback(() => setTick(t => t + 1), []);
+ const layerRef = React.useRef<HTMLDivElement>(null);
+
+ const selectedId = selection?.id ?? null;
+ const editingId = selection?.editing ? selection.id : null;
+
+ React.useEffect(() => {
+ if (!active)
+ return;
+ const onResize = () => forceRender();
+ const img = displayRef.current;
+ const onLoad = () => forceRender();
+ window.addEventListener('resize', onResize);
+ img?.addEventListener('load', onLoad);
+ return () => {
+ window.removeEventListener('resize', onResize);
+ img?.removeEventListener('load', onLoad);
+ };
+ }, [active, displayRef, forceRender]);
+
+ React.useEffect(() => {
+ if (active)
+ layerRef.current?.focus();
+ else
+ setSelection(null);
+ }, [active]);
+
+ React.useEffect(() => {
+ if (!drag)
+ return;
+ const onMove = (e: MouseEvent) => {
+ const layout = getImageLayout(displayRef.current);
+ if (!layout || !viewportWidth || !viewportHeight)
+ return;
+ const vp = clientToViewport(layout, viewportWidth, viewportHeight, e.clientX, e.clientY);
+ const dvx = vp.x - drag.startVx;
+ const dvy = vp.y - drag.startVy;
+ if (dvx === 0 && dvy === 0)
+ return;
+ setAnnotations(prev => prev.map(a => a.id === drag.id ? { ...a, ...applyDrag(drag.orig, drag.kind, dvx, dvy) } : a));
+ };
+ const onUp = () => setDrag(null);
+ window.addEventListener('mousemove', onMove);
+ window.addEventListener('mouseup', onUp);
+ return () => {
+ window.removeEventListener('mousemove', onMove);
+ window.removeEventListener('mouseup', onUp);
+ };
+ }, [drag, displayRef, viewportWidth, viewportHeight]);
+
+ function imgCoords(e: React.MouseEvent): { x: number; y: number } | null {
+ if (!viewportWidth || !viewportHeight)
+ return null;
+ const layout = getImageLayout(displayRef.current);
+ if (!layout)
+ return null;
+ return clientToViewport(layout, viewportWidth, viewportHeight, e.clientX, e.clientY);
+ }
+
+ function hitTestAnnotation(x: number, y: number): Annotation | undefined {
+ for (let i = annotations.length - 1; i >= 0; i--) {
+ const a = annotations[i];
+ if (x >= a.x && x <= a.x + a.width && y >= a.y && y <= a.y + a.height)
+ return a;
+ }
+ return undefined;
+ }
+
+ function onLayerMouseDown(e: React.MouseEvent) {
+ if (e.button !== 0)
+ return;
+ layerRef.current?.focus();
+ const vp = imgCoords(e);
+ if (!vp)
+ return;
+ const hit = hitTestAnnotation(vp.x, vp.y);
+ e.preventDefault();
+ if (hit) {
+ setSelection({ id: hit.id, editing: false });
+ setDrag({ kind: 'move', id: hit.id, orig: { x: hit.x, y: hit.y, width: hit.width, height: hit.height }, startVx: vp.x, startVy: vp.y });
+ return;
+ }
+ setDraft({ startX: vp.x, startY: vp.y, x: vp.x, y: vp.y });
+ setSelection(null);
+ }
+
+ function onLayerMouseMove(e: React.MouseEvent) {
+ if (drag || !draft)
+ return;
+ const vp = imgCoords(e);
+ if (!vp)
+ return;
+ if (draft.x === vp.x && draft.y === vp.y)
+ return;
+ setDraft({ ...draft, x: vp.x, y: vp.y });
+ }
+
+ function onLayerMouseUp(e: React.MouseEvent) {
+ if (!draft)
+ return;
+ e.preventDefault();
+ const rect = normalizeRect(draft);
+ setDraft(null);
+ if (rect.width < MIN_ANNOTATION_SIZE || rect.height < MIN_ANNOTATION_SIZE)
+ return;
+ const id = newAnnotationId();
+ setAnnotations(prev => [...prev, { id, ...rect, text: '' }]);
+ setSelection({ id, editing: true });
+ }
+
+ function startResize(kind: Exclude<DragKind, 'move'>, a: Annotation, e: React.MouseEvent) {
+ e.preventDefault();
+ e.stopPropagation();
+ layerRef.current?.focus();
+ const vp = imgCoords(e);
+ if (!vp)
+ return;
+ setSelection({ id: a.id, editing: false });
+ setDrag({ kind, id: a.id, orig: { x: a.x, y: a.y, width: a.width, height: a.height }, startVx: vp.x, startVy: vp.y });
+ }
+
+ function nudgeSelected(dx: number, dy: number) {
+ if (!selectedId)
+ return;
+ setAnnotations(prev => prev.map(a => a.id === selectedId ? { ...a, x: a.x + dx, y: a.y + dy } : a));
+ }
+
+ function closeEditor() {
+ setSelection(sel => sel?.editing ? { ...sel, editing: false } : sel);
+ layerRef.current?.focus();
+ }
+
+ function onLayerKeyDown(e: React.KeyboardEvent) {
+ if (e.key === 'Escape') {
+ e.preventDefault();
+ if (draft)
+ setDraft(null);
+ else if (editingId)
+ closeEditor();
+ else if (selectedId)
+ setSelection(null);
+ return;
+ }
+ if ((e.key === 'Delete' || e.key === 'Backspace') && selectedId && !editingId) {
+ e.preventDefault();
+ setAnnotations(prev => prev.filter(a => a.id !== selectedId));
+ setSelection(null);
+ return;
+ }
+ if (selectedId && !editingId) {
+ const step = e.shiftKey ? 1 : 5;
+ if (e.key === 'ArrowLeft') { e.preventDefault(); nudgeSelected(-step, 0); return; }
+ if (e.key === 'ArrowRight') { e.preventDefault(); nudgeSelected(step, 0); return; }
+ if (e.key === 'ArrowUp') { e.preventDefault(); nudgeSelected(0, -step); return; }
+ if (e.key === 'ArrowDown') { e.preventDefault(); nudgeSelected(0, step); return; }
+ }
+ }
+
+ function submitAnnotations() {
+ const json = JSON.stringify({ viewportWidth, viewportHeight, annotations }, null, 2);
+ console.log('Annotations:', json);
+ setSubmittedJson(json);
+ }
+
+ if (!active)
+ return null;
+
+ const layout = getImageLayout(displayRef.current);
+ const screenRect = screenRef.current?.getBoundingClientRect() ?? null;
+ const mapRect = (r: Rect): React.CSSProperties | null =>
+ (layout && screenRect && viewportWidth && viewportHeight)
+ ? viewportRectToScreenStyle(layout, screenRect, viewportWidth, viewportHeight, r)
+ : null;
+
+ const editingAnnotation = editingId ? annotations.find(a => a.id === editingId) : undefined;
+
+ return (
+ <div
+ ref={layerRef}
+ className='annotation-layer'
+ tabIndex={0}
+ onMouseDown={onLayerMouseDown}
+ onMouseMove={onLayerMouseMove}
+ onMouseUp={onLayerMouseUp}
+ onKeyDown={onLayerKeyDown}
+ onContextMenu={e => e.preventDefault()}
+ >
+ <div className='annotation-toolbar' onMouseDown={e => e.stopPropagation()}>
+ <button
+ className='annotate-action-btn'
+ onClick={() => { setAnnotations([]); setDraft(null); setSelection(null); }}
+ disabled={annotations.length === 0}
+ title='Remove all annotations'
+ >
+ Clear
+ </button>
+ <button
+ className='annotate-action-btn primary'
+ onClick={submitAnnotations}
+ disabled={annotations.length === 0}
+ title='Submit annotations'
+ >
+ Submit
+ </button>
+ </div>
+
+ {annotations.map(a => {
+ const style = mapRect(a);
+ if (!style)
+ return null;
+ const isSelected = a.id === selectedId;
+ const isEditing = a.id === editingId;
+ return (
+ <div
+ key={a.id}
+ className={'annotation-rect' + (isSelected ? ' selected' : '') + (isEditing ? ' editing' : '') + (a.text ? '' : ' empty')}
+ style={style}
+ onDoubleClick={e => {
+ e.preventDefault();
+ e.stopPropagation();
+ setSelection({ id: a.id, editing: true });
+ }}
+ >
+ {a.text && (
+ <div
+ className='annotation-label'
+ onMouseDown={e => {
+ e.preventDefault();
+ e.stopPropagation();
+ setSelection({ id: a.id, editing: true });
+ }}
+ >
+ {a.text}
+ </div>
+ )}
+ {isSelected && !isEditing && HANDLES.map(h => (
+ <div
+ key={h}
+ className={'annotation-handle annotation-handle-' + h}
+ onMouseDown={e => startResize(h, a, e)}
+ />
+ ))}
+ </div>
+ );
+ })}
+
+ {draft && (() => {
+ const style = mapRect(normalizeRect(draft));
+ return style ? <div className='annotation-rect draft' style={style} /> : null;
+ })()}
+
+ {editingAnnotation && (() => {
+ const style = mapRect(editingAnnotation);
+ if (!style)
+ return null;
+ const popoverStyle: React.CSSProperties = {
+ left: style.left as number,
+ top: (style.top as number) + (style.height as number) + 6,
+ };
+ return (
+ <div
+ className='annotation-popover'
+ style={popoverStyle}
+ onMouseDown={e => e.stopPropagation()}
+ onClick={e => e.stopPropagation()}
+ >
+ <textarea
+ className='annotation-textarea'
+ autoFocus
+ value={editingAnnotation.text}
+ placeholder='Task or comment…'
+ onChange={e => {
+ const text = e.target.value;
+ setAnnotations(prev => prev.map(a => a.id === editingAnnotation.id ? { ...a, text } : a));
+ }}
+ onKeyDown={e => {
+ if (e.key === 'Enter' && !e.shiftKey && !e.ctrlKey && !e.metaKey) {
+ e.preventDefault();
+ closeEditor();
+ } else if (e.key === 'Escape') {
+ e.preventDefault();
+ closeEditor();
+ }
+ e.stopPropagation();
+ }}
+ onKeyUp={e => e.stopPropagation()}
+ />
+ <div className='annotation-popover-actions'>
+ <button
+ className='annotate-action-btn danger'
+ onClick={() => {
+ setAnnotations(prev => prev.filter(a => a.id !== editingAnnotation.id));
+ setSelection(null);
+ layerRef.current?.focus();
+ }}
+ >
+ Delete
+ </button>
+ <button
+ className='annotate-action-btn'
+ onClick={closeEditor}
+ >
+ Done
+ </button>
+ </div>
+ </div>
+ );
+ })()}
+
+ {submittedJson !== null && (
+ <div className='annotation-modal-backdrop' onMouseDown={e => { e.stopPropagation(); setSubmittedJson(null); }}>
+ <div className='annotation-modal' onMouseDown={e => e.stopPropagation()}>
+ <div className='annotation-modal-header'>
+ <span>Annotations JSON</span>
+ <button className='annotate-action-btn' onClick={() => setSubmittedJson(null)}>Close</button>
+ </div>
+ <textarea className='annotation-modal-text' readOnly value={submittedJson} />
+ <div className='annotation-modal-actions'>
+ <button
+ className='annotate-action-btn primary'
+ onClick={() => { navigator.clipboard?.writeText(submittedJson).catch(() => {}); }}
+ >
+ Copy
+ </button>
+ </div>
+ </div>
+ </div>
+ )}
+ </div>
+ );
+};
diff --git a/packages/dashboard/src/dashboard.css b/packages/dashboard/src/dashboard.css
index f3b2269870..d6b411353d 100644
--- a/packages/dashboard/src/dashboard.css
+++ b/packages/dashboard/src/dashboard.css
@@ -568,3 +568,41 @@
color: var(--color-accent-fg);
background: var(--color-accent-subtle);
}
+
+/* -- Annotate mode (dashboard chrome) -- */
+.dashboard-view {
+ --annotate-blue: 54 116 209;
+}
+
+.segmented-control-3 {
+ padding: 2px;
+}
+
+.segmented-control-3::before {
+ width: calc(33.333% - 2px);
+ transform: translateX(0);
+}
+
+.segmented-control-3.interactive::before {
+ transform: translateX(100%);
+ background: rgb(var(--interactive-orange) / 0.95);
+}
+
+.segmented-control-3.annotate::before {
+ transform: translateX(200%);
+ background: rgb(var(--annotate-blue) / 0.95);
+}
+
+.segmented-control-3 .segmented-control-option {
+ width: 88px;
+}
+
+.dashboard-view.annotate .toolbar {
+ background: rgb(var(--annotate-blue));
+ color: var(--color-fg-on-emphasis);
+}
+
+.dashboard-view.annotate .toolbar .nav-btn,
+.dashboard-view.annotate .toolbar .omnibox {
+ color: var(--color-fg-on-emphasis);
+}
diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsx
index 618410f784..e685950da0 100644
--- a/packages/dashboard/src/dashboard.tsx
+++ b/packages/dashboard/src/dashboard.tsx
@@ -21,6 +21,7 @@ import { asLocator } from '@isomorphic/locatorGenerators';
import { SplitView } from '@web/components/splitView';
import { ChevronLeftIcon, ChevronRightIcon, CloseIcon, PlusIcon, ReloadIcon, PickLocatorIcon, InspectorPanelIcon } from './icons';
import { SettingsButton } from './settingsView';
+import { Annotations, getImageLayout, clientToViewport } from './annotations';
import type { Tab, DashboardChannelEvents } from './dashboardChannel';
@@ -36,18 +37,20 @@ function tabFavicon(url: string): string {
const BUTTONS = ['left', 'middle', 'right'] as const;
+type Mode = 'readonly' | 'interactive' | 'annotate';
+
export const Dashboard: React.FC<{
browser: string;
autoInteractive?: boolean;
onAutoInteractiveConsumed?: () => void;
}> = ({ browser, autoInteractive, onAutoInteractiveConsumed }) => {
const client = React.useContext(DashboardClientContext);
- const [interactive, setInteractive] = React.useState(false);
+ const [mode, setMode] = React.useState<Mode>('readonly');
React.useEffect(() => {
if (!autoInteractive)
return;
- setInteractive(true);
+ setMode('interactive');
onAutoInteractiveConsumed?.();
}, [autoInteractive, onAutoInteractiveConsumed]);
const [tabs, setTabs] = React.useState<Tab[] | null>(null);
@@ -63,6 +66,12 @@ export const Dashboard: React.FC<{
const tabbarRef = React.useRef<HTMLDivElement>(null);
const toolbarRef = React.useRef<HTMLDivElement>(null);
const moveThrottleRef = React.useRef(0);
+ const modeRef = React.useRef<Mode>('readonly');
+
+ React.useEffect(() => { modeRef.current = mode; }, [mode]);
+
+ const interactive = mode === 'interactive';
+ const annotating = mode === 'annotate';
React.useEffect(() => {
if (!client)
@@ -81,6 +90,8 @@ export const Dashboard: React.FC<{
const onFrame = (params: DashboardChannelEvents['frame']) => {
if (params.target.browser !== browser)
return;
+ if (modeRef.current === 'annotate')
+ return;
setFrame(params);
const tabbar = tabbarRef.current;
const toolbar = toolbarRef.current;
@@ -124,7 +135,7 @@ export const Dashboard: React.FC<{
setContext(undefined);
setTabs(null);
setFrame(undefined);
- setInteractive(false);
+ setMode('readonly');
setPickingPage(null);
setShowInspector(false);
};
@@ -139,32 +150,10 @@ export const Dashboard: React.FC<{
function imgCoords(e: React.MouseEvent): { x: number; y: number } {
const vw = frame?.viewportWidth ?? 0;
const vh = frame?.viewportHeight ?? 0;
- if (!vw || !vh)
- return { x: 0, y: 0 };
- const display = displayRef.current;
- if (!display)
+ const layout = getImageLayout(displayRef.current);
+ if (!vw || !vh || !layout)
return { x: 0, y: 0 };
- const rect = display.getBoundingClientRect();
- const imgAspect = display.naturalWidth / display.naturalHeight;
- const elemAspect = rect.width / rect.height;
- let renderW: number, renderH: number, offsetX: number, offsetY: number;
- if (imgAspect > elemAspect) {
- renderW = rect.width;
- renderH = rect.width / imgAspect;
- offsetX = 0;
- offsetY = (rect.height - renderH) / 2;
- } else {
- renderH = rect.height;
- renderW = rect.height * imgAspect;
- offsetX = (rect.width - renderW) / 2;
- offsetY = 0;
- }
- const fracX = (e.clientX - rect.left - offsetX) / renderW;
- const fracY = (e.clientY - rect.top - offsetY) / renderH;
- return {
- x: Math.round(fracX * vw),
- y: Math.round(fracY * vh),
- };
+ return clientToViewport(layout, vw, vh, e.clientX, e.clientY);
}
function sendMouseEvent(method: 'mousedown' | 'mouseup', e: React.MouseEvent) {
@@ -175,26 +164,28 @@ export const Dashboard: React.FC<{
}
function onScreenMouseDown(e: React.MouseEvent) {
+ if (annotating)
+ return;
e.preventDefault();
screenRef.current?.focus();
if (!ready)
return;
if (!interactive) {
- setInteractive(true);
+ setMode('interactive');
return;
}
sendMouseEvent('mousedown', e);
}
function onScreenMouseUp(e: React.MouseEvent) {
- if (!interactive)
+ if (annotating || !interactive)
return;
e.preventDefault();
sendMouseEvent('mouseup', e);
}
function onScreenMouseMove(e: React.MouseEvent) {
- if (!interactive || !pageTarget)
+ if (annotating || !interactive || !pageTarget)
return;
const now = Date.now();
if (now - moveThrottleRef.current < 32)
@@ -205,13 +196,15 @@ export const Dashboard: React.FC<{
}
function onScreenWheel(e: React.WheelEvent) {
- if (!interactive || !pageTarget)
+ if (annotating || !interactive || !pageTarget)
return;
e.preventDefault();
client?.wheel({ ...pageTarget, deltaX: e.deltaX, deltaY: e.deltaY });
}
function onScreenKeyDown(e: React.KeyboardEvent) {
+ if (annotating)
+ return;
if (pickingPage !== null && e.key === 'Escape') {
e.preventDefault();
if (pageTarget)
@@ -226,7 +219,7 @@ export const Dashboard: React.FC<{
}
function onScreenKeyUp(e: React.KeyboardEvent) {
- if (!interactive || !pageTarget)
+ if (annotating || !interactive || !pageTarget)
return;
e.preventDefault();
client?.keyup({ ...pageTarget, key: e.key });
@@ -254,7 +247,7 @@ export const Dashboard: React.FC<{
else if (tabs.length === 0)
overlayText = 'No tabs open';
- return (<div className={'dashboard-view' + (interactive ? ' interactive' : '')}
+ return (<div className={'dashboard-view' + (interactive ? ' interactive' : '') + (annotating ? ' annotate' : '')}
>
{/* Tab bar */}
<div ref={tabbarRef} className='tabbar'>
@@ -294,31 +287,51 @@ export const Dashboard: React.FC<{
<PlusIcon />
</button>
<div className='interactive-controls'>
- <div className={'segmented-control' + (interactive ? ' interactive' : '')} role='group' aria-label='Interaction mode' title={interactive ? 'Interactive mode: page input is forwarded' : 'Read-only mode: page input is blocked'}>
+ <div
+ className={'segmented-control segmented-control-3' + (interactive ? ' interactive' : '') + (annotating ? ' annotate' : '')}
+ role='group'
+ aria-label='Interaction mode'
+ title={annotating ? 'Annotate mode: draw rectangular regions and add comments' : interactive ? 'Interactive mode: page input is forwarded' : 'Read-only mode: page input is blocked'}
+ >
<button
- className={'segmented-control-option' + (!interactive ? ' active' : '')}
+ className={'segmented-control-option' + (mode === 'readonly' ? ' active' : '')}
disabled={!ready}
- aria-pressed={!interactive}
+ aria-pressed={mode === 'readonly'}
title='Read-only mode'
onClick={() => {
if (pageTarget)
client?.cancelPickLocator(pageTarget);
setPickingPage(null);
setShowInspector(false);
- setInteractive(false);
+ setMode('readonly');
}}
>
Read-only
</button>
<button
- className={'segmented-control-option' + (interactive ? ' active' : '')}
+ className={'segmented-control-option' + (mode === 'interactive' ? ' active' : '')}
disabled={!ready}
- aria-pressed={interactive}
+ aria-pressed={mode === 'interactive'}
title='Interactive mode'
- onClick={() => setInteractive(true)}
+ onClick={() => setMode('interactive')}
>
Interactive
</button>
+ <button
+ className={'segmented-control-option' + (mode === 'annotate' ? ' active' : '')}
+ disabled={!ready || !frame}
+ aria-pressed={mode === 'annotate'}
+ title='Annotate mode'
+ onClick={() => {
+ if (pageTarget)
+ client?.cancelPickLocator(pageTarget);
+ setPickingPage(null);
+ setShowInspector(false);
+ setMode('annotate');
+ }}
+ >
+ Annotate
+ </button>
</div>
<SettingsButton />
</div>
@@ -359,7 +372,7 @@ export const Dashboard: React.FC<{
client?.cancelPickLocator(pageTarget);
setPickingPage(null);
} else {
- setInteractive(true);
+ setMode('interactive');
setPickingPage(selectedTab?.page ?? null);
screenRef.current?.focus();
client?.pickLocator(pageTarget);
@@ -375,7 +388,7 @@ export const Dashboard: React.FC<{
aria-pressed={showInspector}
disabled={!ready}
onClick={() => {
- setInteractive(true);
+ setMode('interactive');
setShowInspector(!showInspector);
}}
>
@@ -413,6 +426,13 @@ export const Dashboard: React.FC<{
alt='screencast'
src={frame ? 'data:image/jpeg;base64,' + frame.data : undefined}
/>
+ <Annotations
+ active={annotating}
+ displayRef={displayRef}
+ screenRef={screenRef}
+ viewportWidth={frame?.viewportWidth ?? 0}
+ viewportHeight={frame?.viewportHeight ?? 0}
+ />
{locatorToast
? <div className='screen-toast visible'>Copied: <code>{locatorToast.text}</code></div>
: picking