Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

chore(dashboard): move annotate and recording into modal dialogs (#40403)

Pavel Feldman9e954abbfd39
diff --git a/packages/dashboard/src/annotations.css b/packages/dashboard/src/annotations.css
index 47cac41cd3..4052b4ec3b 100644
--- a/packages/dashboard/src/annotations.css
+++ b/packages/dashboard/src/annotations.css
@@ -14,7 +14,11 @@
* limitations under the License.
*/
-.annotation-layer {
+:root {
+ --annotations-blue: 54 116 209;
+}
+
+.annotations-layer {
position: absolute;
inset: 0;
z-index: 5;
@@ -22,19 +26,14 @@
cursor: crosshair;
}
-.annotation-layer::after {
+.annotations-layer::after {
content: '';
position: absolute;
inset: 0;
pointer-events: none;
}
-:root.dark-mode .annotate-action-btn {
- background: var(--color-neutral-muted);
- border-color: var(--color-border-default);
-}
-
-.annotate-action-btn {
+.annotations-action-btn {
height: 26px;
padding: 0 12px;
font-size: 11px;
@@ -49,78 +48,83 @@
line-height: 1;
}
-.annotate-action-btn:hover:not(:disabled) {
+:root.dark-mode .annotations-action-btn {
+ background: var(--color-neutral-muted);
+ border-color: var(--color-border-default);
+}
+
+.annotations-action-btn:hover:not(:disabled) {
background: var(--color-neutral-muted);
}
-.annotate-action-btn:disabled {
+.annotations-action-btn:disabled {
opacity: 0.5;
cursor: default;
}
-.annotate-action-btn.danger {
+.annotations-action-btn.danger {
background: transparent;
color: var(--color-danger-fg);
border-color: var(--color-border-muted);
}
-.annotate-action-btn.danger:hover:not(:disabled) {
+.annotations-action-btn.danger:hover:not(:disabled) {
background: var(--color-danger-subtle);
}
-.annotation-rect {
+.annotations-rect {
position: absolute;
box-sizing: border-box;
- border: 2px solid rgb(var(--annotate-blue));
- background: rgb(var(--annotate-blue) / 0.12);
+ border: 2px solid rgb(var(--annotations-blue));
+ background: rgb(var(--annotations-blue) / 0.12);
cursor: pointer;
transition: box-shadow 120ms ease;
}
-.annotation-rect:hover {
- box-shadow: 0 0 0 3px rgb(var(--annotate-blue) / 0.25);
+.annotations-rect:hover {
+ box-shadow: 0 0 0 3px rgb(var(--annotations-blue) / 0.25);
}
-.annotation-rect.selected {
- box-shadow: 0 0 0 3px rgb(var(--annotate-blue) / 0.4);
+.annotations-rect.selected {
+ box-shadow: 0 0 0 3px rgb(var(--annotations-blue) / 0.4);
border-style: solid;
- border-color: rgb(var(--annotate-blue));
+ border-color: rgb(var(--annotations-blue));
cursor: move;
}
-.annotation-rect.draft {
+.annotations-rect.draft {
border-style: dashed;
- background: rgb(var(--annotate-blue) / 0.18);
+ background: rgb(var(--annotations-blue) / 0.18);
pointer-events: none;
}
-.annotation-handle {
+.annotations-handle {
position: absolute;
width: 10px;
height: 10px;
background: #fff;
- border: 1.5px solid rgb(var(--annotate-blue));
+ border: 1.5px solid rgb(var(--annotations-blue));
border-radius: 2px;
box-sizing: border-box;
z-index: 1;
}
-.annotation-handle-nw { left: -6px; top: -6px; cursor: nwse-resize; }
-.annotation-handle-n { left: 50%; top: -6px; margin-left: -5px; cursor: ns-resize; }
-.annotation-handle-ne { right: -6px; top: -6px; cursor: nesw-resize; }
-.annotation-handle-e { right: -6px; top: 50%; margin-top: -5px; cursor: ew-resize; }
-.annotation-handle-se { right: -6px; bottom: -6px; cursor: nwse-resize; }
-.annotation-handle-s { left: 50%; bottom: -6px; margin-left: -5px; cursor: ns-resize; }
-.annotation-handle-sw { left: -6px; bottom: -6px; cursor: nesw-resize; }
-.annotation-handle-w { left: -6px; top: 50%; margin-top: -5px; cursor: ew-resize; }
+.annotations-handle-nw { left: -6px; top: -6px; cursor: nwse-resize; }
+.annotations-handle-n { left: 50%; top: -6px; margin-left: -5px; cursor: ns-resize; }
+.annotations-handle-ne { right: -6px; top: -6px; cursor: nesw-resize; }
+.annotations-handle-e { right: -6px; top: 50%; margin-top: -5px; cursor: ew-resize; }
+.annotations-handle-se { right: -6px; bottom: -6px; cursor: nwse-resize; }
+.annotations-handle-s { left: 50%; bottom: -6px; margin-left: -5px; cursor: ns-resize; }
+.annotations-handle-sw { left: -6px; bottom: -6px; cursor: nesw-resize; }
+.annotations-handle-w { left: -6px; top: 50%; margin-top: -5px; cursor: ew-resize; }
-.annotation-label {
+.annotations-label {
position: absolute;
top: -2px;
left: -2px;
max-width: calc(100% + 4px);
padding: 2px 6px;
- background: rgb(var(--annotate-blue));
+ background: rgb(var(--annotations-blue));
color: #fff;
font-size: 11px;
font-weight: 500;
@@ -132,11 +136,11 @@
cursor: text;
}
-.annotation-label:hover {
- background: rgb(var(--annotate-blue) / 0.85);
+.annotations-label:hover {
+ background: rgb(var(--annotations-blue) / 0.85);
}
-.annotation-popover {
+.annotations-popover {
position: absolute;
min-width: 220px;
max-width: 320px;
@@ -152,7 +156,7 @@
cursor: default;
}
-.annotation-textarea {
+.annotations-textarea {
width: 100%;
min-height: 64px;
resize: vertical;
@@ -167,68 +171,22 @@
box-sizing: border-box;
}
-.annotation-textarea:focus {
- border-color: rgb(var(--annotate-blue));
+.annotations-textarea:focus {
+ border-color: rgb(var(--annotations-blue));
}
-.annotation-popover-actions {
+.annotations-popover-actions {
display: flex;
gap: 6px;
justify-content: flex-end;
}
-.annotation-modal-backdrop {
- position: absolute;
- inset: 0;
- background: rgba(0, 0, 0, 0.5);
- display: flex;
- align-items: center;
- justify-content: center;
- z-index: 100;
- cursor: default;
-}
-
-.annotation-modal {
- width: min(520px, 92%);
- max-height: 80%;
- background: var(--color-canvas-default);
- border: 1px solid var(--color-border-default);
- border-radius: 10px;
- box-shadow: 0 12px 40px rgba(0, 0, 0, 0.45);
- display: flex;
- flex-direction: column;
- overflow: hidden;
-}
-
-.annotation-modal-header {
- display: flex;
- align-items: center;
- justify-content: space-between;
- padding: 12px 16px;
- border-bottom: 1px solid var(--color-border-muted);
- font-size: 13px;
- font-weight: 600;
- color: var(--color-fg-default);
-}
-
-.annotation-modal-text {
- flex: 1;
- min-height: 200px;
- padding: 12px 16px;
- font-family: 'SF Mono', 'Cascadia Code', 'Fira Code', 'Consolas', monospace;
- font-size: 12px;
- color: var(--color-fg-default);
- background: var(--color-canvas-subtle);
- border: none;
- outline: none;
- resize: none;
- white-space: pre;
-}
-
-.annotation-modal-actions {
- display: flex;
- justify-content: flex-end;
- gap: 8px;
- padding: 10px 16px;
- border-top: 1px solid var(--color-border-muted);
+.annotate-modal-image {
+ max-width: 100%;
+ max-height: 100%;
+ object-fit: contain;
+ display: block;
+ user-select: none;
+ -webkit-user-drag: none;
+ box-shadow: 0 0 0 1px var(--color-border-muted), 0 0 8px var(--color-border-muted);
}
diff --git a/packages/dashboard/src/annotations.tsx b/packages/dashboard/src/annotations.tsx
index 92e1f9e365..973d0b7370 100644
--- a/packages/dashboard/src/annotations.tsx
+++ b/packages/dashboard/src/annotations.tsx
@@ -15,7 +15,14 @@
*/
import React from 'react';
+import './modal.css';
import './annotations.css';
+import { DownloadIcon } from './icons';
+import { ToolbarButton } from '@web/components/toolbarButton';
+import { clientToViewport, getImageLayout } from './imageLayout';
+
+import type { ImageLayout } from './imageLayout';
+import type { AnnotateFrame } from './dashboardModel';
export type Annotation = { id: string; x: number; y: number; width: number; height: number; text: string };
@@ -73,34 +80,6 @@ function applyDrag(orig: Rect, kind: DragKind, dvx: number, dvy: number): Rect {
};
}
-export type ImageLayout = {
- rect: DOMRect;
- renderW: number;
- renderH: number;
- offsetX: number;
- offsetY: number;
-};
-
-export function getImageLayout(display: HTMLImageElement | null): ImageLayout | null {
- if (!display || !display.naturalWidth || !display.naturalHeight)
- return null;
- const rect = display.getBoundingClientRect();
- const imgAspect = display.naturalWidth / display.naturalHeight;
- const elemAspect = rect.width / rect.height;
- if (imgAspect > elemAspect) {
- const renderH = rect.width / imgAspect;
- return { rect, renderW: rect.width, renderH, offsetX: 0, offsetY: (rect.height - renderH) / 2 };
- }
- const renderW = rect.height * imgAspect;
- return { rect, renderW, renderH: rect.height, offsetX: (rect.width - renderW) / 2, offsetY: 0 };
-}
-
-export function clientToViewport(layout: ImageLayout, vw: number, vh: number, clientX: number, clientY: number): { x: number; y: number } {
- const fracX = (clientX - layout.rect.left - layout.offsetX) / layout.renderW;
- const fracY = (clientY - layout.rect.top - layout.offsetY) / layout.renderH;
- return { x: Math.round(fracX * vw), y: Math.round(fracY * vh) };
-}
-
function viewportRectToScreenStyle(layout: ImageLayout, screenRect: DOMRect, vw: number, vh: number, r: Rect): React.CSSProperties {
const baseLeft = layout.rect.left - screenRect.left + layout.offsetX;
const baseTop = layout.rect.top - screenRect.top + layout.offsetY;
@@ -418,7 +397,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
return (
<div
ref={layerRef}
- className='annotation-layer'
+ className='annotations-layer'
tabIndex={0}
onMouseDown={onLayerMouseDown}
onMouseMove={onLayerMouseMove}
@@ -435,7 +414,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
return (
<div
key={a.id}
- className={'annotation-rect' + (isSelected ? ' selected' : '') + (isEditing ? ' editing' : '') + (a.text ? '' : ' empty')}
+ className={'annotations-rect' + (isSelected ? ' selected' : '') + (isEditing ? ' editing' : '') + (a.text ? '' : ' empty')}
style={style}
onDoubleClick={e => {
e.preventDefault();
@@ -445,7 +424,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
>
{a.text && (
<div
- className='annotation-label'
+ className='annotations-label'
onMouseDown={e => {
e.preventDefault();
e.stopPropagation();
@@ -458,7 +437,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
{isSelected && !isEditing && HANDLES.map(h => (
<div
key={h}
- className={'annotation-handle annotation-handle-' + h}
+ className={'annotations-handle annotations-handle-' + h}
onMouseDown={e => startResize(h, a, e)}
/>
))}
@@ -468,7 +447,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
{draft && (() => {
const style = mapRect(normalizeRect(draft));
- return style ? <div className='annotation-rect draft' style={style} /> : null;
+ return style ? <div className='annotations-rect draft' style={style} /> : null;
})()}
{editingAnnotation && (() => {
@@ -481,13 +460,13 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
};
return (
<div
- className='annotation-popover'
+ className='annotations-popover'
style={popoverStyle}
onMouseDown={e => e.stopPropagation()}
onClick={e => e.stopPropagation()}
>
<textarea
- className='annotation-textarea'
+ className='annotations-textarea'
autoFocus
value={editingAnnotation.text}
placeholder='Task or comment…'
@@ -507,9 +486,9 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
}}
onKeyUp={e => e.stopPropagation()}
/>
- <div className='annotation-popover-actions'>
+ <div className='annotations-popover-actions'>
<button
- className='annotate-action-btn danger'
+ className='annotations-action-btn danger'
onClick={() => {
setAnnotations(prev => prev.filter(a => a.id !== editingAnnotation.id));
setSelection(null);
@@ -519,7 +498,7 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
Discard
</button>
<button
- className='annotate-action-btn'
+ className='annotations-action-btn'
onClick={closeEditor}
>
Add
@@ -533,3 +512,70 @@ export const Annotations = React.forwardRef<AnnotationsHandle, AnnotationsProps>
);
});
Annotations.displayName = 'Annotations';
+
+type AnnotateModalProps = {
+ frame: AnnotateFrame;
+ showSubmit: boolean;
+ onSubmit: (blob: Blob, annotations: Annotation[]) => Promise<void>;
+ onClose: () => void;
+};
+
+export const AnnotateModal: React.FC<AnnotateModalProps> = ({ frame, showSubmit, onSubmit, onClose }) => {
+ const [annotationCount, setAnnotationCount] = React.useState(0);
+ const annotationsRef = React.useRef<AnnotationsHandle>(null);
+ const displayRef = React.useRef<HTMLImageElement>(null);
+ const viewRef = React.useRef<HTMLDivElement>(null);
+
+ return (
+ <div className='modal-overlay' role='dialog' aria-modal='true' aria-label='Annotate screenshot'>
+ <div className='modal'>
+ <div className='modal-toolbar'>
+ <div className='modal-title'>Annotate screenshot</div>
+ {showSubmit && (
+ <ToolbarButton
+ title='Submit annotation'
+ icon='check'
+ disabled={annotationCount === 0}
+ onClick={() => annotationsRef.current?.submit()}
+ />
+ )}
+ <ToolbarButton
+ title='Save annotated image'
+ onClick={() => annotationsRef.current?.save()}
+ >
+ <DownloadIcon />
+ </ToolbarButton>
+ <ToolbarButton
+ title='Clear annotations'
+ icon='circle-slash'
+ disabled={annotationCount === 0}
+ onClick={() => annotationsRef.current?.clear()}
+ />
+ <ToolbarButton
+ title='Discard'
+ icon='close'
+ onClick={onClose}
+ />
+ </div>
+ <div ref={viewRef} className='modal-body'>
+ <img
+ ref={displayRef}
+ className='annotate-modal-image'
+ alt='annotation'
+ src={'data:image/png;base64,' + frame.data}
+ />
+ <Annotations
+ ref={annotationsRef}
+ active={true}
+ displayRef={displayRef}
+ screenRef={viewRef}
+ viewportWidth={frame.viewportWidth}
+ viewportHeight={frame.viewportHeight}
+ onSubmit={onSubmit}
+ onAnnotationsChange={setAnnotationCount}
+ />
+ </div>
+ </div>
+ </div>
+ );
+};
diff --git a/packages/dashboard/src/dashboard.css b/packages/dashboard/src/dashboard.css
index 7ba66894e4..4444e70521 100644
--- a/packages/dashboard/src/dashboard.css
+++ b/packages/dashboard/src/dashboard.css
@@ -16,7 +16,6 @@
.dashboard-view {
--interactive-orange: 238 134 12;
- --annotate-blue: 54 116 209;
--recording-red: 239 68 68;
display: flex;
flex-direction: column;
@@ -26,36 +25,7 @@
position: relative;
}
-.dashboard-view.annotate::after,
-.dashboard-view.recording::after {
- position: absolute;
- inset: 0;
- content: '';
- border-radius: 8px;
- pointer-events: none;
-}
-
-.dashboard-view.annotate::after {
- box-shadow: inset 0 0 0 1px rgb(var(--annotate-blue) / 0.72), inset 0 0 8px rgb(var(--annotate-blue) / 0.28);
-}
-
-.dashboard-view.recording::after {
- box-shadow: inset 0 0 0 1px rgb(var(--recording-red) / 0.72), inset 0 0 8px rgb(var(--recording-red) / 0.28);
-}
-
-.dashboard-view.annotate .toolbar,
-.dashboard-view.recording .toolbar {
- background: none !important;
-}
-
-/* -- Mode toggle buttons (interactive / annotate) -- */
-.toolbar-right {
- margin-left: auto;
- display: flex;
- align-items: center;
- gap: 4px;
-}
-
+/* -- Mode toggle buttons (interactive / record) -- */
.mode-toggle.toolbar-button.toggled,
.mode-toggle.toolbar-button.toggled > .codicon {
color: var(--color-fg-on-emphasis);
@@ -66,9 +36,9 @@
box-shadow: 0 0 0 2px rgb(var(--interactive-orange) / 0.35);
}
-.mode-toggle.toolbar-button.toggled.mode-annotate {
- background: rgb(var(--annotate-blue));
- box-shadow: 0 0 0 2px rgb(var(--annotate-blue) / 0.35);
+.mode-toggle.toolbar-button.toggled.mode-record {
+ background: rgb(var(--recording-red));
+ box-shadow: 0 0 0 2px rgb(var(--recording-red) / 0.35);
}
.mode-toggle.flash {
@@ -240,34 +210,6 @@
width: 100%;
}
-.annotate-view, .recording-view {
- position: relative;
- width: calc(100% - 16px);
- height: calc(100% - 16px);
- display: flex;
- align-items: center;
- justify-content: center;
-}
-
-.annotate-image {
- max-width: 100%;
- max-height: 100%;
- object-fit: contain;
- display: block;
- user-select: none;
- -webkit-user-drag: none;
- box-shadow: 0 0 0 1px var(--color-border-muted), 0 0 8px var(--color-border-muted);
-}
-
-.recording-video {
- max-width: 100%;
- max-height: 100%;
- object-fit: contain;
- display: block;
- background: #000;
- box-shadow: 0 0 0 1px var(--color-border-muted), 0 0 8px var(--color-border-muted);
-}
-
.display {
display: block;
width: 100%;
@@ -305,47 +247,11 @@
box-shadow: 0 0 24px 12px rgba(255, 255, 255, 0.2);
}
-.screen-toast {
- position: absolute;
- bottom: 16px;
- left: 50%;
- transform: translateX(-50%);
- background: var(--color-primer-canvas-sticky);
- backdrop-filter: blur(8px);
- color: var(--color-fg-default);
- font-size: 12px;
- padding: 6px 16px;
- border-radius: 20px;
- pointer-events: none;
- opacity: 0;
- transition: opacity 0.2s;
- white-space: nowrap;
- border: 1px solid var(--color-border-subtle);
-}
-
-.screen-toast.visible {
- opacity: 1;
-}
-
-.screen-toast code {
- color: var(--color-accent-fg);
- font-family: 'SF Mono', 'Cascadia Code', 'Fira Code', 'Consolas', monospace;
- margin-left: 4px;
-}
-
-.inspector-frame {
- display: block;
- width: 100%;
- height: 100%;
- border: none;
- background: var(--color-canvas-default);
-}
-
-.recording.toggled > .codicon {
- animation: recording-pulse 1.5s ease-in-out infinite;
+.mode-record.toggled > .codicon {
+ animation: mode-record-pulse 1.5s ease-in-out infinite;
}
-@keyframes recording-pulse {
+@keyframes mode-record-pulse {
0%, 100% {
opacity: 1;
transform: scale(1);
@@ -356,14 +262,14 @@
}
}
-.recording-label {
+.mode-record-label {
display: inline-block;
margin-left: 5px;
white-space: nowrap;
- animation: recording-label-slide-in 300ms ease both;
+ animation: mode-record-label-slide-in 300ms ease both;
}
-@keyframes recording-label-slide-in {
+@keyframes mode-record-label-slide-in {
from {
max-width: 0;
opacity: 0;
diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsx
index f3e97c1a20..8b1d49d11c 100644
--- a/packages/dashboard/src/dashboard.tsx
+++ b/packages/dashboard/src/dashboard.tsx
@@ -16,11 +16,13 @@
import React from 'react';
import './dashboard.css';
-import { ChevronLeftIcon, ChevronRightIcon, DownloadIcon, LockIcon, LockOpenIcon, ReloadIcon, ScreenshotRegionIcon } from './icons';
-import { Annotations, getImageLayout, clientToViewport } from './annotations';
+import { ChevronLeftIcon, ChevronRightIcon, LockIcon, LockOpenIcon, ReloadIcon, ScreenshotRegionIcon } from './icons';
+import { AnnotateModal } from './annotations';
+import { clientToViewport, getImageLayout } from './imageLayout';
+import { Recording } from './recording';
-import type { Annotation, AnnotationsHandle } from './annotations';
-import { ToolbarButton, ToolbarSeparator } from '@web/components/toolbarButton';
+import type { Annotation } from './annotations';
+import { ToolbarButton } from '@web/components/toolbarButton';
import { useMeasureForRef } from '@web/uiUtils';
import type { DashboardModel } from './dashboardModel';
@@ -67,19 +69,15 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
const { tabs, mode, recording, liveFrame, annotateFrame, annotateInitiator, pendingAnnotate } = model.state;
const interactive = mode === 'interactive';
- const annotating = mode === 'annotate';
const [flashTick, setFlashTick] = React.useState(0);
- const [annotationCount, setAnnotationCount] = React.useState(0);
const displayRef = React.useRef<HTMLImageElement>(null);
const screenRef = React.useRef<HTMLDivElement>(null);
- const annotateViewRef = React.useRef<HTMLDivElement>(null);
const toolbarRef = React.useRef<HTMLDivElement>(null);
const viewportMainRef = React.useRef<HTMLDivElement>(null);
const browserChromeRef = React.useRef<HTMLDivElement>(null);
const interactiveBtnRef = React.useRef<HTMLButtonElement>(null);
- const annotationsRef = React.useRef<AnnotationsHandle>(null);
const moveThrottleRef = React.useRef(0);
const aspect = liveFrame && liveFrame.viewportWidth && liveFrame.viewportHeight
@@ -89,7 +87,7 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
const [viewportRect] = useMeasureForRef(viewportMainRef);
// Active recording hides the browser chrome so the viewport matches what's
- // being captured (mirroring annotate mode's chrome-less frame).
+ // being captured.
const showBrowserChrome = recording?.phase !== 'recording';
const windowStyle = React.useMemo<React.CSSProperties | undefined>(() => {
@@ -148,15 +146,21 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
setFlashTick(tick => tick + 1);
}
- const onSaveRecording = React.useCallback(async () => {
- if (recording?.phase !== 'stopped')
- return;
+ const onSaveRecording = React.useCallback(async (blob: Blob) => {
const writable = await pickSaveWritable(`playwright-recording-${Date.now()}.webm`, 'WebM Video', 'video/webm', '.webm');
if (!writable)
return;
- await writable.write(recording.blob);
+ await writable.write(blob);
await writable.close();
- }, [recording]);
+ model.discardRecording();
+ }, [model]);
+
+ const onCloseAnnotate = React.useCallback(() => {
+ if (annotateInitiator === 'cli')
+ model.completeAnnotation();
+ else
+ model.cancelAnnotate();
+ }, [model, annotateInitiator]);
const selectedTab = tabs?.find(t => t.selected);
const ready = !!selectedTab;
@@ -178,8 +182,6 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
}
function onScreenMouseDown(e: React.MouseEvent) {
- if (annotating)
- return;
e.preventDefault();
screenRef.current?.focus();
if (!ready)
@@ -192,14 +194,14 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
}
function onScreenMouseUp(e: React.MouseEvent) {
- if (annotating || !interactive)
+ if (!interactive)
return;
e.preventDefault();
sendMouseEvent('mouseup', e);
}
function onScreenMouseMove(e: React.MouseEvent) {
- if (annotating || !interactive)
+ if (!interactive)
return;
const now = Date.now();
if (now - moveThrottleRef.current < 32)
@@ -210,21 +212,21 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
}
function onScreenWheel(e: React.WheelEvent) {
- if (annotating || !interactive)
+ if (!interactive)
return;
e.preventDefault();
model.wheel(e.deltaX, e.deltaY);
}
function onScreenKeyDown(e: React.KeyboardEvent) {
- if (annotating || !interactive)
+ if (!interactive)
return;
e.preventDefault();
model.keydown(e.key);
}
function onScreenKeyUp(e: React.KeyboardEvent) {
- if (annotating || !interactive)
+ if (!interactive)
return;
e.preventDefault();
model.keyup(e.key);
@@ -239,236 +241,160 @@ export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
}
const overlayText = selectedTab ? undefined : 'Select a session';
- const modeLabel = annotating ? 'Dashboard: annotate' : recording ? 'Dashboard: record' : 'Dashboard';
+ const isRecording = recording?.phase === 'recording';
+ const showAnnotateModal = !!annotateFrame;
+ const showRecording = recording?.phase === 'stopped';
+ const modeLabel = showAnnotateModal ? 'Dashboard: annotate' : isRecording ? 'Dashboard: record' : 'Dashboard';
return (
- <main className={'dashboard-view' + (interactive ? ' interactive' : '') + (annotating ? ' annotate' : '') + (recording ? ' recording' : '')} aria-label={modeLabel}>
+ <main className={'dashboard-view' + (interactive ? ' interactive' : '')} aria-label={modeLabel}>
{/* Toolbar */}
<div ref={toolbarRef} className='toolbar'>
- {annotating ? (
- <>
- {annotateInitiator === 'cli' && (
- <>
- <ToolbarButton
- className='annotate-toolbar-btn'
- title='Submit annotation'
- icon='check'
- disabled={annotationCount === 0}
- onClick={() => annotationsRef.current?.submit()}
- />
- <ToolbarSeparator />
- </>
- )}
- <ToolbarButton
- className='annotate-toolbar-btn'
- title='Save annotated image'
- onClick={() => annotationsRef.current?.save()}
- >
- <DownloadIcon />
- </ToolbarButton>
- <ToolbarButton
- className='annotate-toolbar-btn'
- title='Clear annotations'
- icon='circle-slash'
- disabled={annotationCount === 0}
- onClick={() => annotationsRef.current?.clear()}
- />
- <div className='toolbar-right'>
- <ToolbarButton
- className='annotate-toolbar-btn'
- title='Close annotation mode'
- icon='close'
- onClick={() => model.completeAnnotation()}
- />
- </div>
- </>
- ) : recording ? (
- <>
- <ToolbarButton
- className='recording'
- title={recording.phase === 'recording' ? 'Stop recording' : 'Start new recording'}
- icon='record'
- toggled={recording.phase === 'recording'}
- style={{ color: recording.phase === 'recording' ? 'var(--color-scale-red-5)' : undefined }}
- onClick={() => {
- if (recording.phase === 'recording') {
- model.stopRecording();
- } else {
- URL.revokeObjectURL(recording.blobUrl);
- model.startRecording();
- }
- }}>
- {recording.phase === 'recording' && <span className='recording-label'>Recording...</span>}
- </ToolbarButton>
- <ToolbarButton
- title='Save recording'
- disabled={recording.phase !== 'stopped'}
- onClick={onSaveRecording}
- >
- <DownloadIcon />
- </ToolbarButton>
- <div className='toolbar-right'>
- <ToolbarButton
- title='Close'
- icon='close'
- onClick={() => model.discardRecording()}
- />
- </div>
- </>
- ) : (
- <>
- <ToolbarButton
- ref={interactiveBtnRef}
- className='mode-toggle mode-interactive'
- title={interactive ? 'Disable interactive mode' : 'Enable interactive mode'}
- toggled={interactive}
- disabled={!ready}
- onClick={() => model.toggleInteractive()}
- >
- {interactive ? <LockOpenIcon /> : <LockIcon />}
- </ToolbarButton>
- <ToolbarSeparator />
- <ToolbarButton
- className='mode-toggle mode-annotate'
- title='Enable annotation mode'
- disabled={!ready}
- onClick={() => {
- if (pendingAnnotate)
- model.cancelAnnotate();
- else
- model.enterAnnotate('user');
- }}
- >
- <ScreenshotRegionIcon />
- </ToolbarButton>
- <ToolbarButton
- title='Record video'
- icon='record'
- disabled={!ready}
- onClick={() => model.startRecording()}
- />
- </>
- )}
+ <ToolbarButton
+ ref={interactiveBtnRef}
+ className='mode-toggle mode-interactive'
+ title={interactive ? 'Disable interactive mode' : 'Enable interactive mode'}
+ toggled={interactive}
+ disabled={!ready}
+ onClick={() => {
+ if (interactive)
+ model.toggleInteractive();
+ else
+ model.enterInteractive();
+ }}
+ >
+ {interactive ? <LockOpenIcon /> : <LockIcon />}
+ </ToolbarButton>
+ <ToolbarButton
+ className='mode-annotate'
+ title='Annotate screenshot'
+ disabled={!ready || !!pendingAnnotate || showAnnotateModal}
+ onClick={() => model.enterAnnotate('user')}
+ >
+ <ScreenshotRegionIcon />
+ </ToolbarButton>
+ <ToolbarButton
+ className='mode-toggle mode-record'
+ title={isRecording ? 'Stop recording' : 'Record video'}
+ icon='record'
+ toggled={isRecording}
+ disabled={!ready || showRecording}
+ onClick={() => {
+ if (isRecording)
+ model.stopRecording();
+ else
+ model.startRecording();
+ }}
+ >
+ {isRecording && <span className='mode-record-label'>Recording...</span>}
+ </ToolbarButton>
</div>
{/* Viewport */}
<div className='viewport-wrapper'>
<div ref={viewportMainRef} className='viewport-main'>
- {annotating && annotateFrame ? (
- <div ref={annotateViewRef} className='annotate-view'>
+ <div className='browser-window' style={windowStyle}>
+ {showBrowserChrome && (
+ <div ref={browserChromeRef} className='browser-chrome'>
+ <button className='nav-btn' title='Back' aria-disabled={!interactive || undefined} onClick={() => {
+ if (!interactive) {
+ flashInteractiveHint();
+ return;
+ }
+ model.back();
+ }}>
+ <ChevronLeftIcon />
+ </button>
+ <button className='nav-btn' title='Forward' aria-disabled={!interactive || undefined} onClick={() => {
+ if (!interactive) {
+ flashInteractiveHint();
+ return;
+ }
+ model.forward();
+ }}>
+ <ChevronRightIcon />
+ </button>
+ <button className='nav-btn' title='Reload' aria-disabled={!interactive || undefined} onClick={() => {
+ if (!interactive) {
+ flashInteractiveHint();
+ return;
+ }
+ model.reload();
+ }}>
+ <ReloadIcon />
+ </button>
+ <div className='omnibox-wrap'>
+ <input
+ id='omnibox'
+ className='omnibox'
+ type='text'
+ placeholder='Search or enter URL'
+ spellCheck={false}
+ autoComplete='off'
+ value={selectedTab?.url}
+ onChange={() => {}}
+ onKeyDown={e => {
+ if (!interactive)
+ return;
+ onOmniboxKeyDown(e);
+ }}
+ onFocus={e => {
+ if (!interactive) {
+ flashInteractiveHint();
+ e.target.blur();
+ return;
+ }
+ e.target.select();
+ }}
+ aria-disabled={!interactive || undefined}
+ readOnly={!interactive}
+ />
+ </div>
+ </div>
+ )}
+ <div
+ ref={screenRef}
+ className='screen'
+ tabIndex={0}
+ style={{ display: liveFrame ? '' : 'none' }}
+ onMouseDown={onScreenMouseDown}
+ onMouseUp={onScreenMouseUp}
+ onMouseMove={onScreenMouseMove}
+ onWheel={onScreenWheel}
+ onKeyDown={onScreenKeyDown}
+ onKeyUp={onScreenKeyUp}
+ onContextMenu={e => e.preventDefault()}
+ >
<img
ref={displayRef}
id='display'
- className='annotate-image'
- alt='annotation'
- src={'data:image/png;base64,' + annotateFrame.data}
- />
- <Annotations
- ref={annotationsRef}
- active={true}
- displayRef={displayRef}
- screenRef={annotateViewRef}
- viewportWidth={annotateFrame.viewportWidth ?? 0}
- viewportHeight={annotateFrame.viewportHeight ?? 0}
- onSubmit={onSubmitAnnotations}
- onAnnotationsChange={setAnnotationCount}
+ className='display'
+ alt='screencast'
+ src={liveFrame ? 'data:image/jpeg;base64,' + liveFrame.data : undefined}
/>
</div>
- ) : recording?.phase === 'stopped' ? (
- <div className='recording-view'>
- <video
- className='recording-video'
- src={recording.blobUrl}
- controls
- autoPlay
- />
- </div>
- ) : (
- <div className='browser-window' style={windowStyle}>
- {showBrowserChrome && (
- <div ref={browserChromeRef} className='browser-chrome'>
- <button className='nav-btn' title='Back' aria-disabled={!interactive || undefined} onClick={() => {
- if (!interactive) {
- flashInteractiveHint();
- return;
- }
- model.back();
- }}>
- <ChevronLeftIcon />
- </button>
- <button className='nav-btn' title='Forward' aria-disabled={!interactive || undefined} onClick={() => {
- if (!interactive) {
- flashInteractiveHint();
- return;
- }
- model.forward();
- }}>
- <ChevronRightIcon />
- </button>
- <button className='nav-btn' title='Reload' aria-disabled={!interactive || undefined} onClick={() => {
- if (!interactive) {
- flashInteractiveHint();
- return;
- }
- model.reload();
- }}>
- <ReloadIcon />
- </button>
- <div className='omnibox-wrap'>
- <input
- id='omnibox'
- className='omnibox'
- type='text'
- placeholder='Search or enter URL'
- spellCheck={false}
- autoComplete='off'
- value={selectedTab?.url}
- onChange={() => {}}
- onKeyDown={e => {
- if (!interactive)
- return;
- onOmniboxKeyDown(e);
- }}
- onFocus={e => {
- if (!interactive) {
- flashInteractiveHint();
- e.target.blur();
- return;
- }
- e.target.select();
- }}
- aria-disabled={!interactive || undefined}
- readOnly={!interactive}
- />
- </div>
- </div>
- )}
- <div
- ref={screenRef}
- className='screen'
- tabIndex={0}
- style={{ display: liveFrame ? '' : 'none' }}
- onMouseDown={onScreenMouseDown}
- onMouseUp={onScreenMouseUp}
- onMouseMove={onScreenMouseMove}
- onWheel={onScreenWheel}
- onKeyDown={onScreenKeyDown}
- onKeyUp={onScreenKeyUp}
- onContextMenu={e => e.preventDefault()}
- >
- <img
- ref={displayRef}
- id='display'
- className='display'
- alt='screencast'
- src={liveFrame ? 'data:image/jpeg;base64,' + liveFrame.data : undefined}
- />
- </div>
- {overlayText && <div className={'screen-overlay' + (liveFrame ? ' has-frame' : '')}><span>{overlayText}</span></div>}
- </div>
- )}
+ {overlayText && <div className={'screen-overlay' + (liveFrame ? ' has-frame' : '')}><span>{overlayText}</span></div>}
+ </div>
</div>
</div>
+
+ {showAnnotateModal && annotateFrame && (
+ <AnnotateModal
+ frame={annotateFrame}
+ showSubmit={annotateInitiator === 'cli'}
+ onSubmit={onSubmitAnnotations}
+ onClose={onCloseAnnotate}
+ />
+ )}
+
+ {showRecording && recording?.phase === 'stopped' && (
+ <Recording
+ blob={recording.blob}
+ blobUrl={recording.blobUrl}
+ onSave={onSaveRecording}
+ onClose={() => model.discardRecording()}
+ />
+ )}
</main>
);
};
diff --git a/packages/dashboard/src/dashboardModel.ts b/packages/dashboard/src/dashboardModel.ts
index c8f079af7b..858b89f15b 100644
--- a/packages/dashboard/src/dashboardModel.ts
+++ b/packages/dashboard/src/dashboardModel.ts
@@ -170,6 +170,10 @@ export class DashboardModel {
// variants; intra-model flows `await` the private variants when they
// need to sequence mode switches correctly.
+ enterInteractive() {
+ void this._enterInteractive();
+ }
+
enterAnnotate(initiator: 'cli' | 'user') {
void this._enterAnnotate(initiator);
}
@@ -209,6 +213,13 @@ export class DashboardModel {
this.cancelAnnotate();
}
+ private async _enterInteractive() {
+ // Interactive mode coexists with recording so the user can drive the
+ // page while it is being captured. Only in-flight annotation is cleared.
+ await this._completeAnnotation();
+ this._emit({ mode: 'interactive' });
+ }
+
private async _enterAnnotate(initiator: 'cli' | 'user') {
await this._cleanupOnModeSwitch();
this._requestAnnotate(initiator);
diff --git a/packages/dashboard/src/imageLayout.ts b/packages/dashboard/src/imageLayout.ts
new file mode 100644
index 0000000000..03912d5d60
--- /dev/null
+++ b/packages/dashboard/src/imageLayout.ts
@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Geometry of an `object-fit: contain` <img>: the rendered image's size and
+// offset relative to the element's bounding box.
+export type ImageLayout = {
+ rect: DOMRect;
+ renderW: number;
+ renderH: number;
+ offsetX: number;
+ offsetY: number;
+};
+
+export function getImageLayout(display: HTMLImageElement | null): ImageLayout | null {
+ if (!display || !display.naturalWidth || !display.naturalHeight)
+ return null;
+ const rect = display.getBoundingClientRect();
+ const imgAspect = display.naturalWidth / display.naturalHeight;
+ const elemAspect = rect.width / rect.height;
+ if (imgAspect > elemAspect) {
+ const renderH = rect.width / imgAspect;
+ return { rect, renderW: rect.width, renderH, offsetX: 0, offsetY: (rect.height - renderH) / 2 };
+ }
+ const renderW = rect.height * imgAspect;
+ return { rect, renderW, renderH: rect.height, offsetX: (rect.width - renderW) / 2, offsetY: 0 };
+}
+
+export function clientToViewport(layout: ImageLayout, vw: number, vh: number, clientX: number, clientY: number): { x: number; y: number } {
+ const fracX = (clientX - layout.rect.left - layout.offsetX) / layout.renderW;
+ const fracY = (clientY - layout.rect.top - layout.offsetY) / layout.renderH;
+ return { x: Math.round(fracX * vw), y: Math.round(fracY * vh) };
+}
diff --git a/packages/dashboard/src/modal.css b/packages/dashboard/src/modal.css
new file mode 100644
index 0000000000..edb9da25d6
--- /dev/null
+++ b/packages/dashboard/src/modal.css
@@ -0,0 +1,72 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.modal-overlay {
+ position: fixed;
+ inset: 0;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ background: rgba(0, 0, 0, 0.55);
+ z-index: 1000;
+ padding: 24px;
+}
+
+.modal {
+ display: flex;
+ flex-direction: column;
+ width: 100%;
+ height: 100%;
+ background: var(--color-canvas-default);
+ border: 1px solid var(--color-border-default);
+ border-radius: 10px;
+ box-shadow: 0 10px 40px rgba(0, 0, 0, 0.45);
+ overflow: hidden;
+}
+
+.modal-toolbar {
+ display: flex;
+ align-items: center;
+ justify-content: flex-end;
+ gap: 4px;
+ height: 40px;
+ min-height: 40px;
+ background: var(--color-canvas-overlay);
+ padding: 0 8px;
+ border-bottom: 1px solid var(--color-border-muted);
+}
+
+:root.light-mode .modal-toolbar {
+ background: var(--color-canvas-subtle);
+}
+
+.modal-title {
+ margin-right: auto;
+ padding-left: 4px;
+ font-size: 13px;
+ font-weight: 600;
+ color: var(--color-fg-default);
+}
+
+.modal-body {
+ position: relative;
+ flex: 1;
+ display: flex;
+ align-items: center;
+ justify-content: center;
+ min-height: 0;
+ padding: 16px;
+}
diff --git a/packages/dashboard/src/recording.css b/packages/dashboard/src/recording.css
new file mode 100644
index 0000000000..36487f98bc
--- /dev/null
+++ b/packages/dashboard/src/recording.css
@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+.recording-body {
+ background: #000;
+}
+
+.recording-video {
+ max-width: 100%;
+ max-height: 100%;
+ object-fit: contain;
+ display: block;
+}
diff --git a/packages/dashboard/src/recording.tsx b/packages/dashboard/src/recording.tsx
new file mode 100644
index 0000000000..362acc747e
--- /dev/null
+++ b/packages/dashboard/src/recording.tsx
@@ -0,0 +1,63 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import React from 'react';
+import './modal.css';
+import './recording.css';
+import { DownloadIcon } from './icons';
+import { ToolbarButton } from '@web/components/toolbarButton';
+
+type RecordingProps = {
+ blob: Blob;
+ blobUrl: string;
+ onSave: (blob: Blob) => Promise<void>;
+ onClose: () => void;
+};
+
+export const Recording: React.FC<RecordingProps> = ({ blob, blobUrl, onSave, onClose }) => {
+ const handleSave = React.useCallback(async () => {
+ await onSave(blob);
+ }, [onSave, blob]);
+
+ return (
+ <div className='modal-overlay' role='dialog' aria-modal='true' aria-label='Recording preview'>
+ <div className='modal'>
+ <div className='modal-toolbar'>
+ <div className='modal-title'>Recording</div>
+ <ToolbarButton
+ title='Save recording'
+ onClick={handleSave}
+ >
+ <DownloadIcon />
+ </ToolbarButton>
+ <ToolbarButton
+ title='Discard'
+ icon='close'
+ onClick={onClose}
+ />
+ </div>
+ <div className='modal-body recording-body'>
+ <video
+ className='recording-video'
+ src={blobUrl}
+ controls
+ autoPlay
+ />
+ </div>
+ </div>
+ </div>
+ );
+};
diff --git a/tests/mcp/dashboard.spec.ts b/tests/mcp/dashboard.spec.ts
index c9ca9fceb7..dfdf05f5b6 100644
--- a/tests/mcp/dashboard.spec.ts
+++ b/tests/mcp/dashboard.spec.ts
@@ -146,8 +146,8 @@ async function drawAndSubmitAnnotation(dashboard: import('playwright-core').Page
await dashboard.mouse.down();
await dashboard.mouse.move(x1, y1);
await dashboard.mouse.up();
- await dashboard.locator('.annotation-textarea').fill(text);
- await dashboard.locator('.annotation-textarea').press('Enter');
+ await dashboard.locator('.annotations-textarea').fill(text);
+ await dashboard.locator('.annotations-textarea').press('Enter');
await dashboard.getByRole('button', { name: 'Submit annotation' }).click();
}
@@ -354,7 +354,7 @@ test('save recording streams WebM bytes to the chosen file', async ({ cli, serve
// Enter recording mode from the normal toolbar.
await dashboard.getByRole('button', { name: 'Record video' }).click();
- await expect(dashboard.locator('.recording-label')).toBeVisible();
+ await expect(dashboard.locator('.mode-record-label')).toBeVisible();
// Click the toggled record button again to transition to the 'stopped' phase.
await dashboard.getByRole('button', { name: 'Stop recording' }).click();