Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(dashboard): annotate on fresh page.screenshot and unify state in DashboardModel (#40385)

Pavel Feldmana2113a303e9b
diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsx
index 1613205d19..40c6ec572f 100644
--- a/packages/dashboard/src/dashboard.tsx
+++ b/packages/dashboard/src/dashboard.tsx
@@ -16,7 +16,6 @@
import React from 'react';
import './dashboard.css';
-import { DashboardClientContext } from './dashboardContext';
import { ChevronLeftIcon, ChevronRightIcon, DownloadIcon, LockIcon, LockOpenIcon, ReloadIcon, ScreenshotRegionIcon } from './icons';
import { Annotations, getImageLayout, clientToViewport } from './annotations';
@@ -24,184 +23,9 @@ import type { Annotation, AnnotationsHandle } from './annotations';
import { ToolbarButton, ToolbarSeparator } from '@web/components/toolbarButton';
import { useMeasureForRef } from '@web/uiUtils';
-import type { Tab, DashboardChannelEvents } from './dashboardChannel';
+import type { DashboardModel } from './dashboardModel';
const BUTTONS = ['left', 'middle', 'right'] as const;
-type Mode = 'readonly' | 'interactive' | 'annotate';
-
-// Recording mode is a separate, mode-replacing overlay similar to annotate.
-// While null, the normal toolbar is shown. When non-null, the recording
-// toolbar replaces it.
-type RecordingState =
- | { phase: 'recording' }
- | { phase: 'stopped'; blob: Blob; url: string };
-
-type DashboardState = {
- // Server-driven session state.
- tabs: Tab[] | null;
- url: string;
- // The latest frame received from the server. Cleared on page swap;
- // the server is responsible for emitting a fresh frame for the new
- // page (see _startScreencast).
- liveFrame: DashboardChannelEvents['frame'] | undefined;
- // Snapshot of the frame at the moment we entered annotate mode. The
- // <Annotations> overlay draws on this frozen image so the canvas does
- // not jump around as new live frames arrive.
- annotateFrame: DashboardChannelEvents['frame'] | undefined;
- // Server requested annotate but we have no fresh frame for the active
- // page. The next FRAME action will enter annotate mode.
- cliAnnotatePending: boolean;
- // Whether the current annotate session was initiated by CLI or by the
- // user. Determines whether submit goes to the server or saves locally.
- annotateInitiator: 'cli' | 'user' | null;
- // Interaction mode and ephemeral UI flags.
- mode: Mode;
- recording: RecordingState | null;
-};
-
-type DashboardAction =
- // Server events
- | { type: 'tabs'; tabs: Tab[] }
- | { type: 'frame'; frame: DashboardChannelEvents['frame'] }
- | { type: 'cliAnnotate' }
- | { type: 'cliCancelAnnotate' }
- // User events
- | { type: 'toggleInteractive' }
- | { type: 'toggleAnnotate' }
- | { type: 'startRecording' }
- | { type: 'stoppedRecording'; blob: Blob; url: string }
- | { type: 'exitRecording' }
- | { type: 'submitAnnotation' }
- | { type: 'setUrl'; url: string };
-
-const initialDashboardState: DashboardState = {
- tabs: null,
- url: '',
- liveFrame: undefined,
- annotateFrame: undefined,
- cliAnnotatePending: false,
- annotateInitiator: null,
- mode: 'readonly',
- recording: null,
-};
-
-function dashboardReducer(state: DashboardState, action: DashboardAction): DashboardState {
- switch (action.type) {
- case 'tabs': {
- const newSelected = action.tabs.find(t => t.selected);
- const oldSelected = state.tabs?.find(t => t.selected);
- const url = newSelected ? newSelected.url : state.url;
- // Page change = had a selected tab, now have a different selected
- // tab. Initial selection (none -> some) is not a "change".
- const pageChanged = !!oldSelected && !!newSelected && newSelected.page !== oldSelected.page;
- if (!pageChanged)
- return { ...state, tabs: action.tabs, url };
- // Page swap. Clear frames and rely on the server to emit a fresh
- // frame for the new page (see _startScreencast which awaits a
- // screenshot to force one). If we were annotating, mark pending so
- // the next FRAME re-enters annotate with the new page's image.
- const wasAnnotateActive = state.mode === 'annotate' || state.cliAnnotatePending;
- const newTabIsBrandNew = !!state.tabs && !state.tabs.some(t => t.page === newSelected.page);
- let mode: Mode = state.mode;
- if (!wasAnnotateActive && newTabIsBrandNew && state.tabs)
- mode = 'interactive';
- return {
- ...state,
- tabs: action.tabs,
- url,
- mode,
- recording: null,
- liveFrame: undefined,
- annotateFrame: undefined,
- cliAnnotatePending: wasAnnotateActive,
- };
- }
- case 'frame': {
- const liveFrame = action.frame;
- if (state.cliAnnotatePending) {
- return {
- ...state,
- liveFrame,
- annotateFrame: liveFrame,
- mode: 'annotate',
- cliAnnotatePending: false,
- annotateInitiator: state.annotateInitiator ?? 'cli',
- };
- }
- return { ...state, liveFrame };
- }
- case 'cliAnnotate': {
- if (state.mode === 'annotate') {
- // Already annotating (user-initiated). Mark CLI as the source so
- // submit goes to the server.
- return { ...state, annotateInitiator: 'cli' };
- }
- if (state.liveFrame) {
- return {
- ...state,
- mode: 'annotate',
- annotateFrame: state.liveFrame,
- cliAnnotatePending: false,
- annotateInitiator: 'cli',
- };
- }
- return { ...state, cliAnnotatePending: true, annotateInitiator: 'cli' };
- }
- case 'cliCancelAnnotate': {
- const exitingAnnotate = state.mode === 'annotate';
- return {
- ...state,
- mode: exitingAnnotate ? 'readonly' : state.mode,
- annotateFrame: undefined,
- cliAnnotatePending: false,
- annotateInitiator: null,
- };
- }
- case 'toggleInteractive': {
- const next: Mode = state.mode === 'interactive' ? 'readonly' : 'interactive';
- return { ...state, mode: next };
- }
- case 'toggleAnnotate': {
- if (state.mode === 'annotate') {
- return {
- ...state,
- mode: 'readonly',
- annotateFrame: undefined,
- annotateInitiator: null,
- };
- }
- if (!state.liveFrame)
- return state;
- // Preserve a CLI initiator across mode toggles so that CLI annotate
- // sessions stay engaged when the user switches to interactive and
- // back. Only set 'user' if there was no prior CLI engagement.
- const initiator: 'cli' | 'user' | null = state.annotateInitiator ?? 'user';
- return {
- ...state,
- mode: 'annotate',
- annotateFrame: state.liveFrame,
- cliAnnotatePending: false,
- annotateInitiator: initiator,
- };
- }
- case 'startRecording':
- return { ...state, recording: { phase: 'recording' } };
- case 'stoppedRecording':
- return { ...state, recording: { phase: 'stopped', blob: action.blob, url: action.url } };
- case 'exitRecording':
- return { ...state, recording: null };
- case 'submitAnnotation':
- return {
- ...state,
- mode: 'readonly',
- annotateFrame: undefined,
- cliAnnotatePending: false,
- annotateInitiator: null,
- };
- case 'setUrl':
- return { ...state, url: action.url };
- }
-}
async function pickSaveWritable(suggestedName: string, description: string, mime: string, extension: string): Promise<FileSystemWritableFileStream | null> {
try {
@@ -215,10 +39,6 @@ async function pickSaveWritable(suggestedName: string, description: string, mime
}
}
-function base64ToBlob(base64: string, mime: string): Blob {
- return new Blob([(Uint8Array as any).fromBase64(base64)], { type: mime });
-}
-
function smartUrl(input: string): string {
const value = input.trim();
if (!value)
@@ -237,10 +57,15 @@ function smartUrl(input: string): string {
return 'https://' + host + '.com' + value.slice(host.length);
}
-export const Dashboard: React.FC = () => {
- const client = React.useContext(DashboardClientContext);
- const [state, dispatch] = React.useReducer(dashboardReducer, initialDashboardState);
- const { tabs, url, mode, recording, liveFrame, annotateFrame, annotateInitiator } = state;
+type DashboardProps = {
+ model: DashboardModel;
+};
+
+export const Dashboard: React.FC<DashboardProps> = ({ model }) => {
+ const [, setRevision] = React.useState(0);
+ React.useEffect(() => model.subscribe(() => setRevision(r => r + 1)), [model]);
+
+ const { tabs, mode, recording, liveFrame, annotateFrame, annotateInitiator, pendingAnnotate } = model.state;
const interactive = mode === 'interactive';
const annotating = mode === 'annotate';
@@ -309,8 +134,6 @@ export const Dashboard: React.FC = () => {
}, [interactive]);
const onSubmitAnnotations = React.useCallback(async (blob: Blob, annotations: Annotation[]) => {
- if (!client)
- return;
const dataUrl = await new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => resolve(reader.result as string);
@@ -318,103 +141,25 @@ export const Dashboard: React.FC = () => {
reader.readAsDataURL(blob);
});
const data = dataUrl.slice(dataUrl.indexOf(',') + 1);
- await client.submitAnnotation({
- data,
- annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),
- });
- dispatch({ type: 'submitAnnotation' });
- }, [client]);
+ await model.submitAnnotation(data, annotations);
+ }, [model]);
function flashInteractiveHint() {
setFlashTick(tick => tick + 1);
}
- const onStartRecording = React.useCallback(async () => {
- if (!client)
- return;
- await client.startRecording();
- dispatch({ type: 'startRecording' });
- }, [client]);
-
- const onStopRecording = React.useCallback(async () => {
- if (!client)
- return;
- const { streamId } = await client.stopRecording();
- const chunks: Blob[] = [];
- while (true) {
- const { data, eof } = await client.readStream({ streamId });
- if (data)
- chunks.push(base64ToBlob(data, 'video/webm'));
- if (eof)
- break;
- }
- const blob = new Blob(chunks, { type: 'video/webm' });
- const url = URL.createObjectURL(blob);
- dispatch({ type: 'stoppedRecording', blob, url });
- }, [client]);
-
const onSaveRecording = React.useCallback(async () => {
- if (state.recording?.phase !== 'stopped')
+ if (recording?.phase !== 'stopped')
return;
const writable = await pickSaveWritable(`playwright-recording-${Date.now()}.webm`, 'WebM Video', 'video/webm', '.webm');
if (!writable)
return;
- await writable.write(state.recording.blob);
+ await writable.write(recording.blob);
await writable.close();
- }, [state.recording]);
-
- const onExitRecording = React.useCallback(async () => {
- if (state.recording?.phase === 'recording' && client) {
- // Drain the stream without persisting; keeps the server clean.
- const { streamId } = await client.stopRecording();
- while (true) {
- const { eof } = await client.readStream({ streamId });
- if (eof)
- break;
- }
- }
- if (state.recording?.phase === 'stopped')
- URL.revokeObjectURL(state.recording.url);
- dispatch({ type: 'exitRecording' });
- }, [state.recording, client]);
-
-
- React.useEffect(() => {
- if (!client)
- return;
- let resized = false;
- const onTabs = (params: DashboardChannelEvents['tabs']) => {
- dispatch({ type: 'tabs', tabs: params.tabs });
- };
- const onFrame = (params: DashboardChannelEvents['frame']) => {
- dispatch({ type: 'frame', frame: params });
- const toolbar = toolbarRef.current;
- if (!resized && toolbar && params.viewportWidth && params.viewportHeight) {
- resized = true;
- const chromeHeight = toolbar.offsetHeight;
- const extraW = window.outerWidth - window.innerWidth;
- const extraH = window.outerHeight - window.innerHeight;
- const targetW = Math.min(params.viewportWidth + extraW, screen.availWidth);
- const targetH = Math.min(params.viewportHeight + chromeHeight + extraH, screen.availHeight);
- window.resizeTo(targetW, targetH);
- }
- };
- const onAnnotate = () => dispatch({ type: 'cliAnnotate' });
- const onCancelAnnotate = () => dispatch({ type: 'cliCancelAnnotate' });
- client.on('tabs', onTabs);
- client.on('frame', onFrame);
- client.on('annotate', onAnnotate);
- client.on('cancelAnnotate', onCancelAnnotate);
- return () => {
- client.off('tabs', onTabs);
- client.off('frame', onFrame);
- client.off('annotate', onAnnotate);
- client.off('cancelAnnotate', onCancelAnnotate);
- };
- }, [client]);
+ }, [recording]);
const selectedTab = tabs?.find(t => t.selected);
- const ready = !!client && !!selectedTab;
+ const ready = !!selectedTab;
function imgCoords(e: React.MouseEvent): { x: number; y: number } {
const vw = liveFrame?.viewportWidth ?? 0;
@@ -428,10 +173,8 @@ export const Dashboard: React.FC = () => {
}
function sendMouseEvent(method: 'mousedown' | 'mouseup', e: React.MouseEvent) {
- if (!client)
- return;
const { x, y } = imgCoords(e);
- client[method]({ x, y, button: BUTTONS[e.button] || 'left' });
+ model[method](x, y, BUTTONS[e.button] || 'left');
}
function onScreenMouseDown(e: React.MouseEvent) {
@@ -456,53 +199,46 @@ export const Dashboard: React.FC = () => {
}
function onScreenMouseMove(e: React.MouseEvent) {
- if (annotating || !interactive || !client)
+ if (annotating || !interactive)
return;
const now = Date.now();
if (now - moveThrottleRef.current < 32)
return;
moveThrottleRef.current = now;
const { x, y } = imgCoords(e);
- client.mousemove({ x, y });
+ model.mousemove(x, y);
}
function onScreenWheel(e: React.WheelEvent) {
- if (annotating || !interactive || !client)
+ if (annotating || !interactive)
return;
e.preventDefault();
- client.wheel({ deltaX: e.deltaX, deltaY: e.deltaY });
+ model.wheel(e.deltaX, e.deltaY);
}
function onScreenKeyDown(e: React.KeyboardEvent) {
- if (annotating)
- return;
- if (!interactive || !client)
+ if (annotating || !interactive)
return;
e.preventDefault();
- client.keydown({ key: e.key });
+ model.keydown(e.key);
}
function onScreenKeyUp(e: React.KeyboardEvent) {
- if (annotating || !interactive || !client)
+ if (annotating || !interactive)
return;
e.preventDefault();
- client.keyup({ key: e.key });
+ model.keyup(e.key);
}
function onOmniboxKeyDown(e: React.KeyboardEvent<HTMLInputElement>) {
if (e.key === 'Enter') {
const value = smartUrl((e.target as HTMLInputElement).value);
- dispatch({ type: 'setUrl', url: value });
- client?.navigate({ url: value });
+ model.navigate(value);
e.currentTarget.blur();
}
}
- let overlayText: string | undefined;
- if (!client)
- overlayText = 'Disconnected';
- else if (!selectedTab)
- overlayText = 'Select a session';
+ const overlayText = selectedTab ? undefined : 'Select a session';
return (
<div className={'dashboard-view' + (interactive ? ' interactive' : '') + (annotating ? ' annotate' : '') + (recording ? ' recording' : '')}>
@@ -541,7 +277,7 @@ export const Dashboard: React.FC = () => {
className='annotate-toolbar-btn'
title='Close annotation mode'
icon='close'
- onClick={() => dispatch({ type: 'toggleAnnotate' })}
+ onClick={() => model.completeAnnotation()}
/>
</div>
</>
@@ -553,12 +289,12 @@ export const Dashboard: React.FC = () => {
icon='record'
toggled={recording.phase === 'recording'}
style={{ color: recording.phase === 'recording' ? 'var(--color-scale-red-5)' : undefined }}
- onClick={async () => {
+ onClick={() => {
if (recording.phase === 'recording') {
- await onStopRecording();
+ model.stopRecording();
} else {
- URL.revokeObjectURL(recording.url);
- await onStartRecording();
+ URL.revokeObjectURL(recording.blobUrl);
+ model.startRecording();
}
}}>
{recording.phase === 'recording' && <span className='recording-label'>Recording...</span>}
@@ -574,7 +310,7 @@ export const Dashboard: React.FC = () => {
<ToolbarButton
title='Close'
icon='close'
- onClick={onExitRecording}
+ onClick={() => model.discardRecording()}
/>
</div>
</>
@@ -586,9 +322,7 @@ export const Dashboard: React.FC = () => {
title={interactive ? 'Disable interactive mode' : 'Enable interactive mode'}
toggled={interactive}
disabled={!ready}
- onClick={() => {
- dispatch({ type: 'toggleInteractive' });
- }}
+ onClick={() => model.toggleInteractive()}
>
{interactive ? <LockOpenIcon /> : <LockIcon />}
</ToolbarButton>
@@ -596,9 +330,12 @@ export const Dashboard: React.FC = () => {
<ToolbarButton
className='mode-toggle mode-annotate'
title='Enable annotation mode'
- disabled={!ready || !liveFrame}
+ disabled={!ready}
onClick={() => {
- dispatch({ type: 'toggleAnnotate' });
+ if (pendingAnnotate)
+ model.cancelAnnotate();
+ else
+ model.enterAnnotate('user');
}}
>
<ScreenshotRegionIcon />
@@ -607,7 +344,7 @@ export const Dashboard: React.FC = () => {
title='Record video'
icon='record'
disabled={!ready}
- onClick={onStartRecording}
+ onClick={() => model.startRecording()}
/>
</>
)}
@@ -623,7 +360,7 @@ export const Dashboard: React.FC = () => {
id='display'
className='annotate-image'
alt='annotation'
- src={'data:image/jpeg;base64,' + annotateFrame.data}
+ src={'data:image/png;base64,' + annotateFrame.data}
/>
<Annotations
ref={annotationsRef}
@@ -640,7 +377,7 @@ export const Dashboard: React.FC = () => {
<div className='recording-view'>
<video
className='recording-video'
- src={recording.url}
+ src={recording.blobUrl}
controls
autoPlay
/>
@@ -654,7 +391,7 @@ export const Dashboard: React.FC = () => {
flashInteractiveHint();
return;
}
- client?.back();
+ model.back();
}}>
<ChevronLeftIcon />
</button>
@@ -663,7 +400,7 @@ export const Dashboard: React.FC = () => {
flashInteractiveHint();
return;
}
- client?.forward();
+ model.forward();
}}>
<ChevronRightIcon />
</button>
@@ -672,7 +409,7 @@ export const Dashboard: React.FC = () => {
flashInteractiveHint();
return;
}
- client?.reload();
+ model.reload();
}}>
<ReloadIcon />
</button>
@@ -684,12 +421,8 @@ export const Dashboard: React.FC = () => {
placeholder='Search or enter URL'
spellCheck={false}
autoComplete='off'
- value={url}
- onChange={e => {
- if (!interactive)
- return;
- dispatch({ type: 'setUrl', url: e.target.value });
- }}
+ value={selectedTab?.url}
+ onChange={() => {}}
onKeyDown={e => {
if (!interactive)
return;
diff --git a/packages/dashboard/src/dashboardChannel.ts b/packages/dashboard/src/dashboardChannel.ts
index 3eb7153abe..5f117a8892 100644
--- a/packages/dashboard/src/dashboardChannel.ts
+++ b/packages/dashboard/src/dashboardChannel.ts
@@ -15,7 +15,9 @@
*/
import type { ClientInfo } from '../../playwright-core/src/tools/cli-client/registry';
-import type { SessionStatus } from './sessionModel';
+import type { BrowserStatus } from '../../playwright-core/src/serverRegistry';
+
+export type SessionStatus = BrowserStatus;
export type Tab = {
browser: string;
@@ -62,8 +64,8 @@ export interface DashboardChannel {
startRecording(): Promise<void>;
stopRecording(): Promise<{ streamId: string }>;
readStream(params: { streamId: string }): Promise<{ data: string; eof: boolean }>;
- screenshot(): Promise<string>;
- submitAnnotation(params: { data: string; annotations: AnnotationData[] }): Promise<void>;
+ screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number }>;
+ submitAnnotation(params: { data: string | undefined; annotations: AnnotationData[] }): Promise<void>;
on<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;
off<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;
diff --git a/packages/dashboard/src/dashboardContext.ts b/packages/dashboard/src/dashboardContext.ts
deleted file mode 100644
index 0c8c15c211..0000000000
--- a/packages/dashboard/src/dashboardContext.ts
+++ /dev/null
@@ -1,24 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-// HMR: extracted from index.tsx so index.tsx only exports components and stays
-// a clean Fast Refresh boundary (mixed exports break @vitejs/plugin-react HMR).
-
-import React from 'react';
-
-import type { DashboardClientChannel } from './dashboardClient';
-
-export const DashboardClientContext = React.createContext<DashboardClientChannel | undefined>(undefined);
diff --git a/packages/dashboard/src/dashboardModel.ts b/packages/dashboard/src/dashboardModel.ts
new file mode 100644
index 0000000000..c8f079af7b
--- /dev/null
+++ b/packages/dashboard/src/dashboardModel.ts
@@ -0,0 +1,319 @@
+/**
+ * Copyright (c) Microsoft Corporation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type { Annotation } from './annotations';
+import type { DashboardChannel, DashboardChannelEvents, MouseButton, SessionStatus, Tab } from './dashboardChannel';
+import type { ClientInfo } from '../../playwright-core/src/tools/cli-client/registry';
+import type { BrowserDescriptor } from '../../playwright-core/src/serverRegistry';
+
+export type { SessionStatus } from './dashboardChannel';
+
+export type Mode = 'readonly' | 'interactive' | 'annotate';
+
+export type RecordingState =
+ | { phase: 'recording' }
+ | { phase: 'stopped'; blob: Blob; blobUrl: string };
+
+export type AnnotateFrame = { data: string; viewportWidth: number; viewportHeight: number };
+
+export type DashboardState = {
+ // Session model state.
+ sessions: SessionStatus[];
+ clientInfo: ClientInfo | undefined;
+ loadingSessions: boolean;
+ // Dashboard / page state.
+ tabs: Tab[] | null;
+ liveFrame: DashboardChannelEvents['frame'] | undefined;
+ annotateFrame: AnnotateFrame | undefined;
+ pendingAnnotate: { initiator: 'cli' | 'user' } | null;
+ annotateInitiator: 'cli' | 'user' | null;
+ mode: Mode;
+ recording: RecordingState | null;
+};
+
+type Listener = () => void;
+
+const initialState: DashboardState = {
+ sessions: [],
+ clientInfo: undefined,
+ loadingSessions: true,
+ tabs: null,
+ liveFrame: undefined,
+ annotateFrame: undefined,
+ pendingAnnotate: null,
+ annotateInitiator: null,
+ mode: 'readonly',
+ recording: null,
+};
+
+export class DashboardModel {
+ state: DashboardState = initialState;
+
+ private _client: DashboardChannel;
+ private _listeners = new Set<Listener>();
+ // Monotonic token to invalidate in-flight screenshot requests when
+ // pendingAnnotate is cleared or replaced.
+ private _requestId = 0;
+
+ constructor(client: DashboardChannel) {
+ this._client = client;
+ client.on('sessions', params => this._emit({ sessions: params.sessions, clientInfo: params.clientInfo, loadingSessions: false }));
+ client.on('tabs', params => this._emit({ tabs: params.tabs }));
+ client.on('frame', params => this._emit({ liveFrame: params }));
+ client.on('annotate', () => this.enterAnnotate('cli'));
+ client.on('cancelAnnotate', () => this.cancelAnnotate());
+ }
+
+ subscribe(listener: Listener): () => void {
+ this._listeners.add(listener);
+ return () => { this._listeners.delete(listener); };
+ }
+
+ // Session actions.
+
+ sessionByGuid(guid: string): SessionStatus | undefined {
+ return this.state.sessions.find(s => s.browser.guid === guid);
+ }
+
+ closeSession(descriptor: BrowserDescriptor) {
+ void this._client.closeSession({ browser: descriptor.browser.guid });
+ }
+
+ deleteSessionData(descriptor: BrowserDescriptor) {
+ void this._client.deleteSessionData({ browser: descriptor.browser.guid });
+ }
+
+ setVisible(visible: boolean) {
+ void this._client.setVisible({ visible });
+ }
+
+ // Tab actions.
+
+ selectTab(tab: Tab) {
+ void this._client.selectTab({ browser: tab.browser, context: tab.context, page: tab.page });
+ }
+
+ closeTab(tab: Tab) {
+ void this._client.closeTab({ browser: tab.browser, context: tab.context, page: tab.page });
+ }
+
+ newTab(browser: string, context: string) {
+ void this._client.newTab({ browser, context });
+ }
+
+ // Navigation.
+
+ navigate(url: string) {
+ void this._client.navigate({ url });
+ }
+
+ back() {
+ void this._client.back();
+ }
+
+ forward() {
+ void this._client.forward();
+ }
+
+ reload() {
+ void this._client.reload();
+ }
+
+ // Input.
+
+ mousemove(x: number, y: number) {
+ void this._client.mousemove({ x, y });
+ }
+
+ mousedown(x: number, y: number, button: MouseButton) {
+ void this._client.mousedown({ x, y, button });
+ }
+
+ mouseup(x: number, y: number, button: MouseButton) {
+ void this._client.mouseup({ x, y, button });
+ }
+
+ wheel(deltaX: number, deltaY: number) {
+ void this._client.wheel({ deltaX, deltaY });
+ }
+
+ keydown(key: string) {
+ void this._client.keydown({ key });
+ }
+
+ keyup(key: string) {
+ void this._client.keyup({ key });
+ }
+
+ // Dashboard / page actions.
+
+ toggleInteractive() {
+ const next: Mode = this.state.mode === 'interactive' ? 'readonly' : 'interactive';
+ this._emit({ mode: next });
+ }
+
+ // Public action methods are sync fire-and-forget wrappers around the
+ // async private implementations. Callers from the GUI use the sync
+ // variants; intra-model flows `await` the private variants when they
+ // need to sequence mode switches correctly.
+
+ enterAnnotate(initiator: 'cli' | 'user') {
+ void this._enterAnnotate(initiator);
+ }
+
+ completeAnnotation() {
+ void this._completeAnnotation();
+ }
+
+ startRecording() {
+ void this._startRecording();
+ }
+
+ stopRecording() {
+ void this._stopRecording();
+ }
+
+ discardRecording() {
+ void this._discardRecording();
+ }
+
+ cancelAnnotate() {
+ this._requestId++;
+ const s = this.state;
+ this._emit({
+ mode: s.mode === 'annotate' ? 'readonly' : s.mode,
+ annotateFrame: undefined,
+ pendingAnnotate: null,
+ annotateInitiator: null,
+ });
+ }
+
+ async submitAnnotation(data: string | undefined, annotations: Annotation[]) {
+ await this._client.submitAnnotation({
+ data,
+ annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),
+ });
+ this.cancelAnnotate();
+ }
+
+ private async _enterAnnotate(initiator: 'cli' | 'user') {
+ await this._cleanupOnModeSwitch();
+ this._requestAnnotate(initiator);
+ }
+
+ private async _completeAnnotation() {
+ const s = this.state;
+ if (s.mode === 'annotate' && s.annotateInitiator === 'cli' && s.annotateFrame)
+ await this.submitAnnotation(undefined, []).catch(() => {});
+ this.cancelAnnotate();
+ }
+
+ private async _startRecording() {
+ await this._cleanupOnModeSwitch();
+ await this._client.startRecording();
+ this._emit({ recording: { phase: 'recording' } });
+ }
+
+ private async _stopRecording() {
+ const { streamId } = await this._client.stopRecording();
+ const chunks: Blob[] = [];
+ while (true) {
+ const { data, eof } = await this._client.readStream({ streamId });
+ if (data)
+ chunks.push(base64ToBlob(data, 'video/webm'));
+ if (eof)
+ break;
+ }
+ const blob = new Blob(chunks, { type: 'video/webm' });
+ const blobUrl = URL.createObjectURL(blob);
+ this._emit({ recording: { phase: 'stopped', blob, blobUrl } });
+ }
+
+ private async _discardRecording() {
+ const s = this.state;
+ if (s.recording?.phase === 'recording') {
+ try {
+ const { streamId } = await this._client.stopRecording();
+ while (true) {
+ const { eof } = await this._client.readStream({ streamId });
+ if (eof)
+ break;
+ }
+ } catch {
+ // Ignore; still transition out.
+ }
+ }
+ if (s.recording?.phase === 'stopped')
+ URL.revokeObjectURL(s.recording.blobUrl);
+ this._emit({ recording: null });
+ }
+
+ private async _cleanupOnModeSwitch() {
+ await this._discardRecording();
+ await this._completeAnnotation();
+ }
+
+ private _requestAnnotate(initiator: 'cli' | 'user') {
+ const s = this.state;
+ if (s.mode === 'annotate') {
+ if (initiator === 'cli' && s.annotateInitiator !== 'cli')
+ this._emit({ annotateInitiator: 'cli' });
+ return;
+ }
+ if (s.pendingAnnotate) {
+ if (initiator === 'cli' && s.pendingAnnotate.initiator !== 'cli')
+ this._emit({ pendingAnnotate: { initiator: 'cli' } });
+ return;
+ }
+ const myId = ++this._requestId;
+ this._emit({ pendingAnnotate: { initiator } });
+ void this._fetchScreenshot(myId);
+ }
+
+ private async _fetchScreenshot(id: number) {
+ let frame: AnnotateFrame | undefined;
+ try {
+ frame = await this._client.screenshot();
+ } catch {
+ // frame stays undefined
+ }
+ if (id !== this._requestId)
+ return;
+ const s = this.state;
+ if (!s.pendingAnnotate)
+ return;
+ if (!frame) {
+ this._emit({ pendingAnnotate: null });
+ return;
+ }
+ this._emit({
+ mode: 'annotate',
+ annotateFrame: frame,
+ annotateInitiator: s.pendingAnnotate.initiator,
+ pendingAnnotate: null,
+ });
+ }
+
+ private _emit(partial: Partial<DashboardState>) {
+ this.state = { ...this.state, ...partial };
+ for (const listener of this._listeners)
+ listener();
+ }
+}
+
+function base64ToBlob(base64: string, mime: string): Blob {
+ return new Blob([(Uint8Array as any).fromBase64(base64)], { type: mime });
+}
diff --git a/packages/dashboard/src/index.tsx b/packages/dashboard/src/index.tsx
index 7b1ae3e715..31549512a6 100644
--- a/packages/dashboard/src/index.tsx
+++ b/packages/dashboard/src/index.tsx
@@ -21,8 +21,7 @@ import '@web/common.css';
import './common.css';
import { applyTheme } from '@web/theme';
import { Dashboard } from './dashboard';
-import { DashboardClientContext } from './dashboardContext';
-import { SessionModel } from './sessionModel';
+import { DashboardModel } from './dashboardModel';
import { DashboardClient } from './dashboardClient';
import { SessionSidebar } from './sessionSidebar';
import { SplitView } from '@web/components/splitView';
@@ -30,9 +29,9 @@ import { SplitView } from '@web/components/splitView';
applyTheme();
const client = DashboardClient.create('/ws');
-const model = new SessionModel(client);
+const model = new DashboardModel(client);
-const pushVisibility = () => client.setVisible({ visible: !document.hidden }).catch(() => {});
+const pushVisibility = () => model.setVisible(!document.hidden);
document.addEventListener('visibilitychange', pushVisibility);
if (document.hidden)
pushVisibility();
@@ -41,24 +40,17 @@ const App: React.FC = () => {
const [, setRevision] = React.useState(0);
React.useEffect(() => model.subscribe(() => setRevision(r => r + 1)), []);
- return <DashboardClientContext.Provider value={client}>
- <SplitView
- orientation='horizontal'
- sidebarIsFirst
- sidebarSize={320}
- minSidebarSize={220}
- settingName='dashboardSessionSidebar'
- sidebar={<SessionSidebar
- model={model}
- onSelectTab={tab => { void client.selectTab({ browser: tab.browser, context: tab.context, page: tab.page }); }}
- onCloseTab={tab => { void client.closeTab({ browser: tab.browser, context: tab.context, page: tab.page }); }}
- onNewTab={(browser, context) => { void client.newTab({ browser, context }); }}
- />}
- main={<div className='dashboard-shell-main'>
- <Dashboard />
- </div>}
- />
- </DashboardClientContext.Provider>;
+ return <SplitView
+ orientation='horizontal'
+ sidebarIsFirst
+ sidebarSize={320}
+ minSidebarSize={220}
+ settingName='dashboardSessionSidebar'
+ sidebar={<SessionSidebar model={model} />}
+ main={<div className='dashboard-shell-main'>
+ <Dashboard model={model} />
+ </div>}
+ />;
};
// HMR begin: cache the root on the DOM node so re-running this module during
diff --git a/packages/dashboard/src/sessionModel.ts b/packages/dashboard/src/sessionModel.ts
deleted file mode 100644
index 1213a0bcea..0000000000
--- a/packages/dashboard/src/sessionModel.ts
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Copyright (c) Microsoft Corporation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import type { ClientInfo } from '../../playwright-core/src/tools/cli-client/registry';
-import type { BrowserDescriptor, BrowserStatus } from '../../playwright-core/src/serverRegistry';
-
-export type SessionStatus = BrowserStatus;
-
-type Listener = () => void;
-
-type SessionsChannel = {
- on(event: 'sessions', listener: (params: { sessions: SessionStatus[]; clientInfo: ClientInfo }) => void): void;
- off(event: 'sessions', listener: (params: { sessions: SessionStatus[]; clientInfo: ClientInfo }) => void): void;
- closeSession(params: { browser: string }): Promise<void>;
- deleteSessionData(params: { browser: string }): Promise<void>;
-};
-
-export class SessionModel {
- sessions: SessionStatus[] = [];
- clientInfo: ClientInfo | undefined;
- loading = true;
-
- private _client: SessionsChannel;
- private _listeners = new Set<Listener>();
- private _onSessions = (params: { sessions: SessionStatus[]; clientInfo: ClientInfo }) => {
- this.sessions = params.sessions;
- this.clientInfo = params.clientInfo;
- this.loading = false;
- this._notify();
- };
-
- constructor(client: SessionsChannel) {
- this._client = client;
- this._client.on('sessions', this._onSessions);
- }
-
- subscribe(listener: Listener): () => void {
- this._listeners.add(listener);
- return () => this._listeners.delete(listener);
- }
-
- private _notify() {
- for (const listener of this._listeners)
- listener();
- }
-
- sessionByGuid(guid: string): SessionStatus | undefined {
- return this.sessions.find(s => s.browser.guid === guid);
- }
-
- async closeSession(descriptor: BrowserDescriptor) {
- await this._client.closeSession({ browser: descriptor.browser.guid });
- }
-
- async deleteSessionData(descriptor: BrowserDescriptor) {
- await this._client.deleteSessionData({ browser: descriptor.browser.guid });
- }
-
- dispose() {
- this._client.off('sessions', this._onSessions);
- this._listeners.clear();
- }
-}
diff --git a/packages/dashboard/src/sessionSidebar.tsx b/packages/dashboard/src/sessionSidebar.tsx
index d82667265f..4279ce6a86 100644
--- a/packages/dashboard/src/sessionSidebar.tsx
+++ b/packages/dashboard/src/sessionSidebar.tsx
@@ -16,19 +16,15 @@
import React from 'react';
import './sessionSidebar.css';
-import { DashboardClientContext } from './dashboardContext';
import { SettingsButton } from './settingsView';
import { BrowserIcon } from './icons';
import { ToolbarButton } from '@web/components/toolbarButton';
-import type { Tab, DashboardChannelEvents } from './dashboardChannel';
-import type { SessionModel, SessionStatus } from './sessionModel';
+import type { SessionStatus, Tab } from './dashboardChannel';
+import type { DashboardModel } from './dashboardModel';
type SessionSidebarProps = {
- model: SessionModel;
- onSelectTab: (tab: Tab) => void;
- onCloseTab: (tab: Tab) => void;
- onNewTab: (browser: string, context: string) => void;
+ model: DashboardModel;
};
function tabFavicon(url: string): string {
@@ -56,19 +52,9 @@ function normalizeWorkspacePath(workspace: string, homeDir: string | undefined):
return normalized;
}
-export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectTab, onCloseTab, onNewTab }) => {
- const client = React.useContext(DashboardClientContext);
- const openSessions = React.useMemo(() => model.sessions.filter(session => session.canConnect), [model.sessions]);
- const clientInfo = model.clientInfo;
- const [allTabs, setAllTabs] = React.useState<Tab[] | null>(null);
-
- React.useEffect(() => {
- if (!client)
- return;
- const onTabs = (params: DashboardChannelEvents['tabs']) => setAllTabs(params.tabs);
- client.on('tabs', onTabs);
- return () => client.off('tabs', onTabs);
- }, [client]);
+export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model }) => {
+ const { sessions, clientInfo, loadingSessions, tabs: allTabs } = model.state;
+ const openSessions = React.useMemo(() => sessions.filter(session => session.canConnect), [sessions]);
const tabsByBrowserAndContext = React.useMemo(() => {
const map = new Map<string, Map<string, Tab[]>>();
@@ -117,8 +103,8 @@ export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectT
<SettingsButton />
</div>
<div className='dashboard-shell-sidebar-content'>
- {model.loading && <div className='sidebar-empty' role='status' aria-live='polite'>Loading sessions...</div>}
- {!model.loading && openSessions.length === 0 && <div className='sidebar-empty' role='status' aria-live='polite'>No open sessions.</div>}
+ {loadingSessions && <div className='sidebar-empty' role='status' aria-live='polite'>Loading sessions...</div>}
+ {!loadingSessions && openSessions.length === 0 && <div className='sidebar-empty' role='status' aria-live='polite'>No open sessions.</div>}
{workspaceGroups.map(([workspace, entries]) => {
const workspacePath = normalizeWorkspacePath(workspace, clientInfo?.homeDir);
return <section key={workspace} className='workspace-group'>
@@ -149,7 +135,7 @@ export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectT
className='session-browser-close'
icon='close'
title='Close session'
- onClick={() => void model.closeSession(session)}
+ onClick={() => model.closeSession(session)}
/>
</div>
<span className='session-chip-name'>{session.title}</span>
@@ -158,7 +144,7 @@ export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectT
className='sidebar-session-new-tab'
icon='add'
title='New tab'
- onClick={() => onNewTab(guid, row.contextGuid!)}
+ onClick={() => model.newTab(guid, row.contextGuid!)}
/>}
</div>
</div>
@@ -176,11 +162,11 @@ export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectT
tabIndex={0}
aria-current={tab.context === activeContext && tab.selected ? 'page' : undefined}
title={tab.url || tab.title}
- onClick={() => onSelectTab(tab)}
+ onClick={() => model.selectTab(tab)}
onKeyDown={e => {
if (e.key === 'Enter' || e.key === ' ') {
e.preventDefault();
- onSelectTab(tab);
+ model.selectTab(tab);
}
}}
>
@@ -198,7 +184,7 @@ export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectT
title='Close tab'
onClick={e => {
e.stopPropagation();
- onCloseTab(tab);
+ model.closeTab(tab);
}}
/>
</div>)}
diff --git a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
index 43f486126a..b1374ce983 100644
--- a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
+++ b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts
@@ -53,7 +53,7 @@ async function startDashboardServer(options: DashboardOptions): Promise<Dashboar
let pendingAnnotate = false;
const waitingSockets = new Set<net.Socket>();
- const submitAnnotation = (base64Png: string, annotations: AnnotationData[]) => {
+ const submitAnnotation = (base64Png: string | undefined, annotations: AnnotationData[]) => {
if (waitingSockets.size === 0)
return;
const payload = JSON.stringify({ png: base64Png, annotations });
@@ -407,14 +407,16 @@ async function runAnnotateClient(options: DashboardOptions): Promise<void> {
if (!text)
return;
const { png, annotations } = JSON.parse(text) as { png: string; annotations: AnnotationData[] };
- const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
- const filePath = await saveOutputFile(`annotations-${timestamp}.png`, Buffer.from(png, 'base64'));
for (const a of annotations) {
// eslint-disable-next-line no-console
console.log(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);
}
- // eslint-disable-next-line no-console
- console.log(`image: ${path.relative(process.cwd(), filePath)}`);
+ if (png) {
+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
+ const filePath = await saveOutputFile(`annotations-${timestamp}.png`, Buffer.from(png, 'base64'));
+ // eslint-disable-next-line no-console
+ console.log(`image: ${path.relative(process.cwd(), filePath)}`);
+ }
}
function selfDestructOnParentGone() {
diff --git a/packages/playwright-core/src/tools/dashboard/dashboardController.ts b/packages/playwright-core/src/tools/dashboard/dashboardController.ts
index 38b9cf7d5a..e6029bdfcf 100644
--- a/packages/playwright-core/src/tools/dashboard/dashboardController.ts
+++ b/packages/playwright-core/src/tools/dashboard/dashboardController.ts
@@ -110,7 +110,7 @@ export class DashboardConnection implements Transport {
private _attachedPage: AttachedPage | undefined;
private _onclose: () => void;
private _onconnected?: () => void;
- private _onAnnotationSubmit?: (base64Png: string, annotations: AnnotationData[]) => void;
+ private _onAnnotationSubmit?: (base64Png: string | undefined, annotations: AnnotationData[]) => void;
private _serverRegistryDispose?: () => void;
private _pushSessionsScheduled = false;
private _pushTabsScheduled = false;
@@ -120,7 +120,7 @@ export class DashboardConnection implements Transport {
_recordingDir: string;
_streams = new Map<string, { handle: fs.promises.FileHandle; path: string }>();
- constructor(onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string, annotations: AnnotationData[]) => void) {
+ constructor(onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string | undefined, annotations: AnnotationData[]) => void) {
this._onclose = onclose;
this._onconnected = onconnected;
this._onAnnotationSubmit = onAnnotationSubmit;
@@ -236,7 +236,7 @@ export class DashboardConnection implements Transport {
this._pushTabs();
}
- async submitAnnotation(params: { data: string; annotations: AnnotationData[] }) {
+ async submitAnnotation(params: { data: string | undefined; annotations: AnnotationData[] }) {
this._onAnnotationSubmit?.(params.data, params.annotations);
}
@@ -558,9 +558,14 @@ class AttachedPage {
return { streamId };
}
- async screenshot(): Promise<string> {
+ async screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number }> {
const buffer = await this._page.screenshot({ type: 'png' });
- return buffer.toString('base64');
+ const vp = this._page.viewportSize();
+ return {
+ data: buffer.toString('base64'),
+ viewportWidth: vp?.width ?? 0,
+ viewportHeight: vp?.height ?? 0,
+ };
}
private async _startScreencast(page: api.Page) {