Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(dashboard): annotate on fresh page.screenshot and unify state in DashboardModel (#40385)
Pavel Feldmana2113a303e9b
diff --git a/packages/dashboard/src/dashboard.tsx b/packages/dashboard/src/dashboard.tsxindex 1613205d19..40c6ec572f 100644--- a/packages/dashboard/src/dashboard.tsx+++ b/packages/dashboard/src/dashboard.tsx@@ -16,7 +16,6 @@import React from 'react';import './dashboard.css';-import { DashboardClientContext } from './dashboardContext';import { ChevronLeftIcon, ChevronRightIcon, DownloadIcon, LockIcon, LockOpenIcon, ReloadIcon, ScreenshotRegionIcon } from './icons';import { Annotations, getImageLayout, clientToViewport } from './annotations';@@ -24,184 +23,9 @@ import type { Annotation, AnnotationsHandle } from './annotations';import { ToolbarButton, ToolbarSeparator } from '@web/components/toolbarButton';import { useMeasureForRef } from '@web/uiUtils';-import type { Tab, DashboardChannelEvents } from './dashboardChannel';+import type { DashboardModel } from './dashboardModel';const BUTTONS = ['left', 'middle', 'right'] as const;-type Mode = 'readonly' | 'interactive' | 'annotate';--// Recording mode is a separate, mode-replacing overlay similar to annotate.-// While null, the normal toolbar is shown. When non-null, the recording-// toolbar replaces it.-type RecordingState =- | { phase: 'recording' }- | { phase: 'stopped'; blob: Blob; url: string };--type DashboardState = {- // Server-driven session state.- tabs: Tab[] | null;- url: string;- // The latest frame received from the server. Cleared on page swap;- // the server is responsible for emitting a fresh frame for the new- // page (see _startScreencast).- liveFrame: DashboardChannelEvents['frame'] | undefined;- // Snapshot of the frame at the moment we entered annotate mode. The- // <Annotations> overlay draws on this frozen image so the canvas does- // not jump around as new live frames arrive.- annotateFrame: DashboardChannelEvents['frame'] | undefined;- // Server requested annotate but we have no fresh frame for the active- // page. The next FRAME action will enter annotate mode.- cliAnnotatePending: boolean;- // Whether the current annotate session was initiated by CLI or by the- // user. Determines whether submit goes to the server or saves locally.- annotateInitiator: 'cli' | 'user' | null;- // Interaction mode and ephemeral UI flags.- mode: Mode;- recording: RecordingState | null;-};--type DashboardAction =- // Server events- | { type: 'tabs'; tabs: Tab[] }- | { type: 'frame'; frame: DashboardChannelEvents['frame'] }- | { type: 'cliAnnotate' }- | { type: 'cliCancelAnnotate' }- // User events- | { type: 'toggleInteractive' }- | { type: 'toggleAnnotate' }- | { type: 'startRecording' }- | { type: 'stoppedRecording'; blob: Blob; url: string }- | { type: 'exitRecording' }- | { type: 'submitAnnotation' }- | { type: 'setUrl'; url: string };--const initialDashboardState: DashboardState = {- tabs: null,- url: '',- liveFrame: undefined,- annotateFrame: undefined,- cliAnnotatePending: false,- annotateInitiator: null,- mode: 'readonly',- recording: null,-};--function dashboardReducer(state: DashboardState, action: DashboardAction): DashboardState {- switch (action.type) {- case 'tabs': {- const newSelected = action.tabs.find(t => t.selected);- const oldSelected = state.tabs?.find(t => t.selected);- const url = newSelected ? newSelected.url : state.url;- // Page change = had a selected tab, now have a different selected- // tab. Initial selection (none -> some) is not a "change".- const pageChanged = !!oldSelected && !!newSelected && newSelected.page !== oldSelected.page;- if (!pageChanged)- return { ...state, tabs: action.tabs, url };- // Page swap. Clear frames and rely on the server to emit a fresh- // frame for the new page (see _startScreencast which awaits a- // screenshot to force one). If we were annotating, mark pending so- // the next FRAME re-enters annotate with the new page's image.- const wasAnnotateActive = state.mode === 'annotate' || state.cliAnnotatePending;- const newTabIsBrandNew = !!state.tabs && !state.tabs.some(t => t.page === newSelected.page);- let mode: Mode = state.mode;- if (!wasAnnotateActive && newTabIsBrandNew && state.tabs)- mode = 'interactive';- return {- ...state,- tabs: action.tabs,- url,- mode,- recording: null,- liveFrame: undefined,- annotateFrame: undefined,- cliAnnotatePending: wasAnnotateActive,- };- }- case 'frame': {- const liveFrame = action.frame;- if (state.cliAnnotatePending) {- return {- ...state,- liveFrame,- annotateFrame: liveFrame,- mode: 'annotate',- cliAnnotatePending: false,- annotateInitiator: state.annotateInitiator ?? 'cli',- };- }- return { ...state, liveFrame };- }- case 'cliAnnotate': {- if (state.mode === 'annotate') {- // Already annotating (user-initiated). Mark CLI as the source so- // submit goes to the server.- return { ...state, annotateInitiator: 'cli' };- }- if (state.liveFrame) {- return {- ...state,- mode: 'annotate',- annotateFrame: state.liveFrame,- cliAnnotatePending: false,- annotateInitiator: 'cli',- };- }- return { ...state, cliAnnotatePending: true, annotateInitiator: 'cli' };- }- case 'cliCancelAnnotate': {- const exitingAnnotate = state.mode === 'annotate';- return {- ...state,- mode: exitingAnnotate ? 'readonly' : state.mode,- annotateFrame: undefined,- cliAnnotatePending: false,- annotateInitiator: null,- };- }- case 'toggleInteractive': {- const next: Mode = state.mode === 'interactive' ? 'readonly' : 'interactive';- return { ...state, mode: next };- }- case 'toggleAnnotate': {- if (state.mode === 'annotate') {- return {- ...state,- mode: 'readonly',- annotateFrame: undefined,- annotateInitiator: null,- };- }- if (!state.liveFrame)- return state;- // Preserve a CLI initiator across mode toggles so that CLI annotate- // sessions stay engaged when the user switches to interactive and- // back. Only set 'user' if there was no prior CLI engagement.- const initiator: 'cli' | 'user' | null = state.annotateInitiator ?? 'user';- return {- ...state,- mode: 'annotate',- annotateFrame: state.liveFrame,- cliAnnotatePending: false,- annotateInitiator: initiator,- };- }- case 'startRecording':- return { ...state, recording: { phase: 'recording' } };- case 'stoppedRecording':- return { ...state, recording: { phase: 'stopped', blob: action.blob, url: action.url } };- case 'exitRecording':- return { ...state, recording: null };- case 'submitAnnotation':- return {- ...state,- mode: 'readonly',- annotateFrame: undefined,- cliAnnotatePending: false,- annotateInitiator: null,- };- case 'setUrl':- return { ...state, url: action.url };- }-}async function pickSaveWritable(suggestedName: string, description: string, mime: string, extension: string): Promise<FileSystemWritableFileStream | null> {try {@@ -215,10 +39,6 @@ async function pickSaveWritable(suggestedName: string, description: string, mime}}-function base64ToBlob(base64: string, mime: string): Blob {- return new Blob([(Uint8Array as any).fromBase64(base64)], { type: mime });-}-function smartUrl(input: string): string {const value = input.trim();if (!value)@@ -237,10 +57,15 @@ function smartUrl(input: string): string {return 'https://' + host + '.com' + value.slice(host.length);}-export const Dashboard: React.FC = () => {- const client = React.useContext(DashboardClientContext);- const [state, dispatch] = React.useReducer(dashboardReducer, initialDashboardState);- const { tabs, url, mode, recording, liveFrame, annotateFrame, annotateInitiator } = state;+type DashboardProps = {+ model: DashboardModel;+};++export const Dashboard: React.FC<DashboardProps> = ({ model }) => {+ const [, setRevision] = React.useState(0);+ React.useEffect(() => model.subscribe(() => setRevision(r => r + 1)), [model]);++ const { tabs, mode, recording, liveFrame, annotateFrame, annotateInitiator, pendingAnnotate } = model.state;const interactive = mode === 'interactive';const annotating = mode === 'annotate';@@ -309,8 +134,6 @@ export const Dashboard: React.FC = () => {}, [interactive]);const onSubmitAnnotations = React.useCallback(async (blob: Blob, annotations: Annotation[]) => {- if (!client)- return;const dataUrl = await new Promise<string>((resolve, reject) => {const reader = new FileReader();reader.onload = () => resolve(reader.result as string);@@ -318,103 +141,25 @@ export const Dashboard: React.FC = () => {reader.readAsDataURL(blob);});const data = dataUrl.slice(dataUrl.indexOf(',') + 1);- await client.submitAnnotation({- data,- annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),- });- dispatch({ type: 'submitAnnotation' });- }, [client]);+ await model.submitAnnotation(data, annotations);+ }, [model]);function flashInteractiveHint() {setFlashTick(tick => tick + 1);}- const onStartRecording = React.useCallback(async () => {- if (!client)- return;- await client.startRecording();- dispatch({ type: 'startRecording' });- }, [client]);-- const onStopRecording = React.useCallback(async () => {- if (!client)- return;- const { streamId } = await client.stopRecording();- const chunks: Blob[] = [];- while (true) {- const { data, eof } = await client.readStream({ streamId });- if (data)- chunks.push(base64ToBlob(data, 'video/webm'));- if (eof)- break;- }- const blob = new Blob(chunks, { type: 'video/webm' });- const url = URL.createObjectURL(blob);- dispatch({ type: 'stoppedRecording', blob, url });- }, [client]);-const onSaveRecording = React.useCallback(async () => {- if (state.recording?.phase !== 'stopped')+ if (recording?.phase !== 'stopped')return;const writable = await pickSaveWritable(`playwright-recording-${Date.now()}.webm`, 'WebM Video', 'video/webm', '.webm');if (!writable)return;- await writable.write(state.recording.blob);+ await writable.write(recording.blob);await writable.close();- }, [state.recording]);-- const onExitRecording = React.useCallback(async () => {- if (state.recording?.phase === 'recording' && client) {- // Drain the stream without persisting; keeps the server clean.- const { streamId } = await client.stopRecording();- while (true) {- const { eof } = await client.readStream({ streamId });- if (eof)- break;- }- }- if (state.recording?.phase === 'stopped')- URL.revokeObjectURL(state.recording.url);- dispatch({ type: 'exitRecording' });- }, [state.recording, client]);--- React.useEffect(() => {- if (!client)- return;- let resized = false;- const onTabs = (params: DashboardChannelEvents['tabs']) => {- dispatch({ type: 'tabs', tabs: params.tabs });- };- const onFrame = (params: DashboardChannelEvents['frame']) => {- dispatch({ type: 'frame', frame: params });- const toolbar = toolbarRef.current;- if (!resized && toolbar && params.viewportWidth && params.viewportHeight) {- resized = true;- const chromeHeight = toolbar.offsetHeight;- const extraW = window.outerWidth - window.innerWidth;- const extraH = window.outerHeight - window.innerHeight;- const targetW = Math.min(params.viewportWidth + extraW, screen.availWidth);- const targetH = Math.min(params.viewportHeight + chromeHeight + extraH, screen.availHeight);- window.resizeTo(targetW, targetH);- }- };- const onAnnotate = () => dispatch({ type: 'cliAnnotate' });- const onCancelAnnotate = () => dispatch({ type: 'cliCancelAnnotate' });- client.on('tabs', onTabs);- client.on('frame', onFrame);- client.on('annotate', onAnnotate);- client.on('cancelAnnotate', onCancelAnnotate);- return () => {- client.off('tabs', onTabs);- client.off('frame', onFrame);- client.off('annotate', onAnnotate);- client.off('cancelAnnotate', onCancelAnnotate);- };- }, [client]);+ }, [recording]);const selectedTab = tabs?.find(t => t.selected);- const ready = !!client && !!selectedTab;+ const ready = !!selectedTab;function imgCoords(e: React.MouseEvent): { x: number; y: number } {const vw = liveFrame?.viewportWidth ?? 0;@@ -428,10 +173,8 @@ export const Dashboard: React.FC = () => {}function sendMouseEvent(method: 'mousedown' | 'mouseup', e: React.MouseEvent) {- if (!client)- return;const { x, y } = imgCoords(e);- client[method]({ x, y, button: BUTTONS[e.button] || 'left' });+ model[method](x, y, BUTTONS[e.button] || 'left');}function onScreenMouseDown(e: React.MouseEvent) {@@ -456,53 +199,46 @@ export const Dashboard: React.FC = () => {}function onScreenMouseMove(e: React.MouseEvent) {- if (annotating || !interactive || !client)+ if (annotating || !interactive)return;const now = Date.now();if (now - moveThrottleRef.current < 32)return;moveThrottleRef.current = now;const { x, y } = imgCoords(e);- client.mousemove({ x, y });+ model.mousemove(x, y);}function onScreenWheel(e: React.WheelEvent) {- if (annotating || !interactive || !client)+ if (annotating || !interactive)return;e.preventDefault();- client.wheel({ deltaX: e.deltaX, deltaY: e.deltaY });+ model.wheel(e.deltaX, e.deltaY);}function onScreenKeyDown(e: React.KeyboardEvent) {- if (annotating)- return;- if (!interactive || !client)+ if (annotating || !interactive)return;e.preventDefault();- client.keydown({ key: e.key });+ model.keydown(e.key);}function onScreenKeyUp(e: React.KeyboardEvent) {- if (annotating || !interactive || !client)+ if (annotating || !interactive)return;e.preventDefault();- client.keyup({ key: e.key });+ model.keyup(e.key);}function onOmniboxKeyDown(e: React.KeyboardEvent<HTMLInputElement>) {if (e.key === 'Enter') {const value = smartUrl((e.target as HTMLInputElement).value);- dispatch({ type: 'setUrl', url: value });- client?.navigate({ url: value });+ model.navigate(value);e.currentTarget.blur();}}- let overlayText: string | undefined;- if (!client)- overlayText = 'Disconnected';- else if (!selectedTab)- overlayText = 'Select a session';+ const overlayText = selectedTab ? undefined : 'Select a session';return (<div className={'dashboard-view' + (interactive ? ' interactive' : '') + (annotating ? ' annotate' : '') + (recording ? ' recording' : '')}>@@ -541,7 +277,7 @@ export const Dashboard: React.FC = () => {className='annotate-toolbar-btn'title='Close annotation mode'icon='close'- onClick={() => dispatch({ type: 'toggleAnnotate' })}+ onClick={() => model.completeAnnotation()}/></div></>@@ -553,12 +289,12 @@ export const Dashboard: React.FC = () => {icon='record'toggled={recording.phase === 'recording'}style={{ color: recording.phase === 'recording' ? 'var(--color-scale-red-5)' : undefined }}- onClick={async () => {+ onClick={() => {if (recording.phase === 'recording') {- await onStopRecording();+ model.stopRecording();} else {- URL.revokeObjectURL(recording.url);- await onStartRecording();+ URL.revokeObjectURL(recording.blobUrl);+ model.startRecording();}}}>{recording.phase === 'recording' && <span className='recording-label'>Recording...</span>}@@ -574,7 +310,7 @@ export const Dashboard: React.FC = () => {<ToolbarButtontitle='Close'icon='close'- onClick={onExitRecording}+ onClick={() => model.discardRecording()}/></div></>@@ -586,9 +322,7 @@ export const Dashboard: React.FC = () => {title={interactive ? 'Disable interactive mode' : 'Enable interactive mode'}toggled={interactive}disabled={!ready}- onClick={() => {- dispatch({ type: 'toggleInteractive' });- }}+ onClick={() => model.toggleInteractive()}>{interactive ? <LockOpenIcon /> : <LockIcon />}</ToolbarButton>@@ -596,9 +330,12 @@ export const Dashboard: React.FC = () => {<ToolbarButtonclassName='mode-toggle mode-annotate'title='Enable annotation mode'- disabled={!ready || !liveFrame}+ disabled={!ready}onClick={() => {- dispatch({ type: 'toggleAnnotate' });+ if (pendingAnnotate)+ model.cancelAnnotate();+ else+ model.enterAnnotate('user');}}><ScreenshotRegionIcon />@@ -607,7 +344,7 @@ export const Dashboard: React.FC = () => {title='Record video'icon='record'disabled={!ready}- onClick={onStartRecording}+ onClick={() => model.startRecording()}/></>)}@@ -623,7 +360,7 @@ export const Dashboard: React.FC = () => {id='display'className='annotate-image'alt='annotation'- src={'data:image/jpeg;base64,' + annotateFrame.data}+ src={'data:image/png;base64,' + annotateFrame.data}/><Annotationsref={annotationsRef}@@ -640,7 +377,7 @@ export const Dashboard: React.FC = () => {<div className='recording-view'><videoclassName='recording-video'- src={recording.url}+ src={recording.blobUrl}controlsautoPlay/>@@ -654,7 +391,7 @@ export const Dashboard: React.FC = () => {flashInteractiveHint();return;}- client?.back();+ model.back();}}><ChevronLeftIcon /></button>@@ -663,7 +400,7 @@ export const Dashboard: React.FC = () => {flashInteractiveHint();return;}- client?.forward();+ model.forward();}}><ChevronRightIcon /></button>@@ -672,7 +409,7 @@ export const Dashboard: React.FC = () => {flashInteractiveHint();return;}- client?.reload();+ model.reload();}}><ReloadIcon /></button>@@ -684,12 +421,8 @@ export const Dashboard: React.FC = () => {placeholder='Search or enter URL'spellCheck={false}autoComplete='off'- value={url}- onChange={e => {- if (!interactive)- return;- dispatch({ type: 'setUrl', url: e.target.value });- }}+ value={selectedTab?.url}+ onChange={() => {}}onKeyDown={e => {if (!interactive)return;diff --git a/packages/dashboard/src/dashboardChannel.ts b/packages/dashboard/src/dashboardChannel.tsindex 3eb7153abe..5f117a8892 100644--- a/packages/dashboard/src/dashboardChannel.ts+++ b/packages/dashboard/src/dashboardChannel.ts@@ -15,7 +15,9 @@*/import type { ClientInfo } from '../../playwright-core/src/tools/cli-client/registry';-import type { SessionStatus } from './sessionModel';+import type { BrowserStatus } from '../../playwright-core/src/serverRegistry';++export type SessionStatus = BrowserStatus;export type Tab = {browser: string;@@ -62,8 +64,8 @@ export interface DashboardChannel {startRecording(): Promise<void>;stopRecording(): Promise<{ streamId: string }>;readStream(params: { streamId: string }): Promise<{ data: string; eof: boolean }>;- screenshot(): Promise<string>;- submitAnnotation(params: { data: string; annotations: AnnotationData[] }): Promise<void>;+ screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number }>;+ submitAnnotation(params: { data: string | undefined; annotations: AnnotationData[] }): Promise<void>;on<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;off<K extends keyof DashboardChannelEvents>(event: K, listener: (params: DashboardChannelEvents[K]) => void): void;diff --git a/packages/dashboard/src/dashboardContext.ts b/packages/dashboard/src/dashboardContext.tsdeleted file mode 100644index 0c8c15c211..0000000000--- a/packages/dashboard/src/dashboardContext.ts+++ /dev/null@@ -1,24 +0,0 @@-/**- * Copyright (c) Microsoft Corporation.- *- * Licensed under the Apache License, Version 2.0 (the "License");- * you may not use this file except in compliance with the License.- * You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--// HMR: extracted from index.tsx so index.tsx only exports components and stays-// a clean Fast Refresh boundary (mixed exports break @vitejs/plugin-react HMR).--import React from 'react';--import type { DashboardClientChannel } from './dashboardClient';--export const DashboardClientContext = React.createContext<DashboardClientChannel | undefined>(undefined);diff --git a/packages/dashboard/src/dashboardModel.ts b/packages/dashboard/src/dashboardModel.tsnew file mode 100644index 0000000000..c8f079af7b--- /dev/null+++ b/packages/dashboard/src/dashboardModel.ts@@ -0,0 +1,319 @@+/**+ * Copyright (c) Microsoft Corporation.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++import type { Annotation } from './annotations';+import type { DashboardChannel, DashboardChannelEvents, MouseButton, SessionStatus, Tab } from './dashboardChannel';+import type { ClientInfo } from '../../playwright-core/src/tools/cli-client/registry';+import type { BrowserDescriptor } from '../../playwright-core/src/serverRegistry';++export type { SessionStatus } from './dashboardChannel';++export type Mode = 'readonly' | 'interactive' | 'annotate';++export type RecordingState =+ | { phase: 'recording' }+ | { phase: 'stopped'; blob: Blob; blobUrl: string };++export type AnnotateFrame = { data: string; viewportWidth: number; viewportHeight: number };++export type DashboardState = {+ // Session model state.+ sessions: SessionStatus[];+ clientInfo: ClientInfo | undefined;+ loadingSessions: boolean;+ // Dashboard / page state.+ tabs: Tab[] | null;+ liveFrame: DashboardChannelEvents['frame'] | undefined;+ annotateFrame: AnnotateFrame | undefined;+ pendingAnnotate: { initiator: 'cli' | 'user' } | null;+ annotateInitiator: 'cli' | 'user' | null;+ mode: Mode;+ recording: RecordingState | null;+};++type Listener = () => void;++const initialState: DashboardState = {+ sessions: [],+ clientInfo: undefined,+ loadingSessions: true,+ tabs: null,+ liveFrame: undefined,+ annotateFrame: undefined,+ pendingAnnotate: null,+ annotateInitiator: null,+ mode: 'readonly',+ recording: null,+};++export class DashboardModel {+ state: DashboardState = initialState;++ private _client: DashboardChannel;+ private _listeners = new Set<Listener>();+ // Monotonic token to invalidate in-flight screenshot requests when+ // pendingAnnotate is cleared or replaced.+ private _requestId = 0;++ constructor(client: DashboardChannel) {+ this._client = client;+ client.on('sessions', params => this._emit({ sessions: params.sessions, clientInfo: params.clientInfo, loadingSessions: false }));+ client.on('tabs', params => this._emit({ tabs: params.tabs }));+ client.on('frame', params => this._emit({ liveFrame: params }));+ client.on('annotate', () => this.enterAnnotate('cli'));+ client.on('cancelAnnotate', () => this.cancelAnnotate());+ }++ subscribe(listener: Listener): () => void {+ this._listeners.add(listener);+ return () => { this._listeners.delete(listener); };+ }++ // Session actions.++ sessionByGuid(guid: string): SessionStatus | undefined {+ return this.state.sessions.find(s => s.browser.guid === guid);+ }++ closeSession(descriptor: BrowserDescriptor) {+ void this._client.closeSession({ browser: descriptor.browser.guid });+ }++ deleteSessionData(descriptor: BrowserDescriptor) {+ void this._client.deleteSessionData({ browser: descriptor.browser.guid });+ }++ setVisible(visible: boolean) {+ void this._client.setVisible({ visible });+ }++ // Tab actions.++ selectTab(tab: Tab) {+ void this._client.selectTab({ browser: tab.browser, context: tab.context, page: tab.page });+ }++ closeTab(tab: Tab) {+ void this._client.closeTab({ browser: tab.browser, context: tab.context, page: tab.page });+ }++ newTab(browser: string, context: string) {+ void this._client.newTab({ browser, context });+ }++ // Navigation.++ navigate(url: string) {+ void this._client.navigate({ url });+ }++ back() {+ void this._client.back();+ }++ forward() {+ void this._client.forward();+ }++ reload() {+ void this._client.reload();+ }++ // Input.++ mousemove(x: number, y: number) {+ void this._client.mousemove({ x, y });+ }++ mousedown(x: number, y: number, button: MouseButton) {+ void this._client.mousedown({ x, y, button });+ }++ mouseup(x: number, y: number, button: MouseButton) {+ void this._client.mouseup({ x, y, button });+ }++ wheel(deltaX: number, deltaY: number) {+ void this._client.wheel({ deltaX, deltaY });+ }++ keydown(key: string) {+ void this._client.keydown({ key });+ }++ keyup(key: string) {+ void this._client.keyup({ key });+ }++ // Dashboard / page actions.++ toggleInteractive() {+ const next: Mode = this.state.mode === 'interactive' ? 'readonly' : 'interactive';+ this._emit({ mode: next });+ }++ // Public action methods are sync fire-and-forget wrappers around the+ // async private implementations. Callers from the GUI use the sync+ // variants; intra-model flows `await` the private variants when they+ // need to sequence mode switches correctly.++ enterAnnotate(initiator: 'cli' | 'user') {+ void this._enterAnnotate(initiator);+ }++ completeAnnotation() {+ void this._completeAnnotation();+ }++ startRecording() {+ void this._startRecording();+ }++ stopRecording() {+ void this._stopRecording();+ }++ discardRecording() {+ void this._discardRecording();+ }++ cancelAnnotate() {+ this._requestId++;+ const s = this.state;+ this._emit({+ mode: s.mode === 'annotate' ? 'readonly' : s.mode,+ annotateFrame: undefined,+ pendingAnnotate: null,+ annotateInitiator: null,+ });+ }++ async submitAnnotation(data: string | undefined, annotations: Annotation[]) {+ await this._client.submitAnnotation({+ data,+ annotations: annotations.map(a => ({ x: a.x, y: a.y, width: a.width, height: a.height, text: a.text })),+ });+ this.cancelAnnotate();+ }++ private async _enterAnnotate(initiator: 'cli' | 'user') {+ await this._cleanupOnModeSwitch();+ this._requestAnnotate(initiator);+ }++ private async _completeAnnotation() {+ const s = this.state;+ if (s.mode === 'annotate' && s.annotateInitiator === 'cli' && s.annotateFrame)+ await this.submitAnnotation(undefined, []).catch(() => {});+ this.cancelAnnotate();+ }++ private async _startRecording() {+ await this._cleanupOnModeSwitch();+ await this._client.startRecording();+ this._emit({ recording: { phase: 'recording' } });+ }++ private async _stopRecording() {+ const { streamId } = await this._client.stopRecording();+ const chunks: Blob[] = [];+ while (true) {+ const { data, eof } = await this._client.readStream({ streamId });+ if (data)+ chunks.push(base64ToBlob(data, 'video/webm'));+ if (eof)+ break;+ }+ const blob = new Blob(chunks, { type: 'video/webm' });+ const blobUrl = URL.createObjectURL(blob);+ this._emit({ recording: { phase: 'stopped', blob, blobUrl } });+ }++ private async _discardRecording() {+ const s = this.state;+ if (s.recording?.phase === 'recording') {+ try {+ const { streamId } = await this._client.stopRecording();+ while (true) {+ const { eof } = await this._client.readStream({ streamId });+ if (eof)+ break;+ }+ } catch {+ // Ignore; still transition out.+ }+ }+ if (s.recording?.phase === 'stopped')+ URL.revokeObjectURL(s.recording.blobUrl);+ this._emit({ recording: null });+ }++ private async _cleanupOnModeSwitch() {+ await this._discardRecording();+ await this._completeAnnotation();+ }++ private _requestAnnotate(initiator: 'cli' | 'user') {+ const s = this.state;+ if (s.mode === 'annotate') {+ if (initiator === 'cli' && s.annotateInitiator !== 'cli')+ this._emit({ annotateInitiator: 'cli' });+ return;+ }+ if (s.pendingAnnotate) {+ if (initiator === 'cli' && s.pendingAnnotate.initiator !== 'cli')+ this._emit({ pendingAnnotate: { initiator: 'cli' } });+ return;+ }+ const myId = ++this._requestId;+ this._emit({ pendingAnnotate: { initiator } });+ void this._fetchScreenshot(myId);+ }++ private async _fetchScreenshot(id: number) {+ let frame: AnnotateFrame | undefined;+ try {+ frame = await this._client.screenshot();+ } catch {+ // frame stays undefined+ }+ if (id !== this._requestId)+ return;+ const s = this.state;+ if (!s.pendingAnnotate)+ return;+ if (!frame) {+ this._emit({ pendingAnnotate: null });+ return;+ }+ this._emit({+ mode: 'annotate',+ annotateFrame: frame,+ annotateInitiator: s.pendingAnnotate.initiator,+ pendingAnnotate: null,+ });+ }++ private _emit(partial: Partial<DashboardState>) {+ this.state = { ...this.state, ...partial };+ for (const listener of this._listeners)+ listener();+ }+}++function base64ToBlob(base64: string, mime: string): Blob {+ return new Blob([(Uint8Array as any).fromBase64(base64)], { type: mime });+}diff --git a/packages/dashboard/src/index.tsx b/packages/dashboard/src/index.tsxindex 7b1ae3e715..31549512a6 100644--- a/packages/dashboard/src/index.tsx+++ b/packages/dashboard/src/index.tsx@@ -21,8 +21,7 @@ import '@web/common.css';import './common.css';import { applyTheme } from '@web/theme';import { Dashboard } from './dashboard';-import { DashboardClientContext } from './dashboardContext';-import { SessionModel } from './sessionModel';+import { DashboardModel } from './dashboardModel';import { DashboardClient } from './dashboardClient';import { SessionSidebar } from './sessionSidebar';import { SplitView } from '@web/components/splitView';@@ -30,9 +29,9 @@ import { SplitView } from '@web/components/splitView';applyTheme();const client = DashboardClient.create('/ws');-const model = new SessionModel(client);+const model = new DashboardModel(client);-const pushVisibility = () => client.setVisible({ visible: !document.hidden }).catch(() => {});+const pushVisibility = () => model.setVisible(!document.hidden);document.addEventListener('visibilitychange', pushVisibility);if (document.hidden)pushVisibility();@@ -41,24 +40,17 @@ const App: React.FC = () => {const [, setRevision] = React.useState(0);React.useEffect(() => model.subscribe(() => setRevision(r => r + 1)), []);- return <DashboardClientContext.Provider value={client}>- <SplitView- orientation='horizontal'- sidebarIsFirst- sidebarSize={320}- minSidebarSize={220}- settingName='dashboardSessionSidebar'- sidebar={<SessionSidebar- model={model}- onSelectTab={tab => { void client.selectTab({ browser: tab.browser, context: tab.context, page: tab.page }); }}- onCloseTab={tab => { void client.closeTab({ browser: tab.browser, context: tab.context, page: tab.page }); }}- onNewTab={(browser, context) => { void client.newTab({ browser, context }); }}- />}- main={<div className='dashboard-shell-main'>- <Dashboard />- </div>}- />- </DashboardClientContext.Provider>;+ return <SplitView+ orientation='horizontal'+ sidebarIsFirst+ sidebarSize={320}+ minSidebarSize={220}+ settingName='dashboardSessionSidebar'+ sidebar={<SessionSidebar model={model} />}+ main={<div className='dashboard-shell-main'>+ <Dashboard model={model} />+ </div>}+ />;};// HMR begin: cache the root on the DOM node so re-running this module duringdiff --git a/packages/dashboard/src/sessionModel.ts b/packages/dashboard/src/sessionModel.tsdeleted file mode 100644index 1213a0bcea..0000000000--- a/packages/dashboard/src/sessionModel.ts+++ /dev/null@@ -1,76 +0,0 @@-/**- * Copyright (c) Microsoft Corporation.- *- * Licensed under the Apache License, Version 2.0 (the "License");- * you may not use this file except in compliance with the License.- * You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */--import type { ClientInfo } from '../../playwright-core/src/tools/cli-client/registry';-import type { BrowserDescriptor, BrowserStatus } from '../../playwright-core/src/serverRegistry';--export type SessionStatus = BrowserStatus;--type Listener = () => void;--type SessionsChannel = {- on(event: 'sessions', listener: (params: { sessions: SessionStatus[]; clientInfo: ClientInfo }) => void): void;- off(event: 'sessions', listener: (params: { sessions: SessionStatus[]; clientInfo: ClientInfo }) => void): void;- closeSession(params: { browser: string }): Promise<void>;- deleteSessionData(params: { browser: string }): Promise<void>;-};--export class SessionModel {- sessions: SessionStatus[] = [];- clientInfo: ClientInfo | undefined;- loading = true;-- private _client: SessionsChannel;- private _listeners = new Set<Listener>();- private _onSessions = (params: { sessions: SessionStatus[]; clientInfo: ClientInfo }) => {- this.sessions = params.sessions;- this.clientInfo = params.clientInfo;- this.loading = false;- this._notify();- };-- constructor(client: SessionsChannel) {- this._client = client;- this._client.on('sessions', this._onSessions);- }-- subscribe(listener: Listener): () => void {- this._listeners.add(listener);- return () => this._listeners.delete(listener);- }-- private _notify() {- for (const listener of this._listeners)- listener();- }-- sessionByGuid(guid: string): SessionStatus | undefined {- return this.sessions.find(s => s.browser.guid === guid);- }-- async closeSession(descriptor: BrowserDescriptor) {- await this._client.closeSession({ browser: descriptor.browser.guid });- }-- async deleteSessionData(descriptor: BrowserDescriptor) {- await this._client.deleteSessionData({ browser: descriptor.browser.guid });- }-- dispose() {- this._client.off('sessions', this._onSessions);- this._listeners.clear();- }-}diff --git a/packages/dashboard/src/sessionSidebar.tsx b/packages/dashboard/src/sessionSidebar.tsxindex d82667265f..4279ce6a86 100644--- a/packages/dashboard/src/sessionSidebar.tsx+++ b/packages/dashboard/src/sessionSidebar.tsx@@ -16,19 +16,15 @@import React from 'react';import './sessionSidebar.css';-import { DashboardClientContext } from './dashboardContext';import { SettingsButton } from './settingsView';import { BrowserIcon } from './icons';import { ToolbarButton } from '@web/components/toolbarButton';-import type { Tab, DashboardChannelEvents } from './dashboardChannel';-import type { SessionModel, SessionStatus } from './sessionModel';+import type { SessionStatus, Tab } from './dashboardChannel';+import type { DashboardModel } from './dashboardModel';type SessionSidebarProps = {- model: SessionModel;- onSelectTab: (tab: Tab) => void;- onCloseTab: (tab: Tab) => void;- onNewTab: (browser: string, context: string) => void;+ model: DashboardModel;};function tabFavicon(url: string): string {@@ -56,19 +52,9 @@ function normalizeWorkspacePath(workspace: string, homeDir: string | undefined):return normalized;}-export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectTab, onCloseTab, onNewTab }) => {- const client = React.useContext(DashboardClientContext);- const openSessions = React.useMemo(() => model.sessions.filter(session => session.canConnect), [model.sessions]);- const clientInfo = model.clientInfo;- const [allTabs, setAllTabs] = React.useState<Tab[] | null>(null);-- React.useEffect(() => {- if (!client)- return;- const onTabs = (params: DashboardChannelEvents['tabs']) => setAllTabs(params.tabs);- client.on('tabs', onTabs);- return () => client.off('tabs', onTabs);- }, [client]);+export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model }) => {+ const { sessions, clientInfo, loadingSessions, tabs: allTabs } = model.state;+ const openSessions = React.useMemo(() => sessions.filter(session => session.canConnect), [sessions]);const tabsByBrowserAndContext = React.useMemo(() => {const map = new Map<string, Map<string, Tab[]>>();@@ -117,8 +103,8 @@ export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectT<SettingsButton /></div><div className='dashboard-shell-sidebar-content'>- {model.loading && <div className='sidebar-empty' role='status' aria-live='polite'>Loading sessions...</div>}- {!model.loading && openSessions.length === 0 && <div className='sidebar-empty' role='status' aria-live='polite'>No open sessions.</div>}+ {loadingSessions && <div className='sidebar-empty' role='status' aria-live='polite'>Loading sessions...</div>}+ {!loadingSessions && openSessions.length === 0 && <div className='sidebar-empty' role='status' aria-live='polite'>No open sessions.</div>}{workspaceGroups.map(([workspace, entries]) => {const workspacePath = normalizeWorkspacePath(workspace, clientInfo?.homeDir);return <section key={workspace} className='workspace-group'>@@ -149,7 +135,7 @@ export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectTclassName='session-browser-close'icon='close'title='Close session'- onClick={() => void model.closeSession(session)}+ onClick={() => model.closeSession(session)}/></div><span className='session-chip-name'>{session.title}</span>@@ -158,7 +144,7 @@ export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectTclassName='sidebar-session-new-tab'icon='add'title='New tab'- onClick={() => onNewTab(guid, row.contextGuid!)}+ onClick={() => model.newTab(guid, row.contextGuid!)}/>}</div></div>@@ -176,11 +162,11 @@ export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectTtabIndex={0}aria-current={tab.context === activeContext && tab.selected ? 'page' : undefined}title={tab.url || tab.title}- onClick={() => onSelectTab(tab)}+ onClick={() => model.selectTab(tab)}onKeyDown={e => {if (e.key === 'Enter' || e.key === ' ') {e.preventDefault();- onSelectTab(tab);+ model.selectTab(tab);}}}>@@ -198,7 +184,7 @@ export const SessionSidebar: React.FC<SessionSidebarProps> = ({ model, onSelectTtitle='Close tab'onClick={e => {e.stopPropagation();- onCloseTab(tab);+ model.closeTab(tab);}}/></div>)}diff --git a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts b/packages/playwright-core/src/tools/dashboard/dashboardApp.tsindex 43f486126a..b1374ce983 100644--- a/packages/playwright-core/src/tools/dashboard/dashboardApp.ts+++ b/packages/playwright-core/src/tools/dashboard/dashboardApp.ts@@ -53,7 +53,7 @@ async function startDashboardServer(options: DashboardOptions): Promise<Dashboarlet pendingAnnotate = false;const waitingSockets = new Set<net.Socket>();- const submitAnnotation = (base64Png: string, annotations: AnnotationData[]) => {+ const submitAnnotation = (base64Png: string | undefined, annotations: AnnotationData[]) => {if (waitingSockets.size === 0)return;const payload = JSON.stringify({ png: base64Png, annotations });@@ -407,14 +407,16 @@ async function runAnnotateClient(options: DashboardOptions): Promise<void> {if (!text)return;const { png, annotations } = JSON.parse(text) as { png: string; annotations: AnnotationData[] };- const timestamp = new Date().toISOString().replace(/[:.]/g, '-');- const filePath = await saveOutputFile(`annotations-${timestamp}.png`, Buffer.from(png, 'base64'));for (const a of annotations) {// eslint-disable-next-line no-consoleconsole.log(`{ x: ${a.x}, y: ${a.y}, width: ${a.width}, height: ${a.height} }: ${a.text}`);}- // eslint-disable-next-line no-console- console.log(`image: ${path.relative(process.cwd(), filePath)}`);+ if (png) {+ const timestamp = new Date().toISOString().replace(/[:.]/g, '-');+ const filePath = await saveOutputFile(`annotations-${timestamp}.png`, Buffer.from(png, 'base64'));+ // eslint-disable-next-line no-console+ console.log(`image: ${path.relative(process.cwd(), filePath)}`);+ }}function selfDestructOnParentGone() {diff --git a/packages/playwright-core/src/tools/dashboard/dashboardController.ts b/packages/playwright-core/src/tools/dashboard/dashboardController.tsindex 38b9cf7d5a..e6029bdfcf 100644--- a/packages/playwright-core/src/tools/dashboard/dashboardController.ts+++ b/packages/playwright-core/src/tools/dashboard/dashboardController.ts@@ -110,7 +110,7 @@ export class DashboardConnection implements Transport {private _attachedPage: AttachedPage | undefined;private _onclose: () => void;private _onconnected?: () => void;- private _onAnnotationSubmit?: (base64Png: string, annotations: AnnotationData[]) => void;+ private _onAnnotationSubmit?: (base64Png: string | undefined, annotations: AnnotationData[]) => void;private _serverRegistryDispose?: () => void;private _pushSessionsScheduled = false;private _pushTabsScheduled = false;@@ -120,7 +120,7 @@ export class DashboardConnection implements Transport {_recordingDir: string;_streams = new Map<string, { handle: fs.promises.FileHandle; path: string }>();- constructor(onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string, annotations: AnnotationData[]) => void) {+ constructor(onclose: () => void, onconnected?: () => void, onAnnotationSubmit?: (base64Png: string | undefined, annotations: AnnotationData[]) => void) {this._onclose = onclose;this._onconnected = onconnected;this._onAnnotationSubmit = onAnnotationSubmit;@@ -236,7 +236,7 @@ export class DashboardConnection implements Transport {this._pushTabs();}- async submitAnnotation(params: { data: string; annotations: AnnotationData[] }) {+ async submitAnnotation(params: { data: string | undefined; annotations: AnnotationData[] }) {this._onAnnotationSubmit?.(params.data, params.annotations);}@@ -558,9 +558,14 @@ class AttachedPage {return { streamId };}- async screenshot(): Promise<string> {+ async screenshot(): Promise<{ data: string; viewportWidth: number; viewportHeight: number }> {const buffer = await this._page.screenshot({ type: 'png' });- return buffer.toString('base64');+ const vp = this._page.viewportSize();+ return {+ data: buffer.toString('base64'),+ viewportWidth: vp?.width ?? 0,+ viewportHeight: vp?.height ?? 0,+ };}private async _startScreencast(page: api.Page) {