Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(trace-viewer): show screencast frames during playback (#42751)
Pavel Feldman0ee695b5c3f8
diff --git a/packages/trace-viewer/src/ui/ariaModeView.css b/packages/trace-viewer/src/ui/ariaModeView.cssindex 852e6f8de8..95800f66f2 100644--- a/packages/trace-viewer/src/ui/ariaModeView.css+++ b/packages/trace-viewer/src/ui/ariaModeView.css@@ -28,7 +28,8 @@position: relative;}-.aria-mode-screenshot img {+.aria-mode-screenshot img,+.aria-mode-screenshot .screencast-frame {max-width: 100%;max-height: 100%;object-fit: contain;diff --git a/packages/trace-viewer/src/ui/ariaModeView.tsx b/packages/trace-viewer/src/ui/ariaModeView.tsxindex da0caec2f2..77a9dc4d5b 100644--- a/packages/trace-viewer/src/ui/ariaModeView.tsx+++ b/packages/trace-viewer/src/ui/ariaModeView.tsx@@ -21,7 +21,7 @@ import { renderAriaSnapshotAsYaml } from '@isomorphic/ariaSnapshotRenderer';import { clsx, useMeasure } from '@web/uiUtils';import { PlaceholderPanel } from './placeholderPanel';-import type { ActionPhase, ActionTraceEvent, ScreenshotTraceEvent } from '@isomorphic/trace/trace';+import type { ActionPhase, ActionTraceEvent } from '@isomorphic/trace/trace';import type { AriaNodeJSON, AriaSnapshotJSON } from '@isomorphic/ariaSnapshot';import type { TraceModel } from '@isomorphic/trace/traceModel';@@ -144,7 +144,8 @@ export const AriaModeView: React.FunctionComponent<{target: AriaModeTarget | undefined,point?: Point,box?: Box,-}> = ({ model, target, point, box }) => {+ screencastFrame?: React.ReactNode,+}> = ({ model, target, point, box, screencastFrame }) => {const screenshot = model && target ? model.screenshotForCall(target.callId, target.phase) : undefined;const ariaSnapshot = model && target ? model.ariaSnapshotForCall(target.callId, target.phase) : undefined;const [lines, setLines] = React.useState<AriaSnapshotLine[]>([]);@@ -172,11 +173,16 @@ export const AriaModeView: React.FunctionComponent<{};}, [model, ariaSnapshot]);- if (!screenshot && !ariaSnapshot)+ if (!screenshot && !ariaSnapshot && !screencastFrame)return <PlaceholderPanel text='No aria snapshot' />;return <div className='aria-mode-view hbox'>- <AriaModeScreenshot model={model!} screenshot={screenshot} highlightedBox={highlightedBox} point={point} box={box} />+ <AriaModeScreenshot+ imageUrl={model && screenshot ? model.createRelativeUrl(`file/${screenshot.file}`) : undefined}+ screencastFrame={screencastFrame}+ highlightedBox={screencastFrame ? undefined : highlightedBox}+ point={screencastFrame ? undefined : point}+ box={screencastFrame ? undefined : box} /><div className='aria-mode-snapshot vbox'>{ariaSnapshot && <div className='aria-mode-lines' onMouseLeave={() => setHighlightedBox(undefined)}>{lines.map((line, index) => <div@@ -191,12 +197,12 @@ export const AriaModeView: React.FunctionComponent<{};const AriaModeScreenshot: React.FunctionComponent<{- model: TraceModel,- screenshot: ScreenshotTraceEvent | undefined,+ imageUrl: string | undefined,+ screencastFrame: React.ReactNode,highlightedBox: Box | undefined,point: Point | undefined,box: Box | undefined,-}> = ({ model, screenshot, highlightedBox, point, box }) => {+}> = ({ imageUrl, screencastFrame, highlightedBox, point, box }) => {const [measure, ref] = useMeasure<HTMLDivElement>();const [naturalSize, setNaturalSize] = React.useState<{ width: number, height: number } | undefined>();@@ -204,7 +210,7 @@ const AriaModeScreenshot: React.FunctionComponent<{// coordinates. The image is scaled to fit into the available area, scale the boxes and// the action point to match the rendered image.let overlays: React.ReactNode;- if (screenshot && naturalSize && measure.width) {+ if (imageUrl && naturalSize && measure.width) {const padding = 10;const availableWidth = measure.width - 2 * padding;const availableHeight = measure.height - 2 * padding;@@ -231,13 +237,13 @@ const AriaModeScreenshot: React.FunctionComponent<{}return <div ref={ref} className='aria-mode-screenshot'>- {screenshot && <img- key={screenshot.file}- src={model.createRelativeUrl(`file/${screenshot.file}`)}+ {screencastFrame}+ {!screencastFrame && imageUrl && <img+ src={imageUrl}alt='Screenshot'onLoad={event => setNaturalSize({ width: event.currentTarget.naturalWidth, height: event.currentTarget.naturalHeight })}/>}- {!screenshot && <PlaceholderPanel text='No screenshot' />}+ {!screencastFrame && !imageUrl && <PlaceholderPanel text='No screenshot' />}{overlays}</div>;};diff --git a/packages/trace-viewer/src/ui/filmStrip.tsx b/packages/trace-viewer/src/ui/filmStrip.tsxindex 4529b80d69..3bf5d2b9f0 100644--- a/packages/trace-viewer/src/ui/filmStrip.tsx+++ b/packages/trace-viewer/src/ui/filmStrip.tsx@@ -20,8 +20,10 @@ import * as React from 'react';import { useMeasure, upperBound } from '@web/uiUtils';import type { PageEntry } from '@isomorphic/trace/entries';import { useTraceModel } from './traceModelContext';-import { useVideoThumbnails } from './videoThumbnails';-import type { VideoThumbnail } from './videoThumbnails';+import { useVideoSources } from './videoThumbnails';+import type { VideoSource, VideoThumbnail } from './videoThumbnails';+import { VideoFrame, lastVideoIndex } from './videoFrame';+import type * as trace from '@isomorphic/trace/trace';export type FilmStripPreviewPoint = {x: number;@@ -41,7 +43,7 @@ export const FilmStrip: React.FunctionComponent<{const lanesRef = React.useRef<HTMLDivElement>(null);const videos = (model?.videos ?? []).map(video => ({ video, url: model!.createRelativeUrl(`file/${video.file}`) }));- const videoThumbnails = useVideoThumbnails(videos);+ const videoSources = useVideoSources(videos);let laneIndex = 0;if (lanesRef.current && previewPoint) {@@ -50,24 +52,30 @@ export const FilmStrip: React.FunctionComponent<{}const pageLanes = (model?.pages ?? []).filter(page => page.screencastFrames.length);- const videoLanes = videoLanesByPage(videos.map(({ video }) => video.pageId), videoThumbnails);+ const videoLanes = videoLanesByPage(videos.map(({ video }) => video), videoSources);- let previewFrames: { timestamp: number, width: number, height: number, url: string }[] | undefined;- if (laneIndex < pageLanes.length)- previewFrames = model ? pageLanes[laneIndex]?.screencastFrames.map(frame => ({ ...frame, url: model.createRelativeUrl(`file/${frame.file}`) })) : undefined;- else- previewFrames = videoLanes[laneIndex - pageLanes.length];-- let previewImage = undefined;- let previewSize = undefined;- if (previewPoint !== undefined && previewFrames && previewFrames.length) {+ // The thumbnails only index the lanes, the preview renders the video itself.+ let preview: React.ReactNode | undefined;+ let previewSize: Size | undefined;+ if (model && previewPoint !== undefined) {const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewPoint.x / measure.width;- previewImage = previewFrames[upperBound(previewFrames, previewTime, timeComparator) - 1];const fitInto = {width: Math.min(800, (window.innerWidth / 2) | 0),height: Math.min(800, (window.innerHeight / 2) | 0),};- previewSize = previewImage ? inscribe({ width: previewImage.width, height: previewImage.height }, fitInto) : undefined;+ const frames = pageLanes[laneIndex]?.screencastFrames;+ const frame = frames?.[upperBound(frames, previewTime, timeComparator) - 1];+ const lane = videoLanes[laneIndex - pageLanes.length];+ const videoIndex = lane ? lastVideoIndex(lane.videos, previewTime) : -1;+ const videoUrl = videoIndex === -1 ? undefined : lane.sources[videoIndex].url;+ if (frame) {+ previewSize = inscribe(frame, fitInto);+ preview = <img src={model.createRelativeUrl(`file/${frame.file}`)} width={previewSize.width} height={previewSize.height} />;+ } else if (videoUrl) {+ const video = lane.videos[videoIndex];+ previewSize = inscribe(video, fitInto);+ preview = <VideoFrame url={videoUrl} startTime={video.timestamp} time={previewTime} width={previewSize.width} height={previewSize.height} />;+ }}return <div className='film-strip' ref={ref}>@@ -78,37 +86,48 @@ export const FilmStrip: React.FunctionComponent<{width={measure.width}key={index}/>)}- {videoLanes.map((thumbnails, index) => <VideoFilmStripLane+ {videoLanes.map((lane, index) => <VideoFilmStripLaneboundaries={boundaries}- thumbnails={thumbnails}+ thumbnails={lane.thumbnails}width={measure.width}key={'video-' + index}/>)}</div>- {model && previewPoint && previewImage && previewSize &&+ {previewPoint && preview && previewSize &&<div className='film-strip-hover' style={{top: measure.bottom + 5,left: Math.min(previewPoint.x, measure.width - previewSize.width - 10),width: previewSize.width,height: previewSize.height,}}>- <img src={previewImage.url} width={previewSize.width} height={previewSize.height} />+ {preview}</div>}</div>;};-function videoLanesByPage(pageIds: string[], thumbnails: VideoThumbnail[][]): VideoThumbnail[][] {- const lanes = new Map<string, VideoThumbnail[]>();- pageIds.forEach((pageId, index) => {- let lane = lanes.get(pageId);+type VideoLane = {+ videos: trace.VideoTraceEvent[];+ sources: VideoSource[];+ thumbnails: VideoThumbnail[];+};++function videoLanesByPage(videos: trace.VideoTraceEvent[], sources: VideoSource[]): VideoLane[] {+ const lanes = new Map<string, VideoLane>();+ videos.forEach((video, index) => {+ let lane = lanes.get(video.pageId);if (!lane) {- lane = [];- lanes.set(pageId, lane);+ lane = { videos: [], sources: [], thumbnails: [] };+ lanes.set(video.pageId, lane);}- lane.push(...thumbnails[index]);+ lane.videos.push(video);+ lane.sources.push(sources[index]);+ lane.thumbnails.push(...sources[index].thumbnails);+ });+ return [...lanes.values()].filter(lane => lane.thumbnails.length).map(lane => {+ lane.thumbnails.sort((a, b) => a.timestamp - b.timestamp);+ return lane;});- return [...lanes.values()].filter(lane => lane.length).map(lane => lane.sort((a, b) => a.timestamp - b.timestamp));}const VideoFilmStripLane: React.FunctionComponent<{diff --git a/packages/trace-viewer/src/ui/playbackControl.tsx b/packages/trace-viewer/src/ui/playbackControl.tsxindex 35aea8ef85..3c7a62fd86 100644--- a/packages/trace-viewer/src/ui/playbackControl.tsx+++ b/packages/trace-viewer/src/ui/playbackControl.tsx@@ -28,11 +28,14 @@ export type PlaybackState = {currentIndex: number;percent: number;animating: boolean;+ screencastTime: number | undefined;togglePlay: () => void;stop: () => void;prev: () => void;next: () => void;cycleSpeed: () => void;+ selectAction: (action: ActionEntry) => void;+ showSelectedAction: () => void;onScrubberMouseDown: (e: React.MouseEvent) => void;scrubberRef: React.RefObject<HTMLDivElement | null>;actionsLength: number;@@ -52,8 +55,8 @@ export function usePlayback(const [playing, setPlaying] = React.useState(false);const [speedIndex, setSpeedIndex] = React.useState(1);const [dragging, setDragging] = React.useState(false);- const [dragFraction, setDragFraction] = React.useState<number | undefined>(undefined);- const [cursorTime, setCursorTime] = React.useState<number | undefined>(undefined);+ // Set while playing or positioned between actions: the screencast is shown at this time instead of the action snapshot.+ const [screencastTime, setScreencastTime] = React.useState<number | undefined>(undefined);const speed = speeds[speedIndex];const currentIndex = selectedAction ? actions.indexOf(selectedAction) : -1;@@ -64,7 +67,6 @@ export function usePlayback(const fullDuration = fullMax - fullMin || 1;// Playback boundaries: constrained to time window when selected.- const windowMin = timeWindow ? timeWindow.minimum : fullMin;const windowMax = timeWindow ? timeWindow.maximum : fullMax;// Actions within the effective window.@@ -86,7 +88,12 @@ export function usePlayback(const scrubberRef = React.useRef<HTMLDivElement>(null);- const actionIndexAtTime = React.useCallback((t: number): number => {+ const clampToWindow = React.useCallback((index: number) => {+ return Math.max(firstWindowIndex, Math.min(lastWindowIndex, index));+ }, [firstWindowIndex, lastWindowIndex]);++ // Last action started at or before t, or the first action when none has started yet.+ const lastStartedIndex = React.useCallback((t: number): number => {let lo = 0;let hi = actions.length - 1;while (lo < hi) {@@ -96,29 +103,13 @@ export function usePlayback(elsehi = mid - 1;}- if (lo < actions.length - 1) {- const distPrev = t - actions[lo].startTime;- const distNext = actions[lo + 1].startTime - t;- if (distNext < distPrev)- lo = lo + 1;- }- // Clamp to window bounds.- return Math.max(firstWindowIndex, Math.min(lastWindowIndex, lo));- }, [actions, firstWindowIndex, lastWindowIndex]);+ return lo;+ }, [actions]);const selectedTime = selectedAction ? selectedAction.startTime : fullMin;+ const positionTime = screencastTime ?? selectedTime;+ const percent = Math.max(0, Math.min(100, ((positionTime - fullMin) / fullDuration) * 100));- let percent: number;- if (dragging && dragFraction !== undefined)- percent = dragFraction * 100;- else if (playing && cursorTime !== undefined)- percent = Math.max(0, Math.min(100, ((cursorTime - fullMin) / fullDuration) * 100));- else- percent = Math.max(0, Math.min(100, ((selectedTime - fullMin) / fullDuration) * 100));-- // Refs for raf closure.- const windowMinRef = React.useRef(windowMin);- windowMinRef.current = windowMin;const windowMaxRef = React.useRef(windowMax);windowMaxRef.current = windowMax;@@ -127,13 +118,8 @@ export function usePlayback(return;let rafId: number;let lastFrameTime: number | undefined;- let traceTime = selectedTime;- // If starting from before the window, jump to window start.- if (traceTime < windowMinRef.current)- traceTime = windowMinRef.current;- let lastSelectedIndex = currentIndex;-- setCursorTime(traceTime);+ let traceTime = positionTime;+ let lastSelectedIndex = clampToWindow(lastStartedIndex(traceTime));const tick = (now: number) => {if (lastFrameTime !== undefined) {@@ -141,16 +127,17 @@ export function usePlayback(traceTime = Math.min(traceTime + delta, windowMaxRef.current);}lastFrameTime = now;- setCursorTime(traceTime);+ setScreencastTime(traceTime);- const idx = actionIndexAtTime(traceTime);- if (idx !== lastSelectedIndex) {- lastSelectedIndex = idx;- onActionSelectedRef.current(actionsRef.current[idx]);+ const index = clampToWindow(lastStartedIndex(traceTime));+ if (index !== lastSelectedIndex) {+ lastSelectedIndex = index;+ onActionSelectedRef.current(actionsRef.current[index]);}if (traceTime >= windowMaxRef.current) {setPlaying(false);+ setScreencastTime(undefined);return;}rafId = requestAnimationFrame(tick);@@ -160,38 +147,51 @@ export function usePlayback(// eslint-disable-next-line react-hooks/exhaustive-deps}, [playing, speed]);- React.useEffect(() => {- if (!playing)- setCursorTime(undefined);- }, [playing]);-const togglePlay = React.useCallback(() => {if (!actions.length)return;- // Always restart from the window start when at the end (or beyond the window).- const atEnd = currentIndex >= lastWindowIndex;- if (!playing && atEnd)+ if (playing) {+ setPlaying(false);+ setScreencastTime(undefined);+ return;+ }+ // Restart from the window start when at the end, or outside the window.+ if (currentIndex >= lastWindowIndex || currentIndex < firstWindowIndex)onActionSelected(actions[firstWindowIndex]);- setPlaying(!playing);+ setPlaying(true);}, [playing, actions, currentIndex, onActionSelected, firstWindowIndex, lastWindowIndex]);+ const selectAction = React.useCallback((action: ActionEntry) => {+ setPlaying(false);+ setScreencastTime(undefined);+ onActionSelectedRef.current(action);+ }, []);++ // Leave the between-actions position, so that the selected action's snapshot is shown.+ const showSelectedAction = React.useCallback(() => {+ setPlaying(false);+ setScreencastTime(undefined);+ }, []);+const stop = React.useCallback(() => {setPlaying(false);+ setScreencastTime(undefined);if (actions.length)onActionSelected(actions[firstWindowIndex]);}, [actions, onActionSelected, firstWindowIndex]);+ const canPrev = currentIndex > firstWindowIndex;+ const canNext = currentIndex < lastWindowIndex;+const prev = React.useCallback(() => {- const target = Math.max(currentIndex - 1, firstWindowIndex);- if (target !== currentIndex)- onActionSelected(actions[target]);- }, [actions, currentIndex, onActionSelected, firstWindowIndex]);+ if (canPrev)+ selectAction(actions[currentIndex - 1]);+ }, [actions, canPrev, currentIndex, selectAction]);const next = React.useCallback(() => {- const target = Math.min(currentIndex + 1, lastWindowIndex);- if (target !== currentIndex)- onActionSelected(actions[target]);- }, [actions, currentIndex, onActionSelected, lastWindowIndex]);+ if (canNext)+ selectAction(actions[currentIndex + 1]);+ }, [actions, canNext, currentIndex, selectAction]);const cycleSpeed = React.useCallback(() => {setSpeedIndex(i => (i + 1) % speeds.length);@@ -199,20 +199,21 @@ export function usePlayback(React.useEffect(() => {setPlaying(false);+ setScreencastTime(undefined);}, [actions]);- const fractionFromMouseEvent = React.useCallback((e: MouseEvent | React.MouseEvent) => {+ const seekToMouseEvent = React.useCallback((e: MouseEvent | React.MouseEvent, snap: boolean) => {const rect = scrubberRef.current!.getBoundingClientRect();- return Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width));- }, []);-- const selectActionAtFraction = React.useCallback((fraction: number) => {- if (!actions.length)- return;- const t = fullMin + fraction * fullDuration;- const idx = actionIndexAtTime(t);- onActionSelectedRef.current(actionsRef.current[idx]);- }, [actions, fullMin, fullDuration, actionIndexAtTime]);+ const fraction = Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width));+ const time = fullMin + fraction * fullDuration;+ // The screencast follows the pointer while dragging, releasing snaps to the action snapshot.+ setScreencastTime(snap ? undefined : time);+ let index = lastStartedIndex(time);+ const next = actions[index + 1];+ if (next && next.startTime - time < time - actions[index].startTime)+ ++index;+ onActionSelectedRef.current(actions[clampToWindow(index)]);+ }, [actions, fullMin, fullDuration, clampToWindow, lastStartedIndex]);const dragCleanupRef = React.useRef<(() => void) | null>(null);@@ -228,22 +229,16 @@ export function usePlayback(scrubberRef.current?.focus();setDragging(true);setPlaying(false);- const fraction = fractionFromMouseEvent(e);- setDragFraction(fraction);- selectActionAtFraction(fraction);+ seekToMouseEvent(e, false);const onMouseMove = (me: MouseEvent) => {- const f = fractionFromMouseEvent(me);- setDragFraction(f);- selectActionAtFraction(f);+ seekToMouseEvent(me, false);};const onMouseUp = (me: MouseEvent) => {document.removeEventListener('mousemove', onMouseMove);document.removeEventListener('mouseup', onMouseUp);dragCleanupRef.current = null;- const f = fractionFromMouseEvent(me);- selectActionAtFraction(f);- setDragFraction(undefined);+ seekToMouseEvent(me, true);setDragging(false);};document.addEventListener('mousemove', onMouseMove);@@ -252,18 +247,16 @@ export function usePlayback(document.removeEventListener('mousemove', onMouseMove);document.removeEventListener('mouseup', onMouseUp);};- }, [actions, selectActionAtFraction, fractionFromMouseEvent]);+ }, [actions, seekToMouseEvent]);const animating = !playing && !dragging;const ticks = actions.length > 0 && actions.length <= 200 ? actions.map(a => ((a.startTime - fullMin) / fullDuration) * 100) : undefined;- const canPrev = currentIndex > firstWindowIndex;- const canNext = currentIndex < lastWindowIndex;const canStop = playing || currentIndex > firstWindowIndex;return {- playing, speed, currentIndex, percent, animating,- togglePlay, stop, prev, next, cycleSpeed,+ playing, speed, currentIndex, percent, animating, screencastTime,+ togglePlay, stop, prev, next, cycleSpeed, selectAction, showSelectedAction,onScrubberMouseDown, scrubberRef, actionsLength: actions.length,canPrev, canNext, canStop, ticks,};diff --git a/packages/trace-viewer/src/ui/snapshotTab.css b/packages/trace-viewer/src/ui/snapshotTab.cssindex 5265f4f5f1..c895859d9a 100644--- a/packages/trace-viewer/src/ui/snapshotTab.css+++ b/packages/trace-viewer/src/ui/snapshotTab.css@@ -105,6 +105,16 @@ iframe.snapshot-visible[name=snapshot] {visibility: visible;}+.snapshot-switcher .screencast-frame {+ position: absolute;+ top: 0;+ left: 0;+ width: 100%;+ height: 100%;+ object-fit: contain;+ background: light-dark(white, #333);+}+.no-snapshot {text-align: center;padding: 50px;diff --git a/packages/trace-viewer/src/ui/snapshotTab.tsx b/packages/trace-viewer/src/ui/snapshotTab.tsxindex 6cd00da5eb..7fdf5b9f77 100644--- a/packages/trace-viewer/src/ui/snapshotTab.tsx+++ b/packages/trace-viewer/src/ui/snapshotTab.tsx@@ -21,7 +21,7 @@ import { nextActionByStartTime, previousActionByEndTime } from '@isomorphic/tracimport type { TraceModel } from '@isomorphic/trace/traceModel';import { Toolbar } from '@web/components/toolbar';import { ToolbarButton } from '@web/components/toolbarButton';-import { clsx, useMeasure, useSetting } from '@web/uiUtils';+import { clsx, upperBound, useMeasure, useSetting } from '@web/uiUtils';import { InjectedScript } from '@injected/injectedScript';import { Recorder } from '@injected/recorder/recorder';import { asLocator } from '@isomorphic/locatorGenerators';@@ -36,6 +36,8 @@ import yaml from 'yaml';import { PlaybackButtons } from './playbackControl';import type { PlaybackState } from './playbackControl';import { AriaModeView, collectAriaModeTargets, shouldDisplayAriaMode } from './ariaModeView';+import { useVideoSources } from './videoThumbnails';+import { VideoFrame, lastVideoIndex } from './videoFrame';export type HighlightedElement = {locator?: string,@@ -52,13 +54,16 @@ export const SnapshotTabsView: React.FunctionComponent<{setIsInspecting: (isInspecting: boolean) => void,highlightedElement: HighlightedElement,setHighlightedElement: (element: HighlightedElement) => void,+ // Time to show the screencast at, undefined to show the action snapshot.+ screencastTime?: number,playback: PlaybackState-}> = ({ action, model, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement, playback }) => {+}> = ({ action, model, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement, screencastTime, playback }) => {const [snapshotTab, setSnapshotTab] = React.useState<'action'|'before'|'after'>('action');const [shouldPopulateCanvasFromScreenshot] = useSetting('shouldPopulateCanvasFromScreenshot', false);const [displayAriaModeSetting] = useSetting('displayAriaMode', false);const displayAriaMode = shouldDisplayAriaMode(model, displayAriaModeSetting);+ const screencastFrame = useScreencastFrame(model, screencastTime, playback.playing, playback.speed);const snapshots = React.useMemo(() => {return collectSnapshots(model, action);@@ -107,6 +112,7 @@ export const SnapshotTabsView: React.FunctionComponent<{target={ariaModeTargets[snapshotTab]}point={snapshotTab === 'action' ? action?.point : undefined}box={snapshotTab === 'action' ? action?.box : undefined}+ screencastFrame={screencastFrame}/>}{!displayAriaMode && <SnapshotViewsnapshotUrls={snapshotUrls}@@ -116,6 +122,7 @@ export const SnapshotTabsView: React.FunctionComponent<{setIsInspecting={setIsInspecting}highlightedElement={highlightedElement}setHighlightedElement={setHighlightedElement}+ screencastFrame={screencastFrame}/>}</div>;};@@ -128,7 +135,8 @@ export const SnapshotView: React.FunctionComponent<{setIsInspecting: (isInspecting: boolean) => void,highlightedElement: HighlightedElement,setHighlightedElement: (element: HighlightedElement) => void,-}> = ({ snapshotUrls, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement }) => {+ screencastFrame?: React.ReactNode,+}> = ({ snapshotUrls, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement, screencastFrame }) => {const iframeRef0 = React.useRef<HTMLIFrameElement>(null);const iframeRef1 = React.useRef<HTMLIFrameElement>(null);const [snapshotInfo, setSnapshotInfo] = React.useState<SnapshotInfo>({ viewport: kDefaultViewport, url: '' });@@ -146,7 +154,8 @@ export const SnapshotView: React.FunctionComponent<{if (loadingRef.current.iteration !== thisIteration)return;- const iframe = [iframeRef0, iframeRef1][newVisibleIframe].current;+ // While the screencast frame covers the snapshot, only take the viewport and the url from it.+ const iframe = screencastFrame ? undefined : [iframeRef0, iframeRef1][newVisibleIframe].current;if (iframe) {let loadedCallback = () => {};const loadedPromise = new Promise<void>(f => loadedCallback = f);@@ -172,10 +181,11 @@ export const SnapshotView: React.FunctionComponent<{if (loadingRef.current.iteration !== thisIteration)return;- loadingRef.current.visibleIframe = newVisibleIframe;+ if (iframe)+ loadingRef.current.visibleIframe = newVisibleIframe;setSnapshotInfo(newSnapshotInfo);})();- }, [snapshotUrls]);+ }, [snapshotUrls, screencastFrame]);return <divclassName='vbox'@@ -207,6 +217,7 @@ export const SnapshotView: React.FunctionComponent<{<div className='snapshot-switcher'><iframe ref={iframeRef0} name='snapshot' title='DOM Snapshot' sandbox='allow-same-origin allow-scripts' className={clsx(loadingRef.current.visibleIframe === 0 && 'snapshot-visible')}></iframe><iframe ref={iframeRef1} name='snapshot' title='DOM Snapshot' sandbox='allow-same-origin allow-scripts' className={clsx(loadingRef.current.visibleIframe === 1 && 'snapshot-visible')}></iframe>+ {screencastFrame}</div></SnapshotWrapper></div>;@@ -478,3 +489,27 @@ export async function fetchSnapshotInfo(snapshotInfoUrl: string | undefined) {}export const kDefaultViewport = { width: 1280, height: 720 };++function useScreencastFrame(model: TraceModel | undefined, time: number | undefined, playing: boolean, speed: number): React.ReactNode | undefined {+ const videos = model && time !== undefined ? model.videos.map(video => ({ video, url: model.createRelativeUrl(`file/${video.file}`) })) : [];+ const videoSources = useVideoSources(videos);+ if (!model || time === undefined)+ return undefined;++ // A trace has either screencast frames or videos, never both.+ // Show the page that painted most recently.+ let frame: { timestamp: number, file: string } | undefined;+ for (const page of model.pages) {+ const candidate = page.screencastFrames[upperBound(page.screencastFrames, time, (t, f) => t - f.timestamp) - 1];+ if (candidate && (!frame || candidate.timestamp > frame.timestamp))+ frame = candidate;+ }+ if (frame)+ return <img className='screencast-frame' src={model.createRelativeUrl(`file/${frame.file}`)} alt='Screencast frame' />;++ const index = lastVideoIndex(videos.map(({ video }) => video), time);+ const url = videoSources[index]?.url;+ if (!url)+ return undefined;+ return <VideoFrame className='screencast-frame' url={url} startTime={videos[index].video.timestamp} time={time} playing={playing} speed={speed} />;+}diff --git a/packages/trace-viewer/src/ui/videoFrame.tsx b/packages/trace-viewer/src/ui/videoFrame.tsxnew file mode 100644index 0000000000..651971b32c--- /dev/null+++ b/packages/trace-viewer/src/ui/videoFrame.tsx@@ -0,0 +1,79 @@+/*+ Copyright (c) Microsoft Corporation.++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+*/++import * as React from 'react';++import type * as trace from '@isomorphic/trace/trace';++// Playback runs the video on its own clock and only resyncs when it drifts this far away.+const maxDriftSeconds = 0.25;++// Video that was recording at the given trace time, -1 when none has started yet.+export function lastVideoIndex(videos: trace.VideoTraceEvent[], time: number): number {+ let result = -1;+ videos.forEach((video, index) => {+ if (video.timestamp <= time && (result === -1 || video.timestamp > videos[result].timestamp))+ result = index;+ });+ return result;+}++export const VideoFrame: React.FunctionComponent<{+ url: string,+ // Video start in the trace time.+ startTime: number,+ // Trace time to render.+ time: number,+ playing?: boolean,+ speed?: number,+ className?: string,+ width?: number,+ height?: number,+}> = ({ url, startTime, time, playing, speed, className, width, height }) => {+ const ref = React.useRef<HTMLVideoElement>(null);+ const offset = Math.max(0, (time - startTime) / 1000);+ const offsetRef = React.useRef(offset);+ offsetRef.current = offset;++ React.useEffect(() => {+ const video = ref.current;+ if (!video)+ return;+ if (!playing) {+ video.pause();+ if (Math.abs(video.currentTime - offset) > 0.001)+ video.currentTime = offset;+ return;+ }+ video.playbackRate = speed ?? 1;+ if (Math.abs(video.currentTime - offset) > maxDriftSeconds)+ video.currentTime = offset;+ if (video.paused)+ void video.play().catch(() => {});+ }, [url, offset, playing, speed]);++ return <video+ ref={ref}+ className={className}+ src={url}+ width={width}+ height={height}+ muted+ playsInline+ preload='auto'+ onLoadedMetadata={event => event.currentTarget.currentTime = offsetRef.current}+ />;+};diff --git a/packages/trace-viewer/src/ui/videoThumbnails.ts b/packages/trace-viewer/src/ui/videoThumbnails.tsindex b9abf73d33..7dc4c603c8 100644--- a/packages/trace-viewer/src/ui/videoThumbnails.ts+++ b/packages/trace-viewer/src/ui/videoThumbnails.ts@@ -24,7 +24,15 @@ export type VideoThumbnail = {height: number;};+export type VideoSource = {+ // Blob url of the whole video, so that the player can seek without range requests.+ readonly url: string | undefined;+ // Keyframes for the film strip, the player renders the video itself.+ readonly thumbnails: VideoThumbnail[];+};+type CacheEntry = {+ url: string | undefined;thumbnails: VideoThumbnail[];listeners: Set<() => void>;started: boolean;@@ -35,7 +43,7 @@ const cache = new Map<string, CacheEntry>();const maxThumbnails = 120;const thumbnailsPerSecond = 2;-export function useVideoThumbnails(videos: { video: trace.VideoTraceEvent, url: string }[]): VideoThumbnail[][] {+export function useVideoSources(videos: { video: trace.VideoTraceEvent, url: string }[]): VideoSource[] {const [, setVersion] = React.useState(0);const entries = videos.map(({ video, url }) => ensureEntry(video, url));const urls = videos.map(({ url }) => url).join('\n');@@ -49,13 +57,13 @@ export function useVideoThumbnails(videos: { video: trace.VideoTraceEvent, url:entry.listeners.delete(listener);};}, [urls]);- return entries.map(entry => entry.thumbnails);+ return entries;}function ensureEntry(video: trace.VideoTraceEvent, videoUrl: string): CacheEntry {let entry = cache.get(videoUrl);if (!entry) {- entry = { thumbnails: [], listeners: new Set(), started: false };+ entry = { url: undefined, thumbnails: [], listeners: new Set(), started: false };cache.set(videoUrl, entry);}if (!entry.started) {@@ -69,6 +77,9 @@ async function generateThumbnails(entry: CacheEntry, video: trace.VideoTraceEvenconst response = await fetch(videoUrl);const blob = await response.blob();const objectUrl = URL.createObjectURL(blob);+ entry.url = objectUrl;+ for (const listener of entry.listeners)+ listener();const element = document.createElement('video');element.muted = true;element.preload = 'auto';@@ -113,9 +124,9 @@ async function generateThumbnails(entry: CacheEntry, video: trace.VideoTraceEvenlistener();}} finally {+ // The blob url outlives thumbnail generation, the player seeks in it.element.removeAttribute('src');element.load();- URL.revokeObjectURL(objectUrl);}}diff --git a/packages/trace-viewer/src/ui/workbench.tsx b/packages/trace-viewer/src/ui/workbench.tsxindex ab31fc558b..12c47c5590 100644--- a/packages/trace-viewer/src/ui/workbench.tsx+++ b/packages/trace-viewer/src/ui/workbench.tsx@@ -178,8 +178,11 @@ const PartitionedWorkbench: React.FunctionComponent<WorkbenchProps & { partitionconst setIsInspecting = React.useCallback((value: boolean) => {if (!isInspecting && value)selectPropertiesTab('inspector');+ // Inspecting works against the DOM snapshot, not the screencast frame.+ if (value)+ playback.showSelectedAction();setIsInspectingState(value);- }, [setIsInspectingState, selectPropertiesTab, isInspecting]);+ }, [setIsInspectingState, selectPropertiesTab, isInspecting, playback.showSelectedAction]);const elementPicked = React.useCallback((element: HighlightedElement) => {setHighlightedElement(element);@@ -234,7 +237,7 @@ const PartitionedWorkbench: React.FunctionComponent<WorkbenchProps & { partitionerrorCount: errorsModel.errors.size,render: () => <ErrorsTab errorsModel={errorsModel} testRunMetadata={testRunMetadata} sdkLanguage={sdkLanguage} revealInSource={error => {if (error.action)- setSelectedAction(error.action);+ playback.selectAction(error.action);elsesetRevealedErrorKey(error.message);selectPropertiesTab('source');@@ -347,7 +350,7 @@ const PartitionedWorkbench: React.FunctionComponent<WorkbenchProps & { partitionsetSelectedTime={setSelectedTime}treeState={treeState}setTreeState={setTreeState}- onSelected={onActionSelected}+ onSelected={playback.selectAction}onHighlighted={setHighlightedAction}revealActionAttachment={revealActionAttachment}revealConsole={() => selectPropertiesTab('console')}@@ -368,7 +371,7 @@ const PartitionedWorkbench: React.FunctionComponent<WorkbenchProps & { partition{!hideTimeline && <Timelinemodel={model}boundaries={boundaries}- onSelected={onActionSelected}+ onSelected={playback.selectAction}selectedTime={selectedTime}setSelectedTime={setSelectedTime}highlightedTime={highlightedTime}@@ -391,6 +394,7 @@ const PartitionedWorkbench: React.FunctionComponent<WorkbenchProps & { partitionsetIsInspecting={setIsInspecting}highlightedElement={highlightedElement}setHighlightedElement={elementPicked}+ screencastTime={highlightedAction ? undefined : playback.screencastTime}playback={playback} />}sidebar={<TabbedPanediff --git a/tests/library/trace-viewer-scrub.spec.ts b/tests/library/trace-viewer-scrub.spec.tsindex 0b5856d208..e633720002 100644--- a/tests/library/trace-viewer-scrub.spec.ts+++ b/tests/library/trace-viewer-scrub.spec.ts@@ -192,3 +192,89 @@ test('should drag scrubber to select action', async ({ runAndTrace, page: actionconst scrubberValue = await scrubber.getAttribute('aria-valuenow');expect(Number(scrubberValue)).toBeGreaterThan(50);});++test('should show screencast while playing and snapshot when stopped', async ({ runAndTrace, page: actionPage }) => {+ const traceViewer = await runAndTrace(async () => {+ await actionPage.setContent('<button>Click me</button>');+ await actionPage.click('button');+ await actionPage.setContent('<input/>');+ await actionPage.waitForTimeout(2000);+ });+ const page = traceViewer.page;+ const screencast = page.getByRole('img', { name: 'Screencast frame' });++ await traceViewer.selectAction('Set content');+ await expect(traceViewer.snapshotContainer).toBeVisible();+ await expect(screencast).toHaveCount(0);++ // Selecting an action stops the playback and snaps to its snapshot.+ await page.getByRole('button', { name: 'Play' }).click();+ await expect(screencast).toBeVisible();+ await traceViewer.selectAction('Click');+ await expect(screencast).toHaveCount(0);+ await expect(page.getByRole('button', { name: 'Play' })).toBeVisible();+ await expect(page.frameLocator('iframe.snapshot-visible[name=snapshot]').getByRole('button', { name: 'Click me' })).toBeVisible();++ // Reaching the end snaps to the last action as well.+ await page.getByRole('button', { name: 'Play' }).click();+ await expect(screencast).toBeVisible();+ await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 15000 });+ await expect(screencast).toHaveCount(0);+ await expect(traceViewer.snapshotContainer).toBeVisible();+});++test('should reveal snapshot when hovering an action or picking a locator during playback', async ({ runAndTrace, page: actionPage }) => {+ const traceViewer = await runAndTrace(async () => {+ await actionPage.setContent('<button>Click me</button>');+ await actionPage.click('button');+ await actionPage.setContent('<input/>');+ await actionPage.waitForTimeout(2000);+ });+ const page = traceViewer.page;+ const screencast = page.getByRole('img', { name: 'Screencast frame' });+ const snapshot = page.frameLocator('iframe.snapshot-visible[name=snapshot]');++ // Hovering an action reveals its snapshot.+ await traceViewer.selectAction('Set content');+ await page.getByRole('button', { name: 'Play' }).click();+ await expect(screencast).toBeVisible();+ await traceViewer.hoverAction('Click');+ await expect(screencast).toHaveCount(0);+ await expect(snapshot.getByRole('button', { name: 'Click me' })).toBeVisible();++ // Picking a locator stops the playback, the snapshot receives mouse events.+ await traceViewer.selectAction('Set content');+ await page.getByRole('button', { name: 'Play' }).click();+ await expect(screencast).toBeVisible();+ await page.getByTitle('Pick locator').click();+ await expect(screencast).toHaveCount(0);+ await expect(traceViewer.snapshotContainer).toBeVisible();+ await snapshot.locator('body').click();+ await expect(page.locator('.cm-wrapper').first()).toContainText('locator(');+});++test('should show screencast while dragging and snap to the action on release', async ({ runAndTrace, page: actionPage }) => {+ const traceViewer = await runAndTrace(async () => {+ await actionPage.setContent('<button>Click me</button>');+ await actionPage.click('button');+ await actionPage.setContent('<input/>');+ });+ const page = traceViewer.page;+ const screencast = page.getByRole('img', { name: 'Screencast frame' });+ const scrubber = page.getByRole('slider', { name: 'Playback position' });+ const box = (await scrubber.boundingBox())!;++ // While dragging, the screencast follows the pointer between the action ticks.+ await page.mouse.move(box.x + box.width * 0.5, box.y + box.height / 2);+ await page.mouse.down();+ await page.mouse.move(box.x + box.width * 0.99, box.y + box.height / 2);+ await expect(screencast).toBeVisible();+ await expect(scrubber).toHaveAttribute('aria-valuenow', '99');++ // Releasing snaps to the nearest action and shows its snapshot.+ await page.mouse.up();+ await expect(screencast).toHaveCount(0);+ await expect(traceViewer.snapshotContainer).toBeVisible();+ await expect(traceViewer.actionsTree.getByRole('treeitem', { selected: true })).toHaveText(/Set content/);+ expect(Number(await scrubber.getAttribute('aria-valuenow'))).toBeLessThan(99);+});