Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(trace-viewer): show screencast frames during playback (#42751)

Pavel Feldman0ee695b5c3f8
diff --git a/packages/trace-viewer/src/ui/ariaModeView.css b/packages/trace-viewer/src/ui/ariaModeView.css
index 852e6f8de8..95800f66f2 100644
--- a/packages/trace-viewer/src/ui/ariaModeView.css
+++ b/packages/trace-viewer/src/ui/ariaModeView.css
@@ -28,7 +28,8 @@
position: relative;
}
-.aria-mode-screenshot img {
+.aria-mode-screenshot img,
+.aria-mode-screenshot .screencast-frame {
max-width: 100%;
max-height: 100%;
object-fit: contain;
diff --git a/packages/trace-viewer/src/ui/ariaModeView.tsx b/packages/trace-viewer/src/ui/ariaModeView.tsx
index da0caec2f2..77a9dc4d5b 100644
--- a/packages/trace-viewer/src/ui/ariaModeView.tsx
+++ b/packages/trace-viewer/src/ui/ariaModeView.tsx
@@ -21,7 +21,7 @@ import { renderAriaSnapshotAsYaml } from '@isomorphic/ariaSnapshotRenderer';
import { clsx, useMeasure } from '@web/uiUtils';
import { PlaceholderPanel } from './placeholderPanel';
-import type { ActionPhase, ActionTraceEvent, ScreenshotTraceEvent } from '@isomorphic/trace/trace';
+import type { ActionPhase, ActionTraceEvent } from '@isomorphic/trace/trace';
import type { AriaNodeJSON, AriaSnapshotJSON } from '@isomorphic/ariaSnapshot';
import type { TraceModel } from '@isomorphic/trace/traceModel';
@@ -144,7 +144,8 @@ export const AriaModeView: React.FunctionComponent<{
target: AriaModeTarget | undefined,
point?: Point,
box?: Box,
-}> = ({ model, target, point, box }) => {
+ screencastFrame?: React.ReactNode,
+}> = ({ model, target, point, box, screencastFrame }) => {
const screenshot = model && target ? model.screenshotForCall(target.callId, target.phase) : undefined;
const ariaSnapshot = model && target ? model.ariaSnapshotForCall(target.callId, target.phase) : undefined;
const [lines, setLines] = React.useState<AriaSnapshotLine[]>([]);
@@ -172,11 +173,16 @@ export const AriaModeView: React.FunctionComponent<{
};
}, [model, ariaSnapshot]);
- if (!screenshot && !ariaSnapshot)
+ if (!screenshot && !ariaSnapshot && !screencastFrame)
return <PlaceholderPanel text='No aria snapshot' />;
return <div className='aria-mode-view hbox'>
- <AriaModeScreenshot model={model!} screenshot={screenshot} highlightedBox={highlightedBox} point={point} box={box} />
+ <AriaModeScreenshot
+ imageUrl={model && screenshot ? model.createRelativeUrl(`file/${screenshot.file}`) : undefined}
+ screencastFrame={screencastFrame}
+ highlightedBox={screencastFrame ? undefined : highlightedBox}
+ point={screencastFrame ? undefined : point}
+ box={screencastFrame ? undefined : box} />
<div className='aria-mode-snapshot vbox'>
{ariaSnapshot && <div className='aria-mode-lines' onMouseLeave={() => setHighlightedBox(undefined)}>
{lines.map((line, index) => <div
@@ -191,12 +197,12 @@ export const AriaModeView: React.FunctionComponent<{
};
const AriaModeScreenshot: React.FunctionComponent<{
- model: TraceModel,
- screenshot: ScreenshotTraceEvent | undefined,
+ imageUrl: string | undefined,
+ screencastFrame: React.ReactNode,
highlightedBox: Box | undefined,
point: Point | undefined,
box: Box | undefined,
-}> = ({ model, screenshot, highlightedBox, point, box }) => {
+}> = ({ imageUrl, screencastFrame, highlightedBox, point, box }) => {
const [measure, ref] = useMeasure<HTMLDivElement>();
const [naturalSize, setNaturalSize] = React.useState<{ width: number, height: number } | undefined>();
@@ -204,7 +210,7 @@ const AriaModeScreenshot: React.FunctionComponent<{
// coordinates. The image is scaled to fit into the available area, scale the boxes and
// the action point to match the rendered image.
let overlays: React.ReactNode;
- if (screenshot && naturalSize && measure.width) {
+ if (imageUrl && naturalSize && measure.width) {
const padding = 10;
const availableWidth = measure.width - 2 * padding;
const availableHeight = measure.height - 2 * padding;
@@ -231,13 +237,13 @@ const AriaModeScreenshot: React.FunctionComponent<{
}
return <div ref={ref} className='aria-mode-screenshot'>
- {screenshot && <img
- key={screenshot.file}
- src={model.createRelativeUrl(`file/${screenshot.file}`)}
+ {screencastFrame}
+ {!screencastFrame && imageUrl && <img
+ src={imageUrl}
alt='Screenshot'
onLoad={event => setNaturalSize({ width: event.currentTarget.naturalWidth, height: event.currentTarget.naturalHeight })}
/>}
- {!screenshot && <PlaceholderPanel text='No screenshot' />}
+ {!screencastFrame && !imageUrl && <PlaceholderPanel text='No screenshot' />}
{overlays}
</div>;
};
diff --git a/packages/trace-viewer/src/ui/filmStrip.tsx b/packages/trace-viewer/src/ui/filmStrip.tsx
index 4529b80d69..3bf5d2b9f0 100644
--- a/packages/trace-viewer/src/ui/filmStrip.tsx
+++ b/packages/trace-viewer/src/ui/filmStrip.tsx
@@ -20,8 +20,10 @@ import * as React from 'react';
import { useMeasure, upperBound } from '@web/uiUtils';
import type { PageEntry } from '@isomorphic/trace/entries';
import { useTraceModel } from './traceModelContext';
-import { useVideoThumbnails } from './videoThumbnails';
-import type { VideoThumbnail } from './videoThumbnails';
+import { useVideoSources } from './videoThumbnails';
+import type { VideoSource, VideoThumbnail } from './videoThumbnails';
+import { VideoFrame, lastVideoIndex } from './videoFrame';
+import type * as trace from '@isomorphic/trace/trace';
export type FilmStripPreviewPoint = {
x: number;
@@ -41,7 +43,7 @@ export const FilmStrip: React.FunctionComponent<{
const lanesRef = React.useRef<HTMLDivElement>(null);
const videos = (model?.videos ?? []).map(video => ({ video, url: model!.createRelativeUrl(`file/${video.file}`) }));
- const videoThumbnails = useVideoThumbnails(videos);
+ const videoSources = useVideoSources(videos);
let laneIndex = 0;
if (lanesRef.current && previewPoint) {
@@ -50,24 +52,30 @@ export const FilmStrip: React.FunctionComponent<{
}
const pageLanes = (model?.pages ?? []).filter(page => page.screencastFrames.length);
- const videoLanes = videoLanesByPage(videos.map(({ video }) => video.pageId), videoThumbnails);
+ const videoLanes = videoLanesByPage(videos.map(({ video }) => video), videoSources);
- let previewFrames: { timestamp: number, width: number, height: number, url: string }[] | undefined;
- if (laneIndex < pageLanes.length)
- previewFrames = model ? pageLanes[laneIndex]?.screencastFrames.map(frame => ({ ...frame, url: model.createRelativeUrl(`file/${frame.file}`) })) : undefined;
- else
- previewFrames = videoLanes[laneIndex - pageLanes.length];
-
- let previewImage = undefined;
- let previewSize = undefined;
- if (previewPoint !== undefined && previewFrames && previewFrames.length) {
+ // The thumbnails only index the lanes, the preview renders the video itself.
+ let preview: React.ReactNode | undefined;
+ let previewSize: Size | undefined;
+ if (model && previewPoint !== undefined) {
const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewPoint.x / measure.width;
- previewImage = previewFrames[upperBound(previewFrames, previewTime, timeComparator) - 1];
const fitInto = {
width: Math.min(800, (window.innerWidth / 2) | 0),
height: Math.min(800, (window.innerHeight / 2) | 0),
};
- previewSize = previewImage ? inscribe({ width: previewImage.width, height: previewImage.height }, fitInto) : undefined;
+ const frames = pageLanes[laneIndex]?.screencastFrames;
+ const frame = frames?.[upperBound(frames, previewTime, timeComparator) - 1];
+ const lane = videoLanes[laneIndex - pageLanes.length];
+ const videoIndex = lane ? lastVideoIndex(lane.videos, previewTime) : -1;
+ const videoUrl = videoIndex === -1 ? undefined : lane.sources[videoIndex].url;
+ if (frame) {
+ previewSize = inscribe(frame, fitInto);
+ preview = <img src={model.createRelativeUrl(`file/${frame.file}`)} width={previewSize.width} height={previewSize.height} />;
+ } else if (videoUrl) {
+ const video = lane.videos[videoIndex];
+ previewSize = inscribe(video, fitInto);
+ preview = <VideoFrame url={videoUrl} startTime={video.timestamp} time={previewTime} width={previewSize.width} height={previewSize.height} />;
+ }
}
return <div className='film-strip' ref={ref}>
@@ -78,37 +86,48 @@ export const FilmStrip: React.FunctionComponent<{
width={measure.width}
key={index}
/>)}
- {videoLanes.map((thumbnails, index) => <VideoFilmStripLane
+ {videoLanes.map((lane, index) => <VideoFilmStripLane
boundaries={boundaries}
- thumbnails={thumbnails}
+ thumbnails={lane.thumbnails}
width={measure.width}
key={'video-' + index}
/>)}
</div>
- {model && previewPoint && previewImage && previewSize &&
+ {previewPoint && preview && previewSize &&
<div className='film-strip-hover' style={{
top: measure.bottom + 5,
left: Math.min(previewPoint.x, measure.width - previewSize.width - 10),
width: previewSize.width,
height: previewSize.height,
}}>
- <img src={previewImage.url} width={previewSize.width} height={previewSize.height} />
+ {preview}
</div>
}
</div>;
};
-function videoLanesByPage(pageIds: string[], thumbnails: VideoThumbnail[][]): VideoThumbnail[][] {
- const lanes = new Map<string, VideoThumbnail[]>();
- pageIds.forEach((pageId, index) => {
- let lane = lanes.get(pageId);
+type VideoLane = {
+ videos: trace.VideoTraceEvent[];
+ sources: VideoSource[];
+ thumbnails: VideoThumbnail[];
+};
+
+function videoLanesByPage(videos: trace.VideoTraceEvent[], sources: VideoSource[]): VideoLane[] {
+ const lanes = new Map<string, VideoLane>();
+ videos.forEach((video, index) => {
+ let lane = lanes.get(video.pageId);
if (!lane) {
- lane = [];
- lanes.set(pageId, lane);
+ lane = { videos: [], sources: [], thumbnails: [] };
+ lanes.set(video.pageId, lane);
}
- lane.push(...thumbnails[index]);
+ lane.videos.push(video);
+ lane.sources.push(sources[index]);
+ lane.thumbnails.push(...sources[index].thumbnails);
+ });
+ return [...lanes.values()].filter(lane => lane.thumbnails.length).map(lane => {
+ lane.thumbnails.sort((a, b) => a.timestamp - b.timestamp);
+ return lane;
});
- return [...lanes.values()].filter(lane => lane.length).map(lane => lane.sort((a, b) => a.timestamp - b.timestamp));
}
const VideoFilmStripLane: React.FunctionComponent<{
diff --git a/packages/trace-viewer/src/ui/playbackControl.tsx b/packages/trace-viewer/src/ui/playbackControl.tsx
index 35aea8ef85..3c7a62fd86 100644
--- a/packages/trace-viewer/src/ui/playbackControl.tsx
+++ b/packages/trace-viewer/src/ui/playbackControl.tsx
@@ -28,11 +28,14 @@ export type PlaybackState = {
currentIndex: number;
percent: number;
animating: boolean;
+ screencastTime: number | undefined;
togglePlay: () => void;
stop: () => void;
prev: () => void;
next: () => void;
cycleSpeed: () => void;
+ selectAction: (action: ActionEntry) => void;
+ showSelectedAction: () => void;
onScrubberMouseDown: (e: React.MouseEvent) => void;
scrubberRef: React.RefObject<HTMLDivElement | null>;
actionsLength: number;
@@ -52,8 +55,8 @@ export function usePlayback(
const [playing, setPlaying] = React.useState(false);
const [speedIndex, setSpeedIndex] = React.useState(1);
const [dragging, setDragging] = React.useState(false);
- const [dragFraction, setDragFraction] = React.useState<number | undefined>(undefined);
- const [cursorTime, setCursorTime] = React.useState<number | undefined>(undefined);
+ // Set while playing or positioned between actions: the screencast is shown at this time instead of the action snapshot.
+ const [screencastTime, setScreencastTime] = React.useState<number | undefined>(undefined);
const speed = speeds[speedIndex];
const currentIndex = selectedAction ? actions.indexOf(selectedAction) : -1;
@@ -64,7 +67,6 @@ export function usePlayback(
const fullDuration = fullMax - fullMin || 1;
// Playback boundaries: constrained to time window when selected.
- const windowMin = timeWindow ? timeWindow.minimum : fullMin;
const windowMax = timeWindow ? timeWindow.maximum : fullMax;
// Actions within the effective window.
@@ -86,7 +88,12 @@ export function usePlayback(
const scrubberRef = React.useRef<HTMLDivElement>(null);
- const actionIndexAtTime = React.useCallback((t: number): number => {
+ const clampToWindow = React.useCallback((index: number) => {
+ return Math.max(firstWindowIndex, Math.min(lastWindowIndex, index));
+ }, [firstWindowIndex, lastWindowIndex]);
+
+ // Last action started at or before t, or the first action when none has started yet.
+ const lastStartedIndex = React.useCallback((t: number): number => {
let lo = 0;
let hi = actions.length - 1;
while (lo < hi) {
@@ -96,29 +103,13 @@ export function usePlayback(
else
hi = mid - 1;
}
- if (lo < actions.length - 1) {
- const distPrev = t - actions[lo].startTime;
- const distNext = actions[lo + 1].startTime - t;
- if (distNext < distPrev)
- lo = lo + 1;
- }
- // Clamp to window bounds.
- return Math.max(firstWindowIndex, Math.min(lastWindowIndex, lo));
- }, [actions, firstWindowIndex, lastWindowIndex]);
+ return lo;
+ }, [actions]);
const selectedTime = selectedAction ? selectedAction.startTime : fullMin;
+ const positionTime = screencastTime ?? selectedTime;
+ const percent = Math.max(0, Math.min(100, ((positionTime - fullMin) / fullDuration) * 100));
- let percent: number;
- if (dragging && dragFraction !== undefined)
- percent = dragFraction * 100;
- else if (playing && cursorTime !== undefined)
- percent = Math.max(0, Math.min(100, ((cursorTime - fullMin) / fullDuration) * 100));
- else
- percent = Math.max(0, Math.min(100, ((selectedTime - fullMin) / fullDuration) * 100));
-
- // Refs for raf closure.
- const windowMinRef = React.useRef(windowMin);
- windowMinRef.current = windowMin;
const windowMaxRef = React.useRef(windowMax);
windowMaxRef.current = windowMax;
@@ -127,13 +118,8 @@ export function usePlayback(
return;
let rafId: number;
let lastFrameTime: number | undefined;
- let traceTime = selectedTime;
- // If starting from before the window, jump to window start.
- if (traceTime < windowMinRef.current)
- traceTime = windowMinRef.current;
- let lastSelectedIndex = currentIndex;
-
- setCursorTime(traceTime);
+ let traceTime = positionTime;
+ let lastSelectedIndex = clampToWindow(lastStartedIndex(traceTime));
const tick = (now: number) => {
if (lastFrameTime !== undefined) {
@@ -141,16 +127,17 @@ export function usePlayback(
traceTime = Math.min(traceTime + delta, windowMaxRef.current);
}
lastFrameTime = now;
- setCursorTime(traceTime);
+ setScreencastTime(traceTime);
- const idx = actionIndexAtTime(traceTime);
- if (idx !== lastSelectedIndex) {
- lastSelectedIndex = idx;
- onActionSelectedRef.current(actionsRef.current[idx]);
+ const index = clampToWindow(lastStartedIndex(traceTime));
+ if (index !== lastSelectedIndex) {
+ lastSelectedIndex = index;
+ onActionSelectedRef.current(actionsRef.current[index]);
}
if (traceTime >= windowMaxRef.current) {
setPlaying(false);
+ setScreencastTime(undefined);
return;
}
rafId = requestAnimationFrame(tick);
@@ -160,38 +147,51 @@ export function usePlayback(
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [playing, speed]);
- React.useEffect(() => {
- if (!playing)
- setCursorTime(undefined);
- }, [playing]);
-
const togglePlay = React.useCallback(() => {
if (!actions.length)
return;
- // Always restart from the window start when at the end (or beyond the window).
- const atEnd = currentIndex >= lastWindowIndex;
- if (!playing && atEnd)
+ if (playing) {
+ setPlaying(false);
+ setScreencastTime(undefined);
+ return;
+ }
+ // Restart from the window start when at the end, or outside the window.
+ if (currentIndex >= lastWindowIndex || currentIndex < firstWindowIndex)
onActionSelected(actions[firstWindowIndex]);
- setPlaying(!playing);
+ setPlaying(true);
}, [playing, actions, currentIndex, onActionSelected, firstWindowIndex, lastWindowIndex]);
+ const selectAction = React.useCallback((action: ActionEntry) => {
+ setPlaying(false);
+ setScreencastTime(undefined);
+ onActionSelectedRef.current(action);
+ }, []);
+
+ // Leave the between-actions position, so that the selected action's snapshot is shown.
+ const showSelectedAction = React.useCallback(() => {
+ setPlaying(false);
+ setScreencastTime(undefined);
+ }, []);
+
const stop = React.useCallback(() => {
setPlaying(false);
+ setScreencastTime(undefined);
if (actions.length)
onActionSelected(actions[firstWindowIndex]);
}, [actions, onActionSelected, firstWindowIndex]);
+ const canPrev = currentIndex > firstWindowIndex;
+ const canNext = currentIndex < lastWindowIndex;
+
const prev = React.useCallback(() => {
- const target = Math.max(currentIndex - 1, firstWindowIndex);
- if (target !== currentIndex)
- onActionSelected(actions[target]);
- }, [actions, currentIndex, onActionSelected, firstWindowIndex]);
+ if (canPrev)
+ selectAction(actions[currentIndex - 1]);
+ }, [actions, canPrev, currentIndex, selectAction]);
const next = React.useCallback(() => {
- const target = Math.min(currentIndex + 1, lastWindowIndex);
- if (target !== currentIndex)
- onActionSelected(actions[target]);
- }, [actions, currentIndex, onActionSelected, lastWindowIndex]);
+ if (canNext)
+ selectAction(actions[currentIndex + 1]);
+ }, [actions, canNext, currentIndex, selectAction]);
const cycleSpeed = React.useCallback(() => {
setSpeedIndex(i => (i + 1) % speeds.length);
@@ -199,20 +199,21 @@ export function usePlayback(
React.useEffect(() => {
setPlaying(false);
+ setScreencastTime(undefined);
}, [actions]);
- const fractionFromMouseEvent = React.useCallback((e: MouseEvent | React.MouseEvent) => {
+ const seekToMouseEvent = React.useCallback((e: MouseEvent | React.MouseEvent, snap: boolean) => {
const rect = scrubberRef.current!.getBoundingClientRect();
- return Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width));
- }, []);
-
- const selectActionAtFraction = React.useCallback((fraction: number) => {
- if (!actions.length)
- return;
- const t = fullMin + fraction * fullDuration;
- const idx = actionIndexAtTime(t);
- onActionSelectedRef.current(actionsRef.current[idx]);
- }, [actions, fullMin, fullDuration, actionIndexAtTime]);
+ const fraction = Math.max(0, Math.min(1, (e.clientX - rect.left) / rect.width));
+ const time = fullMin + fraction * fullDuration;
+ // The screencast follows the pointer while dragging, releasing snaps to the action snapshot.
+ setScreencastTime(snap ? undefined : time);
+ let index = lastStartedIndex(time);
+ const next = actions[index + 1];
+ if (next && next.startTime - time < time - actions[index].startTime)
+ ++index;
+ onActionSelectedRef.current(actions[clampToWindow(index)]);
+ }, [actions, fullMin, fullDuration, clampToWindow, lastStartedIndex]);
const dragCleanupRef = React.useRef<(() => void) | null>(null);
@@ -228,22 +229,16 @@ export function usePlayback(
scrubberRef.current?.focus();
setDragging(true);
setPlaying(false);
- const fraction = fractionFromMouseEvent(e);
- setDragFraction(fraction);
- selectActionAtFraction(fraction);
+ seekToMouseEvent(e, false);
const onMouseMove = (me: MouseEvent) => {
- const f = fractionFromMouseEvent(me);
- setDragFraction(f);
- selectActionAtFraction(f);
+ seekToMouseEvent(me, false);
};
const onMouseUp = (me: MouseEvent) => {
document.removeEventListener('mousemove', onMouseMove);
document.removeEventListener('mouseup', onMouseUp);
dragCleanupRef.current = null;
- const f = fractionFromMouseEvent(me);
- selectActionAtFraction(f);
- setDragFraction(undefined);
+ seekToMouseEvent(me, true);
setDragging(false);
};
document.addEventListener('mousemove', onMouseMove);
@@ -252,18 +247,16 @@ export function usePlayback(
document.removeEventListener('mousemove', onMouseMove);
document.removeEventListener('mouseup', onMouseUp);
};
- }, [actions, selectActionAtFraction, fractionFromMouseEvent]);
+ }, [actions, seekToMouseEvent]);
const animating = !playing && !dragging;
const ticks = actions.length > 0 && actions.length <= 200 ? actions.map(a => ((a.startTime - fullMin) / fullDuration) * 100) : undefined;
- const canPrev = currentIndex > firstWindowIndex;
- const canNext = currentIndex < lastWindowIndex;
const canStop = playing || currentIndex > firstWindowIndex;
return {
- playing, speed, currentIndex, percent, animating,
- togglePlay, stop, prev, next, cycleSpeed,
+ playing, speed, currentIndex, percent, animating, screencastTime,
+ togglePlay, stop, prev, next, cycleSpeed, selectAction, showSelectedAction,
onScrubberMouseDown, scrubberRef, actionsLength: actions.length,
canPrev, canNext, canStop, ticks,
};
diff --git a/packages/trace-viewer/src/ui/snapshotTab.css b/packages/trace-viewer/src/ui/snapshotTab.css
index 5265f4f5f1..c895859d9a 100644
--- a/packages/trace-viewer/src/ui/snapshotTab.css
+++ b/packages/trace-viewer/src/ui/snapshotTab.css
@@ -105,6 +105,16 @@ iframe.snapshot-visible[name=snapshot] {
visibility: visible;
}
+.snapshot-switcher .screencast-frame {
+ position: absolute;
+ top: 0;
+ left: 0;
+ width: 100%;
+ height: 100%;
+ object-fit: contain;
+ background: light-dark(white, #333);
+}
+
.no-snapshot {
text-align: center;
padding: 50px;
diff --git a/packages/trace-viewer/src/ui/snapshotTab.tsx b/packages/trace-viewer/src/ui/snapshotTab.tsx
index 6cd00da5eb..7fdf5b9f77 100644
--- a/packages/trace-viewer/src/ui/snapshotTab.tsx
+++ b/packages/trace-viewer/src/ui/snapshotTab.tsx
@@ -21,7 +21,7 @@ import { nextActionByStartTime, previousActionByEndTime } from '@isomorphic/trac
import type { TraceModel } from '@isomorphic/trace/traceModel';
import { Toolbar } from '@web/components/toolbar';
import { ToolbarButton } from '@web/components/toolbarButton';
-import { clsx, useMeasure, useSetting } from '@web/uiUtils';
+import { clsx, upperBound, useMeasure, useSetting } from '@web/uiUtils';
import { InjectedScript } from '@injected/injectedScript';
import { Recorder } from '@injected/recorder/recorder';
import { asLocator } from '@isomorphic/locatorGenerators';
@@ -36,6 +36,8 @@ import yaml from 'yaml';
import { PlaybackButtons } from './playbackControl';
import type { PlaybackState } from './playbackControl';
import { AriaModeView, collectAriaModeTargets, shouldDisplayAriaMode } from './ariaModeView';
+import { useVideoSources } from './videoThumbnails';
+import { VideoFrame, lastVideoIndex } from './videoFrame';
export type HighlightedElement = {
locator?: string,
@@ -52,13 +54,16 @@ export const SnapshotTabsView: React.FunctionComponent<{
setIsInspecting: (isInspecting: boolean) => void,
highlightedElement: HighlightedElement,
setHighlightedElement: (element: HighlightedElement) => void,
+ // Time to show the screencast at, undefined to show the action snapshot.
+ screencastTime?: number,
playback: PlaybackState
-}> = ({ action, model, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement, playback }) => {
+}> = ({ action, model, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement, screencastTime, playback }) => {
const [snapshotTab, setSnapshotTab] = React.useState<'action'|'before'|'after'>('action');
const [shouldPopulateCanvasFromScreenshot] = useSetting('shouldPopulateCanvasFromScreenshot', false);
const [displayAriaModeSetting] = useSetting('displayAriaMode', false);
const displayAriaMode = shouldDisplayAriaMode(model, displayAriaModeSetting);
+ const screencastFrame = useScreencastFrame(model, screencastTime, playback.playing, playback.speed);
const snapshots = React.useMemo(() => {
return collectSnapshots(model, action);
@@ -107,6 +112,7 @@ export const SnapshotTabsView: React.FunctionComponent<{
target={ariaModeTargets[snapshotTab]}
point={snapshotTab === 'action' ? action?.point : undefined}
box={snapshotTab === 'action' ? action?.box : undefined}
+ screencastFrame={screencastFrame}
/>}
{!displayAriaMode && <SnapshotView
snapshotUrls={snapshotUrls}
@@ -116,6 +122,7 @@ export const SnapshotTabsView: React.FunctionComponent<{
setIsInspecting={setIsInspecting}
highlightedElement={highlightedElement}
setHighlightedElement={setHighlightedElement}
+ screencastFrame={screencastFrame}
/>}
</div>;
};
@@ -128,7 +135,8 @@ export const SnapshotView: React.FunctionComponent<{
setIsInspecting: (isInspecting: boolean) => void,
highlightedElement: HighlightedElement,
setHighlightedElement: (element: HighlightedElement) => void,
-}> = ({ snapshotUrls, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement }) => {
+ screencastFrame?: React.ReactNode,
+}> = ({ snapshotUrls, sdkLanguage, testIdAttributeName, isInspecting, setIsInspecting, highlightedElement, setHighlightedElement, screencastFrame }) => {
const iframeRef0 = React.useRef<HTMLIFrameElement>(null);
const iframeRef1 = React.useRef<HTMLIFrameElement>(null);
const [snapshotInfo, setSnapshotInfo] = React.useState<SnapshotInfo>({ viewport: kDefaultViewport, url: '' });
@@ -146,7 +154,8 @@ export const SnapshotView: React.FunctionComponent<{
if (loadingRef.current.iteration !== thisIteration)
return;
- const iframe = [iframeRef0, iframeRef1][newVisibleIframe].current;
+ // While the screencast frame covers the snapshot, only take the viewport and the url from it.
+ const iframe = screencastFrame ? undefined : [iframeRef0, iframeRef1][newVisibleIframe].current;
if (iframe) {
let loadedCallback = () => {};
const loadedPromise = new Promise<void>(f => loadedCallback = f);
@@ -172,10 +181,11 @@ export const SnapshotView: React.FunctionComponent<{
if (loadingRef.current.iteration !== thisIteration)
return;
- loadingRef.current.visibleIframe = newVisibleIframe;
+ if (iframe)
+ loadingRef.current.visibleIframe = newVisibleIframe;
setSnapshotInfo(newSnapshotInfo);
})();
- }, [snapshotUrls]);
+ }, [snapshotUrls, screencastFrame]);
return <div
className='vbox'
@@ -207,6 +217,7 @@ export const SnapshotView: React.FunctionComponent<{
<div className='snapshot-switcher'>
<iframe ref={iframeRef0} name='snapshot' title='DOM Snapshot' sandbox='allow-same-origin allow-scripts' className={clsx(loadingRef.current.visibleIframe === 0 && 'snapshot-visible')}></iframe>
<iframe ref={iframeRef1} name='snapshot' title='DOM Snapshot' sandbox='allow-same-origin allow-scripts' className={clsx(loadingRef.current.visibleIframe === 1 && 'snapshot-visible')}></iframe>
+ {screencastFrame}
</div>
</SnapshotWrapper>
</div>;
@@ -478,3 +489,27 @@ export async function fetchSnapshotInfo(snapshotInfoUrl: string | undefined) {
}
export const kDefaultViewport = { width: 1280, height: 720 };
+
+function useScreencastFrame(model: TraceModel | undefined, time: number | undefined, playing: boolean, speed: number): React.ReactNode | undefined {
+ const videos = model && time !== undefined ? model.videos.map(video => ({ video, url: model.createRelativeUrl(`file/${video.file}`) })) : [];
+ const videoSources = useVideoSources(videos);
+ if (!model || time === undefined)
+ return undefined;
+
+ // A trace has either screencast frames or videos, never both.
+ // Show the page that painted most recently.
+ let frame: { timestamp: number, file: string } | undefined;
+ for (const page of model.pages) {
+ const candidate = page.screencastFrames[upperBound(page.screencastFrames, time, (t, f) => t - f.timestamp) - 1];
+ if (candidate && (!frame || candidate.timestamp > frame.timestamp))
+ frame = candidate;
+ }
+ if (frame)
+ return <img className='screencast-frame' src={model.createRelativeUrl(`file/${frame.file}`)} alt='Screencast frame' />;
+
+ const index = lastVideoIndex(videos.map(({ video }) => video), time);
+ const url = videoSources[index]?.url;
+ if (!url)
+ return undefined;
+ return <VideoFrame className='screencast-frame' url={url} startTime={videos[index].video.timestamp} time={time} playing={playing} speed={speed} />;
+}
diff --git a/packages/trace-viewer/src/ui/videoFrame.tsx b/packages/trace-viewer/src/ui/videoFrame.tsx
new file mode 100644
index 0000000000..651971b32c
--- /dev/null
+++ b/packages/trace-viewer/src/ui/videoFrame.tsx
@@ -0,0 +1,79 @@
+/*
+ Copyright (c) Microsoft Corporation.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+import * as React from 'react';
+
+import type * as trace from '@isomorphic/trace/trace';
+
+// Playback runs the video on its own clock and only resyncs when it drifts this far away.
+const maxDriftSeconds = 0.25;
+
+// Video that was recording at the given trace time, -1 when none has started yet.
+export function lastVideoIndex(videos: trace.VideoTraceEvent[], time: number): number {
+ let result = -1;
+ videos.forEach((video, index) => {
+ if (video.timestamp <= time && (result === -1 || video.timestamp > videos[result].timestamp))
+ result = index;
+ });
+ return result;
+}
+
+export const VideoFrame: React.FunctionComponent<{
+ url: string,
+ // Video start in the trace time.
+ startTime: number,
+ // Trace time to render.
+ time: number,
+ playing?: boolean,
+ speed?: number,
+ className?: string,
+ width?: number,
+ height?: number,
+}> = ({ url, startTime, time, playing, speed, className, width, height }) => {
+ const ref = React.useRef<HTMLVideoElement>(null);
+ const offset = Math.max(0, (time - startTime) / 1000);
+ const offsetRef = React.useRef(offset);
+ offsetRef.current = offset;
+
+ React.useEffect(() => {
+ const video = ref.current;
+ if (!video)
+ return;
+ if (!playing) {
+ video.pause();
+ if (Math.abs(video.currentTime - offset) > 0.001)
+ video.currentTime = offset;
+ return;
+ }
+ video.playbackRate = speed ?? 1;
+ if (Math.abs(video.currentTime - offset) > maxDriftSeconds)
+ video.currentTime = offset;
+ if (video.paused)
+ void video.play().catch(() => {});
+ }, [url, offset, playing, speed]);
+
+ return <video
+ ref={ref}
+ className={className}
+ src={url}
+ width={width}
+ height={height}
+ muted
+ playsInline
+ preload='auto'
+ onLoadedMetadata={event => event.currentTarget.currentTime = offsetRef.current}
+ />;
+};
diff --git a/packages/trace-viewer/src/ui/videoThumbnails.ts b/packages/trace-viewer/src/ui/videoThumbnails.ts
index b9abf73d33..7dc4c603c8 100644
--- a/packages/trace-viewer/src/ui/videoThumbnails.ts
+++ b/packages/trace-viewer/src/ui/videoThumbnails.ts
@@ -24,7 +24,15 @@ export type VideoThumbnail = {
height: number;
};
+export type VideoSource = {
+ // Blob url of the whole video, so that the player can seek without range requests.
+ readonly url: string | undefined;
+ // Keyframes for the film strip, the player renders the video itself.
+ readonly thumbnails: VideoThumbnail[];
+};
+
type CacheEntry = {
+ url: string | undefined;
thumbnails: VideoThumbnail[];
listeners: Set<() => void>;
started: boolean;
@@ -35,7 +43,7 @@ const cache = new Map<string, CacheEntry>();
const maxThumbnails = 120;
const thumbnailsPerSecond = 2;
-export function useVideoThumbnails(videos: { video: trace.VideoTraceEvent, url: string }[]): VideoThumbnail[][] {
+export function useVideoSources(videos: { video: trace.VideoTraceEvent, url: string }[]): VideoSource[] {
const [, setVersion] = React.useState(0);
const entries = videos.map(({ video, url }) => ensureEntry(video, url));
const urls = videos.map(({ url }) => url).join('\n');
@@ -49,13 +57,13 @@ export function useVideoThumbnails(videos: { video: trace.VideoTraceEvent, url:
entry.listeners.delete(listener);
};
}, [urls]);
- return entries.map(entry => entry.thumbnails);
+ return entries;
}
function ensureEntry(video: trace.VideoTraceEvent, videoUrl: string): CacheEntry {
let entry = cache.get(videoUrl);
if (!entry) {
- entry = { thumbnails: [], listeners: new Set(), started: false };
+ entry = { url: undefined, thumbnails: [], listeners: new Set(), started: false };
cache.set(videoUrl, entry);
}
if (!entry.started) {
@@ -69,6 +77,9 @@ async function generateThumbnails(entry: CacheEntry, video: trace.VideoTraceEven
const response = await fetch(videoUrl);
const blob = await response.blob();
const objectUrl = URL.createObjectURL(blob);
+ entry.url = objectUrl;
+ for (const listener of entry.listeners)
+ listener();
const element = document.createElement('video');
element.muted = true;
element.preload = 'auto';
@@ -113,9 +124,9 @@ async function generateThumbnails(entry: CacheEntry, video: trace.VideoTraceEven
listener();
}
} finally {
+ // The blob url outlives thumbnail generation, the player seeks in it.
element.removeAttribute('src');
element.load();
- URL.revokeObjectURL(objectUrl);
}
}
diff --git a/packages/trace-viewer/src/ui/workbench.tsx b/packages/trace-viewer/src/ui/workbench.tsx
index ab31fc558b..12c47c5590 100644
--- a/packages/trace-viewer/src/ui/workbench.tsx
+++ b/packages/trace-viewer/src/ui/workbench.tsx
@@ -178,8 +178,11 @@ const PartitionedWorkbench: React.FunctionComponent<WorkbenchProps & { partition
const setIsInspecting = React.useCallback((value: boolean) => {
if (!isInspecting && value)
selectPropertiesTab('inspector');
+ // Inspecting works against the DOM snapshot, not the screencast frame.
+ if (value)
+ playback.showSelectedAction();
setIsInspectingState(value);
- }, [setIsInspectingState, selectPropertiesTab, isInspecting]);
+ }, [setIsInspectingState, selectPropertiesTab, isInspecting, playback.showSelectedAction]);
const elementPicked = React.useCallback((element: HighlightedElement) => {
setHighlightedElement(element);
@@ -234,7 +237,7 @@ const PartitionedWorkbench: React.FunctionComponent<WorkbenchProps & { partition
errorCount: errorsModel.errors.size,
render: () => <ErrorsTab errorsModel={errorsModel} testRunMetadata={testRunMetadata} sdkLanguage={sdkLanguage} revealInSource={error => {
if (error.action)
- setSelectedAction(error.action);
+ playback.selectAction(error.action);
else
setRevealedErrorKey(error.message);
selectPropertiesTab('source');
@@ -347,7 +350,7 @@ const PartitionedWorkbench: React.FunctionComponent<WorkbenchProps & { partition
setSelectedTime={setSelectedTime}
treeState={treeState}
setTreeState={setTreeState}
- onSelected={onActionSelected}
+ onSelected={playback.selectAction}
onHighlighted={setHighlightedAction}
revealActionAttachment={revealActionAttachment}
revealConsole={() => selectPropertiesTab('console')}
@@ -368,7 +371,7 @@ const PartitionedWorkbench: React.FunctionComponent<WorkbenchProps & { partition
{!hideTimeline && <Timeline
model={model}
boundaries={boundaries}
- onSelected={onActionSelected}
+ onSelected={playback.selectAction}
selectedTime={selectedTime}
setSelectedTime={setSelectedTime}
highlightedTime={highlightedTime}
@@ -391,6 +394,7 @@ const PartitionedWorkbench: React.FunctionComponent<WorkbenchProps & { partition
setIsInspecting={setIsInspecting}
highlightedElement={highlightedElement}
setHighlightedElement={elementPicked}
+ screencastTime={highlightedAction ? undefined : playback.screencastTime}
playback={playback} />}
sidebar={
<TabbedPane
diff --git a/tests/library/trace-viewer-scrub.spec.ts b/tests/library/trace-viewer-scrub.spec.ts
index 0b5856d208..e633720002 100644
--- a/tests/library/trace-viewer-scrub.spec.ts
+++ b/tests/library/trace-viewer-scrub.spec.ts
@@ -192,3 +192,89 @@ test('should drag scrubber to select action', async ({ runAndTrace, page: action
const scrubberValue = await scrubber.getAttribute('aria-valuenow');
expect(Number(scrubberValue)).toBeGreaterThan(50);
});
+
+test('should show screencast while playing and snapshot when stopped', async ({ runAndTrace, page: actionPage }) => {
+ const traceViewer = await runAndTrace(async () => {
+ await actionPage.setContent('<button>Click me</button>');
+ await actionPage.click('button');
+ await actionPage.setContent('<input/>');
+ await actionPage.waitForTimeout(2000);
+ });
+ const page = traceViewer.page;
+ const screencast = page.getByRole('img', { name: 'Screencast frame' });
+
+ await traceViewer.selectAction('Set content');
+ await expect(traceViewer.snapshotContainer).toBeVisible();
+ await expect(screencast).toHaveCount(0);
+
+ // Selecting an action stops the playback and snaps to its snapshot.
+ await page.getByRole('button', { name: 'Play' }).click();
+ await expect(screencast).toBeVisible();
+ await traceViewer.selectAction('Click');
+ await expect(screencast).toHaveCount(0);
+ await expect(page.getByRole('button', { name: 'Play' })).toBeVisible();
+ await expect(page.frameLocator('iframe.snapshot-visible[name=snapshot]').getByRole('button', { name: 'Click me' })).toBeVisible();
+
+ // Reaching the end snaps to the last action as well.
+ await page.getByRole('button', { name: 'Play' }).click();
+ await expect(screencast).toBeVisible();
+ await expect(page.getByRole('button', { name: 'Play' })).toBeVisible({ timeout: 15000 });
+ await expect(screencast).toHaveCount(0);
+ await expect(traceViewer.snapshotContainer).toBeVisible();
+});
+
+test('should reveal snapshot when hovering an action or picking a locator during playback', async ({ runAndTrace, page: actionPage }) => {
+ const traceViewer = await runAndTrace(async () => {
+ await actionPage.setContent('<button>Click me</button>');
+ await actionPage.click('button');
+ await actionPage.setContent('<input/>');
+ await actionPage.waitForTimeout(2000);
+ });
+ const page = traceViewer.page;
+ const screencast = page.getByRole('img', { name: 'Screencast frame' });
+ const snapshot = page.frameLocator('iframe.snapshot-visible[name=snapshot]');
+
+ // Hovering an action reveals its snapshot.
+ await traceViewer.selectAction('Set content');
+ await page.getByRole('button', { name: 'Play' }).click();
+ await expect(screencast).toBeVisible();
+ await traceViewer.hoverAction('Click');
+ await expect(screencast).toHaveCount(0);
+ await expect(snapshot.getByRole('button', { name: 'Click me' })).toBeVisible();
+
+ // Picking a locator stops the playback, the snapshot receives mouse events.
+ await traceViewer.selectAction('Set content');
+ await page.getByRole('button', { name: 'Play' }).click();
+ await expect(screencast).toBeVisible();
+ await page.getByTitle('Pick locator').click();
+ await expect(screencast).toHaveCount(0);
+ await expect(traceViewer.snapshotContainer).toBeVisible();
+ await snapshot.locator('body').click();
+ await expect(page.locator('.cm-wrapper').first()).toContainText('locator(');
+});
+
+test('should show screencast while dragging and snap to the action on release', async ({ runAndTrace, page: actionPage }) => {
+ const traceViewer = await runAndTrace(async () => {
+ await actionPage.setContent('<button>Click me</button>');
+ await actionPage.click('button');
+ await actionPage.setContent('<input/>');
+ });
+ const page = traceViewer.page;
+ const screencast = page.getByRole('img', { name: 'Screencast frame' });
+ const scrubber = page.getByRole('slider', { name: 'Playback position' });
+ const box = (await scrubber.boundingBox())!;
+
+ // While dragging, the screencast follows the pointer between the action ticks.
+ await page.mouse.move(box.x + box.width * 0.5, box.y + box.height / 2);
+ await page.mouse.down();
+ await page.mouse.move(box.x + box.width * 0.99, box.y + box.height / 2);
+ await expect(screencast).toBeVisible();
+ await expect(scrubber).toHaveAttribute('aria-valuenow', '99');
+
+ // Releasing snaps to the nearest action and shows its snapshot.
+ await page.mouse.up();
+ await expect(screencast).toHaveCount(0);
+ await expect(traceViewer.snapshotContainer).toBeVisible();
+ await expect(traceViewer.actionsTree.getByRole('treeitem', { selected: true })).toHaveText(/Set content/);
+ expect(Number(await scrubber.getAttribute('aria-valuenow'))).toBeLessThan(99);
+});