Skip to content

Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.

feat(trace-viewer): webm-based film strip (#42319)

Pavel Feldman0d86401890ab
diff --git a/packages/isomorphic/trace/entries.ts b/packages/isomorphic/trace/entries.ts
index 09c71959f3..a762d285bd 100644
--- a/packages/isomorphic/trace/entries.ts
+++ b/packages/isomorphic/trace/entries.ts
@@ -39,6 +39,7 @@ export type ContextEntry = {
actions: ActionEntry[];
screenshots: trace.ScreenshotTraceEvent[];
ariaSnapshots: trace.AriaSnapshotTraceEvent[];
+ videos: trace.VideoTraceEvent[];
events: (trace.EventTraceEvent | trace.ConsoleMessageTraceEvent)[];
stdio: trace.StdioTraceEvent[];
errors: trace.ErrorTraceEvent[];
diff --git a/packages/isomorphic/trace/traceLoader.ts b/packages/isomorphic/trace/traceLoader.ts
index 40c0b0e4ba..9daa159473 100644
--- a/packages/isomorphic/trace/traceLoader.ts
+++ b/packages/isomorphic/trace/traceLoader.ts
@@ -152,6 +152,7 @@ function createEmptyContext(): ContextEntry {
actions: [],
screenshots: [],
ariaSnapshots: [],
+ videos: [],
events: [],
errors: [],
stdio: [],
diff --git a/packages/isomorphic/trace/traceModel.ts b/packages/isomorphic/trace/traceModel.ts
index 27185f56e7..459088a7e8 100644
--- a/packages/isomorphic/trace/traceModel.ts
+++ b/packages/isomorphic/trace/traceModel.ts
@@ -66,6 +66,7 @@ export class TraceModel {
readonly title?: string;
readonly options: trace.BrowserContextEventOptions;
readonly pages: PageEntry[];
+ readonly videos: trace.VideoTraceEvent[];
readonly actions: ActionEntry[];
readonly attachments: Attachment[];
readonly visibleAttachments: Attachment[];
@@ -105,6 +106,7 @@ export class TraceModel {
// Next call updates all timestamps for all events in library contexts, so it must be done first.
this.actions = mergeActionsAndUpdateTiming(contexts);
this.pages = ([] as PageEntry[]).concat(...contexts.map(c => c.pages));
+ this.videos = [];
this.wallTime = contexts.map(c => c.wallTime).reduce((prev, cur) => Math.min(prev || Number.MAX_VALUE, cur!), Number.MAX_VALUE);
this.startTime = contexts.map(c => c.startTime).reduce((prev, cur) => Math.min(prev, cur), Number.MAX_VALUE);
this.endTime = contexts.map(c => c.endTime).reduce((prev, cur) => Math.max(prev, cur), Number.MIN_VALUE);
@@ -126,6 +128,7 @@ export class TraceModel {
this._screenshots.set(`${event.callId}/${event.phase}`, event);
for (const event of context.ariaSnapshots || [])
this._ariaSnapshots.set(`${event.callId}/${event.phase}`, event);
+ this.videos.push(...(context.videos || []));
}
this.attachments = this.actions.flatMap(action => action.attachments?.map(attachment => ({ ...attachment, callId: action.callId, traceUri })) ?? []);
this.visibleAttachments = this.attachments.filter(attachment => !attachment.name.startsWith('_'));
@@ -344,6 +347,8 @@ function adjustMonotonicTime(context: ContextEntry, monotonicTimeDelta: number)
for (const frame of page.screencastFrames)
frame.timestamp += monotonicTimeDelta;
}
+ for (const video of context.videos || [])
+ video.timestampOrigin += monotonicTimeDelta;
for (const resource of context.resources) {
if (resource._monotonicTime)
resource._monotonicTime += monotonicTimeDelta;
diff --git a/packages/isomorphic/trace/traceModernizer.ts b/packages/isomorphic/trace/traceModernizer.ts
index 48012f9b0d..f883aaa37f 100644
--- a/packages/isomorphic/trace/traceModernizer.ts
+++ b/packages/isomorphic/trace/traceModernizer.ts
@@ -110,6 +110,10 @@ export class TraceModernizer {
contextEntry.screenshots.push(event);
break;
}
+ case 'video': {
+ contextEntry.videos.push(event);
+ break;
+ }
case 'aria-snapshot': {
contextEntry.ariaSnapshots.push(event);
break;
diff --git a/packages/trace-viewer/src/ui/filmStrip.tsx b/packages/trace-viewer/src/ui/filmStrip.tsx
index a264b28d4c..37a704ad61 100644
--- a/packages/trace-viewer/src/ui/filmStrip.tsx
+++ b/packages/trace-viewer/src/ui/filmStrip.tsx
@@ -20,6 +20,8 @@ import * as React from 'react';
import { useMeasure, upperBound } from '@web/uiUtils';
import type { PageEntry } from '@isomorphic/trace/entries';
import { useTraceModel } from './traceModelContext';
+import { useVideoThumbnails } from './videoThumbnails';
+import type { VideoThumbnail } from './videoThumbnails';
export type FilmStripPreviewPoint = {
x: number;
@@ -38,18 +40,29 @@ export const FilmStrip: React.FunctionComponent<{
const [measure, ref] = useMeasure<HTMLDivElement>();
const lanesRef = React.useRef<HTMLDivElement>(null);
- let pageIndex = 0;
+ const video = model?.videos?.[0];
+ const videoThumbnails = useVideoThumbnails(video, video && model ? model.createRelativeUrl(`file/${video.file}`) : undefined);
+
+ let laneIndex = 0;
if (lanesRef.current && previewPoint) {
const bounds = lanesRef.current.getBoundingClientRect();
- pageIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0;
+ laneIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0;
}
- const screencastFrames = model?.pages?.[pageIndex]?.screencastFrames;
+ const pageLanes = (model?.pages ?? []).filter(page => page.screencastFrames.length);
+ const videoLanes: VideoThumbnail[][] = videoThumbnails.length ? [videoThumbnails] : [];
+
+ let previewFrames: { timestamp: number, width: number, height: number, url: string }[] | undefined;
+ if (laneIndex < pageLanes.length)
+ previewFrames = model ? pageLanes[laneIndex]?.screencastFrames.map(frame => ({ ...frame, url: model.createRelativeUrl(`file/${frame.file}`) })) : undefined;
+ else
+ previewFrames = videoLanes[laneIndex - pageLanes.length];
+
let previewImage = undefined;
let previewSize = undefined;
- if (previewPoint !== undefined && screencastFrames && screencastFrames.length) {
+ if (previewPoint !== undefined && previewFrames && previewFrames.length) {
const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewPoint.x / measure.width;
- previewImage = screencastFrames[upperBound(screencastFrames, previewTime, timeComparator) - 1];
+ previewImage = previewFrames[upperBound(previewFrames, previewTime, timeComparator) - 1];
const fitInto = {
width: Math.min(800, (window.innerWidth / 2) | 0),
height: Math.min(800, (window.innerHeight / 2) | 0),
@@ -58,14 +71,20 @@ export const FilmStrip: React.FunctionComponent<{
}
return <div className='film-strip' ref={ref}>
- <div className='film-strip-lanes' ref={lanesRef}>{
- model?.pages.map((page, index) => page.screencastFrames.length ? <FilmStripLane
+ <div className='film-strip-lanes' ref={lanesRef}>
+ {pageLanes.map((page, index) => <FilmStripLane
boundaries={boundaries}
page={page}
width={measure.width}
key={index}
- /> : null)
- }</div>
+ />)}
+ {videoLanes.map((thumbnails, index) => <VideoFilmStripLane
+ boundaries={boundaries}
+ thumbnails={thumbnails}
+ width={measure.width}
+ key={'video-' + index}
+ />)}
+ </div>
{model && previewPoint && previewImage && previewSize &&
<div className='film-strip-hover' style={{
top: measure.bottom + 5,
@@ -73,12 +92,61 @@ export const FilmStrip: React.FunctionComponent<{
width: previewSize.width,
height: previewSize.height,
}}>
- <img src={model.createRelativeUrl(`file/${previewImage.file}`)} width={previewSize.width} height={previewSize.height} />
+ <img src={previewImage.url} width={previewSize.width} height={previewSize.height} />
</div>
}
</div>;
};
+const VideoFilmStripLane: React.FunctionComponent<{
+ boundaries: Boundaries,
+ thumbnails: VideoThumbnail[],
+ width: number,
+}> = ({ boundaries, thumbnails, width }) => {
+ const viewportSize = { width: 0, height: 0 };
+ for (const thumbnail of thumbnails) {
+ viewportSize.width = Math.max(viewportSize.width, thumbnail.width);
+ viewportSize.height = Math.max(viewportSize.height, thumbnail.height);
+ }
+ const frameSize = inscribe(viewportSize, tileSize);
+ const startTime = thumbnails[0].timestamp;
+ const endTime = thumbnails[thumbnails.length - 1].timestamp;
+
+ const boundariesDuration = boundaries.maximum - boundaries.minimum;
+ const gapLeft = (startTime - boundaries.minimum) / boundariesDuration * width;
+ const gapRight = (boundaries.maximum - endTime) / boundariesDuration * width;
+ const effectiveWidth = (endTime - startTime) / boundariesDuration * width;
+ const frameCount = (effectiveWidth / (frameSize.width + 2 * frameMargin)) | 0;
+ const frameDuration = (endTime - startTime) / frameCount;
+
+ const frames: React.JSX.Element[] = [];
+ for (let i = 0; startTime && frameDuration && i < frameCount; ++i) {
+ const time = startTime + frameDuration * i;
+ const index = upperBound(thumbnails, time, timeComparator) - 1;
+ frames.push(<div className='film-strip-frame' key={i} style={{
+ width: frameSize.width,
+ height: frameSize.height,
+ backgroundImage: `url(${thumbnails[index].url})`,
+ backgroundSize: `${frameSize.width}px ${frameSize.height}px`,
+ margin: frameMargin,
+ marginRight: frameMargin,
+ }} />);
+ }
+ frames.push(<div className='film-strip-frame' key={frames.length} style={{
+ width: frameSize.width,
+ height: frameSize.height,
+ backgroundImage: `url(${thumbnails[thumbnails.length - 1].url})`,
+ backgroundSize: `${frameSize.width}px ${frameSize.height}px`,
+ margin: frameMargin,
+ marginRight: frameMargin,
+ }} />);
+
+ return <div className='film-strip-lane' style={{
+ marginLeft: gapLeft + 'px',
+ marginRight: gapRight + 'px',
+ }}>{frames}</div>;
+};
+
const FilmStripLane: React.FunctionComponent<{
boundaries: Boundaries,
page: PageEntry,
diff --git a/packages/trace-viewer/src/ui/videoThumbnails.ts b/packages/trace-viewer/src/ui/videoThumbnails.ts
new file mode 100644
index 0000000000..ebaac09bf6
--- /dev/null
+++ b/packages/trace-viewer/src/ui/videoThumbnails.ts
@@ -0,0 +1,127 @@
+/*
+ Copyright (c) Microsoft Corporation.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+import * as React from 'react';
+import type * as trace from '@trace/trace';
+
+export type VideoThumbnail = {
+ timestamp: number;
+ url: string;
+ width: number;
+ height: number;
+};
+
+type CacheEntry = {
+ thumbnails: VideoThumbnail[];
+ listeners: Set<() => void>;
+ started: boolean;
+};
+
+const cache = new Map<string, CacheEntry>();
+
+const maxThumbnails = 120;
+const thumbnailsPerSecond = 2;
+
+export function useVideoThumbnails(video: trace.VideoTraceEvent | undefined, videoUrl: string | undefined): VideoThumbnail[] {
+ const [, setVersion] = React.useState(0);
+ const entry = video && videoUrl ? ensureEntry(video, videoUrl) : undefined;
+ React.useEffect(() => {
+ if (!entry)
+ return;
+ const listener = () => setVersion(version => version + 1);
+ entry.listeners.add(listener);
+ return () => {
+ entry.listeners.delete(listener);
+ };
+ }, [entry]);
+ return entry?.thumbnails ?? [];
+}
+
+function ensureEntry(video: trace.VideoTraceEvent, videoUrl: string): CacheEntry {
+ let entry = cache.get(videoUrl);
+ if (!entry) {
+ entry = { thumbnails: [], listeners: new Set(), started: false };
+ cache.set(videoUrl, entry);
+ }
+ if (!entry.started) {
+ entry.started = true;
+ void generateThumbnails(entry, video, videoUrl).catch(() => {});
+ }
+ return entry;
+}
+
+async function generateThumbnails(entry: CacheEntry, video: trace.VideoTraceEvent, videoUrl: string): Promise<void> {
+ const response = await fetch(videoUrl);
+ const blob = await response.blob();
+ const objectUrl = URL.createObjectURL(blob);
+ const element = document.createElement('video');
+ element.muted = true;
+ element.preload = 'auto';
+ element.src = objectUrl;
+ try {
+ await new Promise<void>((resolve, reject) => {
+ element.addEventListener('loadedmetadata', () => resolve(), { once: true });
+ element.addEventListener('error', () => reject(new Error('video failed to load')), { once: true });
+ });
+ let duration = element.duration;
+ if (!isFinite(duration) || duration <= 0) {
+ element.currentTime = Number.MAX_SAFE_INTEGER;
+ await new Promise<void>(resolve => element.addEventListener('seeked', () => resolve(), { once: true }));
+ duration = element.duration;
+ if (!isFinite(duration) || duration <= 0)
+ return;
+ }
+ const width = element.videoWidth || video.width;
+ const height = element.videoHeight || video.height;
+ const canvas = document.createElement('canvas');
+ canvas.width = width;
+ canvas.height = height;
+ const context = canvas.getContext('2d');
+ if (!context)
+ return;
+ const count = Math.max(1, Math.min(maxThumbnails, Math.ceil(duration * thumbnailsPerSecond)));
+ const step = duration / count;
+ for (let i = 0; i <= count; ++i) {
+ const time = Math.min(i * step, Math.max(0, duration - 0.001));
+ await seek(element, time);
+ context.drawImage(element, 0, 0, width, height);
+ const frame = await new Promise<Blob | null>(resolve => canvas.toBlob(resolve, 'image/jpeg', 0.8));
+ if (!frame)
+ continue;
+ entry.thumbnails.push({
+ timestamp: video.timestampOrigin + time * 1000,
+ url: URL.createObjectURL(frame),
+ width,
+ height,
+ });
+ for (const listener of entry.listeners)
+ listener();
+ }
+ } finally {
+ element.removeAttribute('src');
+ element.load();
+ URL.revokeObjectURL(objectUrl);
+ }
+}
+
+async function seek(element: HTMLVideoElement, time: number): Promise<void> {
+ if (Math.abs(element.currentTime - time) < 0.001 && element.readyState >= 2)
+ return;
+ await new Promise<void>(resolve => {
+ element.addEventListener('seeked', () => resolve(), { once: true });
+ element.currentTime = time;
+ });
+}
diff --git a/packages/trace/src/trace.ts b/packages/trace/src/trace.ts
index 46aa729f63..e5317b8b08 100644
--- a/packages/trace/src/trace.ts
+++ b/packages/trace/src/trace.ts
@@ -106,6 +106,14 @@ export type ScreencastFrameTraceEvent = {
frameSwapWallTime?: number,
};
+export type VideoTraceEvent = {
+ type: 'video',
+ file: string,
+ width: number,
+ height: number,
+ timestampOrigin: number,
+};
+
export type ActionPhase = 'before' | 'action' | 'after';
export type ScreenshotTraceEvent = {
@@ -233,6 +241,7 @@ export type ErrorTraceEvent = {
export type TraceEvent =
ContextCreatedTraceEvent |
ScreencastFrameTraceEvent |
+ VideoTraceEvent |
ScreenshotTraceEvent |
AriaSnapshotTraceEvent |
ActionTraceEvent |