Playwright is a framework for Web Testing and Automation. It allows testing Chromium, Firefox and WebKit with a single API.
TypeScript automationchromechromiume2e-testingelectronend-to-end-testingfirefoxjavascriptplaywrighttesttest-automationtestingtesting-toolswebwebkit
feat(trace-viewer): webm-based film strip (#42319)
Pavel Feldman0d86401890ab
diff --git a/packages/isomorphic/trace/entries.ts b/packages/isomorphic/trace/entries.tsindex 09c71959f3..a762d285bd 100644--- a/packages/isomorphic/trace/entries.ts+++ b/packages/isomorphic/trace/entries.ts@@ -39,6 +39,7 @@ export type ContextEntry = {actions: ActionEntry[];screenshots: trace.ScreenshotTraceEvent[];ariaSnapshots: trace.AriaSnapshotTraceEvent[];+ videos: trace.VideoTraceEvent[];events: (trace.EventTraceEvent | trace.ConsoleMessageTraceEvent)[];stdio: trace.StdioTraceEvent[];errors: trace.ErrorTraceEvent[];diff --git a/packages/isomorphic/trace/traceLoader.ts b/packages/isomorphic/trace/traceLoader.tsindex 40c0b0e4ba..9daa159473 100644--- a/packages/isomorphic/trace/traceLoader.ts+++ b/packages/isomorphic/trace/traceLoader.ts@@ -152,6 +152,7 @@ function createEmptyContext(): ContextEntry {actions: [],screenshots: [],ariaSnapshots: [],+ videos: [],events: [],errors: [],stdio: [],diff --git a/packages/isomorphic/trace/traceModel.ts b/packages/isomorphic/trace/traceModel.tsindex 27185f56e7..459088a7e8 100644--- a/packages/isomorphic/trace/traceModel.ts+++ b/packages/isomorphic/trace/traceModel.ts@@ -66,6 +66,7 @@ export class TraceModel {readonly title?: string;readonly options: trace.BrowserContextEventOptions;readonly pages: PageEntry[];+ readonly videos: trace.VideoTraceEvent[];readonly actions: ActionEntry[];readonly attachments: Attachment[];readonly visibleAttachments: Attachment[];@@ -105,6 +106,7 @@ export class TraceModel {// Next call updates all timestamps for all events in library contexts, so it must be done first.this.actions = mergeActionsAndUpdateTiming(contexts);this.pages = ([] as PageEntry[]).concat(...contexts.map(c => c.pages));+ this.videos = [];this.wallTime = contexts.map(c => c.wallTime).reduce((prev, cur) => Math.min(prev || Number.MAX_VALUE, cur!), Number.MAX_VALUE);this.startTime = contexts.map(c => c.startTime).reduce((prev, cur) => Math.min(prev, cur), Number.MAX_VALUE);this.endTime = contexts.map(c => c.endTime).reduce((prev, cur) => Math.max(prev, cur), Number.MIN_VALUE);@@ -126,6 +128,7 @@ export class TraceModel {this._screenshots.set(`${event.callId}/${event.phase}`, event);for (const event of context.ariaSnapshots || [])this._ariaSnapshots.set(`${event.callId}/${event.phase}`, event);+ this.videos.push(...(context.videos || []));}this.attachments = this.actions.flatMap(action => action.attachments?.map(attachment => ({ ...attachment, callId: action.callId, traceUri })) ?? []);this.visibleAttachments = this.attachments.filter(attachment => !attachment.name.startsWith('_'));@@ -344,6 +347,8 @@ function adjustMonotonicTime(context: ContextEntry, monotonicTimeDelta: number)for (const frame of page.screencastFrames)frame.timestamp += monotonicTimeDelta;}+ for (const video of context.videos || [])+ video.timestampOrigin += monotonicTimeDelta;for (const resource of context.resources) {if (resource._monotonicTime)resource._monotonicTime += monotonicTimeDelta;diff --git a/packages/isomorphic/trace/traceModernizer.ts b/packages/isomorphic/trace/traceModernizer.tsindex 48012f9b0d..f883aaa37f 100644--- a/packages/isomorphic/trace/traceModernizer.ts+++ b/packages/isomorphic/trace/traceModernizer.ts@@ -110,6 +110,10 @@ export class TraceModernizer {contextEntry.screenshots.push(event);break;}+ case 'video': {+ contextEntry.videos.push(event);+ break;+ }case 'aria-snapshot': {contextEntry.ariaSnapshots.push(event);break;diff --git a/packages/trace-viewer/src/ui/filmStrip.tsx b/packages/trace-viewer/src/ui/filmStrip.tsxindex a264b28d4c..37a704ad61 100644--- a/packages/trace-viewer/src/ui/filmStrip.tsx+++ b/packages/trace-viewer/src/ui/filmStrip.tsx@@ -20,6 +20,8 @@ import * as React from 'react';import { useMeasure, upperBound } from '@web/uiUtils';import type { PageEntry } from '@isomorphic/trace/entries';import { useTraceModel } from './traceModelContext';+import { useVideoThumbnails } from './videoThumbnails';+import type { VideoThumbnail } from './videoThumbnails';export type FilmStripPreviewPoint = {x: number;@@ -38,18 +40,29 @@ export const FilmStrip: React.FunctionComponent<{const [measure, ref] = useMeasure<HTMLDivElement>();const lanesRef = React.useRef<HTMLDivElement>(null);- let pageIndex = 0;+ const video = model?.videos?.[0];+ const videoThumbnails = useVideoThumbnails(video, video && model ? model.createRelativeUrl(`file/${video.file}`) : undefined);++ let laneIndex = 0;if (lanesRef.current && previewPoint) {const bounds = lanesRef.current.getBoundingClientRect();- pageIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0;+ laneIndex = ((previewPoint.clientY - bounds.top + lanesRef.current.scrollTop) / rowHeight) | 0;}- const screencastFrames = model?.pages?.[pageIndex]?.screencastFrames;+ const pageLanes = (model?.pages ?? []).filter(page => page.screencastFrames.length);+ const videoLanes: VideoThumbnail[][] = videoThumbnails.length ? [videoThumbnails] : [];++ let previewFrames: { timestamp: number, width: number, height: number, url: string }[] | undefined;+ if (laneIndex < pageLanes.length)+ previewFrames = model ? pageLanes[laneIndex]?.screencastFrames.map(frame => ({ ...frame, url: model.createRelativeUrl(`file/${frame.file}`) })) : undefined;+ else+ previewFrames = videoLanes[laneIndex - pageLanes.length];+let previewImage = undefined;let previewSize = undefined;- if (previewPoint !== undefined && screencastFrames && screencastFrames.length) {+ if (previewPoint !== undefined && previewFrames && previewFrames.length) {const previewTime = boundaries.minimum + (boundaries.maximum - boundaries.minimum) * previewPoint.x / measure.width;- previewImage = screencastFrames[upperBound(screencastFrames, previewTime, timeComparator) - 1];+ previewImage = previewFrames[upperBound(previewFrames, previewTime, timeComparator) - 1];const fitInto = {width: Math.min(800, (window.innerWidth / 2) | 0),height: Math.min(800, (window.innerHeight / 2) | 0),@@ -58,14 +71,20 @@ export const FilmStrip: React.FunctionComponent<{}return <div className='film-strip' ref={ref}>- <div className='film-strip-lanes' ref={lanesRef}>{- model?.pages.map((page, index) => page.screencastFrames.length ? <FilmStripLane+ <div className='film-strip-lanes' ref={lanesRef}>+ {pageLanes.map((page, index) => <FilmStripLaneboundaries={boundaries}page={page}width={measure.width}key={index}- /> : null)- }</div>+ />)}+ {videoLanes.map((thumbnails, index) => <VideoFilmStripLane+ boundaries={boundaries}+ thumbnails={thumbnails}+ width={measure.width}+ key={'video-' + index}+ />)}+ </div>{model && previewPoint && previewImage && previewSize &&<div className='film-strip-hover' style={{top: measure.bottom + 5,@@ -73,12 +92,61 @@ export const FilmStrip: React.FunctionComponent<{width: previewSize.width,height: previewSize.height,}}>- <img src={model.createRelativeUrl(`file/${previewImage.file}`)} width={previewSize.width} height={previewSize.height} />+ <img src={previewImage.url} width={previewSize.width} height={previewSize.height} /></div>}</div>;};+const VideoFilmStripLane: React.FunctionComponent<{+ boundaries: Boundaries,+ thumbnails: VideoThumbnail[],+ width: number,+}> = ({ boundaries, thumbnails, width }) => {+ const viewportSize = { width: 0, height: 0 };+ for (const thumbnail of thumbnails) {+ viewportSize.width = Math.max(viewportSize.width, thumbnail.width);+ viewportSize.height = Math.max(viewportSize.height, thumbnail.height);+ }+ const frameSize = inscribe(viewportSize, tileSize);+ const startTime = thumbnails[0].timestamp;+ const endTime = thumbnails[thumbnails.length - 1].timestamp;++ const boundariesDuration = boundaries.maximum - boundaries.minimum;+ const gapLeft = (startTime - boundaries.minimum) / boundariesDuration * width;+ const gapRight = (boundaries.maximum - endTime) / boundariesDuration * width;+ const effectiveWidth = (endTime - startTime) / boundariesDuration * width;+ const frameCount = (effectiveWidth / (frameSize.width + 2 * frameMargin)) | 0;+ const frameDuration = (endTime - startTime) / frameCount;++ const frames: React.JSX.Element[] = [];+ for (let i = 0; startTime && frameDuration && i < frameCount; ++i) {+ const time = startTime + frameDuration * i;+ const index = upperBound(thumbnails, time, timeComparator) - 1;+ frames.push(<div className='film-strip-frame' key={i} style={{+ width: frameSize.width,+ height: frameSize.height,+ backgroundImage: `url(${thumbnails[index].url})`,+ backgroundSize: `${frameSize.width}px ${frameSize.height}px`,+ margin: frameMargin,+ marginRight: frameMargin,+ }} />);+ }+ frames.push(<div className='film-strip-frame' key={frames.length} style={{+ width: frameSize.width,+ height: frameSize.height,+ backgroundImage: `url(${thumbnails[thumbnails.length - 1].url})`,+ backgroundSize: `${frameSize.width}px ${frameSize.height}px`,+ margin: frameMargin,+ marginRight: frameMargin,+ }} />);++ return <div className='film-strip-lane' style={{+ marginLeft: gapLeft + 'px',+ marginRight: gapRight + 'px',+ }}>{frames}</div>;+};+const FilmStripLane: React.FunctionComponent<{boundaries: Boundaries,page: PageEntry,diff --git a/packages/trace-viewer/src/ui/videoThumbnails.ts b/packages/trace-viewer/src/ui/videoThumbnails.tsnew file mode 100644index 0000000000..ebaac09bf6--- /dev/null+++ b/packages/trace-viewer/src/ui/videoThumbnails.ts@@ -0,0 +1,127 @@+/*+ Copyright (c) Microsoft Corporation.++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+*/++import * as React from 'react';+import type * as trace from '@trace/trace';++export type VideoThumbnail = {+ timestamp: number;+ url: string;+ width: number;+ height: number;+};++type CacheEntry = {+ thumbnails: VideoThumbnail[];+ listeners: Set<() => void>;+ started: boolean;+};++const cache = new Map<string, CacheEntry>();++const maxThumbnails = 120;+const thumbnailsPerSecond = 2;++export function useVideoThumbnails(video: trace.VideoTraceEvent | undefined, videoUrl: string | undefined): VideoThumbnail[] {+ const [, setVersion] = React.useState(0);+ const entry = video && videoUrl ? ensureEntry(video, videoUrl) : undefined;+ React.useEffect(() => {+ if (!entry)+ return;+ const listener = () => setVersion(version => version + 1);+ entry.listeners.add(listener);+ return () => {+ entry.listeners.delete(listener);+ };+ }, [entry]);+ return entry?.thumbnails ?? [];+}++function ensureEntry(video: trace.VideoTraceEvent, videoUrl: string): CacheEntry {+ let entry = cache.get(videoUrl);+ if (!entry) {+ entry = { thumbnails: [], listeners: new Set(), started: false };+ cache.set(videoUrl, entry);+ }+ if (!entry.started) {+ entry.started = true;+ void generateThumbnails(entry, video, videoUrl).catch(() => {});+ }+ return entry;+}++async function generateThumbnails(entry: CacheEntry, video: trace.VideoTraceEvent, videoUrl: string): Promise<void> {+ const response = await fetch(videoUrl);+ const blob = await response.blob();+ const objectUrl = URL.createObjectURL(blob);+ const element = document.createElement('video');+ element.muted = true;+ element.preload = 'auto';+ element.src = objectUrl;+ try {+ await new Promise<void>((resolve, reject) => {+ element.addEventListener('loadedmetadata', () => resolve(), { once: true });+ element.addEventListener('error', () => reject(new Error('video failed to load')), { once: true });+ });+ let duration = element.duration;+ if (!isFinite(duration) || duration <= 0) {+ element.currentTime = Number.MAX_SAFE_INTEGER;+ await new Promise<void>(resolve => element.addEventListener('seeked', () => resolve(), { once: true }));+ duration = element.duration;+ if (!isFinite(duration) || duration <= 0)+ return;+ }+ const width = element.videoWidth || video.width;+ const height = element.videoHeight || video.height;+ const canvas = document.createElement('canvas');+ canvas.width = width;+ canvas.height = height;+ const context = canvas.getContext('2d');+ if (!context)+ return;+ const count = Math.max(1, Math.min(maxThumbnails, Math.ceil(duration * thumbnailsPerSecond)));+ const step = duration / count;+ for (let i = 0; i <= count; ++i) {+ const time = Math.min(i * step, Math.max(0, duration - 0.001));+ await seek(element, time);+ context.drawImage(element, 0, 0, width, height);+ const frame = await new Promise<Blob | null>(resolve => canvas.toBlob(resolve, 'image/jpeg', 0.8));+ if (!frame)+ continue;+ entry.thumbnails.push({+ timestamp: video.timestampOrigin + time * 1000,+ url: URL.createObjectURL(frame),+ width,+ height,+ });+ for (const listener of entry.listeners)+ listener();+ }+ } finally {+ element.removeAttribute('src');+ element.load();+ URL.revokeObjectURL(objectUrl);+ }+}++async function seek(element: HTMLVideoElement, time: number): Promise<void> {+ if (Math.abs(element.currentTime - time) < 0.001 && element.readyState >= 2)+ return;+ await new Promise<void>(resolve => {+ element.addEventListener('seeked', () => resolve(), { once: true });+ element.currentTime = time;+ });+}diff --git a/packages/trace/src/trace.ts b/packages/trace/src/trace.tsindex 46aa729f63..e5317b8b08 100644--- a/packages/trace/src/trace.ts+++ b/packages/trace/src/trace.ts@@ -106,6 +106,14 @@ export type ScreencastFrameTraceEvent = {frameSwapWallTime?: number,};+export type VideoTraceEvent = {+ type: 'video',+ file: string,+ width: number,+ height: number,+ timestampOrigin: number,+};+export type ActionPhase = 'before' | 'action' | 'after';export type ScreenshotTraceEvent = {@@ -233,6 +241,7 @@ export type ErrorTraceEvent = {export type TraceEvent =ContextCreatedTraceEvent |ScreencastFrameTraceEvent |+ VideoTraceEvent |ScreenshotTraceEvent |AriaSnapshotTraceEvent |ActionTraceEvent |