π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Fix Firefox Android video covers and processing
Anand Baburajan942b4bc2deb1
diff --git a/web/apps/space/src/components/PostComposer.tsx b/web/apps/space/src/components/PostComposer.tsxindex 25a44a600f..f14035f6b2 100644--- a/web/apps/space/src/components/PostComposer.tsx+++ b/web/apps/space/src/components/PostComposer.tsx@@ -70,7 +70,7 @@ export const SpacePostComposer: React.FC<{const inputRef = React.useRef<HTMLInputElement>(null);const previewURLsRef = React.useRef(new Set<string>());const publishedPreviewURLRef = React.useRef<string>(undefined);- const preparingRef = React.useRef(new Set<number>());+ const preparingRef = React.useRef(new Map<number, AbortController>());const draftsRef = React.useRef(drafts);draftsRef.current = drafts;const mountedRef = React.useRef(false);@@ -96,8 +96,11 @@ export const SpacePostComposer: React.FC<{React.useEffect(() => {mountedRef.current = true;const urls = previewURLsRef.current;+ const preparing = preparingRef.current;return () => {mountedRef.current = false;+ preparing.forEach((controller) => controller.abort());+ preparing.clear();urls.forEach((url) => {if (url != publishedPreviewURLRef.current)URL.revokeObjectURL(url);@@ -116,7 +119,12 @@ export const SpacePostComposer: React.FC<{!draft.error &&!preparingRef.current.has(draft.id),);- pending.forEach((draft) => preparingRef.current.add(draft.id));+ const controllers = new Map(+ pending.map((draft) => [draft.id, new AbortController()]),+ );+ controllers.forEach((controller, id) =>+ preparingRef.current.set(id, controller),+ );if (pending.some((draft) => isSpaceVideoFile(draft.file))) {void import("utils/video-encoding/web").then(({ preloadVideoEncoderWeb }) => preloadVideoEncoderWeb())@@ -127,15 +135,19 @@ export const SpacePostComposer: React.FC<{void (async () => {for (const draft of pending) {if (!isActiveDraft(draft.id)) continue;+ const controller = controllers.get(draft.id)!;+ const { signal } = controller;try {+ signal.throwIfAborted();let photo: SpaceViewerPhoto;let videoEdit: SpacePostVideoEdit | undefined;if (isSpaceVideoFile(draft.file)) {- const info = await spaceVideoInfo(draft.file);+ const info = await spaceVideoInfo(draft.file, signal);videoEdit = initialSpaceVideoEdit(info.duration);const cover = await spaceVideoCover(draft.file,videoEdit.coverTime,+ signal,);const url = URL.createObjectURL(draft.file);previewURLsRef.current.add(url);@@ -191,6 +203,7 @@ export const SpacePostComposer: React.FC<{),);} catch (error) {+ if (signal.aborted) continue;log.error("Failed to prepare post preview", error);if (isActiveDraft(draft.id))setDrafts((current) =>@@ -210,7 +223,8 @@ export const SpacePostComposer: React.FC<{),);} finally {- preparingRef.current.delete(draft.id);+ if (preparingRef.current.get(draft.id) == controller)+ preparingRef.current.delete(draft.id);}}})();@@ -221,6 +235,7 @@ export const SpacePostComposer: React.FC<{setDrafts((current) => [...current, ...draftPhotos(files)]);};const releasePreview = (draft: DraftPhoto) => {+ preparingRef.current.get(draft.id)?.abort();for (const url of new Set([draft.photo?.imageUrl,draft.originalPhoto?.imageUrl,diff --git a/web/apps/space/src/components/PostVideoContent.ts b/web/apps/space/src/components/PostVideoContent.tsindex dbc5c8c268..554c7fa691 100644--- a/web/apps/space/src/components/PostVideoContent.ts+++ b/web/apps/space/src/components/PostVideoContent.ts@@ -1,4 +1,5 @@import type { SpaceViewerPhoto } from "components/FileViewer";+import { logToDisk } from "ente-base/log-web";import type { SpacePostAssetURLLoader } from "services/space";export const createSpaceVideoContent = (@@ -63,19 +64,40 @@ export const createSpaceVideoContent = (loading.setAttribute("aria-label", "Loading video");loading.innerHTML = '<span class="space-video-spinner"></span>';Object.assign(loading.style, { ...controlStyle, pointerEvents: "none" });- element.append(video, button, loading);+ const errorMessage = document.createElement("span");+ errorMessage.textContent = "Couldn't play this video. Tap to retry.";+ errorMessage.hidden = true;+ errorMessage.setAttribute("role", "status");+ Object.assign(errorMessage.style, {+ position: "absolute",+ bottom: "12px",+ left: "12px",+ right: "12px",+ background: "rgba(0, 0, 0, 0.65)",+ color: "white",+ borderRadius: "6px",+ padding: "8px",+ fontSize: "13px",+ textAlign: "center",+ pointerEvents: "none",+ });+ element.append(video, button, loading, errorMessage);let generation = 0;let ownedURL: string | undefined;let disposed = false;let active = inline;let loadingTimer: ReturnType<typeof setTimeout> | undefined;+ let playbackTimeout: ReturnType<typeof setTimeout> | undefined;const hideLoading = () => {clearTimeout(loadingTimer);loadingTimer = undefined;+ clearTimeout(playbackTimeout);+ playbackTimeout = undefined;loading.setAttribute("aria-hidden", "true");element.removeAttribute("aria-busy");};const showLoading = () => {+ playbackTimeout ??= setTimeout(() => failPlayback("timeout"), 30_000);if (document.activeElement == button)video.focus({ preventScroll: true });element.setAttribute("aria-busy", "true");@@ -112,8 +134,16 @@ export const createSpaceVideoContent = (video.removeAttribute("src");video.load();};+ const failPlayback = (reason: "timeout" | "media" | "play") => {+ logToDisk(+ `[error] Space video playback failed reason=${reason} code=${video.error?.code ?? 0} readyState=${video.readyState} networkState=${video.networkState}`,+ );+ reset();+ errorMessage.hidden = false;+ };const clear = () => {active = inline;+ errorMessage.hidden = true;reset();if (ownedURL) URL.revokeObjectURL(ownedURL);ownedURL = undefined;@@ -125,6 +155,7 @@ export const createSpaceVideoContent = (const play = () => {if (button.disabled || !video.paused) return;active = true;+ errorMessage.hidden = true;const current = ++generation;showLoading();button.disabled = true;@@ -142,7 +173,6 @@ export const createSpaceVideoContent = (if (disposed || generation != current) return;if (!video.getAttribute("src")) {video.src = url;- video.currentTime = media.start ?? 0;} else if (video.ended ||(media.end != undefined && video.currentTime >= media.end)@@ -152,7 +182,7 @@ export const createSpaceVideoContent = (await video.play();if (generation == current) showPlaying();})().catch(() => {- if (!disposed && generation == current) showPlayButton();+ if (!disposed && generation == current) failPlayback("play");});};const togglePlayback = () => {@@ -189,7 +219,10 @@ export const createSpaceVideoContent = (if (!video.paused) showLoading();};video.onplaying = showPlaying;- video.onerror = pause;+ video.onerror = () => failPlayback("media");+ video.onloadedmetadata = () => {+ if (media.start) video.currentTime = media.start;+ };video.ontimeupdate = () => {if (media.end != undefined && video.currentTime >= media.end) {if (inline) video.currentTime = media.start ?? 0;diff --git a/web/apps/space/src/components/PostVideoEditor.tsx b/web/apps/space/src/components/PostVideoEditor.tsxindex 213eeed270..2eed1c0842 100644--- a/web/apps/space/src/components/PostVideoEditor.tsx+++ b/web/apps/space/src/components/PostVideoEditor.tsx@@ -11,8 +11,7 @@ import {clampVideoCover,initialSpaceVideoEdit,maxSpaceVideoDuration,- openSpaceVideo,- seekSpaceVideo,+ spaceVideoFrames,type SpacePostVideoEdit,} from "utils/post-video";@@ -122,38 +121,30 @@ export const SpacePostVideoEditor: React.FC<{const urls: string[] = [];const timer = window.setTimeout(() => {void (async () => {- const { video, dispose } = await openSpaceVideo(file, signal);- try {- const times = Array.from(- { length: 8 },- (_, i) => visibleStart + (span * (i + 0.5)) / 8,- );- for (const time of times) {- await seekSpaceVideo(video, time, signal);- const canvas = document.createElement("canvas");- canvas.width = 80;- canvas.height = 64;- const scale = Math.max(- 80 / video.videoWidth,- 64 / video.videoHeight,+ const times = Array.from(+ { length: 8 },+ (_, i) => visibleStart + (span * (i + 0.5)) / 8,+ );+ const frames = await spaceVideoFrames(file, times, 128, signal);+ for (const frame of frames) {+ const canvas = document.createElement("canvas");+ canvas.width = 80;+ canvas.height = 64;+ const scale = Math.max(80 / frame.width, 64 / frame.height);+ const width = frame.width * scale;+ const height = frame.height * scale;+ canvas+ .getContext("2d")!+ .drawImage(+ frame,+ (80 - width) / 2,+ (64 - height) / 2,+ width,+ height,);- const width = video.videoWidth * scale;- const height = video.videoHeight * scale;- canvas- .getContext("2d")!- .drawImage(- video,- (80 - width) / 2,- (64 - height) / 2,- width,- height,- );- urls.push(canvas.toDataURL("image/jpeg", 0.65));- }- if (!signal.aborted) setFrames(urls);- } finally {- dispose();+ urls.push(canvas.toDataURL("image/jpeg", 0.65));}+ if (!signal.aborted) setFrames(urls);})().catch(() => {if (!signal.aborted) setFrames([]);});diff --git a/web/apps/space/src/components/video-trim/playback.ts b/web/apps/space/src/components/video-trim/playback.tsindex ba2e8edf5d..2d5bbdd423 100644--- a/web/apps/space/src/components/video-trim/playback.ts+++ b/web/apps/space/src/components/video-trim/playback.ts@@ -1,3 +1,5 @@+import { logToDisk } from "ente-base/log-web";+export const createVideoTrimPlayback = (video: HTMLVideoElement,getRange: () => { start: number; end: number },@@ -8,6 +10,12 @@ export const createVideoTrimPlayback = (let restartOnPlay = true;let playRequested = false;let generation = 0;+ let loadingTimer: ReturnType<typeof setTimeout> | undefined;++ const clearLoadingTimer = () => {+ clearTimeout(loadingTimer);+ loadingTimer = undefined;+ };const cancelFrame = () => {if (frame != undefined) cancelAnimationFrame(frame);@@ -17,6 +25,7 @@ export const createVideoTrimPlayback = (const pause = () => {generation++;playRequested = false;+ clearLoadingTimer();video.pause();cancelFrame();};@@ -63,11 +72,23 @@ export const createVideoTrimPlayback = (if (!video.paused) return;generation++;playRequested = false;+ clearLoadingTimer();cancelFrame();};const onEnded = () => seek(getRange().start);+ const failed = () => {+ logToDisk(+ `[error] Space video editor playback failed code=${video.error?.code ?? 0} readyState=${video.readyState} networkState=${video.networkState}`,+ );+ pause();+ onError();+ };+ const waiting = () => {+ if (playRequested) loadingTimer ??= setTimeout(failed, 30_000);+ };+const toggle = () => {if (playRequested || !video.paused) {pause();@@ -87,10 +108,10 @@ export const createVideoTrimPlayback = (video.currentTime = time;const current = ++generation;playRequested = true;+ waiting();void video.play().catch(() => {if (generation != current) return;- pause();- onError();+ failed();});};@@ -100,6 +121,9 @@ export const createVideoTrimPlayback = (video.addEventListener("timeupdate", checkRange);video.addEventListener("seeked", applySeek);video.addEventListener("loadedmetadata", applySeek);+ video.addEventListener("playing", clearLoadingTimer);+ video.addEventListener("waiting", waiting);+ video.addEventListener("error", failed);return {toggle,@@ -112,6 +136,9 @@ export const createVideoTrimPlayback = (video.removeEventListener("timeupdate", checkRange);video.removeEventListener("seeked", applySeek);video.removeEventListener("loadedmetadata", applySeek);+ video.removeEventListener("playing", clearLoadingTimer);+ video.removeEventListener("waiting", waiting);+ video.removeEventListener("error", failed);},};};diff --git a/web/apps/space/src/screens/HomeScreen.tsx b/web/apps/space/src/screens/HomeScreen.tsxindex 53d2cb725e..930fa8f786 100644--- a/web/apps/space/src/screens/HomeScreen.tsx+++ b/web/apps/space/src/screens/HomeScreen.tsx@@ -374,6 +374,7 @@ type FeedPhotoSource = SpacePostPhoto & Pick<SpacePost, "postId" | "spaceId">;interface FeedItemProps {onRetry?: () => void;+ processing?: string;photoCount?: number;photoIndex?: number;photos?: FeedPostPhoto[];@@ -871,6 +872,7 @@ const FeedItem: React.FC<FeedItemProps> = ({onOpenProfile,onSetPostLiked,onRetry,+ processing,postId,spaceId,thumbHash,@@ -1566,7 +1568,7 @@ const FeedItem: React.FC<FeedItemProps> = ({role="status"aria-label={timestampStatus == "posting"- ? "Posting"+ ? (processing ?? "Posting"): timestampStatus == "failed"? "Failed": "Posted"@@ -1642,7 +1644,9 @@ const FeedItem: React.FC<FeedItemProps> = ({</>) : (<>- <Box component="span">Posting</Box>+ <Box component="span">+ {processing ?? "Posting"}+ </Box><Boxcomponent="span"aria-hidden@@ -2221,6 +2225,7 @@ export const HomeScreen: React.FC<HomeScreenProps> = ({onRetry={onRetryPost ? () => void onRetryPost(item.id) : undefined}+ processing={item.processing}timestampStatus={item.status == "failed"? "failed"diff --git a/web/apps/space/src/state/AppStateProvider.tsx b/web/apps/space/src/state/AppStateProvider.tsxindex 199ba650db..6a6b8510bf 100644--- a/web/apps/space/src/state/AppStateProvider.tsx+++ b/web/apps/space/src/state/AppStateProvider.tsx@@ -89,6 +89,7 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({const profileLoadGenerationRef = useRef(0);const postPublishGenerationRef = useRef(0);const postReadbacksRef = useRef(new Map<string, () => Promise<void>>());+ const postPreparationsRef = useRef(new Map<string, AbortController>());const postUploadsRef = useRef(new Map<string, () => ReturnType<SpaceAppState["publishPost"]>>(),);@@ -96,6 +97,7 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({useEffect(() => {const readbacks = postReadbacksRef.current;const uploads = postUploadsRef.current;+ const preparations = postPreparationsRef.current;const retryReadbacks = () => {readbacks.forEach((readback) => void readback());};@@ -106,6 +108,8 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({window.removeEventListener("focus", retryReadbacks);readbacks.clear();uploads.clear();+ preparations.forEach((controller) => controller.abort());+ preparations.clear();};}, []);@@ -178,6 +182,11 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({},};const prepared: PreparedSpacePostMedia[] = [];+ const videoCount = images.filter((image) => image.video).length;+ const mediaCounts = {+ photo: images.length - videoCount,+ video: videoCount,+ };const session: SpacePostUploadSession = {requestId: crypto.randomUUID(),uploads: [],@@ -189,23 +198,50 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({const generation = ++postPublishGenerationRef.current;const startedAt = Date.now();attempt += 1;+ const controller = new AbortController();+ const { signal } = controller;+ postPreparationsRef.current.set(localPostId, controller);setPostPublication(publication);setLocalFeedPosts((current) =>current.map((item) =>item.id == localPostId && item.status == "failed"- ? { ...item, status: "pending" }+ ? {+ ...item,+ status: "pending",+ processing: undefined,+ }: item,),);pending = (async () => {try {+ const processingCounts = { photo: 0, video: 0 };for (const [index, image] of images.entries()) {+ signal.throwIfAborted();+ const mediaType = image.video ? "video" : "photo";+ const mediaIndex = ++processingCounts[mediaType];if (prepared[index]) continue;+ if (videoCount > 0 || images.length >= 3) {+ const count = mediaCounts[mediaType];+ const processing =+ images.length == 1+ ? "Processing"+ : `Processing ${mediaType}${count > 1 ? ` ${mediaIndex}/${count}` : ""}`;+ setLocalFeedPosts((current) =>+ current.map((item) =>+ item.id == localPostId &&+ item.status == "pending"+ ? { ...item, processing }+ : item,+ ),+ );+ }try {prepared[index] = image.video? await prepareSpaceVideo(image.file,image.video,+ signal,): await prepareSpacePostImageFromEdit(image.file,@@ -219,6 +255,16 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({throw error;}}+ signal.throwIfAborted();+ postPreparationsRef.current.delete(localPostId);+ setLocalFeedPosts((current) =>+ current.map((item) =>+ item.id == localPostId &&+ item.status == "pending"+ ? { ...item, processing: undefined }+ : item,+ ),+ );if (!postUploadsRef.current.has(localPostId))throw new DOMException("Canceled", "AbortError");const postId = await createCurrentMediaPost({@@ -328,6 +374,7 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({throw error;}})().finally(() => {+ postPreparationsRef.current.delete(localPostId);pending = undefined;});return pending;@@ -514,6 +561,8 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({postPublishGenerationRef.current += 1;postReadbacksRef.current.clear();postUploadsRef.current.clear();+ postPreparationsRef.current.forEach((controller) => controller.abort());+ postPreparationsRef.current.clear();setPostPublication(null);setPendingLoginCredentials(null);setPendingPasskeyVerification(null);diff --git a/web/apps/space/src/state/app-state.tsx b/web/apps/space/src/state/app-state.tsxindex 0d20253d5d..84b29635c0 100644--- a/web/apps/space/src/state/app-state.tsx+++ b/web/apps/space/src/state/app-state.tsx@@ -38,6 +38,7 @@ interface PendingSpaceFeedPost {spaceId: string;photoCount: number;postId?: number;+ processing?: string;status: "pending";timestampMs: number;width?: number;diff --git a/web/apps/space/src/utils/post-video.ts b/web/apps/space/src/utils/post-video.tsindex 1148b34b76..3435b0232f 100644--- a/web/apps/space/src/utils/post-video.ts+++ b/web/apps/space/src/utils/post-video.ts@@ -1,3 +1,4 @@+import { logToDisk } from "ente-base/log-web";import type { FFmpegCommand } from "ente-base/types/ipc";import type { PreparedSpacePostImage } from "utils/post-image";import { thumbHashBase64FromCanvas } from "utils/thumbhash";@@ -21,6 +22,10 @@ export interface SpaceVideoInfo {export const maxSpaceVideoDuration = 10;+export const useSoftwareVideoFrames = () =>+ navigator.userAgent.includes("Android") &&+ navigator.userAgent.includes("Firefox/");+export const isSpaceVideoFile = (file: File) =>file.type.startsWith("video/") || /\.(mp4|mov|m4v|webm)$/i.test(file.name);@@ -49,8 +54,13 @@ const waitForVideo = (video.removeEventListener(event, ready);video.removeEventListener("error", failed);signal?.removeEventListener("abort", abort);- if (error) reject(error);- else resolve();+ if (error) {+ if (!signal?.aborted)+ logToDisk(+ `[error] Space video event=${event} failed code=${video.error?.code ?? 0} readyState=${video.readyState} networkState=${video.networkState}`,+ );+ reject(error);+ } else resolve();};const ready = () => finish();const failed = () =>@@ -104,8 +114,15 @@ export const seekSpaceVideo = async (await ready;};-export const spaceVideoInfo = async (file: File): Promise<SpaceVideoInfo> => {- const { video, dispose } = await openSpaceVideo(file);+export const spaceVideoInfo = async (+ file: File,+ signal?: AbortSignal,+): Promise<SpaceVideoInfo> => {+ if (useSoftwareVideoFrames()) {+ const { determineVideoInfoWeb } = await import("./video-encoding/web");+ return determineVideoInfoWeb(file, signal);+ }+ const { video, dispose } = await openSpaceVideo(file, signal);try {if (!Number.isFinite(video.duration) || video.duration <= 0)throw new Error("This video has no readable duration.");@@ -119,48 +136,104 @@ export const spaceVideoInfo = async (file: File): Promise<SpaceVideoInfo> => {}};-export const spaceVideoCover = async (+export const spaceVideoFrames = async (file: Blob,- time: number,+ times: number[],+ maxDimension: number,signal?: AbortSignal,-): Promise<PreparedSpacePostImage> => {- const { video, dispose } = await openSpaceVideo(file, signal);- try {- await seekSpaceVideo(video, time, signal);- const scale = Math.min(- 1,- 1280 / Math.max(video.videoWidth, video.videoHeight),- );+) => {+ const startedAt = Date.now();+ const software = useSoftwareVideoFrames();+ const frames: HTMLCanvasElement[] = [];+ const draw = (source: CanvasImageSource, width: number, height: number) => {+ const scale = Math.min(1, maxDimension / Math.max(width, height));const canvas = document.createElement("canvas");- canvas.width = Math.round(video.videoWidth * scale);- canvas.height = Math.round(video.videoHeight * scale);+ canvas.width = Math.max(1, Math.round(width * scale));+ canvas.height = Math.max(1, Math.round(height * scale));const context = canvas.getContext("2d");if (!context) throw new Error("Couldn't prepare the video cover.");- context.drawImage(video, 0, 0, canvas.width, canvas.height);- const thumbHash = thumbHashBase64FromCanvas(canvas);- const blob = await new Promise<Blob>((resolve, reject) =>- canvas.toBlob(- (blob) =>- blob- ? resolve(blob)- : reject(- new Error("Couldn't prepare the video cover."),- ),- "image/webp",- 0.85,- ),+ context.drawImage(source, 0, 0, canvas.width, canvas.height);+ const sample = document.createElement("canvas");+ sample.width = sample.height = 1;+ const sampleContext = sample.getContext("2d")!;+ sampleContext.drawImage(canvas, 0, 0, 1, 1);+ if (sampleContext.getImageData(0, 0, 1, 1).data[3] == 0)+ throw new Error("Couldn't read a video frame. Please try again.");+ frames.push(canvas);+ };+ try {+ if (software) {+ const { extractVideoFramesWeb } =+ await import("./video-encoding/web");+ const blobs = await extractVideoFramesWeb(+ file,+ times,+ maxDimension,+ signal,+ );+ for (const blob of blobs) {+ signal?.throwIfAborted();+ const bitmap = await createImageBitmap(blob);+ try {+ draw(bitmap, bitmap.width, bitmap.height);+ } finally {+ bitmap.close();+ }+ }+ } else {+ const { video, dispose } = await openSpaceVideo(file, signal);+ try {+ for (const time of times) {+ await seekSpaceVideo(video, time, signal);+ draw(video, video.videoWidth, video.videoHeight);+ }+ } finally {+ dispose();+ }+ }+ logToDisk(+ `[info] Space video frames backend=${software ? "wasm" : "browser"} count=${frames.length} bytes=${file.size} elapsedMs=${Date.now() - startedAt}`,);- return {- file: new File([blob], "cover.webp", { type: blob.type }),- width: canvas.width,- height: canvas.height,- thumbHash,- };- } finally {- dispose();+ return frames;+ } catch (error) {+ if (!signal?.aborted)+ logToDisk(+ `[error] Space video frames failed backend=${software ? "wasm" : "browser"} count=${frames.length}/${times.length} bytes=${file.size} elapsedMs=${Date.now() - startedAt}`,+ );+ throw error;}};+export const spaceVideoCover = async (+ file: Blob,+ time: number,+ signal?: AbortSignal,+): Promise<PreparedSpacePostImage> => {+ const [canvas] = await spaceVideoFrames(file, [time], 960, signal);+ if (!canvas) throw new Error("Couldn't prepare the video cover.");+ const thumbHash = thumbHashBase64FromCanvas(canvas);+ const blob = await new Promise<Blob>((resolve, reject) =>+ canvas.toBlob(+ (blob) =>+ blob+ ? resolve(blob)+ : reject(new Error("Couldn't prepare the video cover.")),+ "image/webp",+ 0.8,+ ),+ );+ signal?.throwIfAborted();+ logToDisk(+ `[info] Space video cover prepared width=${canvas.width} height=${canvas.height} bytes=${blob.size}`,+ );+ return {+ file: new File([blob], "cover.webp", { type: blob.type }),+ width: canvas.width,+ height: canvas.height,+ thumbHash,+ };+};+const videoExportCommand = (edit: SpacePostVideoEdit,hdr: boolean,@@ -175,7 +248,7 @@ const videoExportCommand = ("0:v:0",...(edit.muted? ["-an"]- : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "192k"]),+ : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "128k"]),"-map_metadata","-1","-map_metadata:s",@@ -185,7 +258,7 @@ const videoExportCommand = ("-vf",["fps=30:start_time=0",- "scale=w='trunc(iw*min(1,min(1920/max(iw,ih),1080/min(iw,ih)))/2)*2':h='trunc(ih*min(1,min(1920/max(iw,ih),1080/min(iw,ih)))/2)*2'",+ "scale=w='trunc(iw*min(1,min(1280/max(iw,ih),720/min(iw,ih)))/2)*2':h='trunc(ih*min(1,min(1280/max(iw,ih),720/min(iw,ih)))/2)*2'",...(hdr? ["zscale=transfer=linear",@@ -201,11 +274,11 @@ const videoExportCommand = ("-preset","veryfast","-crf",- "20",+ "23","-maxrate",- "8M",+ "4M","-bufsize",- "16M",+ "8M","-movflags","+faststart",outputPathPlaceholder,diff --git a/web/apps/space/src/utils/video-encoding/web-codecs.ts b/web/apps/space/src/utils/video-encoding/web-codecs.tsindex cb13b4d30c..70f0cd5ec9 100644--- a/web/apps/space/src/utils/video-encoding/web-codecs.ts+++ b/web/apps/space/src/utils/video-encoding/web-codecs.ts@@ -214,7 +214,7 @@ export const encodeVideoWithWebCodecs = async (codec: "avc1.640028",width: Math.floor((width * scale) / 2) * 2,height: Math.floor((height * scale) / 2) * 2,- bitrate: 8_000_000,+ bitrate: 4_000_000,bitrateMode: "variable",framerate: frameRate,latencyMode: "realtime",@@ -267,7 +267,7 @@ export const encodeVideoWithWebCodecs = async ("30000",...(edit.muted? ["-an"]- : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "192k"]),+ : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "128k"]),"-map_metadata","-1","-map_metadata:s",diff --git a/web/apps/space/src/utils/video-encoding/web.ts b/web/apps/space/src/utils/video-encoding/web.tsindex 91f7d387e8..862ac0d6d6 100644--- a/web/apps/space/src/utils/video-encoding/web.ts+++ b/web/apps/space/src/utils/video-encoding/web.ts@@ -2,6 +2,7 @@ import { FFFSType, FFmpeg } from "@ffmpeg/ffmpeg";import { joinPath } from "ente-base/file-name";import { newID } from "ente-base/id";import log from "ente-base/log";+import { logToDisk } from "ente-base/log-web";import type { FFmpegCommand } from "ente-base/types/ipc";import { ensureArrayBufferBacked } from "ente-utils/bytes";import { PromiseQueue } from "ente-utils/promise";@@ -9,42 +10,242 @@ import { z } from "zod";import { inputPathPlaceholder, outputPathPlaceholder } from "./constants";import type { VideoTranscodeEdit } from "./web-codecs";-let _ffmpeg: Promise<FFmpeg> | undefined;+let _ffmpeg: { instance: FFmpeg; ready: Promise<void> } | undefined;const _ffmpegTaskQueue = new PromiseQueue<unknown>();-const ffmpegLazy = (signal?: AbortSignal): Promise<FFmpeg> =>- (_ffmpeg ??= createFFmpeg(signal));+const ffmpegLazy = () => {+ if (!_ffmpeg) {+ const instance = new FFmpeg();+ _ffmpeg = { instance, ready: loadFFmpeg(instance) };+ }+ return _ffmpeg;+};-const createFFmpeg = async (signal?: AbortSignal) => {- const ffmpeg = new FFmpeg();+const loadFFmpeg = async (ffmpeg: FFmpeg) => {+ const startedAt = Date.now();+ const timeout = new AbortController();+ const timer = setTimeout(() => {+ timeout.abort();+ ffmpeg.terminate();+ }, 60_000);+ logToDisk("[info] Space video encoder loading");try {- await ffmpeg.load(- {- coreURL:- "https://assets.ente.com/ffmpeg-core-0.12.10/ffmpeg-core.js",- wasmURL:- "https://assets.ente.com/ffmpeg-core-0.12.10/ffmpeg-core.wasm",- },- { signal },+ await ffmpeg.load({+ coreURL:+ "https://assets.ente.com/ffmpeg-core-0.12.10/ffmpeg-core.js",+ wasmURL:+ "https://assets.ente.com/ffmpeg-core-0.12.10/ffmpeg-core.wasm",+ });+ logToDisk(+ `[info] Space video encoder loaded elapsedMs=${Date.now() - startedAt}`,);- return ffmpeg;} catch (error) {ffmpeg.terminate();- _ffmpeg = undefined;+ if (_ffmpeg?.instance == ffmpeg) _ffmpeg = undefined;+ logToDisk(+ `[error] Space video encoder load ${timeout.signal.aborted ? "timeout" : "failed"} elapsedMs=${Date.now() - startedAt}`,+ );+ if (timeout.signal.aborted)+ throw new Error(+ "The video processor took too long to load. Please try again.",+ { cause: error },+ );throw error;+ } finally {+ clearTimeout(timer);}};export const preloadVideoEncoderWeb = async () => {- await ffmpegLazy();+ await ffmpegLazy().ready;};export const determineVideoDurationWeb = async (blob: Blob,signal?: AbortSignal,): Promise<number> =>- runFFmpegTask((ffmpeg) => ffprobeExecVideoDuration(ffmpeg, blob), signal);+ runFFmpegTask(+ "duration",+ blob.size,+ (ffmpeg) => ffprobeExecVideoDuration(ffmpeg, blob),+ signal,+ );++export const determineVideoInfoWeb = (blob: Blob, signal?: AbortSignal) =>+ runFFmpegTask(+ "metadata",+ blob.size,+ (ffmpeg) =>+ withInputMount(ffmpeg, blob, async (inputPath) => {+ const json = await ffprobeOutput(+ ffmpeg,+ [+ "-v",+ "error",+ "-select_streams",+ "v:0",+ "-show_entries",+ "format=duration:stream=width,height:stream_side_data=rotation",+ "-of",+ "json",+ "-o",+ "output.json",+ inputPath,+ ],+ "output.json",+ );+ const info = z+ .object({+ format: z.object({+ duration: z.coerce.number().positive(),+ }),+ streams: z+ .array(+ z.object({+ width: z.number().positive(),+ height: z.number().positive(),+ side_data_list: z+ .array(+ z.object({+ rotation: z.number().optional(),+ }),+ )+ .optional(),+ }),+ )+ .nonempty(),+ })+ .parse(JSON.parse(json));+ const stream = info.streams[0]!;+ const rotation =+ stream.side_data_list?.find(+ (data) => data.rotation != undefined,+ )?.rotation ?? 0;+ const sideways = Math.abs(rotation % 180) == 90;+ return {+ duration: info.format.duration,+ width: sideways ? stream.height : stream.width,+ height: sideways ? stream.width : stream.height,+ };+ }),+ signal,+ );++export const extractVideoFramesWeb = (+ blob: Blob,+ times: number[],+ maxDimension: number,+ signal?: AbortSignal,+) =>+ runFFmpegTask(+ "frames",+ blob.size,+ async (ffmpeg, signal) => {+ const frames: Blob[] = [];+ await withInputMount(ffmpeg, blob, async (inputPath) => {+ const outputPath = newID("frame_") + ".png";+ const hdr = await isHDRVideo(ffmpeg, inputPath);+ try {+ for (const time of times) {+ signal.throwIfAborted();+ const json = await ffprobeOutput(+ ffmpeg,+ [+ "-v",+ "error",+ "-select_streams",+ "v:0",+ "-read_intervals",+ `${time}%${time + 1}`,+ "-show_entries",+ "packet=pts_time:format=start_time",+ "-of",+ "json",+ "-o",+ "frame.json",+ inputPath,+ ],+ "frame.json",+ );+ const probe = z+ .object({+ format: z.object({+ start_time: z.coerce.number().optional(),+ }),+ packets: z+ .array(+ z.object({+ pts_time: z.coerce.number(),+ }),+ )+ .nonempty(),+ })+ .parse(JSON.parse(json));+ const start = probe.format.start_time ?? 0;+ const timestamps = probe.packets.map(+ (packet) => packet.pts_time - start,+ );+ const frameTime = timestamps.reduce(+ (closest, timestamp) =>+ Math.abs(timestamp - time) <+ Math.abs(closest - time)+ ? timestamp+ : closest,+ );+ const status = await ffmpeg.exec([+ "-ss",+ String(Math.max(0, frameTime - 0.000001)),+ "-i",+ inputPath,+ "-map",+ "0:v:0",+ "-frames:v",+ "1",+ "-an",+ "-vf",+ [+ `scale=w='min(${maxDimension},iw)':h='min(${maxDimension},ih)':force_original_aspect_ratio=decrease`,+ ...(hdr+ ? [+ "zscale=transfer=linear",+ "tonemap=tonemap=hable:desat=0",+ "zscale=primaries=709:transfer=709:matrix=709",+ ]+ : []),+ "format=rgb24",+ "setsar=1",+ ].join(","),+ "-map_metadata",+ "-1",+ "-update",+ "1",+ outputPath,+ ]);+ if (status != 0)+ throw new Error("Couldn't extract a video frame.");+ const bytes = await ffmpeg.readFile(outputPath);+ if (typeof bytes == "string")+ throw new Error("Expected a video frame.");+ frames.push(+ new Blob([ensureArrayBufferBacked(bytes)], {+ type: "image/png",+ }),+ );+ await ffmpeg.deleteFile(outputPath);+ }+ } finally {+ if (ffmpeg.loaded) {+ const files = await ffmpeg.listDir("/");+ if (files.some((file) => file.name == outputPath))+ await ffmpeg.deleteFile(outputPath);+ }+ }+ });+ return frames;+ },+ signal,+ );export const transcodeVideoWeb = (blob: Blob,@@ -52,56 +253,108 @@ export const transcodeVideoWeb = (fallbackCommand: FFmpegCommand,signal?: AbortSignal,) =>- runFFmpegTask(async (ffmpeg) => {- if (- typeof VideoDecoder !== "undefined" &&- typeof VideoEncoder !== "undefined"- ) {- try {- const { encodeVideoWithWebCodecs } =- await import("./web-codecs");- const result = await withInputMount(ffmpeg, blob, (inputPath) =>- encodeVideoWithWebCodecs(+ runFFmpegTask(+ "export",+ blob.size,+ async (ffmpeg, signal) => {+ if (+ typeof VideoDecoder !== "undefined" &&+ typeof VideoEncoder !== "undefined"+ ) {+ try {+ const { encodeVideoWithWebCodecs } =+ await import("./web-codecs");+ const result = await withInputMount(ffmpeg,- inputPath,blob,- edit,- signal,- ),- );- if (result) {- log.debug(() => "Encoded video with WebCodecs");- return result;+ (inputPath) =>+ encodeVideoWithWebCodecs(+ ffmpeg,+ inputPath,+ blob,+ edit,+ signal,+ ),+ );+ if (result) {+ logToDisk(+ `[info] Space video encoded encoder=webcodecs bytes=${result.byteLength}`,+ );+ return result;+ }+ } catch {+ signal.throwIfAborted();+ logToDisk(+ "[warn] Space video WebCodecs export failed, using software encoder",+ );}- } catch (error) {- signal?.throwIfAborted();- log.warn("WebCodecs export failed, using FFmpeg", error);}- }- signal?.throwIfAborted();- return ffmpegExec(ffmpeg, fallbackCommand, "mp4", blob);- }, signal);+ signal.throwIfAborted();+ logToDisk("[info] Space video encoding encoder=wasm");+ const result = await ffmpegExec(+ ffmpeg,+ fallbackCommand,+ "mp4",+ blob,+ );+ logToDisk(+ `[info] Space video encoded encoder=wasm bytes=${result.byteLength}`,+ );+ return result;+ },+ signal,+ 300_000,+ );const runFFmpegTask = <T>(- task: (ffmpeg: FFmpeg) => Promise<T>,+ stage: string,+ bytes: number,+ task: (ffmpeg: FFmpeg, signal: AbortSignal) => Promise<T>,signal?: AbortSignal,+ timeout = 60_000,): Promise<T> => {const pending = _ffmpegTaskQueue.add(async () => {signal?.throwIfAborted();- const ffmpeg = await ffmpegLazy(signal);- signal?.throwIfAborted();+ const engine = ffmpegLazy();+ const ffmpeg = engine.instance;+ const controller = new AbortController();+ const cancel = () => controller.abort(signal?.reason);const abort = () => {ffmpeg.terminate();- _ffmpeg = undefined;+ if (_ffmpeg == engine) _ffmpeg = undefined;};- signal?.addEventListener("abort", abort, { once: true });+ const timer = setTimeout(+ () =>+ controller.abort(+ new DOMException(+ "Video processing took too long. Try a shorter clip.",+ "TimeoutError",+ ),+ ),+ timeout,+ );+ signal?.addEventListener("abort", cancel, { once: true });+ controller.signal.addEventListener("abort", abort, { once: true });+ const startedAt = Date.now();+ logToDisk(`[info] Space video stage=${stage} started bytes=${bytes}`);try {- return await task(ffmpeg);+ await engine.ready;+ controller.signal.throwIfAborted();+ const result = await task(ffmpeg, controller.signal);+ logToDisk(+ `[info] Space video stage=${stage} completed elapsedMs=${Date.now() - startedAt}`,+ );+ return result;} catch (error) {- signal?.throwIfAborted();+ logToDisk(+ `[error] Space video stage=${stage} ${signal?.aborted ? "canceled" : controller.signal.aborted ? "timeout" : "failed"} elapsedMs=${Date.now() - startedAt}`,+ );+ controller.signal.throwIfAborted();throw error;} finally {- signal?.removeEventListener("abort", abort);+ clearTimeout(timer);+ signal?.removeEventListener("abort", cancel);+ controller.signal.removeEventListener("abort", abort);}}) as Promise<T>;if (!signal) return pending;@@ -229,7 +482,7 @@ const isHDRVideo = async (ffmpeg: FFmpeg, inputFilePath: string) => {ffmpeg,[["-i", inputFilePath],- "-show_streams",+ ["-show_entries", "stream=color_transfer"],["-select_streams", "v:0"],["-of", "json"],["-o", "output.json"],diff --git a/web/apps/space/tests/video-cover.test.ts b/web/apps/space/tests/video-cover.test.tsnew file mode 100644index 0000000000..98a414c285--- /dev/null+++ b/web/apps/space/tests/video-cover.test.ts@@ -0,0 +1,139 @@+import { afterEach, beforeEach, expect, test, vi } from "vitest";+import {+ spaceVideoCover,+ spaceVideoFrames,+ spaceVideoInfo,+ useSoftwareVideoFrames,+} from "../src/utils/post-video";++const mocks = vi.hoisted(() => ({+ frames: vi.fn(),+ info: vi.fn(),+ log: vi.fn(),+}));+vi.mock("../src/utils/video-encoding/web", () => ({+ extractVideoFramesWeb: mocks.frames,+ determineVideoInfoWeb: mocks.info,+}));+vi.mock("ente-base/log-web", () => ({ logToDisk: mocks.log }));+vi.mock("utils/thumbhash", () => ({ thumbHashBase64FromCanvas: () => "hash" }));++let alpha: number;+let bitmapClose: ReturnType<typeof vi.fn>;+let createElement: ReturnType<typeof vi.fn>;++beforeEach(() => {+ vi.resetAllMocks();+ alpha = 255;+ bitmapClose = vi.fn();+ vi.stubGlobal("navigator", {+ userAgent:+ "Mozilla/5.0 (Android 16; Mobile; rv:143.0) Gecko/143.0 Firefox/143.0",+ });+ mocks.frames.mockResolvedValue([new Blob(["png"], { type: "image/png" })]);+ mocks.info.mockResolvedValue({ duration: 3.45, width: 1080, height: 1920 });+ vi.stubGlobal(+ "createImageBitmap",+ vi+ .fn()+ .mockImplementation(() =>+ Promise.resolve({+ width: 540,+ height: 960,+ close: bitmapClose,+ }),+ ),+ );+ createElement = vi.fn((tag: string) => {+ if (tag != "canvas")+ throw new Error(+ "Must not decode through a video element on Firefox Android",+ );+ return {+ width: 0,+ height: 0,+ getContext: () => ({+ drawImage: vi.fn(),+ getImageData: () => ({+ data: new Uint8ClampedArray([0, 0, 0, alpha]),+ }),+ }),+ toBlob: (callback: BlobCallback) =>+ callback(new Blob(["cover"], { type: "image/webp" })),+ };+ });+ vi.stubGlobal("document", { createElement });+});+afterEach(() => vi.unstubAllGlobals());++test.each([+ [+ "Mozilla/5.0 (Android 16; Mobile; rv:143.0) Gecko/143.0 Firefox/143.0",+ true,+ ],+ ["Mozilla/5.0 (Android 16) Chrome/140.0.0.0 Mobile Safari/537.36", false],+ [+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:143.0) Gecko/20100101 Firefox/143.0",+ false,+ ],+])(+ "selects software frames only for Firefox Android (%s)",+ (userAgent, expected) => {+ vi.stubGlobal("navigator", { userAgent });+ expect(useSoftwareVideoFrames()).toBe(expected);+ },+);++test("Firefox Android gets metadata and a cover without creating a video element", async () => {+ const file = new File(["video"], "private-video.mp4");+ const controller = new AbortController();+ expect(await spaceVideoInfo(file, controller.signal)).toMatchObject({+ width: 1080,+ height: 1920,+ });+ const cover = await spaceVideoCover(file, 1.25, controller.signal);+ expect(mocks.frames).toHaveBeenCalledWith(+ file,+ [1.25],+ 960,+ controller.signal,+ );+ expect(cover).toMatchObject({ width: 540, height: 960, thumbHash: "hash" });+ expect(cover.file.type).toBe("image/webp");+ expect(bitmapClose).toHaveBeenCalledOnce();+ expect(JSON.stringify(mocks.log.mock.calls)).not.toContain("private-video");+});++test("opaque black frames are valid covers, but unreadable transparent frames fail", async () => {+ const file = new Blob(["video"]);+ await expect(spaceVideoCover(file, 0)).resolves.toBeDefined();+ alpha = 0;+ await expect(spaceVideoCover(file, 0)).rejects.toThrow(+ "Couldn't read a video frame",+ );+ expect(bitmapClose).toHaveBeenCalledTimes(2);+ expect(mocks.log).toHaveBeenCalledWith(+ expect.stringContaining("frames failed backend=wasm"),+ );+});++test("timeline frame extraction is batched and releases each decoded image", async () => {+ const file = new Blob(["video"]);+ const times = [0.5, 1.5, 2.5];+ mocks.frames.mockResolvedValue(times.map(() => new Blob(["png"])));+ expect(await spaceVideoFrames(file, times, 128)).toHaveLength(3);+ expect(mocks.frames).toHaveBeenCalledWith(file, times, 128, undefined);+ expect(bitmapClose).toHaveBeenCalledTimes(3);+});++test("canceling frame extraction does not leave decoded bitmaps or return a cover", async () => {+ const controller = new AbortController();+ mocks.frames.mockImplementation(() => {+ controller.abort();+ return Promise.resolve([new Blob(["png"])]);+ });+ await expect(+ spaceVideoCover(new Blob(["video"]), 0, controller.signal),+ ).rejects.toMatchObject({ name: "AbortError" });+ expect(createImageBitmap).not.toHaveBeenCalled();+});diff --git a/web/apps/space/tests/video-export.test.ts b/web/apps/space/tests/video-export.test.tsindex 199c17bdf9..65a279fa65 100644--- a/web/apps/space/tests/video-export.test.ts+++ b/web/apps/space/tests/video-export.test.ts@@ -3,16 +3,19 @@ import { afterEach, beforeEach, expect, test, vi } from "vitest";const mocks = vi.hoisted(() => ({encode: vi.fn(),load: vi.fn(),+ logToDisk: vi.fn(),workers: [] as {loaded: boolean;terminate: ReturnType<typeof vi.fn>;exec: ReturnType<typeof vi.fn>;unmount: ReturnType<typeof vi.fn>;+ readFile: ReturnType<typeof vi.fn>;}[],}));vi.mock("ente-base/log", () => ({default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },}));+vi.mock("ente-base/log-web", () => ({ logToDisk: mocks.logToDisk }));vi.mock("../src/utils/video-encoding/web-codecs", () => ({encodeVideoWithWebCodecs: mocks.encode,}));@@ -23,9 +26,15 @@ vi.mock("@ffmpeg/ffmpeg", () => ({constructor() {mocks.workers.push(this);}- load = mocks.load;+ rejectLoad: ((error: Error) => void) | undefined;+ load = (...args: unknown[]) =>+ new Promise((resolve, reject) => {+ this.rejectLoad = reject;+ Promise.resolve(mocks.load(...args)).then(resolve, reject);+ });terminate = vi.fn(() => {this.loaded = false;+ this.rejectLoad?.(new Error("Terminated"));});on = vi.fn();off = vi.fn();@@ -34,6 +43,8 @@ vi.mock("@ffmpeg/ffmpeg", () => ({mount = vi.fn();unmount = vi.fn();deleteFile = vi.fn();+ listDir = vi.fn().mockResolvedValue([]);+ ffprobe = vi.fn().mockResolvedValue(0);exec = vi.fn().mockResolvedValue(0);readFile = vi.fn().mockResolvedValue(new Uint8Array([7]));},@@ -44,10 +55,14 @@ beforeEach(() => {mocks.encode.mockReset();mocks.load.mockReset().mockResolvedValue(true);mocks.workers.length = 0;+ mocks.logToDisk.mockClear();vi.stubGlobal("VideoEncoder", {});vi.stubGlobal("VideoDecoder", {});});-afterEach(() => vi.unstubAllGlobals());+afterEach(() => {+ vi.unstubAllGlobals();+ vi.useRealTimers();+});const exporter = async () => {const { transcodeVideoWeb } =@@ -165,16 +180,7 @@ test("a queued export can be cancelled without waiting for or stopping another e});test("cancelling engine loading terminates it and allows the next export to load again", async () => {- mocks.load.mockImplementationOnce(- (_config, { signal }: { signal: AbortSignal }) =>- new Promise((_, reject) =>- signal.addEventListener(- "abort",- () => reject(new DOMException("Canceled", "AbortError")),- { once: true },- ),- ),- );+ mocks.load.mockImplementationOnce(() => new Promise(() => undefined));const run = await exporter();const controller = new AbortController();const pending = run(controller.signal);@@ -189,3 +195,79 @@ test("cancelling engine loading terminates it and allows the next export to loadexpect(mocks.workers[0]!.terminate).toHaveBeenCalled();expect(mocks.workers).toHaveLength(2);});++test("a stalled engine load times out and can be retried", async () => {+ vi.useFakeTimers();+ mocks.load.mockImplementationOnce(() => new Promise(() => undefined));+ const { preloadVideoEncoderWeb } =+ await import("../src/utils/video-encoding/web");+ const rejected = expect(preloadVideoEncoderWeb()).rejects.toThrow(+ "took too long to load",+ );+ await vi.advanceTimersByTimeAsync(60_000);+ await rejected;+ await preloadVideoEncoderWeb();+ expect(mocks.workers).toHaveLength(2);+ expect(mocks.workers[0]!.terminate).toHaveBeenCalled();+});++test("a stalled export times out, closes the worker, and releases the queue for retry", async () => {+ vi.useFakeTimers();+ mocks.encode+ .mockImplementationOnce(+ (_ffmpeg, _path, _blob, _edit, signal: AbortSignal) =>+ new Promise((_, reject) =>+ signal.addEventListener("abort", () =>+ reject(signal.reason as Error),+ ),+ ),+ )+ .mockResolvedValue(new Uint8Array([3]));+ const run = await exporter();+ const first = run();+ const rejected = expect(first).rejects.toMatchObject({+ name: "TimeoutError",+ });+ await vi.advanceTimersByTimeAsync(300_000);+ await rejected;+ expect(await run()).toEqual(new Uint8Array([3]));+ expect(mocks.workers[0]!.terminate).toHaveBeenCalled();+ expect(mocks.workers[0]!.exec).not.toHaveBeenCalled();+ expect(mocks.workers).toHaveLength(2);+ expect(mocks.logToDisk).toHaveBeenCalledWith(+ expect.stringContaining("stage=export timeout"),+ );+});++test("cover extraction near the clip end seeks to the last available frame", async () => {+ const { preloadVideoEncoderWeb, extractVideoFramesWeb } =+ await import("../src/utils/video-encoding/web");+ await preloadVideoEncoderWeb();+ const worker = mocks.workers[0]!;+ worker.readFile.mockImplementation((path: string) => {+ if (path == "output.json") return JSON.stringify({ streams: [{}] });+ if (path == "frame.json")+ return JSON.stringify({+ format: { start_time: "0" },+ packets: [+ { pts_time: "3.38" },+ { pts_time: "3.413667" },+ { pts_time: "3.430322" },+ ],+ });+ return new Uint8Array([1, 2, 3]);+ });+ const frames = await extractVideoFramesWeb(+ new Blob(["video"]),+ [3.4499],+ 960,+ );+ expect(frames).toHaveLength(1);+ expect(frames[0]!.type).toBe("image/png");+ const command = worker.exec.mock.calls[0]![0] as string[];+ expect(Number(command[command.indexOf("-ss") + 1])).toBeCloseTo(+ 3.430321,+ 6,+ );+ expect(worker.unmount).toHaveBeenCalled();+});diff --git a/web/apps/space/tests/video-trim-playback.test.ts b/web/apps/space/tests/video-trim-playback.test.tsindex 5a61e73304..b21d40a373 100644--- a/web/apps/space/tests/video-trim-playback.test.ts+++ b/web/apps/space/tests/video-trim-playback.test.ts@@ -1,6 +1,8 @@import { afterEach, beforeEach, expect, test, vi } from "vitest";import { createVideoTrimPlayback } from "../src/components/video-trim/playback";+vi.mock("ente-base/log-web", () => ({ logToDisk: vi.fn() }));+class Video extends EventTarget {currentTime = 0;paused = true;@@ -47,6 +49,7 @@ beforeEach(() => {afterEach(() => {playback.dispose();vi.unstubAllGlobals();+ vi.useRealTimers();});const renderFrame = () => {@@ -187,3 +190,29 @@ test("closing the editor pauses playback and removes the frame monitor", () => {video.dispatchEvent(new Event("timeupdate"));expect(video.currentTime).toBe(12);});++test("a stuck preview reports an error and can be played again", async () => {+ vi.useFakeTimers();+ vi.spyOn(video, "play").mockImplementationOnce(+ () => new Promise(() => undefined),+ );+ playback.toggle();+ await vi.advanceTimersByTimeAsync(30_000);+ expect(onError).toHaveBeenCalledTimes(1);+ playback.toggle();+ expect(video.paused).toBe(false);+ video.dispatchEvent(new Event("playing"));+ await vi.advanceTimersByTimeAsync(30_000);+ expect(onError).toHaveBeenCalledTimes(1);+});++test("closing or pausing a loading preview cancels its timeout", async () => {+ vi.useFakeTimers();+ vi.spyOn(video, "play").mockImplementationOnce(+ () => new Promise(() => undefined),+ );+ playback.toggle();+ playback.dispose();+ await vi.advanceTimersByTimeAsync(30_000);+ expect(onError).not.toHaveBeenCalled();+});