Skip to content

armgit / ente

Public Imported from GitHub
0

πŸ’š End-to-end encrypted cloud for everything.

Fix Firefox Android video covers and processing

Anand Baburajan942b4bc2deb1
diff --git a/web/apps/space/src/components/PostComposer.tsx b/web/apps/space/src/components/PostComposer.tsx
index 25a44a600f..f14035f6b2 100644
--- a/web/apps/space/src/components/PostComposer.tsx
+++ b/web/apps/space/src/components/PostComposer.tsx
@@ -70,7 +70,7 @@ export const SpacePostComposer: React.FC<{
const inputRef = React.useRef<HTMLInputElement>(null);
const previewURLsRef = React.useRef(new Set<string>());
const publishedPreviewURLRef = React.useRef<string>(undefined);
- const preparingRef = React.useRef(new Set<number>());
+ const preparingRef = React.useRef(new Map<number, AbortController>());
const draftsRef = React.useRef(drafts);
draftsRef.current = drafts;
const mountedRef = React.useRef(false);
@@ -96,8 +96,11 @@ export const SpacePostComposer: React.FC<{
React.useEffect(() => {
mountedRef.current = true;
const urls = previewURLsRef.current;
+ const preparing = preparingRef.current;
return () => {
mountedRef.current = false;
+ preparing.forEach((controller) => controller.abort());
+ preparing.clear();
urls.forEach((url) => {
if (url != publishedPreviewURLRef.current)
URL.revokeObjectURL(url);
@@ -116,7 +119,12 @@ export const SpacePostComposer: React.FC<{
!draft.error &&
!preparingRef.current.has(draft.id),
);
- pending.forEach((draft) => preparingRef.current.add(draft.id));
+ const controllers = new Map(
+ pending.map((draft) => [draft.id, new AbortController()]),
+ );
+ controllers.forEach((controller, id) =>
+ preparingRef.current.set(id, controller),
+ );
if (pending.some((draft) => isSpaceVideoFile(draft.file))) {
void import("utils/video-encoding/web")
.then(({ preloadVideoEncoderWeb }) => preloadVideoEncoderWeb())
@@ -127,15 +135,19 @@ export const SpacePostComposer: React.FC<{
void (async () => {
for (const draft of pending) {
if (!isActiveDraft(draft.id)) continue;
+ const controller = controllers.get(draft.id)!;
+ const { signal } = controller;
try {
+ signal.throwIfAborted();
let photo: SpaceViewerPhoto;
let videoEdit: SpacePostVideoEdit | undefined;
if (isSpaceVideoFile(draft.file)) {
- const info = await spaceVideoInfo(draft.file);
+ const info = await spaceVideoInfo(draft.file, signal);
videoEdit = initialSpaceVideoEdit(info.duration);
const cover = await spaceVideoCover(
draft.file,
videoEdit.coverTime,
+ signal,
);
const url = URL.createObjectURL(draft.file);
previewURLsRef.current.add(url);
@@ -191,6 +203,7 @@ export const SpacePostComposer: React.FC<{
),
);
} catch (error) {
+ if (signal.aborted) continue;
log.error("Failed to prepare post preview", error);
if (isActiveDraft(draft.id))
setDrafts((current) =>
@@ -210,7 +223,8 @@ export const SpacePostComposer: React.FC<{
),
);
} finally {
- preparingRef.current.delete(draft.id);
+ if (preparingRef.current.get(draft.id) == controller)
+ preparingRef.current.delete(draft.id);
}
}
})();
@@ -221,6 +235,7 @@ export const SpacePostComposer: React.FC<{
setDrafts((current) => [...current, ...draftPhotos(files)]);
};
const releasePreview = (draft: DraftPhoto) => {
+ preparingRef.current.get(draft.id)?.abort();
for (const url of new Set([
draft.photo?.imageUrl,
draft.originalPhoto?.imageUrl,
diff --git a/web/apps/space/src/components/PostVideoContent.ts b/web/apps/space/src/components/PostVideoContent.ts
index dbc5c8c268..554c7fa691 100644
--- a/web/apps/space/src/components/PostVideoContent.ts
+++ b/web/apps/space/src/components/PostVideoContent.ts
@@ -1,4 +1,5 @@
import type { SpaceViewerPhoto } from "components/FileViewer";
+import { logToDisk } from "ente-base/log-web";
import type { SpacePostAssetURLLoader } from "services/space";
export const createSpaceVideoContent = (
@@ -63,19 +64,40 @@ export const createSpaceVideoContent = (
loading.setAttribute("aria-label", "Loading video");
loading.innerHTML = '<span class="space-video-spinner"></span>';
Object.assign(loading.style, { ...controlStyle, pointerEvents: "none" });
- element.append(video, button, loading);
+ const errorMessage = document.createElement("span");
+ errorMessage.textContent = "Couldn't play this video. Tap to retry.";
+ errorMessage.hidden = true;
+ errorMessage.setAttribute("role", "status");
+ Object.assign(errorMessage.style, {
+ position: "absolute",
+ bottom: "12px",
+ left: "12px",
+ right: "12px",
+ background: "rgba(0, 0, 0, 0.65)",
+ color: "white",
+ borderRadius: "6px",
+ padding: "8px",
+ fontSize: "13px",
+ textAlign: "center",
+ pointerEvents: "none",
+ });
+ element.append(video, button, loading, errorMessage);
let generation = 0;
let ownedURL: string | undefined;
let disposed = false;
let active = inline;
let loadingTimer: ReturnType<typeof setTimeout> | undefined;
+ let playbackTimeout: ReturnType<typeof setTimeout> | undefined;
const hideLoading = () => {
clearTimeout(loadingTimer);
loadingTimer = undefined;
+ clearTimeout(playbackTimeout);
+ playbackTimeout = undefined;
loading.setAttribute("aria-hidden", "true");
element.removeAttribute("aria-busy");
};
const showLoading = () => {
+ playbackTimeout ??= setTimeout(() => failPlayback("timeout"), 30_000);
if (document.activeElement == button)
video.focus({ preventScroll: true });
element.setAttribute("aria-busy", "true");
@@ -112,8 +134,16 @@ export const createSpaceVideoContent = (
video.removeAttribute("src");
video.load();
};
+ const failPlayback = (reason: "timeout" | "media" | "play") => {
+ logToDisk(
+ `[error] Space video playback failed reason=${reason} code=${video.error?.code ?? 0} readyState=${video.readyState} networkState=${video.networkState}`,
+ );
+ reset();
+ errorMessage.hidden = false;
+ };
const clear = () => {
active = inline;
+ errorMessage.hidden = true;
reset();
if (ownedURL) URL.revokeObjectURL(ownedURL);
ownedURL = undefined;
@@ -125,6 +155,7 @@ export const createSpaceVideoContent = (
const play = () => {
if (button.disabled || !video.paused) return;
active = true;
+ errorMessage.hidden = true;
const current = ++generation;
showLoading();
button.disabled = true;
@@ -142,7 +173,6 @@ export const createSpaceVideoContent = (
if (disposed || generation != current) return;
if (!video.getAttribute("src")) {
video.src = url;
- video.currentTime = media.start ?? 0;
} else if (
video.ended ||
(media.end != undefined && video.currentTime >= media.end)
@@ -152,7 +182,7 @@ export const createSpaceVideoContent = (
await video.play();
if (generation == current) showPlaying();
})().catch(() => {
- if (!disposed && generation == current) showPlayButton();
+ if (!disposed && generation == current) failPlayback("play");
});
};
const togglePlayback = () => {
@@ -189,7 +219,10 @@ export const createSpaceVideoContent = (
if (!video.paused) showLoading();
};
video.onplaying = showPlaying;
- video.onerror = pause;
+ video.onerror = () => failPlayback("media");
+ video.onloadedmetadata = () => {
+ if (media.start) video.currentTime = media.start;
+ };
video.ontimeupdate = () => {
if (media.end != undefined && video.currentTime >= media.end) {
if (inline) video.currentTime = media.start ?? 0;
diff --git a/web/apps/space/src/components/PostVideoEditor.tsx b/web/apps/space/src/components/PostVideoEditor.tsx
index 213eeed270..2eed1c0842 100644
--- a/web/apps/space/src/components/PostVideoEditor.tsx
+++ b/web/apps/space/src/components/PostVideoEditor.tsx
@@ -11,8 +11,7 @@ import {
clampVideoCover,
initialSpaceVideoEdit,
maxSpaceVideoDuration,
- openSpaceVideo,
- seekSpaceVideo,
+ spaceVideoFrames,
type SpacePostVideoEdit,
} from "utils/post-video";
@@ -122,38 +121,30 @@ export const SpacePostVideoEditor: React.FC<{
const urls: string[] = [];
const timer = window.setTimeout(() => {
void (async () => {
- const { video, dispose } = await openSpaceVideo(file, signal);
- try {
- const times = Array.from(
- { length: 8 },
- (_, i) => visibleStart + (span * (i + 0.5)) / 8,
- );
- for (const time of times) {
- await seekSpaceVideo(video, time, signal);
- const canvas = document.createElement("canvas");
- canvas.width = 80;
- canvas.height = 64;
- const scale = Math.max(
- 80 / video.videoWidth,
- 64 / video.videoHeight,
+ const times = Array.from(
+ { length: 8 },
+ (_, i) => visibleStart + (span * (i + 0.5)) / 8,
+ );
+ const frames = await spaceVideoFrames(file, times, 128, signal);
+ for (const frame of frames) {
+ const canvas = document.createElement("canvas");
+ canvas.width = 80;
+ canvas.height = 64;
+ const scale = Math.max(80 / frame.width, 64 / frame.height);
+ const width = frame.width * scale;
+ const height = frame.height * scale;
+ canvas
+ .getContext("2d")!
+ .drawImage(
+ frame,
+ (80 - width) / 2,
+ (64 - height) / 2,
+ width,
+ height,
);
- const width = video.videoWidth * scale;
- const height = video.videoHeight * scale;
- canvas
- .getContext("2d")!
- .drawImage(
- video,
- (80 - width) / 2,
- (64 - height) / 2,
- width,
- height,
- );
- urls.push(canvas.toDataURL("image/jpeg", 0.65));
- }
- if (!signal.aborted) setFrames(urls);
- } finally {
- dispose();
+ urls.push(canvas.toDataURL("image/jpeg", 0.65));
}
+ if (!signal.aborted) setFrames(urls);
})().catch(() => {
if (!signal.aborted) setFrames([]);
});
diff --git a/web/apps/space/src/components/video-trim/playback.ts b/web/apps/space/src/components/video-trim/playback.ts
index ba2e8edf5d..2d5bbdd423 100644
--- a/web/apps/space/src/components/video-trim/playback.ts
+++ b/web/apps/space/src/components/video-trim/playback.ts
@@ -1,3 +1,5 @@
+import { logToDisk } from "ente-base/log-web";
+
export const createVideoTrimPlayback = (
video: HTMLVideoElement,
getRange: () => { start: number; end: number },
@@ -8,6 +10,12 @@ export const createVideoTrimPlayback = (
let restartOnPlay = true;
let playRequested = false;
let generation = 0;
+ let loadingTimer: ReturnType<typeof setTimeout> | undefined;
+
+ const clearLoadingTimer = () => {
+ clearTimeout(loadingTimer);
+ loadingTimer = undefined;
+ };
const cancelFrame = () => {
if (frame != undefined) cancelAnimationFrame(frame);
@@ -17,6 +25,7 @@ export const createVideoTrimPlayback = (
const pause = () => {
generation++;
playRequested = false;
+ clearLoadingTimer();
video.pause();
cancelFrame();
};
@@ -63,11 +72,23 @@ export const createVideoTrimPlayback = (
if (!video.paused) return;
generation++;
playRequested = false;
+ clearLoadingTimer();
cancelFrame();
};
const onEnded = () => seek(getRange().start);
+ const failed = () => {
+ logToDisk(
+ `[error] Space video editor playback failed code=${video.error?.code ?? 0} readyState=${video.readyState} networkState=${video.networkState}`,
+ );
+ pause();
+ onError();
+ };
+ const waiting = () => {
+ if (playRequested) loadingTimer ??= setTimeout(failed, 30_000);
+ };
+
const toggle = () => {
if (playRequested || !video.paused) {
pause();
@@ -87,10 +108,10 @@ export const createVideoTrimPlayback = (
video.currentTime = time;
const current = ++generation;
playRequested = true;
+ waiting();
void video.play().catch(() => {
if (generation != current) return;
- pause();
- onError();
+ failed();
});
};
@@ -100,6 +121,9 @@ export const createVideoTrimPlayback = (
video.addEventListener("timeupdate", checkRange);
video.addEventListener("seeked", applySeek);
video.addEventListener("loadedmetadata", applySeek);
+ video.addEventListener("playing", clearLoadingTimer);
+ video.addEventListener("waiting", waiting);
+ video.addEventListener("error", failed);
return {
toggle,
@@ -112,6 +136,9 @@ export const createVideoTrimPlayback = (
video.removeEventListener("timeupdate", checkRange);
video.removeEventListener("seeked", applySeek);
video.removeEventListener("loadedmetadata", applySeek);
+ video.removeEventListener("playing", clearLoadingTimer);
+ video.removeEventListener("waiting", waiting);
+ video.removeEventListener("error", failed);
},
};
};
diff --git a/web/apps/space/src/screens/HomeScreen.tsx b/web/apps/space/src/screens/HomeScreen.tsx
index 53d2cb725e..930fa8f786 100644
--- a/web/apps/space/src/screens/HomeScreen.tsx
+++ b/web/apps/space/src/screens/HomeScreen.tsx
@@ -374,6 +374,7 @@ type FeedPhotoSource = SpacePostPhoto & Pick<SpacePost, "postId" | "spaceId">;
interface FeedItemProps {
onRetry?: () => void;
+ processing?: string;
photoCount?: number;
photoIndex?: number;
photos?: FeedPostPhoto[];
@@ -871,6 +872,7 @@ const FeedItem: React.FC<FeedItemProps> = ({
onOpenProfile,
onSetPostLiked,
onRetry,
+ processing,
postId,
spaceId,
thumbHash,
@@ -1566,7 +1568,7 @@ const FeedItem: React.FC<FeedItemProps> = ({
role="status"
aria-label={
timestampStatus == "posting"
- ? "Posting"
+ ? (processing ?? "Posting")
: timestampStatus == "failed"
? "Failed"
: "Posted"
@@ -1642,7 +1644,9 @@ const FeedItem: React.FC<FeedItemProps> = ({
</>
) : (
<>
- <Box component="span">Posting</Box>
+ <Box component="span">
+ {processing ?? "Posting"}
+ </Box>
<Box
component="span"
aria-hidden
@@ -2221,6 +2225,7 @@ export const HomeScreen: React.FC<HomeScreenProps> = ({
onRetry={
onRetryPost ? () => void onRetryPost(item.id) : undefined
}
+ processing={item.processing}
timestampStatus={
item.status == "failed"
? "failed"
diff --git a/web/apps/space/src/state/AppStateProvider.tsx b/web/apps/space/src/state/AppStateProvider.tsx
index 199ba650db..6a6b8510bf 100644
--- a/web/apps/space/src/state/AppStateProvider.tsx
+++ b/web/apps/space/src/state/AppStateProvider.tsx
@@ -89,6 +89,7 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({
const profileLoadGenerationRef = useRef(0);
const postPublishGenerationRef = useRef(0);
const postReadbacksRef = useRef(new Map<string, () => Promise<void>>());
+ const postPreparationsRef = useRef(new Map<string, AbortController>());
const postUploadsRef = useRef(
new Map<string, () => ReturnType<SpaceAppState["publishPost"]>>(),
);
@@ -96,6 +97,7 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({
useEffect(() => {
const readbacks = postReadbacksRef.current;
const uploads = postUploadsRef.current;
+ const preparations = postPreparationsRef.current;
const retryReadbacks = () => {
readbacks.forEach((readback) => void readback());
};
@@ -106,6 +108,8 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({
window.removeEventListener("focus", retryReadbacks);
readbacks.clear();
uploads.clear();
+ preparations.forEach((controller) => controller.abort());
+ preparations.clear();
};
}, []);
@@ -178,6 +182,11 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({
},
};
const prepared: PreparedSpacePostMedia[] = [];
+ const videoCount = images.filter((image) => image.video).length;
+ const mediaCounts = {
+ photo: images.length - videoCount,
+ video: videoCount,
+ };
const session: SpacePostUploadSession = {
requestId: crypto.randomUUID(),
uploads: [],
@@ -189,23 +198,50 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({
const generation = ++postPublishGenerationRef.current;
const startedAt = Date.now();
attempt += 1;
+ const controller = new AbortController();
+ const { signal } = controller;
+ postPreparationsRef.current.set(localPostId, controller);
setPostPublication(publication);
setLocalFeedPosts((current) =>
current.map((item) =>
item.id == localPostId && item.status == "failed"
- ? { ...item, status: "pending" }
+ ? {
+ ...item,
+ status: "pending",
+ processing: undefined,
+ }
: item,
),
);
pending = (async () => {
try {
+ const processingCounts = { photo: 0, video: 0 };
for (const [index, image] of images.entries()) {
+ signal.throwIfAborted();
+ const mediaType = image.video ? "video" : "photo";
+ const mediaIndex = ++processingCounts[mediaType];
if (prepared[index]) continue;
+ if (videoCount > 0 || images.length >= 3) {
+ const count = mediaCounts[mediaType];
+ const processing =
+ images.length == 1
+ ? "Processing"
+ : `Processing ${mediaType}${count > 1 ? ` ${mediaIndex}/${count}` : ""}`;
+ setLocalFeedPosts((current) =>
+ current.map((item) =>
+ item.id == localPostId &&
+ item.status == "pending"
+ ? { ...item, processing }
+ : item,
+ ),
+ );
+ }
try {
prepared[index] = image.video
? await prepareSpaceVideo(
image.file,
image.video,
+ signal,
)
: await prepareSpacePostImageFromEdit(
image.file,
@@ -219,6 +255,16 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({
throw error;
}
}
+ signal.throwIfAborted();
+ postPreparationsRef.current.delete(localPostId);
+ setLocalFeedPosts((current) =>
+ current.map((item) =>
+ item.id == localPostId &&
+ item.status == "pending"
+ ? { ...item, processing: undefined }
+ : item,
+ ),
+ );
if (!postUploadsRef.current.has(localPostId))
throw new DOMException("Canceled", "AbortError");
const postId = await createCurrentMediaPost({
@@ -328,6 +374,7 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({
throw error;
}
})().finally(() => {
+ postPreparationsRef.current.delete(localPostId);
pending = undefined;
});
return pending;
@@ -514,6 +561,8 @@ export const SpaceAppStateProvider: React.FC<React.PropsWithChildren> = ({
postPublishGenerationRef.current += 1;
postReadbacksRef.current.clear();
postUploadsRef.current.clear();
+ postPreparationsRef.current.forEach((controller) => controller.abort());
+ postPreparationsRef.current.clear();
setPostPublication(null);
setPendingLoginCredentials(null);
setPendingPasskeyVerification(null);
diff --git a/web/apps/space/src/state/app-state.tsx b/web/apps/space/src/state/app-state.tsx
index 0d20253d5d..84b29635c0 100644
--- a/web/apps/space/src/state/app-state.tsx
+++ b/web/apps/space/src/state/app-state.tsx
@@ -38,6 +38,7 @@ interface PendingSpaceFeedPost {
spaceId: string;
photoCount: number;
postId?: number;
+ processing?: string;
status: "pending";
timestampMs: number;
width?: number;
diff --git a/web/apps/space/src/utils/post-video.ts b/web/apps/space/src/utils/post-video.ts
index 1148b34b76..3435b0232f 100644
--- a/web/apps/space/src/utils/post-video.ts
+++ b/web/apps/space/src/utils/post-video.ts
@@ -1,3 +1,4 @@
+import { logToDisk } from "ente-base/log-web";
import type { FFmpegCommand } from "ente-base/types/ipc";
import type { PreparedSpacePostImage } from "utils/post-image";
import { thumbHashBase64FromCanvas } from "utils/thumbhash";
@@ -21,6 +22,10 @@ export interface SpaceVideoInfo {
export const maxSpaceVideoDuration = 10;
+export const useSoftwareVideoFrames = () =>
+ navigator.userAgent.includes("Android") &&
+ navigator.userAgent.includes("Firefox/");
+
export const isSpaceVideoFile = (file: File) =>
file.type.startsWith("video/") || /\.(mp4|mov|m4v|webm)$/i.test(file.name);
@@ -49,8 +54,13 @@ const waitForVideo = (
video.removeEventListener(event, ready);
video.removeEventListener("error", failed);
signal?.removeEventListener("abort", abort);
- if (error) reject(error);
- else resolve();
+ if (error) {
+ if (!signal?.aborted)
+ logToDisk(
+ `[error] Space video event=${event} failed code=${video.error?.code ?? 0} readyState=${video.readyState} networkState=${video.networkState}`,
+ );
+ reject(error);
+ } else resolve();
};
const ready = () => finish();
const failed = () =>
@@ -104,8 +114,15 @@ export const seekSpaceVideo = async (
await ready;
};
-export const spaceVideoInfo = async (file: File): Promise<SpaceVideoInfo> => {
- const { video, dispose } = await openSpaceVideo(file);
+export const spaceVideoInfo = async (
+ file: File,
+ signal?: AbortSignal,
+): Promise<SpaceVideoInfo> => {
+ if (useSoftwareVideoFrames()) {
+ const { determineVideoInfoWeb } = await import("./video-encoding/web");
+ return determineVideoInfoWeb(file, signal);
+ }
+ const { video, dispose } = await openSpaceVideo(file, signal);
try {
if (!Number.isFinite(video.duration) || video.duration <= 0)
throw new Error("This video has no readable duration.");
@@ -119,48 +136,104 @@ export const spaceVideoInfo = async (file: File): Promise<SpaceVideoInfo> => {
}
};
-export const spaceVideoCover = async (
+export const spaceVideoFrames = async (
file: Blob,
- time: number,
+ times: number[],
+ maxDimension: number,
signal?: AbortSignal,
-): Promise<PreparedSpacePostImage> => {
- const { video, dispose } = await openSpaceVideo(file, signal);
- try {
- await seekSpaceVideo(video, time, signal);
- const scale = Math.min(
- 1,
- 1280 / Math.max(video.videoWidth, video.videoHeight),
- );
+) => {
+ const startedAt = Date.now();
+ const software = useSoftwareVideoFrames();
+ const frames: HTMLCanvasElement[] = [];
+ const draw = (source: CanvasImageSource, width: number, height: number) => {
+ const scale = Math.min(1, maxDimension / Math.max(width, height));
const canvas = document.createElement("canvas");
- canvas.width = Math.round(video.videoWidth * scale);
- canvas.height = Math.round(video.videoHeight * scale);
+ canvas.width = Math.max(1, Math.round(width * scale));
+ canvas.height = Math.max(1, Math.round(height * scale));
const context = canvas.getContext("2d");
if (!context) throw new Error("Couldn't prepare the video cover.");
- context.drawImage(video, 0, 0, canvas.width, canvas.height);
- const thumbHash = thumbHashBase64FromCanvas(canvas);
- const blob = await new Promise<Blob>((resolve, reject) =>
- canvas.toBlob(
- (blob) =>
- blob
- ? resolve(blob)
- : reject(
- new Error("Couldn't prepare the video cover."),
- ),
- "image/webp",
- 0.85,
- ),
+ context.drawImage(source, 0, 0, canvas.width, canvas.height);
+ const sample = document.createElement("canvas");
+ sample.width = sample.height = 1;
+ const sampleContext = sample.getContext("2d")!;
+ sampleContext.drawImage(canvas, 0, 0, 1, 1);
+ if (sampleContext.getImageData(0, 0, 1, 1).data[3] == 0)
+ throw new Error("Couldn't read a video frame. Please try again.");
+ frames.push(canvas);
+ };
+ try {
+ if (software) {
+ const { extractVideoFramesWeb } =
+ await import("./video-encoding/web");
+ const blobs = await extractVideoFramesWeb(
+ file,
+ times,
+ maxDimension,
+ signal,
+ );
+ for (const blob of blobs) {
+ signal?.throwIfAborted();
+ const bitmap = await createImageBitmap(blob);
+ try {
+ draw(bitmap, bitmap.width, bitmap.height);
+ } finally {
+ bitmap.close();
+ }
+ }
+ } else {
+ const { video, dispose } = await openSpaceVideo(file, signal);
+ try {
+ for (const time of times) {
+ await seekSpaceVideo(video, time, signal);
+ draw(video, video.videoWidth, video.videoHeight);
+ }
+ } finally {
+ dispose();
+ }
+ }
+ logToDisk(
+ `[info] Space video frames backend=${software ? "wasm" : "browser"} count=${frames.length} bytes=${file.size} elapsedMs=${Date.now() - startedAt}`,
);
- return {
- file: new File([blob], "cover.webp", { type: blob.type }),
- width: canvas.width,
- height: canvas.height,
- thumbHash,
- };
- } finally {
- dispose();
+ return frames;
+ } catch (error) {
+ if (!signal?.aborted)
+ logToDisk(
+ `[error] Space video frames failed backend=${software ? "wasm" : "browser"} count=${frames.length}/${times.length} bytes=${file.size} elapsedMs=${Date.now() - startedAt}`,
+ );
+ throw error;
}
};
+export const spaceVideoCover = async (
+ file: Blob,
+ time: number,
+ signal?: AbortSignal,
+): Promise<PreparedSpacePostImage> => {
+ const [canvas] = await spaceVideoFrames(file, [time], 960, signal);
+ if (!canvas) throw new Error("Couldn't prepare the video cover.");
+ const thumbHash = thumbHashBase64FromCanvas(canvas);
+ const blob = await new Promise<Blob>((resolve, reject) =>
+ canvas.toBlob(
+ (blob) =>
+ blob
+ ? resolve(blob)
+ : reject(new Error("Couldn't prepare the video cover.")),
+ "image/webp",
+ 0.8,
+ ),
+ );
+ signal?.throwIfAborted();
+ logToDisk(
+ `[info] Space video cover prepared width=${canvas.width} height=${canvas.height} bytes=${blob.size}`,
+ );
+ return {
+ file: new File([blob], "cover.webp", { type: blob.type }),
+ width: canvas.width,
+ height: canvas.height,
+ thumbHash,
+ };
+};
+
const videoExportCommand = (
edit: SpacePostVideoEdit,
hdr: boolean,
@@ -175,7 +248,7 @@ const videoExportCommand = (
"0:v:0",
...(edit.muted
? ["-an"]
- : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "192k"]),
+ : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "128k"]),
"-map_metadata",
"-1",
"-map_metadata:s",
@@ -185,7 +258,7 @@ const videoExportCommand = (
"-vf",
[
"fps=30:start_time=0",
- "scale=w='trunc(iw*min(1,min(1920/max(iw,ih),1080/min(iw,ih)))/2)*2':h='trunc(ih*min(1,min(1920/max(iw,ih),1080/min(iw,ih)))/2)*2'",
+ "scale=w='trunc(iw*min(1,min(1280/max(iw,ih),720/min(iw,ih)))/2)*2':h='trunc(ih*min(1,min(1280/max(iw,ih),720/min(iw,ih)))/2)*2'",
...(hdr
? [
"zscale=transfer=linear",
@@ -201,11 +274,11 @@ const videoExportCommand = (
"-preset",
"veryfast",
"-crf",
- "20",
+ "23",
"-maxrate",
- "8M",
+ "4M",
"-bufsize",
- "16M",
+ "8M",
"-movflags",
"+faststart",
outputPathPlaceholder,
diff --git a/web/apps/space/src/utils/video-encoding/web-codecs.ts b/web/apps/space/src/utils/video-encoding/web-codecs.ts
index cb13b4d30c..70f0cd5ec9 100644
--- a/web/apps/space/src/utils/video-encoding/web-codecs.ts
+++ b/web/apps/space/src/utils/video-encoding/web-codecs.ts
@@ -214,7 +214,7 @@ export const encodeVideoWithWebCodecs = async (
codec: "avc1.640028",
width: Math.floor((width * scale) / 2) * 2,
height: Math.floor((height * scale) / 2) * 2,
- bitrate: 8_000_000,
+ bitrate: 4_000_000,
bitrateMode: "variable",
framerate: frameRate,
latencyMode: "realtime",
@@ -267,7 +267,7 @@ export const encodeVideoWithWebCodecs = async (
"30000",
...(edit.muted
? ["-an"]
- : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "192k"]),
+ : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "128k"]),
"-map_metadata",
"-1",
"-map_metadata:s",
diff --git a/web/apps/space/src/utils/video-encoding/web.ts b/web/apps/space/src/utils/video-encoding/web.ts
index 91f7d387e8..862ac0d6d6 100644
--- a/web/apps/space/src/utils/video-encoding/web.ts
+++ b/web/apps/space/src/utils/video-encoding/web.ts
@@ -2,6 +2,7 @@ import { FFFSType, FFmpeg } from "@ffmpeg/ffmpeg";
import { joinPath } from "ente-base/file-name";
import { newID } from "ente-base/id";
import log from "ente-base/log";
+import { logToDisk } from "ente-base/log-web";
import type { FFmpegCommand } from "ente-base/types/ipc";
import { ensureArrayBufferBacked } from "ente-utils/bytes";
import { PromiseQueue } from "ente-utils/promise";
@@ -9,42 +10,242 @@ import { z } from "zod";
import { inputPathPlaceholder, outputPathPlaceholder } from "./constants";
import type { VideoTranscodeEdit } from "./web-codecs";
-let _ffmpeg: Promise<FFmpeg> | undefined;
+let _ffmpeg: { instance: FFmpeg; ready: Promise<void> } | undefined;
const _ffmpegTaskQueue = new PromiseQueue<unknown>();
-const ffmpegLazy = (signal?: AbortSignal): Promise<FFmpeg> =>
- (_ffmpeg ??= createFFmpeg(signal));
+const ffmpegLazy = () => {
+ if (!_ffmpeg) {
+ const instance = new FFmpeg();
+ _ffmpeg = { instance, ready: loadFFmpeg(instance) };
+ }
+ return _ffmpeg;
+};
-const createFFmpeg = async (signal?: AbortSignal) => {
- const ffmpeg = new FFmpeg();
+const loadFFmpeg = async (ffmpeg: FFmpeg) => {
+ const startedAt = Date.now();
+ const timeout = new AbortController();
+ const timer = setTimeout(() => {
+ timeout.abort();
+ ffmpeg.terminate();
+ }, 60_000);
+ logToDisk("[info] Space video encoder loading");
try {
- await ffmpeg.load(
- {
- coreURL:
- "https://assets.ente.com/ffmpeg-core-0.12.10/ffmpeg-core.js",
- wasmURL:
- "https://assets.ente.com/ffmpeg-core-0.12.10/ffmpeg-core.wasm",
- },
- { signal },
+ await ffmpeg.load({
+ coreURL:
+ "https://assets.ente.com/ffmpeg-core-0.12.10/ffmpeg-core.js",
+ wasmURL:
+ "https://assets.ente.com/ffmpeg-core-0.12.10/ffmpeg-core.wasm",
+ });
+ logToDisk(
+ `[info] Space video encoder loaded elapsedMs=${Date.now() - startedAt}`,
);
- return ffmpeg;
} catch (error) {
ffmpeg.terminate();
- _ffmpeg = undefined;
+ if (_ffmpeg?.instance == ffmpeg) _ffmpeg = undefined;
+ logToDisk(
+ `[error] Space video encoder load ${timeout.signal.aborted ? "timeout" : "failed"} elapsedMs=${Date.now() - startedAt}`,
+ );
+ if (timeout.signal.aborted)
+ throw new Error(
+ "The video processor took too long to load. Please try again.",
+ { cause: error },
+ );
throw error;
+ } finally {
+ clearTimeout(timer);
}
};
export const preloadVideoEncoderWeb = async () => {
- await ffmpegLazy();
+ await ffmpegLazy().ready;
};
export const determineVideoDurationWeb = async (
blob: Blob,
signal?: AbortSignal,
): Promise<number> =>
- runFFmpegTask((ffmpeg) => ffprobeExecVideoDuration(ffmpeg, blob), signal);
+ runFFmpegTask(
+ "duration",
+ blob.size,
+ (ffmpeg) => ffprobeExecVideoDuration(ffmpeg, blob),
+ signal,
+ );
+
+export const determineVideoInfoWeb = (blob: Blob, signal?: AbortSignal) =>
+ runFFmpegTask(
+ "metadata",
+ blob.size,
+ (ffmpeg) =>
+ withInputMount(ffmpeg, blob, async (inputPath) => {
+ const json = await ffprobeOutput(
+ ffmpeg,
+ [
+ "-v",
+ "error",
+ "-select_streams",
+ "v:0",
+ "-show_entries",
+ "format=duration:stream=width,height:stream_side_data=rotation",
+ "-of",
+ "json",
+ "-o",
+ "output.json",
+ inputPath,
+ ],
+ "output.json",
+ );
+ const info = z
+ .object({
+ format: z.object({
+ duration: z.coerce.number().positive(),
+ }),
+ streams: z
+ .array(
+ z.object({
+ width: z.number().positive(),
+ height: z.number().positive(),
+ side_data_list: z
+ .array(
+ z.object({
+ rotation: z.number().optional(),
+ }),
+ )
+ .optional(),
+ }),
+ )
+ .nonempty(),
+ })
+ .parse(JSON.parse(json));
+ const stream = info.streams[0]!;
+ const rotation =
+ stream.side_data_list?.find(
+ (data) => data.rotation != undefined,
+ )?.rotation ?? 0;
+ const sideways = Math.abs(rotation % 180) == 90;
+ return {
+ duration: info.format.duration,
+ width: sideways ? stream.height : stream.width,
+ height: sideways ? stream.width : stream.height,
+ };
+ }),
+ signal,
+ );
+
+export const extractVideoFramesWeb = (
+ blob: Blob,
+ times: number[],
+ maxDimension: number,
+ signal?: AbortSignal,
+) =>
+ runFFmpegTask(
+ "frames",
+ blob.size,
+ async (ffmpeg, signal) => {
+ const frames: Blob[] = [];
+ await withInputMount(ffmpeg, blob, async (inputPath) => {
+ const outputPath = newID("frame_") + ".png";
+ const hdr = await isHDRVideo(ffmpeg, inputPath);
+ try {
+ for (const time of times) {
+ signal.throwIfAborted();
+ const json = await ffprobeOutput(
+ ffmpeg,
+ [
+ "-v",
+ "error",
+ "-select_streams",
+ "v:0",
+ "-read_intervals",
+ `${time}%${time + 1}`,
+ "-show_entries",
+ "packet=pts_time:format=start_time",
+ "-of",
+ "json",
+ "-o",
+ "frame.json",
+ inputPath,
+ ],
+ "frame.json",
+ );
+ const probe = z
+ .object({
+ format: z.object({
+ start_time: z.coerce.number().optional(),
+ }),
+ packets: z
+ .array(
+ z.object({
+ pts_time: z.coerce.number(),
+ }),
+ )
+ .nonempty(),
+ })
+ .parse(JSON.parse(json));
+ const start = probe.format.start_time ?? 0;
+ const timestamps = probe.packets.map(
+ (packet) => packet.pts_time - start,
+ );
+ const frameTime = timestamps.reduce(
+ (closest, timestamp) =>
+ Math.abs(timestamp - time) <
+ Math.abs(closest - time)
+ ? timestamp
+ : closest,
+ );
+ const status = await ffmpeg.exec([
+ "-ss",
+ String(Math.max(0, frameTime - 0.000001)),
+ "-i",
+ inputPath,
+ "-map",
+ "0:v:0",
+ "-frames:v",
+ "1",
+ "-an",
+ "-vf",
+ [
+ `scale=w='min(${maxDimension},iw)':h='min(${maxDimension},ih)':force_original_aspect_ratio=decrease`,
+ ...(hdr
+ ? [
+ "zscale=transfer=linear",
+ "tonemap=tonemap=hable:desat=0",
+ "zscale=primaries=709:transfer=709:matrix=709",
+ ]
+ : []),
+ "format=rgb24",
+ "setsar=1",
+ ].join(","),
+ "-map_metadata",
+ "-1",
+ "-update",
+ "1",
+ outputPath,
+ ]);
+ if (status != 0)
+ throw new Error("Couldn't extract a video frame.");
+ const bytes = await ffmpeg.readFile(outputPath);
+ if (typeof bytes == "string")
+ throw new Error("Expected a video frame.");
+ frames.push(
+ new Blob([ensureArrayBufferBacked(bytes)], {
+ type: "image/png",
+ }),
+ );
+ await ffmpeg.deleteFile(outputPath);
+ }
+ } finally {
+ if (ffmpeg.loaded) {
+ const files = await ffmpeg.listDir("/");
+ if (files.some((file) => file.name == outputPath))
+ await ffmpeg.deleteFile(outputPath);
+ }
+ }
+ });
+ return frames;
+ },
+ signal,
+ );
export const transcodeVideoWeb = (
blob: Blob,
@@ -52,56 +253,108 @@ export const transcodeVideoWeb = (
fallbackCommand: FFmpegCommand,
signal?: AbortSignal,
) =>
- runFFmpegTask(async (ffmpeg) => {
- if (
- typeof VideoDecoder !== "undefined" &&
- typeof VideoEncoder !== "undefined"
- ) {
- try {
- const { encodeVideoWithWebCodecs } =
- await import("./web-codecs");
- const result = await withInputMount(ffmpeg, blob, (inputPath) =>
- encodeVideoWithWebCodecs(
+ runFFmpegTask(
+ "export",
+ blob.size,
+ async (ffmpeg, signal) => {
+ if (
+ typeof VideoDecoder !== "undefined" &&
+ typeof VideoEncoder !== "undefined"
+ ) {
+ try {
+ const { encodeVideoWithWebCodecs } =
+ await import("./web-codecs");
+ const result = await withInputMount(
ffmpeg,
- inputPath,
blob,
- edit,
- signal,
- ),
- );
- if (result) {
- log.debug(() => "Encoded video with WebCodecs");
- return result;
+ (inputPath) =>
+ encodeVideoWithWebCodecs(
+ ffmpeg,
+ inputPath,
+ blob,
+ edit,
+ signal,
+ ),
+ );
+ if (result) {
+ logToDisk(
+ `[info] Space video encoded encoder=webcodecs bytes=${result.byteLength}`,
+ );
+ return result;
+ }
+ } catch {
+ signal.throwIfAborted();
+ logToDisk(
+ "[warn] Space video WebCodecs export failed, using software encoder",
+ );
}
- } catch (error) {
- signal?.throwIfAborted();
- log.warn("WebCodecs export failed, using FFmpeg", error);
}
- }
- signal?.throwIfAborted();
- return ffmpegExec(ffmpeg, fallbackCommand, "mp4", blob);
- }, signal);
+ signal.throwIfAborted();
+ logToDisk("[info] Space video encoding encoder=wasm");
+ const result = await ffmpegExec(
+ ffmpeg,
+ fallbackCommand,
+ "mp4",
+ blob,
+ );
+ logToDisk(
+ `[info] Space video encoded encoder=wasm bytes=${result.byteLength}`,
+ );
+ return result;
+ },
+ signal,
+ 300_000,
+ );
const runFFmpegTask = <T>(
- task: (ffmpeg: FFmpeg) => Promise<T>,
+ stage: string,
+ bytes: number,
+ task: (ffmpeg: FFmpeg, signal: AbortSignal) => Promise<T>,
signal?: AbortSignal,
+ timeout = 60_000,
): Promise<T> => {
const pending = _ffmpegTaskQueue.add(async () => {
signal?.throwIfAborted();
- const ffmpeg = await ffmpegLazy(signal);
- signal?.throwIfAborted();
+ const engine = ffmpegLazy();
+ const ffmpeg = engine.instance;
+ const controller = new AbortController();
+ const cancel = () => controller.abort(signal?.reason);
const abort = () => {
ffmpeg.terminate();
- _ffmpeg = undefined;
+ if (_ffmpeg == engine) _ffmpeg = undefined;
};
- signal?.addEventListener("abort", abort, { once: true });
+ const timer = setTimeout(
+ () =>
+ controller.abort(
+ new DOMException(
+ "Video processing took too long. Try a shorter clip.",
+ "TimeoutError",
+ ),
+ ),
+ timeout,
+ );
+ signal?.addEventListener("abort", cancel, { once: true });
+ controller.signal.addEventListener("abort", abort, { once: true });
+ const startedAt = Date.now();
+ logToDisk(`[info] Space video stage=${stage} started bytes=${bytes}`);
try {
- return await task(ffmpeg);
+ await engine.ready;
+ controller.signal.throwIfAborted();
+ const result = await task(ffmpeg, controller.signal);
+ logToDisk(
+ `[info] Space video stage=${stage} completed elapsedMs=${Date.now() - startedAt}`,
+ );
+ return result;
} catch (error) {
- signal?.throwIfAborted();
+ logToDisk(
+ `[error] Space video stage=${stage} ${signal?.aborted ? "canceled" : controller.signal.aborted ? "timeout" : "failed"} elapsedMs=${Date.now() - startedAt}`,
+ );
+ controller.signal.throwIfAborted();
throw error;
} finally {
- signal?.removeEventListener("abort", abort);
+ clearTimeout(timer);
+ signal?.removeEventListener("abort", cancel);
+ controller.signal.removeEventListener("abort", abort);
}
}) as Promise<T>;
if (!signal) return pending;
@@ -229,7 +482,7 @@ const isHDRVideo = async (ffmpeg: FFmpeg, inputFilePath: string) => {
ffmpeg,
[
["-i", inputFilePath],
- "-show_streams",
+ ["-show_entries", "stream=color_transfer"],
["-select_streams", "v:0"],
["-of", "json"],
["-o", "output.json"],
diff --git a/web/apps/space/tests/video-cover.test.ts b/web/apps/space/tests/video-cover.test.ts
new file mode 100644
index 0000000000..98a414c285
--- /dev/null
+++ b/web/apps/space/tests/video-cover.test.ts
@@ -0,0 +1,139 @@
+import { afterEach, beforeEach, expect, test, vi } from "vitest";
+import {
+ spaceVideoCover,
+ spaceVideoFrames,
+ spaceVideoInfo,
+ useSoftwareVideoFrames,
+} from "../src/utils/post-video";
+
+const mocks = vi.hoisted(() => ({
+ frames: vi.fn(),
+ info: vi.fn(),
+ log: vi.fn(),
+}));
+vi.mock("../src/utils/video-encoding/web", () => ({
+ extractVideoFramesWeb: mocks.frames,
+ determineVideoInfoWeb: mocks.info,
+}));
+vi.mock("ente-base/log-web", () => ({ logToDisk: mocks.log }));
+vi.mock("utils/thumbhash", () => ({ thumbHashBase64FromCanvas: () => "hash" }));
+
+let alpha: number;
+let bitmapClose: ReturnType<typeof vi.fn>;
+let createElement: ReturnType<typeof vi.fn>;
+
+beforeEach(() => {
+ vi.resetAllMocks();
+ alpha = 255;
+ bitmapClose = vi.fn();
+ vi.stubGlobal("navigator", {
+ userAgent:
+ "Mozilla/5.0 (Android 16; Mobile; rv:143.0) Gecko/143.0 Firefox/143.0",
+ });
+ mocks.frames.mockResolvedValue([new Blob(["png"], { type: "image/png" })]);
+ mocks.info.mockResolvedValue({ duration: 3.45, width: 1080, height: 1920 });
+ vi.stubGlobal(
+ "createImageBitmap",
+ vi
+ .fn()
+ .mockImplementation(() =>
+ Promise.resolve({
+ width: 540,
+ height: 960,
+ close: bitmapClose,
+ }),
+ ),
+ );
+ createElement = vi.fn((tag: string) => {
+ if (tag != "canvas")
+ throw new Error(
+ "Must not decode through a video element on Firefox Android",
+ );
+ return {
+ width: 0,
+ height: 0,
+ getContext: () => ({
+ drawImage: vi.fn(),
+ getImageData: () => ({
+ data: new Uint8ClampedArray([0, 0, 0, alpha]),
+ }),
+ }),
+ toBlob: (callback: BlobCallback) =>
+ callback(new Blob(["cover"], { type: "image/webp" })),
+ };
+ });
+ vi.stubGlobal("document", { createElement });
+});
+afterEach(() => vi.unstubAllGlobals());
+
+test.each([
+ [
+ "Mozilla/5.0 (Android 16; Mobile; rv:143.0) Gecko/143.0 Firefox/143.0",
+ true,
+ ],
+ ["Mozilla/5.0 (Android 16) Chrome/140.0.0.0 Mobile Safari/537.36", false],
+ [
+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:143.0) Gecko/20100101 Firefox/143.0",
+ false,
+ ],
+])(
+ "selects software frames only for Firefox Android (%s)",
+ (userAgent, expected) => {
+ vi.stubGlobal("navigator", { userAgent });
+ expect(useSoftwareVideoFrames()).toBe(expected);
+ },
+);
+
+test("Firefox Android gets metadata and a cover without creating a video element", async () => {
+ const file = new File(["video"], "private-video.mp4");
+ const controller = new AbortController();
+ expect(await spaceVideoInfo(file, controller.signal)).toMatchObject({
+ width: 1080,
+ height: 1920,
+ });
+ const cover = await spaceVideoCover(file, 1.25, controller.signal);
+ expect(mocks.frames).toHaveBeenCalledWith(
+ file,
+ [1.25],
+ 960,
+ controller.signal,
+ );
+ expect(cover).toMatchObject({ width: 540, height: 960, thumbHash: "hash" });
+ expect(cover.file.type).toBe("image/webp");
+ expect(bitmapClose).toHaveBeenCalledOnce();
+ expect(JSON.stringify(mocks.log.mock.calls)).not.toContain("private-video");
+});
+
+test("opaque black frames are valid covers, but unreadable transparent frames fail", async () => {
+ const file = new Blob(["video"]);
+ await expect(spaceVideoCover(file, 0)).resolves.toBeDefined();
+ alpha = 0;
+ await expect(spaceVideoCover(file, 0)).rejects.toThrow(
+ "Couldn't read a video frame",
+ );
+ expect(bitmapClose).toHaveBeenCalledTimes(2);
+ expect(mocks.log).toHaveBeenCalledWith(
+ expect.stringContaining("frames failed backend=wasm"),
+ );
+});
+
+test("timeline frame extraction is batched and releases each decoded image", async () => {
+ const file = new Blob(["video"]);
+ const times = [0.5, 1.5, 2.5];
+ mocks.frames.mockResolvedValue(times.map(() => new Blob(["png"])));
+ expect(await spaceVideoFrames(file, times, 128)).toHaveLength(3);
+ expect(mocks.frames).toHaveBeenCalledWith(file, times, 128, undefined);
+ expect(bitmapClose).toHaveBeenCalledTimes(3);
+});
+
+test("canceling frame extraction does not leave decoded bitmaps or return a cover", async () => {
+ const controller = new AbortController();
+ mocks.frames.mockImplementation(() => {
+ controller.abort();
+ return Promise.resolve([new Blob(["png"])]);
+ });
+ await expect(
+ spaceVideoCover(new Blob(["video"]), 0, controller.signal),
+ ).rejects.toMatchObject({ name: "AbortError" });
+ expect(createImageBitmap).not.toHaveBeenCalled();
+});
diff --git a/web/apps/space/tests/video-export.test.ts b/web/apps/space/tests/video-export.test.ts
index 199c17bdf9..65a279fa65 100644
--- a/web/apps/space/tests/video-export.test.ts
+++ b/web/apps/space/tests/video-export.test.ts
@@ -3,16 +3,19 @@ import { afterEach, beforeEach, expect, test, vi } from "vitest";
const mocks = vi.hoisted(() => ({
encode: vi.fn(),
load: vi.fn(),
+ logToDisk: vi.fn(),
workers: [] as {
loaded: boolean;
terminate: ReturnType<typeof vi.fn>;
exec: ReturnType<typeof vi.fn>;
unmount: ReturnType<typeof vi.fn>;
+ readFile: ReturnType<typeof vi.fn>;
}[],
}));
vi.mock("ente-base/log", () => ({
default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
+vi.mock("ente-base/log-web", () => ({ logToDisk: mocks.logToDisk }));
vi.mock("../src/utils/video-encoding/web-codecs", () => ({
encodeVideoWithWebCodecs: mocks.encode,
}));
@@ -23,9 +26,15 @@ vi.mock("@ffmpeg/ffmpeg", () => ({
constructor() {
mocks.workers.push(this);
}
- load = mocks.load;
+ rejectLoad: ((error: Error) => void) | undefined;
+ load = (...args: unknown[]) =>
+ new Promise((resolve, reject) => {
+ this.rejectLoad = reject;
+ Promise.resolve(mocks.load(...args)).then(resolve, reject);
+ });
terminate = vi.fn(() => {
this.loaded = false;
+ this.rejectLoad?.(new Error("Terminated"));
});
on = vi.fn();
off = vi.fn();
@@ -34,6 +43,8 @@ vi.mock("@ffmpeg/ffmpeg", () => ({
mount = vi.fn();
unmount = vi.fn();
deleteFile = vi.fn();
+ listDir = vi.fn().mockResolvedValue([]);
+ ffprobe = vi.fn().mockResolvedValue(0);
exec = vi.fn().mockResolvedValue(0);
readFile = vi.fn().mockResolvedValue(new Uint8Array([7]));
},
@@ -44,10 +55,14 @@ beforeEach(() => {
mocks.encode.mockReset();
mocks.load.mockReset().mockResolvedValue(true);
mocks.workers.length = 0;
+ mocks.logToDisk.mockClear();
vi.stubGlobal("VideoEncoder", {});
vi.stubGlobal("VideoDecoder", {});
});
-afterEach(() => vi.unstubAllGlobals());
+afterEach(() => {
+ vi.unstubAllGlobals();
+ vi.useRealTimers();
+});
const exporter = async () => {
const { transcodeVideoWeb } =
@@ -165,16 +180,7 @@ test("a queued export can be cancelled without waiting for or stopping another e
});
test("cancelling engine loading terminates it and allows the next export to load again", async () => {
- mocks.load.mockImplementationOnce(
- (_config, { signal }: { signal: AbortSignal }) =>
- new Promise((_, reject) =>
- signal.addEventListener(
- "abort",
- () => reject(new DOMException("Canceled", "AbortError")),
- { once: true },
- ),
- ),
- );
+ mocks.load.mockImplementationOnce(() => new Promise(() => undefined));
const run = await exporter();
const controller = new AbortController();
const pending = run(controller.signal);
@@ -189,3 +195,79 @@ test("cancelling engine loading terminates it and allows the next export to load
expect(mocks.workers[0]!.terminate).toHaveBeenCalled();
expect(mocks.workers).toHaveLength(2);
});
+
+test("a stalled engine load times out and can be retried", async () => {
+ vi.useFakeTimers();
+ mocks.load.mockImplementationOnce(() => new Promise(() => undefined));
+ const { preloadVideoEncoderWeb } =
+ await import("../src/utils/video-encoding/web");
+ const rejected = expect(preloadVideoEncoderWeb()).rejects.toThrow(
+ "took too long to load",
+ );
+ await vi.advanceTimersByTimeAsync(60_000);
+ await rejected;
+ await preloadVideoEncoderWeb();
+ expect(mocks.workers).toHaveLength(2);
+ expect(mocks.workers[0]!.terminate).toHaveBeenCalled();
+});
+
+test("a stalled export times out, closes the worker, and releases the queue for retry", async () => {
+ vi.useFakeTimers();
+ mocks.encode
+ .mockImplementationOnce(
+ (_ffmpeg, _path, _blob, _edit, signal: AbortSignal) =>
+ new Promise((_, reject) =>
+ signal.addEventListener("abort", () =>
+ reject(signal.reason as Error),
+ ),
+ ),
+ )
+ .mockResolvedValue(new Uint8Array([3]));
+ const run = await exporter();
+ const first = run();
+ const rejected = expect(first).rejects.toMatchObject({
+ name: "TimeoutError",
+ });
+ await vi.advanceTimersByTimeAsync(300_000);
+ await rejected;
+ expect(await run()).toEqual(new Uint8Array([3]));
+ expect(mocks.workers[0]!.terminate).toHaveBeenCalled();
+ expect(mocks.workers[0]!.exec).not.toHaveBeenCalled();
+ expect(mocks.workers).toHaveLength(2);
+ expect(mocks.logToDisk).toHaveBeenCalledWith(
+ expect.stringContaining("stage=export timeout"),
+ );
+});
+
+test("cover extraction near the clip end seeks to the last available frame", async () => {
+ const { preloadVideoEncoderWeb, extractVideoFramesWeb } =
+ await import("../src/utils/video-encoding/web");
+ await preloadVideoEncoderWeb();
+ const worker = mocks.workers[0]!;
+ worker.readFile.mockImplementation((path: string) => {
+ if (path == "output.json") return JSON.stringify({ streams: [{}] });
+ if (path == "frame.json")
+ return JSON.stringify({
+ format: { start_time: "0" },
+ packets: [
+ { pts_time: "3.38" },
+ { pts_time: "3.413667" },
+ { pts_time: "3.430322" },
+ ],
+ });
+ return new Uint8Array([1, 2, 3]);
+ });
+ const frames = await extractVideoFramesWeb(
+ new Blob(["video"]),
+ [3.4499],
+ 960,
+ );
+ expect(frames).toHaveLength(1);
+ expect(frames[0]!.type).toBe("image/png");
+ const command = worker.exec.mock.calls[0]![0] as string[];
+ expect(Number(command[command.indexOf("-ss") + 1])).toBeCloseTo(
+ 3.430321,
+ 6,
+ );
+ expect(worker.unmount).toHaveBeenCalled();
+});
diff --git a/web/apps/space/tests/video-trim-playback.test.ts b/web/apps/space/tests/video-trim-playback.test.ts
index 5a61e73304..b21d40a373 100644
--- a/web/apps/space/tests/video-trim-playback.test.ts
+++ b/web/apps/space/tests/video-trim-playback.test.ts
@@ -1,6 +1,8 @@
import { afterEach, beforeEach, expect, test, vi } from "vitest";
import { createVideoTrimPlayback } from "../src/components/video-trim/playback";
+vi.mock("ente-base/log-web", () => ({ logToDisk: vi.fn() }));
+
class Video extends EventTarget {
currentTime = 0;
paused = true;
@@ -47,6 +49,7 @@ beforeEach(() => {
afterEach(() => {
playback.dispose();
vi.unstubAllGlobals();
+ vi.useRealTimers();
});
const renderFrame = () => {
@@ -187,3 +190,29 @@ test("closing the editor pauses playback and removes the frame monitor", () => {
video.dispatchEvent(new Event("timeupdate"));
expect(video.currentTime).toBe(12);
});
+
+test("a stuck preview reports an error and can be played again", async () => {
+ vi.useFakeTimers();
+ vi.spyOn(video, "play").mockImplementationOnce(
+ () => new Promise(() => undefined),
+ );
+ playback.toggle();
+ await vi.advanceTimersByTimeAsync(30_000);
+ expect(onError).toHaveBeenCalledTimes(1);
+ playback.toggle();
+ expect(video.paused).toBe(false);
+ video.dispatchEvent(new Event("playing"));
+ await vi.advanceTimersByTimeAsync(30_000);
+ expect(onError).toHaveBeenCalledTimes(1);
+});
+
+test("closing or pausing a loading preview cancels its timeout", async () => {
+ vi.useFakeTimers();
+ vi.spyOn(video, "play").mockImplementationOnce(
+ () => new Promise(() => undefined),
+ );
+ playback.toggle();
+ playback.dispose();
+ await vi.advanceTimersByTimeAsync(30_000);
+ expect(onError).not.toHaveBeenCalled();
+});