π End-to-end encrypted cloud for everything.
Dart 2faandroidauthye2eeencryptionend-to-end-encryptionfluttergolanggoogle-photosgoogle-photos-alternativeiosopensourcephotosprivacyprivacy-toolstypescriptzero-knowledge
Use native video processing with codec fallbacks
Anand Baburajan59b256fee64a
diff --git a/web/apps/space/package.json b/web/apps/space/package.jsonindex 0ab86e4cdf..97600cc135 100644--- a/web/apps/space/package.json+++ b/web/apps/space/package.json@@ -12,12 +12,14 @@"@fontsource/nunito": "5.3.0","@hugeicons/core-free-icons": "4.3.5","@hugeicons/react": "1.1.10",+ "@mediabunny/aac-encoder": "1.61.6","@rive-app/canvas-lite": "2.43.1","ente-accounts": "*","ente-base": "*","ente-media": "*","ente-prelogin-wasm": "*","ente-space-wasm": "*",+ "mediabunny": "1.61.6","photoswipe": "5.4.4","react-easy-crop": "5.5.7","thumbhash": "0.1.1"diff --git a/web/apps/space/src/components/PostComposer.tsx b/web/apps/space/src/components/PostComposer.tsxindex 8e7686143b..b6df8e0592 100644--- a/web/apps/space/src/components/PostComposer.tsx+++ b/web/apps/space/src/components/PostComposer.tsx@@ -132,13 +132,6 @@ const SpacePostComposer: React.FC<{controllers.forEach((controller, id) =>preparingRef.current.set(id, controller),);- if (pending.some((draft) => isSpaceVideoFile(draft.file))) {- void import("utils/video-encoding/web")- .then(({ preloadVideoEncoderWeb }) => preloadVideoEncoderWeb())- .catch((error: unknown) =>- log.warn("Failed to preload video encoder", error),- );- }void (async () => {for (const draft of pending) {if (!isActiveDraft(draft.id)) continue;diff --git a/web/apps/space/src/utils/post-video.ts b/web/apps/space/src/utils/post-video.tsindex 6a32a3418d..44c4efcd47 100644--- a/web/apps/space/src/utils/post-video.ts+++ b/web/apps/space/src/utils/post-video.ts@@ -1,11 +1,6 @@import { logToDisk } from "ente-base/log-web";-import type { FFmpegCommand } from "ente-base/types/ipc";import type { PreparedSpacePostImage } from "utils/post-image";import { thumbHashBase64FromCanvas } from "utils/thumbhash";-import {- inputPathPlaceholder,- outputPathPlaceholder,-} from "./video-encoding/constants";export interface SpacePostVideoEdit {start: number;@@ -22,10 +17,6 @@ export interface SpaceVideoInfo {export const maxSpaceVideoDuration = 10;-const useSoftwareVideoFrames = () =>- navigator.userAgent.includes("Android") &&- navigator.userAgent.includes("Firefox/");-export const isSpaceVideoFile = (file: File) =>file.type.startsWith("video/") || /\.(mp4|mov|m4v|webm)$/i.test(file.name);@@ -43,105 +34,12 @@ export const clampVideoCover = (edit: SpacePostVideoEdit) => ({coverTime: Math.max(edit.start, Math.min(edit.coverTime, edit.end - 0.001)),});-const waitForVideo = (- video: HTMLVideoElement,- event: "loadeddata" | "seeked",- signal?: AbortSignal,-) =>- new Promise<void>((resolve, reject) => {- let frameCallback: number | undefined;- const finish = (error?: Error) => {- window.clearTimeout(timer);- if (frameCallback != undefined)- video.cancelVideoFrameCallback(frameCallback);- video.removeEventListener(event, ready);- video.removeEventListener("error", failed);- signal?.removeEventListener("abort", abort);- if (error) {- if (!signal?.aborted)- logToDisk(- `[error] Space video event=${event} failed code=${video.error?.code ?? 0} readyState=${video.readyState} networkState=${video.networkState}`,- );- reject(error);- } else resolve();- };- const ready = () => finish();- const failed = () =>- finish(new Error("This browser can't preview this video."));- const abort = () => finish(new DOMException("Canceled", "AbortError"));- const timer = window.setTimeout(- () => finish(new Error("The video took too long to load.")),- 30_000,- );- if (- event == "loadeddata" &&- typeof video.requestVideoFrameCallback == "function"- )- frameCallback = video.requestVideoFrameCallback(ready);- else video.addEventListener(event, ready, { once: true });- video.addEventListener("error", failed, { once: true });- signal?.addEventListener("abort", abort, { once: true });- if (signal?.aborted) abort();- });--const openSpaceVideo = async (file: Blob, signal?: AbortSignal) => {- signal?.throwIfAborted();- const video = document.createElement("video");- const url = URL.createObjectURL(file);- video.muted = true;- video.playsInline = true;- video.preload = "auto";- const dispose = () => {- video.pause();- video.removeAttribute("src");- video.load();- URL.revokeObjectURL(url);- };- try {- const ready = waitForVideo(video, "loadeddata", signal);- video.src = url;- video.load();- await ready;- return { video, dispose };- } catch (error) {- dispose();- throw error;- }-};--const seekSpaceVideo = async (- video: HTMLVideoElement,- time: number,- signal?: AbortSignal,-) => {- signal?.throwIfAborted();- const target = Math.max(0, Math.min(time, video.duration - 0.001));- if (video.currentTime == target) return;- const ready = waitForVideo(video, "seeked", signal);- video.currentTime = target;- await ready;-};-export const spaceVideoInfo = async (file: File,signal?: AbortSignal,): Promise<SpaceVideoInfo> => {- if (useSoftwareVideoFrames()) {- const { determineVideoInfoWeb } = await import("./video-encoding/web");- return determineVideoInfoWeb(file, signal);- }- const { video, dispose } = await openSpaceVideo(file, signal);- try {- if (!Number.isFinite(video.duration) || video.duration <= 0)- throw new Error("This video has no readable duration.");- return {- duration: video.duration,- width: video.videoWidth,- height: video.videoHeight,- };- } finally {- dispose();- }+ const { determineVideoInfoWeb } = await import("./video-encoding/web");+ return determineVideoInfoWeb(file, signal);};export const spaceVideoFrames = async (@@ -150,66 +48,8 @@ export const spaceVideoFrames = async (maxDimension: number,signal?: AbortSignal,) => {- const startedAt = Date.now();- const software = useSoftwareVideoFrames();- const frames: HTMLCanvasElement[] = [];- const draw = (source: CanvasImageSource, width: number, height: number) => {- const scale = Math.min(1, maxDimension / Math.max(width, height));- const canvas = document.createElement("canvas");- canvas.width = Math.max(1, Math.round(width * scale));- canvas.height = Math.max(1, Math.round(height * scale));- const context = canvas.getContext("2d");- if (!context) throw new Error("Couldn't prepare the video cover.");- context.drawImage(source, 0, 0, canvas.width, canvas.height);- const sample = document.createElement("canvas");- sample.width = sample.height = 1;- const sampleContext = sample.getContext("2d")!;- sampleContext.drawImage(canvas, 0, 0, 1, 1);- if (sampleContext.getImageData(0, 0, 1, 1).data[3] == 0)- throw new Error("Couldn't read a video frame. Please try again.");- frames.push(canvas);- };- try {- if (software) {- const { extractVideoFramesWeb } =- await import("./video-encoding/web");- const blobs = await extractVideoFramesWeb(- file,- times,- maxDimension,- signal,- );- for (const blob of blobs) {- signal?.throwIfAborted();- const bitmap = await createImageBitmap(blob);- try {- draw(bitmap, bitmap.width, bitmap.height);- } finally {- bitmap.close();- }- }- } else {- const { video, dispose } = await openSpaceVideo(file, signal);- try {- for (const time of times) {- await seekSpaceVideo(video, time, signal);- draw(video, video.videoWidth, video.videoHeight);- }- } finally {- dispose();- }- }- logToDisk(- `[info] Space video frames backend=${software ? "wasm" : "browser"} count=${frames.length} bytes=${file.size} elapsedMs=${Date.now() - startedAt}`,- );- return frames;- } catch (error) {- if (!signal?.aborted)- logToDisk(- `[error] Space video frames failed backend=${software ? "wasm" : "browser"} count=${frames.length}/${times.length} bytes=${file.size} elapsedMs=${Date.now() - startedAt}`,- );- throw error;- }+ const { extractVideoFramesWeb } = await import("./video-encoding/web");+ return extractVideoFramesWeb(file, times, maxDimension, signal);};export const spaceVideoCover = async (@@ -242,56 +82,6 @@ export const spaceVideoCover = async (};};-const videoExportCommand = (- edit: SpacePostVideoEdit,- hdr: boolean,-): string[] => [- "-ss",- String(edit.start),- "-i",- inputPathPlaceholder,- "-t",- String(Math.min(maxSpaceVideoDuration, edit.end - edit.start)),- "-map",- "0:v:0",- ...(edit.muted- ? ["-an"]- : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "128k"]),- "-map_metadata",- "-1",- "-map_metadata:s",- "-1",- "-map_chapters",- "-1",- "-vf",- [- "fps=30:start_time=0",- "scale=w='trunc(iw*min(1,min(1280/max(iw,ih),720/min(iw,ih)))/2)*2':h='trunc(ih*min(1,min(1280/max(iw,ih),720/min(iw,ih)))/2)*2'",- ...(hdr- ? [- "zscale=transfer=linear",- "tonemap=tonemap=hable:desat=0",- "zscale=primaries=709:transfer=709:matrix=709",- ]- : []),- "format=yuv420p",- "setsar=1",- ].join(","),- "-c:v",- "libx264",- "-preset",- "veryfast",- "-crf",- "23",- "-maxrate",- "4M",- "-bufsize",- "8M",- "-movflags",- "+faststart",- outputPathPlaceholder,-];-const exports = new WeakMap<File,{ key: string; file: File; durationMs: number }@@ -316,11 +106,7 @@ const exportSpaceVideo = async (if (exported?.key != key) {const { transcodeVideoWeb, determineVideoDurationWeb } =await import("./video-encoding/web");- const command: FFmpegCommand = {- default: videoExportCommand(edit, false),- hdr: videoExportCommand(edit, true),- };- const bytes = await transcodeVideoWeb(file, edit, command, signal);+ const bytes = await transcodeVideoWeb(file, edit, signal);signal?.throwIfAborted();const output = new File([bytes], "video.mp4", { type: "video/mp4" });if (output.size > 15 * 1024 * 1024 - 42)diff --git a/web/apps/space/src/utils/video-encoding/constants.ts b/web/apps/space/src/utils/video-encoding/constants.tsdeleted file mode 100644index 2efe7e0a88..0000000000--- a/web/apps/space/src/utils/video-encoding/constants.ts+++ /dev/null@@ -1,2 +0,0 @@-export const inputPathPlaceholder = "INPUT";-export const outputPathPlaceholder = "OUTPUT";diff --git a/web/apps/space/src/utils/video-encoding/software.ts b/web/apps/space/src/utils/video-encoding/software.tsnew file mode 100644index 0000000000..3a3ea2df0b--- /dev/null+++ b/web/apps/space/src/utils/video-encoding/software.ts@@ -0,0 +1,203 @@+import { FFFSType, FFmpeg } from "@ffmpeg/ffmpeg";+import { logToDisk } from "ente-base/log-web";+import { ensureArrayBufferBacked } from "ente-utils/bytes";+import { z } from "zod";+import type { SpacePostVideoEdit } from "../post-video";++const hdrFilters = [+ "zscale=transfer=linear",+ "tonemap=tonemap=hable:desat=0",+ "zscale=primaries=709:transfer=709:matrix=709",+];++export const extractSoftwareVideoFrames = (+ blob: Blob,+ times: number[],+ maxDimension: number,+ hdr: boolean,+ signal: AbortSignal,+) =>+ withSoftwareVideo(blob, signal, async (ffmpeg, inputPath) => {+ const frames: HTMLCanvasElement[] = [];+ for (const time of times) {+ signal.throwIfAborted();+ const status = await ffmpeg.ffprobe([+ "-v",+ "error",+ "-select_streams",+ "v:0",+ "-read_intervals",+ `${time}%${time + 1}`,+ "-show_entries",+ "packet=pts_time:format=start_time",+ "-of",+ "json",+ "-o",+ "frame.json",+ inputPath,+ ]);+ if (status !== 0) throw new Error("Couldn't locate a video frame.");+ const json = await ffmpeg.readFile("frame.json", "utf8");+ await ffmpeg.deleteFile("frame.json");+ if (typeof json !== "string")+ throw new Error("Expected video timestamps.");+ const probe = z+ .object({+ format: z.object({+ start_time: z.coerce.number().optional(),+ }),+ packets: z+ .array(z.object({ pts_time: z.coerce.number() }))+ .nonempty(),+ })+ .parse(JSON.parse(json));+ const start = probe.format.start_time ?? 0;+ const frameTime = probe.packets+ .map((packet) => packet.pts_time - start)+ .reduce((closest, timestamp) =>+ Math.abs(timestamp - time) < Math.abs(closest - time)+ ? timestamp+ : closest,+ );+ const bytes = await execute(+ ffmpeg,+ [+ "-ss",+ String(Math.max(0, frameTime - 0.000001)),+ "-i",+ inputPath,+ "-map",+ "0:v:0",+ "-frames:v",+ "1",+ "-an",+ "-vf",+ [+ `scale=w='min(${maxDimension},iw)':h='min(${maxDimension},ih)':force_original_aspect_ratio=decrease`,+ ...(hdr ? hdrFilters : []),+ "format=rgb24",+ "setsar=1",+ ].join(","),+ "-map_metadata",+ "-1",+ "-update",+ "1",+ ],+ "frame.png",+ );+ const bitmap = await createImageBitmap(+ new Blob([bytes], { type: "image/png" }),+ );+ try {+ signal.throwIfAborted();+ const canvas = document.createElement("canvas");+ canvas.width = bitmap.width;+ canvas.height = bitmap.height;+ const context = canvas.getContext("2d");+ if (!context)+ throw new Error("Couldn't prepare the video frames.");+ context.drawImage(bitmap, 0, 0);+ frames.push(canvas);+ } finally {+ bitmap.close();+ }+ }+ return frames;+ });++export const transcodeSoftwareVideo = (+ blob: Blob,+ edit: SpacePostVideoEdit,+ hdr: boolean,+ signal: AbortSignal,+) =>+ withSoftwareVideo(blob, signal, (ffmpeg, inputPath) =>+ execute(+ ffmpeg,+ [+ "-ss",+ String(edit.start),+ "-i",+ inputPath,+ "-t",+ String(edit.end - edit.start),+ "-map",+ "0:v:0",+ ...(edit.muted+ ? ["-an"]+ : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "128k"]),+ "-map_metadata",+ "-1",+ "-map_metadata:s",+ "-1",+ "-map_chapters",+ "-1",+ "-vf",+ [+ "fps=30:start_time=0",+ "scale=w='trunc(iw*min(1,min(1280/max(iw,ih),720/min(iw,ih)))/2)*2':h='trunc(ih*min(1,min(1280/max(iw,ih),720/min(iw,ih)))/2)*2'",+ ...(hdr ? hdrFilters : []),+ "format=yuv420p",+ "setsar=1",+ ].join(","),+ "-c:v",+ "libx264",+ "-preset",+ "veryfast",+ "-crf",+ "23",+ "-maxrate",+ "4M",+ "-bufsize",+ "8M",+ "-movflags",+ "+faststart",+ ],+ "video.mp4",+ ),+ );++const execute = async (ffmpeg: FFmpeg, args: string[], outputPath: string) => {+ const status = await ffmpeg.exec([...args, outputPath]);+ if (status !== 0) throw new Error("Couldn't process this video.");+ const bytes = await ffmpeg.readFile(outputPath);+ await ffmpeg.deleteFile(outputPath);+ if (typeof bytes === "string")+ throw new Error("Expected encoded video data.");+ return ensureArrayBufferBacked(bytes);+};++const withSoftwareVideo = async <T>(+ blob: Blob,+ signal: AbortSignal,+ task: (ffmpeg: FFmpeg, inputPath: string) => Promise<T>,+): Promise<T> => {+ signal.throwIfAborted();+ const ffmpeg = new FFmpeg();+ const cancel = () => ffmpeg.terminate();+ signal.addEventListener("abort", cancel, { once: true });+ try {+ logToDisk("[info] Space video software processor loading");+ await ffmpeg.load({+ coreURL: "https://assets.ente.com/ffmpeg-core-9.0.2/ffmpeg-core.js",+ wasmURL:+ "https://assets.ente.com/ffmpeg-core-9.0.2/ffmpeg-core.wasm",+ });+ signal.throwIfAborted();+ await ffmpeg.createDir("/input");+ await ffmpeg.mount(+ FFFSType.WORKERFS,+ { files: [new File([blob], "video")] },+ "/input",+ );+ const result = await task(ffmpeg, "/input/video");+ signal.throwIfAborted();+ return result;+ } catch (error) {+ signal.throwIfAborted();+ throw error;+ } finally {+ signal.removeEventListener("abort", cancel);+ ffmpeg.terminate();+ }+};diff --git a/web/apps/space/src/utils/video-encoding/web-codecs.ts b/web/apps/space/src/utils/video-encoding/web-codecs.tsdeleted file mode 100644index 9cd712f499..0000000000--- a/web/apps/space/src/utils/video-encoding/web-codecs.ts+++ /dev/null@@ -1,476 +0,0 @@-import type { FFmpeg } from "@ffmpeg/ffmpeg";-import { newID } from "ente-base/id";-import { ensureArrayBufferBacked } from "ente-utils/bytes";-import { z } from "zod";--export interface VideoTranscodeEdit {- start: number;- end: number;- muted: boolean;-}--const Probe = z.object({- format: z.object({ format_name: z.string() }),- streams: z.array(- z.object({- codec_name: z.string(),- codec_tag_string: z.string(),- width: z.number().int().positive(),- height: z.number().int().positive(),- pix_fmt: z.string(),- color_transfer: z.string().optional(),- color_primaries: z.string().optional(),- sample_aspect_ratio: z.string().optional(),- start_time: z.string().optional(),- side_data_list: z- .array(- z.object({- rotation: z.number().optional(),- displaymatrix: z.string().optional(),- }),- )- .optional(),- }),- ),- packets: z.array(- z.object({- pts_time: z.coerce.number(),- duration_time: z.coerce.number().positive(),- pos: z.coerce.number().int().nonnegative(),- size: z.coerce.number().int().positive(),- flags: z.string(),- }),- ),-});--type Packet = z.infer<typeof Probe>["packets"][number];--const frameRate = 30;-const frameTime = (index: number) =>- Math.round((index * 1_000_000) / frameRate);--export const encodeVideoWithWebCodecs = async (- ffmpeg: FFmpeg,- inputPath: string,- blob: Blob,- edit: VideoTranscodeEdit,- signal?: AbortSignal,-): Promise<Uint8Array<ArrayBuffer> | undefined> => {- signal?.throwIfAborted();- const frameCount = Math.floor(- (edit.end - edit.start) * frameRate + 0.000001,- );- if (frameCount < 1) return;- const directory = newID("webcodecs_");- await ffmpeg.createDir(directory);- const probePath = `${directory}/probe.json`;- const descriptionPath = `${directory}/codec.bin`;- const videoPath = `${directory}/video.h264`;- const outputPath = `${directory}/video.mp4`;- const exec = async (command: string[]) => {- signal?.throwIfAborted();- const status = await ffmpeg.exec(command);- if (status !== 0) throw new Error(`FFmpeg exited with ${status}`);- };- const readBytes = async (path: string) => {- const bytes = await ffmpeg.readFile(path);- if (typeof bytes === "string") throw new Error("Expected binary data");- return ensureArrayBufferBacked(bytes);- };- try {- const status = await ffmpeg.ffprobe([- "-v",- "error",- "-select_streams",- "v:0",- "-show_entries",- "format=format_name:stream=codec_name,codec_tag_string,width,height,pix_fmt,color_transfer,color_primaries,sample_aspect_ratio,start_time:stream_side_data=rotation,displaymatrix:packet=pts_time,duration_time,pos,size,flags",- "-of",- "json",- "-o",- probePath,- inputPath,- ]);- if (status !== 0) throw new Error("Couldn't read video packets");- const json = await ffmpeg.readFile(probePath, "utf8");- if (typeof json !== "string")- throw new Error("Expected video information");- const probe = Probe.parse(JSON.parse(json));- const stream = probe.streams[0];- if (- !probe.format.format_name.split(",").includes("mov") ||- !stream ||- !(stream.codec_name === "h264"- ? ["avc1", "avc3"].includes(stream.codec_tag_string)- : stream.codec_name === "hevc" &&- ["hvc1", "hev1"].includes(stream.codec_tag_string)) ||- !["yuv420p", "yuvj420p", "yuv420p10le"].includes(stream.pix_fmt) ||- (stream.sample_aspect_ratio &&- stream.sample_aspect_ratio !== "1:1") ||- Number(stream.start_time ?? 0) !== 0- )- return;- const display = stream.side_data_list?.find(- (data) => data.displaymatrix,- );- const rotation = display?.rotation ?? 0;- if (![0, 90, -90, 180, -180].includes(rotation)) return;- if (display?.displaymatrix) {- const matrix = display.displaymatrix- .trim()- .split("\n")- .flatMap((line) =>- line- .slice(line.indexOf(":") + 1)- .trim()- .split(/\s+/)- .map(Number),- );- const angle = (rotation * Math.PI) / 180;- const expected = [- Math.round(Math.cos(angle) * 65536),- -Math.round(Math.sin(angle) * 65536),- 0,- Math.round(Math.sin(angle) * 65536),- Math.round(Math.cos(angle) * 65536),- 0,- 0,- 0,- 1073741824,- ];- if (- matrix.length !== 9 ||- matrix.some((value, index) => value !== expected[index])- )- return;- }- if (- !probe.packets.length ||- probe.packets.some((packet) => packet.pos + packet.size > blob.size)- )- return;- await exec([- "-dump_attachment:v:0",- descriptionPath,- "-i",- inputPath,- "-t",- "0",- "-c",- "copy",- "-f",- "null",- "-",- ]);- const description = await readBytes(descriptionPath);- const hevc = stream.codec_name === "hevc";- if (description.length < (hevc ? 23 : 7) || description[0] !== 1)- return;- let codec: string;- if (hevc) {- const profile = description[1]!;- const compatibility = new DataView(- description.buffer,- description.byteOffset,- description.byteLength,- ).getUint32(2);- const reversedCompatibility = parseInt(- compatibility- .toString(2)- .padStart(32, "0")- .split("")- .reverse()- .join(""),- 2,- );- const constraints = Array.from(description.slice(6, 12), (value) =>- value.toString(16).padStart(2, "0"),- )- .join(".")- .replace(/(\.00)+$/, "");- codec = `${stream.codec_tag_string}.${["", "A", "B", "C"][profile >> 6]}${profile & 31}.${reversedCompatibility.toString(16)}.${profile & 32 ? "H" : "L"}${description[12]}.${constraints}`;- } else {- codec =- "avc1." +- Array.from(description.slice(1, 4), (value) =>- value.toString(16).padStart(2, "0"),- ).join("");- }- const decoderConfig: VideoDecoderConfig = {- codec,- description,- hardwareAcceleration: "prefer-hardware",- };- const sideways = Math.abs(rotation) === 90;- const width = sideways ? stream.height : stream.width;- const height = sideways ? stream.width : stream.height;- const scale = Math.min(- 1,- 1920 / Math.max(width, height),- 1080 / Math.min(width, height),- );- const encoderConfig: VideoEncoderConfig = {- codec: "avc1.640028",- width: Math.floor((width * scale) / 2) * 2,- height: Math.floor((height * scale) / 2) * 2,- bitrate: 4_000_000,- bitrateMode: "variable",- framerate: frameRate,- latencyMode: "realtime",- hardwareAcceleration: "prefer-hardware",- avc: { format: "annexb" },- };- const [decoderSupport, encoderSupport] = await Promise.all([- VideoDecoder.isConfigSupported(decoderConfig),- VideoEncoder.isConfigSupported(encoderConfig),- ]);- if (!decoderSupport.supported || !encoderSupport.supported) return;- const video = await encodeFrames(- blob,- probe.packets,- edit.start,- frameCount,- rotation,- stream.pix_fmt === "yuv420p10le" ||- stream.color_primaries === "bt2020" ||- ["smpte2084", "arib-std-b67"].includes(- stream.color_transfer ?? "",- ),- decoderConfig,- encoderConfig,- signal,- );- signal?.throwIfAborted();- await ffmpeg.writeFile(- videoPath,- new Uint8Array(await video.arrayBuffer()),- );- await exec([- "-ss",- String(edit.start),- "-i",- inputPath,- "-r",- String(frameRate),- "-i",- videoPath,- "-t",- String(frameCount / frameRate),- "-map",- "1:v:0",- "-c:v",- "copy",- "-bsf:v",- `setts=ts=N:duration=1:time_base=1/${frameRate}`,- "-video_track_timescale",- "30000",- ...(edit.muted- ? ["-an"]- : ["-map", "0:a:0?", "-c:a", "aac", "-b:a", "128k"]),- "-map_metadata",- "-1",- "-map_metadata:s",- "-1",- "-map_chapters",- "-1",- "-movflags",- "+faststart",- outputPath,- ]);- return await readBytes(outputPath);- } finally {- if (ffmpeg.loaded) {- for (const file of await ffmpeg.listDir(directory)) {- if (!file.isDir)- await ffmpeg.deleteFile(`${directory}/${file.name}`);- }- await ffmpeg.deleteDir(directory);- }- }-};--const encodeFrames = async (- blob: Blob,- packets: Packet[],- start: number,- frameCount: number,- rotation: number,- convertColor: boolean,- decoderConfig: VideoDecoderConfig,- encoderConfig: VideoEncoderConfig,- signal?: AbortSignal,-) => {- signal?.throwIfAborted();- const startTime = Math.round(start * 1_000_000);- const durations = new Map(- packets.map((packet) => [- Math.round(packet.pts_time * 1_000_000),- Math.round(packet.duration_time * 1_000_000),- ]),- );- let first = -1;- for (const [index, packet] of packets.entries()) {- if (- packet.flags.includes("K") &&- Math.round(packet.pts_time * 1_000_000) <= startTime- )- first = index;- }- if (first < 0) throw new Error("No keyframe before the selected clip");- let failure: Error | undefined;- let wake: (() => void) | undefined;- const notify = () => {- wake?.();- wake = undefined;- };- const fail = (error: Error) => {- failure = error;- notify();- };- const checkError = () => {- signal?.throwIfAborted();- if (failure) throw failure;- };- let previous: VideoFrame | undefined;- let encoded = 0;- const chunks: Uint8Array<ArrayBuffer>[] = [];- const canvas =- rotation || convertColor- ? new OffscreenCanvas(encoderConfig.width, encoderConfig.height)- : undefined;- const context = canvas?.getContext("2d", { colorSpace: "srgb" });- if (canvas && !context)- throw new Error("Couldn't prepare the video frames");- const encoder = new VideoEncoder({- output(chunk) {- if (chunk.timestamp !== frameTime(chunks.length)) {- fail(new Error("Video encoder reordered the output frames"));- return;- }- const bytes = new Uint8Array(chunk.byteLength);- chunk.copyTo(bytes);- chunks.push(bytes);- notify();- },- error: fail,- });- const encodeUntil = (time: number) => {- if (!previous) return;- while (encoded < frameCount && startTime + frameTime(encoded) < time) {- checkError();- const timestamp = frameTime(encoded);- if (canvas && context) {- const sideways = Math.abs(rotation) === 90;- const width = sideways ? canvas.height : canvas.width;- const height = sideways ? canvas.width : canvas.height;- context.setTransform(- 1,- 0,- 0,- 1,- canvas.width / 2,- canvas.height / 2,- );- context.rotate((-rotation * Math.PI) / 180);- context.drawImage(- previous,- -width / 2,- -height / 2,- width,- height,- );- }- const frame = new VideoFrame(canvas ?? previous, {- timestamp,- duration: frameTime(encoded + 1) - timestamp,- });- try {- encoder.encode(frame, { keyFrame: encoded % 60 === 0 });- encoded++;- } finally {- frame.close();- }- }- };- const decoder = new VideoDecoder({- output(frame) {- if (failure || encoded === frameCount) {- frame.close();- return;- }- try {- if (!previous && frame.timestamp > startTime)- throw new Error("Video starts after the selected clip");- encodeUntil(frame.timestamp);- previous?.close();- previous = frame;- } catch (error) {- frame.close();- fail(error as Error);- }- notify();- },- error: fail,- });- const abort = () => {- fail(new DOMException("Canceled", "AbortError"));- if (decoder.state !== "closed") decoder.close();- if (encoder.state !== "closed") encoder.close();- };- decoder.addEventListener("dequeue", notify);- encoder.addEventListener("dequeue", notify);- signal?.addEventListener("abort", abort, { once: true });- try {- signal?.throwIfAborted();- encoder.configure(encoderConfig);- decoder.configure(decoderConfig);- for (- let index = first;- index < packets.length && encoded < frameCount;- index++- ) {- while (decoder.decodeQueueSize > 8 || encoder.encodeQueueSize > 8) {- checkError();- await new Promise<void>((resolve) => {- wake = resolve;- });- }- checkError();- const packet = packets[index]!;- const data = await blob- .slice(packet.pos, packet.pos + packet.size)- .arrayBuffer();- checkError();- decoder.decode(- new EncodedVideoChunk({- type: packet.flags.includes("K") ? "key" : "delta",- timestamp: Math.round(packet.pts_time * 1_000_000),- duration: Math.round(packet.duration_time * 1_000_000),- data,- }),- );- }- await decoder.flush();- checkError();- if (previous)- encodeUntil(- previous.timestamp + (durations.get(previous.timestamp) ?? 0),- );- await encoder.flush();- checkError();- if (chunks.length !== frameCount)- throw new Error("Video export is missing frames");- return new Blob(chunks);- } finally {- signal?.removeEventListener("abort", abort);- decoder.removeEventListener("dequeue", notify);- encoder.removeEventListener("dequeue", notify);- previous?.close();- if (decoder.state !== "closed") decoder.close();- if (encoder.state !== "closed") encoder.close();- if (canvas) {- canvas.width = 1;- canvas.height = 1;- }- }-};diff --git a/web/apps/space/src/utils/video-encoding/web.ts b/web/apps/space/src/utils/video-encoding/web.tsindex fbe6ac26a7..892d437c47 100644--- a/web/apps/space/src/utils/video-encoding/web.ts+++ b/web/apps/space/src/utils/video-encoding/web.ts@@ -1,144 +1,47 @@-import { FFFSType, FFmpeg } from "@ffmpeg/ffmpeg";-import { joinPath } from "ente-base/file-name";-import { newID } from "ente-base/id";-import log from "ente-base/log";import { logToDisk } from "ente-base/log-web";-import type { FFmpegCommand } from "ente-base/types/ipc";-import { ensureArrayBufferBacked } from "ente-utils/bytes";import { PromiseQueue } from "ente-utils/promise";-import { z } from "zod";-import { inputPathPlaceholder, outputPathPlaceholder } from "./constants";-import type { VideoTranscodeEdit } from "./web-codecs";--let _ffmpeg: { instance: FFmpeg; ready: Promise<void> } | undefined;--const _ffmpegTaskQueue = new PromiseQueue<unknown>();--const ffmpegLazy = () => {- if (!_ffmpeg) {- const instance = new FFmpeg();- const failed = (error: unknown): never => {- if (_ffmpeg?.instance == instance) {- instance.terminate();- _ffmpeg = undefined;- }- throw error;- };- const exec = instance.exec.bind(instance);- const probe = instance.ffprobe.bind(instance);- instance.exec = (...args) => exec(...args).catch(failed);- instance.ffprobe = (...args) => probe(...args).catch(failed);- _ffmpeg = { instance, ready: loadFFmpeg(instance) };- }- return _ffmpeg;-};--const loadFFmpeg = async (ffmpeg: FFmpeg) => {- const startedAt = Date.now();- const timeout = new AbortController();- const timer = setTimeout(() => {- timeout.abort();- ffmpeg.terminate();- }, 60_000);- logToDisk("[info] Space video encoder loading");- try {- await ffmpeg.load({- coreURL: "https://assets.ente.com/ffmpeg-core-9.0.2/ffmpeg-core.js",- wasmURL:- "https://assets.ente.com/ffmpeg-core-9.0.2/ffmpeg-core.wasm",- });- logToDisk(- `[info] Space video encoder loaded elapsedMs=${Date.now() - startedAt}`,- );- } catch (error) {- ffmpeg.terminate();- if (_ffmpeg?.instance == ffmpeg) _ffmpeg = undefined;- logToDisk(- `[error] Space video encoder load ${timeout.signal.aborted ? "timeout" : "failed"} elapsedMs=${Date.now() - startedAt} error=${JSON.stringify(error instanceof Error ? error.toString() : error)}`,- );- if (timeout.signal.aborted)- throw new Error(- "The video processor took too long to load. Please try again.",- { cause: error },- );- throw error;- } finally {- clearTimeout(timer);- }-};--export const preloadVideoEncoderWeb = async () => {- await ffmpegLazy().ready;-};--export const determineVideoDurationWeb = async (- blob: Blob,- signal?: AbortSignal,-): Promise<number> =>- runFFmpegTask(- "duration",- blob.size,- (ffmpeg) => ffprobeExecVideoDuration(ffmpeg, blob),- signal,- );+import {+ ALL_FORMATS,+ BlobSource,+ BufferSource,+ BufferTarget,+ canEncodeAudio,+ CanvasSink,+ Conversion,+ Input,+ Mp4OutputFormat,+ Output,+ Quality,+ type ConversionAudioOptions,+ type ConversionVideoOptions,+} from "mediabunny";+import type { SpacePostVideoEdit } from "../post-video";++const videoTaskQueue = new PromiseQueue<unknown>();++const canUseSoftwareVideo = () =>+ !navigator.userAgent.includes("AppleWebKit/") ||+ navigator.userAgent.includes("Chrome/");++export const determineVideoDurationWeb = (blob: Blob, signal?: AbortSignal) =>+ runVideoTask("duration", blob, (input) => input.computeDuration(), signal);export const determineVideoInfoWeb = (blob: Blob, signal?: AbortSignal) =>- runFFmpegTask(+ runVideoTask("metadata",- blob.size,- (ffmpeg) =>- withInputMount(ffmpeg, blob, async (inputPath) => {- const json = await ffprobeOutput(- ffmpeg,- [- "-v",- "error",- "-select_streams",- "v:0",- "-show_entries",- "format=duration:stream=width,height:stream_side_data=rotation",- "-of",- "json",- "-o",- "output.json",- inputPath,- ],- "output.json",- );- const info = z- .object({- format: z.object({- duration: z.coerce.number().positive(),- }),- streams: z- .array(- z.object({- width: z.number().positive(),- height: z.number().positive(),- side_data_list: z- .array(- z.object({- rotation: z.number().optional(),- }),- )- .optional(),- }),- )- .nonempty(),- })- .parse(JSON.parse(json));- const stream = info.streams[0]!;- const rotation =- stream.side_data_list?.find(- (data) => data.rotation != undefined,- )?.rotation ?? 0;- const sideways = Math.abs(rotation % 180) == 90;- return {- duration: info.format.duration,- width: sideways ? stream.height : stream.width,- height: sideways ? stream.width : stream.height,- };- }),+ blob,+ async (input) => {+ const track = await input.getPrimaryVideoTrack();+ if (!track) throw new Error("This file has no readable video.");+ const [duration, width, height] = await Promise.all([+ track.computeDuration(),+ track.getDisplayWidth(),+ track.getDisplayHeight(),+ ]);+ if (!Number.isFinite(duration) || duration <= 0)+ throw new Error("This video has no readable duration.");+ return { duration, width, height };+ },signal,);@@ -148,110 +51,52 @@ export const extractVideoFramesWeb = (maxDimension: number,signal?: AbortSignal,) =>- runFFmpegTask(+ runVideoTask("frames",- blob.size,- async (ffmpeg, signal) => {- const frames: Blob[] = [];- await withInputMount(ffmpeg, blob, async (inputPath) => {- const outputPath = newID("frame_") + ".png";- const hdr = await isHDRVideo(ffmpeg, inputPath);- try {- for (const time of times) {- signal.throwIfAborted();- const json = await ffprobeOutput(- ffmpeg,- [- "-v",- "error",- "-select_streams",- "v:0",- "-read_intervals",- `${time}%${time + 1}`,- "-show_entries",- "packet=pts_time:format=start_time",- "-of",- "json",- "-o",- "frame.json",- inputPath,- ],- "frame.json",- );- const probe = z- .object({- format: z.object({- start_time: z.coerce.number().optional(),- }),- packets: z- .array(- z.object({- pts_time: z.coerce.number(),- }),- )- .nonempty(),- })- .parse(JSON.parse(json));- const start = probe.format.start_time ?? 0;- const timestamps = probe.packets.map(- (packet) => packet.pts_time - start,- );- const frameTime = timestamps.reduce(- (closest, timestamp) =>- Math.abs(timestamp - time) <- Math.abs(closest - time)- ? timestamp- : closest,- );- const status = await ffmpeg.exec([- "-ss",- String(Math.max(0, frameTime - 0.000001)),- "-i",- inputPath,- "-map",- "0:v:0",- "-frames:v",- "1",- "-an",- "-vf",- [- `scale=w='min(${maxDimension},iw)':h='min(${maxDimension},ih)':force_original_aspect_ratio=decrease`,- ...(hdr- ? [- "zscale=transfer=linear",- "tonemap=tonemap=hable:desat=0",- "zscale=primaries=709:transfer=709:matrix=709",- ]- : []),- "format=rgb24",- "setsar=1",- ].join(","),- "-map_metadata",- "-1",- "-update",- "1",- outputPath,- ]);- if (status != 0)- throw new Error("Couldn't extract a video frame.");- const bytes = await ffmpeg.readFile(outputPath);- if (typeof bytes == "string")- throw new Error("Expected a video frame.");- frames.push(- new Blob([ensureArrayBufferBacked(bytes)], {- type: "image/png",- }),- );- await ffmpeg.deleteFile(outputPath);- }- } finally {- if (ffmpeg.loaded) {- const files = await ffmpeg.listDir("/");- if (files.some((file) => file.name == outputPath))- await ffmpeg.deleteFile(outputPath);- }- }+ blob,+ async (input, signal) => {+ const track = await input.getPrimaryVideoTrack();+ if (!track)+ throw new Error("This browser can't preview this video.");+ if (!(await track.canDecode())) {+ if (!canUseSoftwareVideo())+ throw new Error("This browser can't preview this video.");+ const { extractSoftwareVideoFrames } =+ await import("./software");+ return extractSoftwareVideoFrames(+ blob,+ times,+ maxDimension,+ await track.hasHighDynamicRange(),+ signal,+ );+ }+ const [width, height, firstTimestamp, duration] = await Promise.all(+ [+ track.getDisplayWidth(),+ track.getDisplayHeight(),+ track.getFirstTimestamp(),+ track.computeDuration(),+ ],+ );+ const scale = Math.min(1, maxDimension / Math.max(width, height));+ const sink = new CanvasSink(track, {+ width: Math.max(1, Math.round(width * scale)),+ height: Math.max(1, Math.round(height * scale)),+ fit: "contain",});+ const timestamps = times.map((time) =>+ Math.max(firstTimestamp, Math.min(time, duration - 0.001)),+ );+ const frames: HTMLCanvasElement[] = [];+ for await (const frame of sink.canvasesAtTimestamps(timestamps)) {+ signal.throwIfAborted();+ if (!frame)+ throw new Error(+ "Couldn't read a video frame. Please try again.",+ );+ frames.push(frame.canvas as HTMLCanvasElement);+ }return frames;},signal,@@ -259,81 +104,206 @@ export const extractVideoFramesWeb = (export const transcodeVideoWeb = (blob: Blob,- edit: VideoTranscodeEdit,- fallbackCommand: FFmpegCommand,+ edit: SpacePostVideoEdit,signal?: AbortSignal,) =>- runFFmpegTask(+ runVideoTask("export",- blob.size,- async (ffmpeg, signal) => {- if (- typeof VideoDecoder !== "undefined" &&- typeof VideoEncoder !== "undefined"- ) {- try {- const { encodeVideoWithWebCodecs } =- await import("./web-codecs");- const result = await withInputMount(- ffmpeg,- blob,- (inputPath) =>- encodeVideoWithWebCodecs(- ffmpeg,- inputPath,- blob,- edit,- signal,- ),- );- if (result) {- logToDisk(- `[info] Space video encoded encoder=webcodecs bytes=${result.byteLength}`,+ blob,+ async (input, signal) => {+ const videoTrack = await input.getPrimaryVideoTrack();+ if (!videoTrack)+ throw new Error("This file has no readable video.");+ const softwareExport = async () => {+ const { transcodeSoftwareVideo } = await import("./software");+ return transcodeSoftwareVideo(+ blob,+ edit,+ await videoTrack.hasHighDynamicRange(),+ signal,+ );+ };+ if (!(await videoTrack.canDecode())) {+ if (canUseSoftwareVideo()) return softwareExport();+ throw new Error("This browser can't decode this video.");+ }+ let audio: ConversionAudioOptions = edit.muted+ ? { discard: true }+ : { codec: "aac" };+ const audioTrack = edit.muted+ ? null+ : await input.getPrimaryAudioTrack();+ if (audioTrack && (await audioTrack.getCodec()) !== "aac") {+ const quality = new Quality({ bitrate: 128_000 });+ audio = { codec: "aac", quality };+ if (+ !(await canEncodeAudio("aac", {+ numberOfChannels:+ await audioTrack.getNumberOfChannels(),+ sampleRate: await audioTrack.getSampleRate(),+ quality,+ }))+ ) {+ const { registerAacEncoder } =+ await import("@mediabunny/aac-encoder");+ signal.throwIfAborted();+ registerAacEncoder();+ }+ }+ let output = new Output({+ format: new Mp4OutputFormat({ fastStart: "in-memory" }),+ target: new BufferTarget(),+ });+ let conversion: Conversion | undefined;+ let encodedInput: Input | undefined;+ const cancel = () => {+ void (conversion ? conversion.cancel() : output.cancel());+ encodedInput?.dispose();+ };+ signal.addEventListener("abort", cancel, { once: true });+ try {+ signal.throwIfAborted();+ conversion = await Conversion.init({+ input,+ output,+ tracks: "primary",+ trim: { start: edit.start, end: edit.end },+ copy: { boundaryPolicy: "shrink" },+ tags: {},+ showWarnings: false,+ video: async (track): Promise<ConversionVideoOptions> => {+ const [width, height, hdr] = await Promise.all([+ track.getDisplayWidth(),+ track.getDisplayHeight(),+ track.hasHighDynamicRange(),+ ]);+ const scale = Math.min(+ 1,+ 1920 / Math.max(width, height),+ 1080 / Math.min(width, height),+ );+ const outputWidth = Math.max(+ 2,+ Math.floor((width * scale) / 2) * 2,);- return result;+ const outputHeight = Math.max(+ 2,+ Math.floor((height * scale) / 2) * 2,+ );+ let context: CanvasRenderingContext2D | null = null;+ if (hdr) {+ const canvas = document.createElement("canvas");+ canvas.width = outputWidth;+ canvas.height = outputHeight;+ context = canvas.getContext("2d", {+ colorSpace: "srgb",+ });+ if (!context)+ throw new Error(+ "Couldn't prepare the video frames.",+ );+ }+ const colorContext = context;+ return {+ codec: "avc",+ width: outputWidth,+ height: outputHeight,+ fit: "contain",+ frameRate: 30,+ quality: new Quality({ bitrate: 4_000_000 }),+ keyFrameInterval: 2,+ allowTransformationMetadata: false,+ process: colorContext+ ? (sample) => {+ sample.draw(+ colorContext,+ 0,+ 0,+ outputWidth,+ outputHeight,+ );+ return colorContext.canvas;+ }+ : undefined,+ };+ },+ audio,+ });+ signal.throwIfAborted();+ if (+ !conversion.isValid ||+ conversion.discardedTracks.some(+ (track) => track.reason !== "discarded_by_user",+ )+ ) {+ if (canUseSoftwareVideo()) {+ await conversion.cancel();+ return await softwareExport();}- } catch (error) {- signal.throwIfAborted();- if (!ffmpeg.loaded) throw error;- logToDisk(- "[warn] Space video WebCodecs export failed, using software encoder",+ throw new Error(+ "This browser can't export this video's format. Try another video or browser.",);}+ await conversion.execute();+ signal.throwIfAborted();+ if (!output.target.buffer)+ throw new Error("Couldn't export this video.");+ encodedInput = new Input({+ formats: ALL_FORMATS,+ source: new BufferSource(output.target.buffer),+ });+ output = new Output({+ format: new Mp4OutputFormat({ fastStart: "in-memory" }),+ target: new BufferTarget(),+ });+ conversion = await Conversion.init({+ input: encodedInput,+ output,+ tracks: "primary",+ trim: { start: 0, end: edit.end - edit.start },+ copy: { mode: "forced", boundaryPolicy: "shrink" },+ tags: {},+ showWarnings: false,+ });+ signal.throwIfAborted();+ if (!conversion.isValid || conversion.discardedTracks.length)+ throw new Error("Couldn't trim the exported video.");+ for (const track of output.tracks) delete track.metadata.name;+ await conversion.execute();+ signal.throwIfAborted();+ if (!output.target.buffer)+ throw new Error("Couldn't export this video.");+ return new Uint8Array(output.target.buffer);+ } finally {+ signal.removeEventListener("abort", cancel);+ if (output.state !== "finalized") {+ if (conversion?.state !== "canceled")+ await conversion?.cancel();+ await output.cancel();+ }+ encodedInput?.dispose();}- signal.throwIfAborted();- logToDisk("[info] Space video encoding encoder=wasm");- const result = await ffmpegExec(- ffmpeg,- fallbackCommand,- "mp4",- blob,- );- logToDisk(- `[info] Space video encoded encoder=wasm bytes=${result.byteLength}`,- );- return result;},signal,300_000,);-const runFFmpegTask = <T>(+const runVideoTask = <T>(stage: string,- bytes: number,- task: (ffmpeg: FFmpeg, signal: AbortSignal) => Promise<T>,+ blob: Blob,+ task: (input: Input, signal: AbortSignal) => Promise<T>,signal?: AbortSignal,timeout = 60_000,): Promise<T> => {- const pending = _ffmpegTaskQueue.add(async () => {+ const pending = videoTaskQueue.add(async () => {signal?.throwIfAborted();- const engine = ffmpegLazy();- const ffmpeg = engine.instance;+ const input = new Input({+ formats: ALL_FORMATS,+ source: new BlobSource(blob),+ });const controller = new AbortController();const cancel = () => controller.abort(signal?.reason);- const abort = () => {- ffmpeg.terminate();- if (_ffmpeg == engine) _ffmpeg = undefined;- };+ const dispose = () => input.dispose();const timer = setTimeout(() =>controller.abort(@@ -345,32 +315,37 @@ const runFFmpegTask = <T>(timeout,);signal?.addEventListener("abort", cancel, { once: true });- controller.signal.addEventListener("abort", abort, { once: true });+ controller.signal.addEventListener("abort", dispose, { once: true });const startedAt = Date.now();- logToDisk(`[info] Space video stage=${stage} started bytes=${bytes}`);+ logToDisk(+ `[info] Space video stage=${stage} started bytes=${blob.size}`,+ );try {- await engine.ready;+ const result = await task(input, controller.signal);controller.signal.throwIfAborted();- const result = await task(ffmpeg, controller.signal);logToDisk(`[info] Space video stage=${stage} completed elapsedMs=${Date.now() - startedAt}`,);return result;} catch (error) {logToDisk(- `[error] Space video stage=${stage} ${signal?.aborted ? "canceled" : controller.signal.aborted ? "timeout" : "failed"} elapsedMs=${Date.now() - startedAt}`,+ `[error] Space video stage=${stage} ${signal?.aborted ? "canceled" : controller.signal.aborted ? "timeout" : "failed"} elapsedMs=${Date.now() - startedAt} error=${JSON.stringify(error instanceof Error ? error.toString() : error)}`,);controller.signal.throwIfAborted();throw error;} finally {clearTimeout(timer);signal?.removeEventListener("abort", cancel);- controller.signal.removeEventListener("abort", abort);+ controller.signal.removeEventListener("abort", dispose);+ input.dispose();}}) as Promise<T>;if (!signal) return pending;return new Promise<T>((resolve, reject) => {- const abort = () => reject(new DOMException("Canceled", "AbortError"));+ const abort = () =>+ reject(+ new DOMException("Video processing canceled.", "AbortError"),+ );signal.addEventListener("abort", abort, { once: true });void pending.then(resolve, reject)@@ -378,202 +353,3 @@ const runFFmpegTask = <T>(if (signal.aborted) abort();});};--const ffmpegExec = async (- ffmpeg: FFmpeg,- command: FFmpegCommand,- outputFileExtension: string,- blob: Blob,-): Promise<Uint8Array<ArrayBuffer>> => {- const outputSuffix = outputFileExtension ? "." + outputFileExtension : "";- const outputPath = newID("out_") + outputSuffix;-- let status: number | undefined;-- return withInputMount(ffmpeg, blob, async (inputPath) => {- try {- const startTime = Date.now();-- let resolvedCommand: string[];- if (Array.isArray(command)) {- resolvedCommand = command;- } else {- const isHDR = await isHDRVideo(ffmpeg, inputPath);- resolvedCommand = isHDR ? command.hdr : command.default;- }-- const cmd = substitutePlaceholders(- resolvedCommand,- inputPath,- outputPath,- );-- status = await ffmpeg.exec(cmd);- if (status !== 0) {- log.info(- `[wasm] ffmpeg command failed with exit code ${status}: ${cmd.join(" ")}`,- );- throw new Error(- `ffmpeg command failed with exit code ${status}`,- );- }-- const result = await ffmpeg.readFile(outputPath);- if (typeof result == "string")- throw new Error("Expected binary data");-- const ms = Date.now() - startTime;- log.debug(() => `[wasm] ffmpeg ${cmd.join(" ")} (${ms} ms)`);- return ensureArrayBufferBacked(result);- } finally {- try {- if (ffmpeg.loaded) await ffmpeg.deleteFile(outputPath);- } catch (e) {- if (status === 0) {- log.error(`Failed to remove output ${outputPath}`, e);- }- }- }- });-};--const withInputMount = async <T>(- ffmpeg: FFmpeg,- blob: Blob,- f: (inputPath: string) => Promise<T>,-): Promise<T> => {- const mountDir = "/mount";- const inputFileName = newID("in_");- const inputPath = joinPath(mountDir, inputFileName);-- const inputFile = new File([blob], inputFileName);-- try {- await ffmpeg.createDir(mountDir);- await ffmpeg.mount(FFFSType.WORKERFS, { files: [inputFile] }, mountDir);-- return await f(inputPath);- } finally {- try {- if (ffmpeg.loaded) await ffmpeg.unmount(mountDir);- } catch (e) {- log.error(`Failed to remove mount ${mountDir}`, e);- }- try {- if (ffmpeg.loaded) await ffmpeg.deleteDir(mountDir);- } catch (e) {- log.error(`Failed to delete mount directory ${mountDir}`, e);- }- }-};--const substitutePlaceholders = (- command: string[],- inputFilePath: string,- outputFilePath: string,-) =>- command.map((segment) => {- if (segment == inputPathPlaceholder) {- return inputFilePath;- } else if (segment == outputPathPlaceholder) {- return outputFilePath;- } else {- return segment;- }- });--const FFProbeOutputIsHDR = z.object({- streams: z.array(z.object({ color_transfer: z.string().optional() })),-});--const isHDRVideo = async (ffmpeg: FFmpeg, inputFilePath: string) => {- let jsonString: string | undefined;- try {- jsonString = await ffprobeOutput(- ffmpeg,- [- ["-i", inputFilePath],- ["-show_entries", "stream=color_transfer"],- ["-select_streams", "v:0"],- ["-of", "json"],- ["-o", "output.json"],- ].flat(),- "output.json",- );-- const output = FFProbeOutputIsHDR.parse(JSON.parse(jsonString));- switch (output.streams[0]?.color_transfer) {- case "smpte2084":- case "arib-std-b67":- return true;- default:- return false;- }- } catch (e) {- log.warn("Could not detect HDR status", e);- if (jsonString) log.debug(() => ["ffprobe-output", jsonString]);- return false;- }-};--const ffprobeOutput = async (- ffmpeg: FFmpeg,- cmd: string[],- outputPath: string,-) => {- let status: number | undefined;-- try {- status = await ffmpeg.ffprobe(cmd);- if (status !== 0) {- log.info(- `[wasm] ffprobe command failed with exit code ${status}: ${cmd.join(" ")}`,- );- throw new Error(`ffprobe command failed with exit code ${status}`);- }-- const result = await ffmpeg.readFile(outputPath, "utf8");- if (typeof result != "string") throw new Error("Expected text data");-- return result;- } finally {- try {- if (ffmpeg.loaded) await ffmpeg.deleteFile(outputPath);- } catch (e) {- if (status === 0) {- log.error(`Failed to remove output ${outputPath}`, e);- }- }- }-};--const FFProbeOutputDuration = z.object({- format: z.object({ duration: z.string() }),-});--const ffprobeExecVideoDuration = async (ffmpeg: FFmpeg, blob: Blob) =>- withInputMount(ffmpeg, blob, async (inputPath) => {- const jsonString = await ffprobeOutput(- ffmpeg,- [- ["-i", inputPath],- ["-v", "error"],- ["-show_entries", "format=duration"],- ["-of", "json"],- ["-o", "output.json"],- ].flat(),- "output.json",- );-- const durationString = FFProbeOutputDuration.parse(- JSON.parse(jsonString),- ).format.duration;-- const duration = parseFloat(durationString);- if (isNaN(duration)) {- const msg = "Could not parse video duration";- log.warn(msg, durationString);- throw new Error(msg);- }- return duration;- });diff --git a/web/apps/space/tests/video-cover.test.ts b/web/apps/space/tests/video-cover.test.tsindex e332990132..e5d63fc817 100644--- a/web/apps/space/tests/video-cover.test.ts+++ b/web/apps/space/tests/video-cover.test.ts@@ -23,194 +23,29 @@ vi.mock("../src/utils/video-encoding/web", () => ({vi.mock("ente-base/log-web", () => ({ logToDisk: mocks.log }));vi.mock("utils/thumbhash", () => ({ thumbHashBase64FromCanvas: () => "hash" }));-let alpha: number;-let bitmapClose: ReturnType<typeof vi.fn>;-let createElement: ReturnType<typeof vi.fn>;+const canvas = () => ({+ width: 540,+ height: 960,+ toBlob: (callback: BlobCallback) =>+ callback(new Blob(["cover"], { type: "image/webp" })),+});beforeEach(() => {vi.resetAllMocks();- alpha = 255;- bitmapClose = vi.fn();- vi.stubGlobal("navigator", {- userAgent:- "Mozilla/5.0 (Android 16; Mobile; rv:143.0) Gecko/143.0 Firefox/143.0",- });- mocks.frames.mockResolvedValue([new Blob(["png"], { type: "image/png" })]);+ mocks.frames.mockImplementation(() => Promise.resolve([canvas()]));mocks.info.mockResolvedValue({ duration: 3.45, width: 1080, height: 1920 });mocks.transcode.mockResolvedValue(new Uint8Array([1, 2, 3]));mocks.duration.mockResolvedValue(3.45);- vi.stubGlobal(- "createImageBitmap",- vi- .fn()- .mockImplementation(() =>- Promise.resolve({- width: 540,- height: 960,- close: bitmapClose,- }),- ),- );- createElement = vi.fn((tag: string) => {- if (tag != "canvas")- throw new Error(- "Must not decode through a video element on Firefox Android",- );- return {- width: 0,- height: 0,- getContext: () => ({- drawImage: vi.fn(),- getImageData: () => ({- data: new Uint8ClampedArray([0, 0, 0, alpha]),- }),- }),- toBlob: (callback: BlobCallback) =>- callback(new Blob(["cover"], { type: "image/webp" })),- };- });- vi.stubGlobal("document", { createElement });});afterEach(() => {- vi.unstubAllGlobals();vi.restoreAllMocks();- vi.useRealTimers();});-const nativeVideo = () => {- vi.stubGlobal("navigator", {- userAgent:- "Mozilla/5.0 (Android 16) Chrome/140.0.0.0 Mobile Safari/537.36",- });- vi.stubGlobal("window", { setTimeout, clearTimeout });- let frameReady!: VideoFrameRequestCallback;- const video = Object.assign(new EventTarget(), {- duration: 3.45,- videoWidth: 540,- videoHeight: 960,- currentTime: 0,- pause: vi.fn(),- removeAttribute: vi.fn(),- load: vi.fn(),- requestVideoFrameCallback: vi.fn(- (callback: VideoFrameRequestCallback) => {- frameReady = callback;- return 7;- },- ),- cancelVideoFrameCallback: vi.fn(),- presentFrame: () => {- alpha = 255;- frameReady(0, {} as VideoFrameCallbackMetadata);- },- });- createElement.mockImplementationOnce(() => video);- return video;-};--test("cellular metadata preload does not draw an empty cover before the first frame is decoded", async () => {- const video = nativeVideo();- const revoke = vi.spyOn(URL, "revokeObjectURL");- alpha = 0;- const pending = spaceVideoCover(new Blob(["video"]), 0);- const settled = vi.fn();- void pending.then(settled, settled);- video.dispatchEvent(new Event("loadeddata"));- await vi.waitFor(() =>- expect(video.requestVideoFrameCallback).toHaveBeenCalledOnce(),- );- expect(settled).not.toHaveBeenCalled();- expect(createElement).not.toHaveBeenCalledWith("canvas");- video.presentFrame();- await expect(pending).resolves.toMatchObject({ width: 540, height: 960 });- expect(mocks.frames).not.toHaveBeenCalled();- expect(video.pause).toHaveBeenCalledOnce();- expect(revoke).toHaveBeenCalledOnce();-});--test("registers the first-frame callback before loading a video on Wi-Fi", async () => {- const video = nativeVideo();- video.load.mockImplementationOnce(() => {- expect(video.requestVideoFrameCallback).toHaveBeenCalledOnce();- video.presentFrame();- video.dispatchEvent(new Event("loadeddata"));- });- await expect(- spaceVideoCover(new Blob(["video"]), 0),- ).resolves.toBeDefined();-});--test.each(["abort", "error", "timeout"])(- "releases a pending frame callback and local video URL on %s",- async (reason) => {- vi.useFakeTimers();- const video = nativeVideo();- const revoke = vi.spyOn(URL, "revokeObjectURL");- const controller = new AbortController();- const pending = spaceVideoCover(- new Blob(["video"]),- 0,- controller.signal,- );- const rejected = expect(pending).rejects.toThrow(- reason == "abort"- ? "Canceled"- : reason == "error"- ? "can't preview"- : "too long to load",- );- if (reason == "abort") controller.abort();- else if (reason == "error") video.dispatchEvent(new Event("error"));- else await vi.advanceTimersByTimeAsync(30_000);- await rejected;- expect(video.cancelVideoFrameCallback).toHaveBeenCalledWith(7);- expect(video.pause).toHaveBeenCalledOnce();- expect(revoke).toHaveBeenCalledOnce();- expect(createElement).not.toHaveBeenCalledWith("canvas");- expect(vi.getTimerCount()).toBe(0);- },-);--test.each([- [- "Mozilla/5.0 (Android 16; Mobile; rv:143.0) Gecko/143.0 Firefox/143.0",- true,- ],- ["Mozilla/5.0 (Android 16) Chrome/140.0.0.0 Mobile Safari/537.36", false],- [- "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:143.0) Gecko/20100101 Firefox/143.0",- false,- ],-])(- "selects software frames only for Firefox Android (%s)",- async (userAgent, software) => {- vi.stubGlobal("navigator", { userAgent });- if (!software) {- vi.stubGlobal("window", { setTimeout, clearTimeout });- const video = Object.assign(new EventTarget(), {- duration: 3.45,- videoWidth: 540,- videoHeight: 960,- currentTime: 0,- pause: vi.fn(),- removeAttribute: vi.fn(),- load: () => video.dispatchEvent(new Event("loadeddata")),- });- createElement.mockImplementationOnce(() => video);- }- const file = new Blob(["video"]);- expect(await spaceVideoFrames(file, [0], 960)).toHaveLength(1);- expect(mocks.frames).toHaveBeenCalledTimes(software ? 1 : 0);- expect(createElement).toHaveBeenCalledWith(- software ? "canvas" : "video",- );- },-);--test("Firefox Android gets metadata and a cover without creating a video element", async () => {+test("prepares metadata and a cover from decoded media without exposing file names in logs", async () => {const file = new File(["video"], "private-video.mp4");const controller = new AbortController();- expect(await spaceVideoInfo(file, controller.signal)).toMatchObject({+ expect(await spaceVideoInfo(file, controller.signal)).toEqual({+ duration: 3.45,width: 1080,height: 1920,});@@ -223,42 +58,26 @@ test("Firefox Android gets metadata and a cover without creating a video element);expect(cover).toMatchObject({ width: 540, height: 960, thumbHash: "hash" });expect(cover.file.type).toBe("image/webp");- expect(bitmapClose).toHaveBeenCalledOnce();expect(JSON.stringify(mocks.log.mock.calls)).not.toContain("private-video");});-test("opaque black frames are valid covers, but unreadable transparent frames fail", async () => {- const file = new Blob(["video"]);- await expect(spaceVideoCover(file, 0)).resolves.toBeDefined();- alpha = 0;- await expect(spaceVideoCover(file, 0)).rejects.toThrow(- "Couldn't read a video frame",- );- expect(bitmapClose).toHaveBeenCalledTimes(2);- expect(mocks.log).toHaveBeenCalledWith(- expect.stringContaining("frames failed backend=wasm"),- );-});--test("timeline frame extraction is batched and releases each decoded image", async () => {+test("timeline frame extraction is batched", async () => {const file = new Blob(["video"]);const times = [0.5, 1.5, 2.5];- mocks.frames.mockResolvedValue(times.map(() => new Blob(["png"])));+ mocks.frames.mockResolvedValue(times.map(canvas));expect(await spaceVideoFrames(file, times, 128)).toHaveLength(3);expect(mocks.frames).toHaveBeenCalledWith(file, times, 128, undefined);- expect(bitmapClose).toHaveBeenCalledTimes(3);});-test("canceling frame extraction does not leave decoded bitmaps or return a cover", async () => {+test("canceling cover generation does not return a prepared cover", async () => {const controller = new AbortController();mocks.frames.mockImplementation(() => {controller.abort();- return Promise.resolve([new Blob(["png"])]);+ return Promise.resolve([canvas()]);});await expect(spaceVideoCover(new Blob(["video"]), 0, controller.signal),).rejects.toMatchObject({ name: "AbortError" });- expect(createImageBitmap).not.toHaveBeenCalled();});const videoEdit = { start: 0, end: 3.45, coverTime: 0, muted: false };@@ -309,7 +128,7 @@ test("completed exports survive cover changes while trim and mute edits producetest("discarding a draft cancels its export and allows a fresh attempt", async () => {mocks.transcode.mockImplementationOnce(- (_file, _edit, _command, signal: AbortSignal) =>+ (_file, _edit, signal: AbortSignal) =>new Promise((_, reject) => {signal.addEventListener("abort", () =>reject(signal.reason as Error),@@ -348,3 +167,23 @@ test("canceling during output validation does not cache an abandoned export", asawait startSpaceVideoExport(file, videoEdit).promise;expect(mocks.transcode).toHaveBeenCalledTimes(2);});++test.each([0, Number.NaN, 10.04])(+ "rejects an invalid exported duration (%s)",+ async (duration) => {+ mocks.duration.mockResolvedValue(duration);+ const file = new File(["source"], "source.mov");+ await expect(+ startSpaceVideoExport(file, videoEdit).promise,+ ).rejects.toThrow("Couldn't export a video within 10 seconds");+ },+);++test("rejects an oversized export before preparing its cover", async () => {+ mocks.transcode.mockResolvedValue(new Uint8Array(15 * 1024 * 1024));+ const file = new File(["source"], "source.mov");+ await expect(prepareSpaceVideo(file, videoEdit)).rejects.toThrow(+ "too large",+ );+ expect(mocks.frames).not.toHaveBeenCalled();+});diff --git a/web/apps/space/tests/video-export.test.ts b/web/apps/space/tests/video-export.test.tsindex 0da1b00e4a..8515ffd691 100644--- a/web/apps/space/tests/video-export.test.ts+++ b/web/apps/space/tests/video-export.test.ts@@ -1,340 +1,544 @@+import type { ConversionOptions } from "mediabunny";import { afterEach, beforeEach, expect, test, vi } from "vitest";const mocks = vi.hoisted(() => ({- encode: vi.fn(),- load: vi.fn(),- logToDisk: vi.fn(),- workers: [] as {- loaded: boolean;- terminate: ReturnType<typeof vi.fn<() => void>>;- exec: ReturnType<typeof vi.fn>;- unmount: ReturnType<typeof vi.fn>;- readFile: ReturnType<typeof vi.fn>;+ init: vi.fn(),+ execute: vi.fn(),+ duration: vi.fn(),+ primaryVideo: vi.fn(),+ primaryAudio: vi.fn(),+ canEncodeAudio: vi.fn(),+ registerAacEncoder: vi.fn(),+ softwareFrames: vi.fn(),+ softwareExport: vi.fn(),+ frames: vi.fn(),+ log: vi.fn(),+ inputs: [] as { dispose: ReturnType<typeof vi.fn> }[],+ outputs: [] as {+ state: string;+ cancel: ReturnType<typeof vi.fn>;+ tracks: { metadata: { name?: string } }[];}[],+ conversions: [] as { state: string; cancel: ReturnType<typeof vi.fn> }[],+ sinkOptions: [] as { width: number; height: number }[],+ track: {+ computeDuration: vi.fn(),+ getDisplayWidth: vi.fn(),+ getDisplayHeight: vi.fn(),+ getFirstTimestamp: vi.fn(),+ hasHighDynamicRange: vi.fn(),+ canDecode: vi.fn(),+ },+ audio: {+ getCodec: vi.fn(),+ getNumberOfChannels: vi.fn(),+ getSampleRate: vi.fn(),+ },}));-vi.mock("ente-base/log", () => ({- default: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() },+vi.mock("ente-base/log-web", () => ({ logToDisk: mocks.log }));+vi.mock("utils/thumbhash", () => ({ thumbHashBase64FromCanvas: () => "hash" }));+vi.mock("@mediabunny/aac-encoder", () => ({+ registerAacEncoder: mocks.registerAacEncoder,}));-vi.mock("ente-base/log-web", () => ({ logToDisk: mocks.logToDisk }));-vi.mock("../src/utils/video-encoding/web-codecs", () => ({- encodeVideoWithWebCodecs: mocks.encode,+vi.mock("../src/utils/video-encoding/software", () => ({+ extractSoftwareVideoFrames: mocks.softwareFrames,+ transcodeSoftwareVideo: mocks.softwareExport,}));-vi.mock("@ffmpeg/ffmpeg", () => ({- FFFSType: { WORKERFS: "WORKERFS" },- FFmpeg: class {- loaded = true;+vi.mock("mediabunny", () => ({+ ALL_FORMATS: [],+ BlobSource: vi.fn(),+ BufferSource: vi.fn(),+ canEncodeAudio: mocks.canEncodeAudio,+ Input: class {constructor() {- mocks.workers.push({- loaded: this.loaded,- terminate: this.terminate,- exec: this.exec,- unmount: this.unmount,- readFile: this.readFile,- });+ mocks.inputs.push(this);}- rejectLoad: ((error: Error) => void) | undefined;- load = (...args: unknown[]) =>- new Promise((resolve, reject) => {- this.rejectLoad = reject;- Promise.resolve(mocks.load(...args)).then(resolve, reject);- });- terminate = vi.fn(() => {- this.loaded = false;- this.rejectLoad?.(new Error("Terminated"));+ dispose = vi.fn();+ getPrimaryVideoTrack = mocks.primaryVideo;+ getPrimaryAudioTrack = mocks.primaryAudio;+ computeDuration = mocks.duration;+ },+ BufferTarget: class {+ buffer = new Uint8Array([1, 2, 3]).buffer;+ },+ Mp4OutputFormat: vi.fn(),+ Quality: class {+ constructor(public options: unknown) {}+ },+ Output: class {+ state = "pending";+ tracks = [{ metadata: { name: "Private recording" } }];+ target: { buffer: ArrayBuffer };+ constructor(options: { target: { buffer: ArrayBuffer } }) {+ this.target = options.target;+ mocks.outputs.push(this);+ }+ cancel = vi.fn(() => {+ this.state = "canceled";+ return Promise.resolve();});- on = vi.fn();- off = vi.fn();- createDir = vi.fn();- deleteDir = vi.fn();- mount = vi.fn();- unmount = vi.fn();- deleteFile = vi.fn();- listDir = vi.fn().mockResolvedValue([]);- ffprobe = vi.fn().mockResolvedValue(0);- exec = vi.fn().mockResolvedValue(0);- readFile = vi.fn().mockResolvedValue(new Uint8Array([7]));},+ CanvasSink: class {+ constructor(+ _track: unknown,+ options: { width: number; height: number },+ ) {+ mocks.sinkOptions.push(options);+ }+ canvasesAtTimestamps = mocks.frames;+ },+ Conversion: { init: mocks.init },}));beforeEach(() => {vi.resetModules();- mocks.encode.mockReset();- mocks.load.mockReset().mockResolvedValue(true);- mocks.workers.length = 0;- mocks.logToDisk.mockClear();- vi.stubGlobal("VideoEncoder", {});- vi.stubGlobal("VideoDecoder", {});+ vi.resetAllMocks();+ mocks.inputs.length = 0;+ mocks.outputs.length = 0;+ mocks.conversions.length = 0;+ mocks.sinkOptions.length = 0;+ mocks.primaryVideo.mockResolvedValue(mocks.track);+ mocks.primaryAudio.mockResolvedValue(mocks.audio);+ mocks.audio.getCodec.mockResolvedValue("aac");+ mocks.audio.getNumberOfChannels.mockResolvedValue(2);+ mocks.audio.getSampleRate.mockResolvedValue(48_000);+ mocks.canEncodeAudio.mockResolvedValue(true);+ mocks.softwareFrames.mockResolvedValue([+ {+ width: 90,+ height: 160,+ toBlob: (callback: BlobCallback) =>+ callback(new Blob(["cover"], { type: "image/png" })),+ },+ ]);+ mocks.softwareExport.mockResolvedValue(new Uint8Array([4, 5, 6]));+ vi.stubGlobal("navigator", {+ userAgent:+ "Mozilla/5.0 (iPhone; CPU iPhone OS 26_5 like Mac OS X) AppleWebKit/605.1.15 Version/26.5 Mobile Safari/604.1",+ });+ mocks.duration.mockResolvedValue(10);+ mocks.track.computeDuration.mockResolvedValue(10.833333);+ mocks.track.getDisplayWidth.mockResolvedValue(1080);+ mocks.track.getDisplayHeight.mockResolvedValue(1920);+ mocks.track.getFirstTimestamp.mockResolvedValue(0);+ mocks.track.hasHighDynamicRange.mockResolvedValue(false);+ mocks.track.canDecode.mockResolvedValue(true);+ mocks.execute.mockResolvedValue(undefined);+ mocks.frames.mockImplementation(function* (times: number[]) {+ for (const time of times)+ yield { canvas: { width: 90, height: 160, time } };+ });+ mocks.init.mockImplementation((options: ConversionOptions) => {+ let rejectExecution: (error: Error) => void = () => undefined;+ const conversion = {+ isValid: true,+ discardedTracks: [],+ state: "ready",+ execute: async () => {+ await Promise.race([+ mocks.execute(options),+ new Promise((_, reject) => {+ rejectExecution = reject;+ }),+ ]);+ conversion.state = "done";+ options.output.state = "finalized";+ },+ cancel: vi.fn(async () => {+ conversion.state = "canceled";+ rejectExecution(new Error("Conversion canceled"));+ await options.output.cancel();+ }),+ };+ mocks.conversions.push(conversion);+ return Promise.resolve(conversion);+ });});afterEach(() => {- vi.unstubAllGlobals();vi.useRealTimers();+ vi.unstubAllGlobals();});-const exporter = async () => {- const { transcodeVideoWeb } =- await import("../src/utils/video-encoding/web");- return (signal?: AbortSignal) =>- transcodeVideoWeb(- new Blob(["video"]),- { start: 0, end: 10, muted: false },- ["fallback"],- signal,- );-};+const file = () => new File(["video"], "private.mov");+const edit = { start: 1.125, end: 9.875, muted: false, coverTime: 1.125 };+const exporter = async () =>+ (await import("../src/utils/video-encoding/web")).transcodeVideoWeb;-test("uses WebCodecs output without running the software encoder", async () => {- const bytes = new Uint8Array([1, 2]);- mocks.encode.mockResolvedValue(bytes);- expect(await (await exporter())()).toBe(bytes);- expect(mocks.workers[0]!.exec).not.toHaveBeenCalled();- expect(mocks.workers[0]!.unmount).toHaveBeenCalled();+test("exports the selected primary tracks without copying location or other source tags", async () => {+ expect(await (await exporter())(file(), edit)).toEqual(+ new Uint8Array([1, 2, 3]),+ );+ const options = mocks.init.mock.calls[0]![0] as ConversionOptions;+ expect(options).toMatchObject({+ tracks: "primary",+ trim: { start: 1.125, end: 9.875 },+ copy: { boundaryPolicy: "shrink" },+ tags: {},+ audio: { codec: "aac" },+ });+ expect(mocks.inputs[0]!.dispose).toHaveBeenCalled();+ expect(mocks.outputs[0]!.cancel).not.toHaveBeenCalled();+ expect(JSON.stringify(mocks.log.mock.calls)).not.toContain("private.mov");});-test("preloading and posting share the same in-flight engine load", async () => {- let finish!: () => void;- mocks.load.mockImplementationOnce(- () =>- new Promise<void>((resolve) => {- finish = resolve;- }),+test("remuxes the encoded packets within the trim to remove encoder padding", async () => {+ await (+ await exporter()+ )(file(), edit);+ expect(mocks.init.mock.calls[1]![0]).toMatchObject({+ tracks: "primary",+ trim: { start: 0, end: 8.75 },+ copy: { mode: "forced", boundaryPolicy: "shrink" },+ tags: {},+ });+ expect(mocks.inputs[1]!.dispose).toHaveBeenCalled();+ expect(mocks.outputs[1]!.tracks[0]!.metadata.name).toBeUndefined();+});++test("canceling the final packet copy disposes both inputs and permits another export", async () => {+ mocks.execute+ .mockResolvedValueOnce(undefined)+ .mockImplementationOnce(() => new Promise(() => undefined));+ const run = await exporter();+ const controller = new AbortController();+ const pending = run(file(), edit, controller.signal);+ const canceled = expect(pending).rejects.toMatchObject({+ name: "AbortError",+ });+ await vi.waitFor(() => expect(mocks.execute).toHaveBeenCalledTimes(2));+ controller.abort();+ await canceled;+ await run(file(), edit);+ expect(mocks.inputs[0]!.dispose).toHaveBeenCalled();+ expect(mocks.inputs[1]!.dispose).toHaveBeenCalled();+ expect(mocks.conversions[1]!.cancel).toHaveBeenCalledOnce();+});++test("bakes portrait orientation into even 1080p dimensions and renders HDR through sRGB", async () => {+ mocks.track.getDisplayWidth.mockResolvedValue(2160);+ mocks.track.getDisplayHeight.mockResolvedValue(3840);+ mocks.track.hasHighDynamicRange.mockResolvedValue(true);+ const draw = vi.fn();+ const context = { canvas: {} };+ const getContext = vi.fn(() => context);+ vi.stubGlobal("document", { createElement: () => ({ getContext }) });+ await (+ await exporter()+ )(file(), edit);+ const options = mocks.init.mock.calls[0]![0] as ConversionOptions;+ const configure = options.video as (+ track: unknown,+ ) => Promise<Record<string, unknown>>;+ const video = await configure(mocks.track);+ expect(video).toMatchObject({+ width: 1080,+ height: 1920,+ frameRate: 30,+ codec: "avc",+ allowTransformationMetadata: false,+ });+ expect(getContext).toHaveBeenCalledWith("2d", { colorSpace: "srgb" });+ expect((video.process as (sample: unknown) => unknown)({ draw })).toBe(+ context.canvas,);- const { preloadVideoEncoderWeb } =- await import("../src/utils/video-encoding/web");- const first = preloadVideoEncoderWeb();- const second = preloadVideoEncoderWeb();- const bytes = new Uint8Array([1, 2]);- mocks.encode.mockResolvedValue(bytes);- const posted = (await exporter())();- expect(mocks.load).toHaveBeenCalledTimes(1);- expect(mocks.encode).not.toHaveBeenCalled();- finish();- await Promise.all([first, second]);- expect(await posted).toBe(bytes);- expect(mocks.load).toHaveBeenCalledTimes(1);- expect(mocks.workers).toHaveLength(1);+ expect(draw).toHaveBeenCalledWith(context, 0, 0, 1080, 1920);});-test("posting can retry after a failed preload", async () => {- mocks.load.mockRejectedValueOnce(new Error("Offline"));- const { preloadVideoEncoderWeb } =- await import("../src/utils/video-encoding/web");- await expect(preloadVideoEncoderWeb()).rejects.toThrow("Offline");- mocks.encode.mockResolvedValue(new Uint8Array([2]));- expect(await (await exporter())()).toEqual(new Uint8Array([2]));- expect(mocks.workers[0]!.terminate).toHaveBeenCalled();- expect(mocks.load).toHaveBeenCalledTimes(2);+test("muting explicitly discards audio", async () => {+ await (+ await exporter()+ )(file(), { ...edit, muted: true });+ expect(mocks.init.mock.calls[0]![0]).toMatchObject({+ audio: { discard: true },+ });});-test.each(["unsupported", "failed", "missing API"])(- "keeps the software exporter available when WebCodecs is %s",- async (mode) => {- if (mode === "failed")- mocks.encode.mockRejectedValue(new Error("Codec unavailable"));- else mocks.encode.mockResolvedValue(undefined);- if (mode === "missing API") vi.stubGlobal("VideoEncoder", undefined);- expect(await (await exporter())()).toEqual(new Uint8Array([7]));- expect(mocks.workers[0]!.exec).toHaveBeenCalledWith(["fallback"]);- if (mode === "missing API") expect(mocks.encode).not.toHaveBeenCalled();+test.each(["undecodable_source_codec", "no_encodable_target_codec"])(+ "rejects unsupported audio instead of silently publishing a muted video (%s)",+ async (reason) => {+ mocks.init.mockResolvedValueOnce({+ isValid: true,+ discardedTracks: [{ reason }],+ state: "ready",+ cancel: vi.fn(),+ });+ await expect((await exporter())(file(), edit)).rejects.toThrow(+ "can't export",+ );+ expect(mocks.execute).not.toHaveBeenCalled();+ expect(mocks.inputs[0]!.dispose).toHaveBeenCalled();},);-test("cancellation stops the active export without starting the software fallback", async () => {- mocks.encode.mockImplementation(- (_ffmpeg, _path, _blob, _edit, signal: AbortSignal) =>- new Promise((_, reject) =>- signal.addEventListener(- "abort",- () => reject(new DOMException("Canceled", "AbortError")),- { once: true },- ),- ),+test("rejects a file without video and releases the input", async () => {+ mocks.primaryVideo.mockResolvedValueOnce(null);+ await expect((await exporter())(file(), edit)).rejects.toThrow(+ "no readable video",);+ expect(mocks.init).not.toHaveBeenCalled();+ expect(mocks.inputs[0]!.dispose).toHaveBeenCalled();+});++test("canceling an active export closes the codecs and allows the next job to run", async () => {+ mocks.execute.mockImplementationOnce(() => new Promise(() => undefined));+ const run = await exporter();const controller = new AbortController();- const pending = (await exporter())(controller.signal);- const rejected = expect(pending).rejects.toMatchObject({+ const pending = run(file(), edit, controller.signal);+ const canceled = expect(pending).rejects.toMatchObject({name: "AbortError",});- await vi.waitFor(() => expect(mocks.encode).toHaveBeenCalled());+ await vi.waitFor(() => expect(mocks.execute).toHaveBeenCalledOnce());controller.abort();- await rejected;- expect(mocks.workers[0]!.terminate).toHaveBeenCalled();- expect(mocks.workers[0]!.exec).not.toHaveBeenCalled();+ await canceled;+ expect(await run(file(), edit)).toEqual(new Uint8Array([1, 2, 3]));+ expect(mocks.conversions[0]!.cancel).toHaveBeenCalledOnce();+ expect(mocks.outputs[0]!.cancel).toHaveBeenCalled();+ expect(mocks.inputs[0]!.dispose).toHaveBeenCalled();});-test("a queued export can be cancelled without waiting for or stopping another export", async () => {- let finish!: (bytes: Uint8Array) => void;- mocks.encode.mockImplementation(+test("canceling a queued export returns promptly without canceling the active export", async () => {+ let finish!: () => void;+ mocks.execute.mockImplementationOnce(() =>- new Promise((resolve) => {+ new Promise<void>((resolve) => {finish = resolve;}),);const run = await exporter();- const first = run();- await vi.waitFor(() => expect(mocks.encode).toHaveBeenCalledTimes(1));+ const first = run(file(), edit);+ await vi.waitFor(() => expect(mocks.execute).toHaveBeenCalledOnce());const controller = new AbortController();- const second = run(controller.signal);- const rejected = expect(second).rejects.toMatchObject({+ const second = run(file(), edit, controller.signal);+ const canceled = expect(second).rejects.toMatchObject({name: "AbortError",});controller.abort();- await rejected;- expect(mocks.workers[0]!.terminate).not.toHaveBeenCalled();- finish(new Uint8Array([1]));+ await canceled;+ expect(mocks.inputs).toHaveLength(1);+ expect(mocks.conversions[0]!.cancel).not.toHaveBeenCalled();+ finish();await first;- expect(mocks.encode).toHaveBeenCalledTimes(1);+ await run(file(), edit);+ expect(mocks.inputs).toHaveLength(4);+ expect(mocks.execute).toHaveBeenCalledTimes(4);});-test("canceling active and queued draft exports frees the software encoder for editor frames and restart", async () => {- vi.stubGlobal("VideoEncoder", undefined);- mocks.load.mockImplementation(() => {- const worker = mocks.workers.at(-1)!;- worker.readFile.mockImplementation((path: string) => {- if (path == "output.json")- return JSON.stringify({- format: { duration: "2" },- streams: [{}],- });- if (path == "frame.json")- return JSON.stringify({- format: {},- packets: [{ pts_time: "0" }, { pts_time: "1" }],- });- return new Uint8Array([7]);- });- if (mocks.workers.length == 1) {- const terminate = worker.terminate.getMockImplementation()!;- worker.exec.mockImplementationOnce(- () =>- new Promise<number>((_, reject) => {- worker.terminate.mockImplementation(() => {- terminate();- reject(new Error("Terminated"));- });- }),- );- }- return Promise.resolve(true);+test("a stalled conversion times out and releases the queue", async () => {+ vi.useFakeTimers();+ mocks.execute.mockImplementationOnce(() => new Promise(() => undefined));+ const run = await exporter();+ const pending = run(file(), edit);+ const timedOut = expect(pending).rejects.toMatchObject({+ name: "TimeoutError",});- const { startSpaceVideoExport } = await import("../src/utils/post-video");- const { extractVideoFramesWeb } =- await import("../src/utils/video-encoding/web");- const files = [- new File(["first"], "first.mp4"),- new File(["second"], "second.mp4"),- ];- const edit = { start: 0, end: 2, coverTime: 0, muted: false };- const jobs = files.map((file) => startSpaceVideoExport(file, edit));- const canceled = Promise.all(- jobs.map((job) =>- expect(job.promise).rejects.toMatchObject({ name: "AbortError" }),- ),- );- await vi.waitFor(() =>- expect(mocks.workers[0]?.exec).toHaveBeenCalledOnce(),- );- jobs.forEach((job) => job.cancel());- const frames = extractVideoFramesWeb(files[0]!, [0, 1], 128);- await canceled;- expect(await frames).toHaveLength(2);- expect(mocks.workers).toHaveLength(2);- expect(mocks.workers[0]!.terminate).toHaveBeenCalledOnce();- expect(mocks.workers[1]!.exec).toHaveBeenCalledTimes(2);- const restarted = await Promise.all(- files.map((file) => startSpaceVideoExport(file, edit).promise),- );- expect(restarted.map((video) => video.durationMs)).toEqual([2000, 2000]);+ await vi.advanceTimersByTimeAsync(300_000);+ await timedOut;+ expect(await run(file(), edit)).toEqual(new Uint8Array([1, 2, 3]));+ expect(mocks.conversions[0]!.cancel).toHaveBeenCalledOnce();});-test("cancelling engine loading terminates it and allows the next export to load again", async () => {- mocks.load.mockImplementationOnce(() => new Promise(() => undefined));+test("a conversion failure releases its resources and permits retry", async () => {+ mocks.execute.mockRejectedValueOnce(new Error("Encoder failed"));const run = await exporter();- const controller = new AbortController();- const pending = run(controller.signal);- const rejected = expect(pending).rejects.toMatchObject({- name: "AbortError",+ await expect(run(file(), edit)).rejects.toThrow("Encoder failed");+ expect(mocks.outputs[0]!.cancel).toHaveBeenCalled();+ expect(mocks.inputs[0]!.dispose).toHaveBeenCalled();+ await expect(run(file(), edit)).resolves.toBeDefined();+});++test("reads oriented metadata without requiring a video decoder", async () => {+ mocks.track.canDecode.mockResolvedValue(false);+ const { determineVideoInfoWeb } =+ await import("../src/utils/video-encoding/web");+ expect(await determineVideoInfoWeb(file())).toEqual({+ duration: 10.833333,+ width: 1080,+ height: 1920,});- await vi.waitFor(() => expect(mocks.load).toHaveBeenCalled());- controller.abort();- await rejected;- mocks.encode.mockResolvedValue(new Uint8Array([2]));- expect(await run()).toEqual(new Uint8Array([2]));- expect(mocks.workers[0]!.terminate).toHaveBeenCalled();- expect(mocks.workers).toHaveLength(2);+ expect(mocks.track.canDecode).not.toHaveBeenCalled();+ expect(mocks.inputs[0]!.dispose).toHaveBeenCalled();});-test("a stalled engine load times out and can be retried", async () => {- vi.useFakeTimers();- mocks.load.mockImplementationOnce(() => new Promise(() => undefined));- const { preloadVideoEncoderWeb } =+test("extracts a batch of portrait thumbnails and clamps requests past the end", async () => {+ const { extractVideoFramesWeb } =await import("../src/utils/video-encoding/web");- const rejected = expect(preloadVideoEncoderWeb()).rejects.toThrow(- "took too long to load",+ expect(await extractVideoFramesWeb(file(), [0, 3, 11], 160)).toHaveLength(+ 3,);- await vi.advanceTimersByTimeAsync(60_000);- await rejected;- await preloadVideoEncoderWeb();- expect(mocks.workers).toHaveLength(2);- expect(mocks.workers[0]!.terminate).toHaveBeenCalled();+ expect(mocks.sinkOptions[0]).toMatchObject({ width: 90, height: 160 });+ const times = mocks.frames.mock.calls[0]![0] as number[];+ expect(times.slice(0, 2)).toEqual([0, 3]);+ expect(times[2]).toBeCloseTo(10.832333, 6);+ expect(mocks.inputs[0]!.dispose).toHaveBeenCalled();});-test("a stalled export times out, closes the worker, and releases the queue for retry", async () => {- vi.useFakeTimers();- mocks.encode- .mockImplementationOnce(- (_ffmpeg, _path, _blob, _edit, signal: AbortSignal) =>- new Promise((_, reject) =>- signal.addEventListener("abort", () =>- reject(signal.reason as Error),- ),- ),- )- .mockResolvedValue(new Uint8Array([3]));- const run = await exporter();- const first = run();- const rejected = expect(first).rejects.toMatchObject({- name: "TimeoutError",- });- await vi.advanceTimersByTimeAsync(300_000);- await rejected;- expect(await run()).toEqual(new Uint8Array([3]));- expect(mocks.workers[0]!.terminate).toHaveBeenCalled();- expect(mocks.workers[0]!.exec).not.toHaveBeenCalled();- expect(mocks.workers).toHaveLength(2);- expect(mocks.logToDisk).toHaveBeenCalledWith(- expect.stringContaining("stage=export timeout"),+test("fails cleanly when the browser cannot decode a cover", async () => {+ mocks.track.canDecode.mockResolvedValue(false);+ const { extractVideoFramesWeb } =+ await import("../src/utils/video-encoding/web");+ await expect(extractVideoFramesWeb(file(), [0], 160)).rejects.toThrow(+ "can't preview",);+ expect(mocks.frames).not.toHaveBeenCalled();+ expect(mocks.inputs[0]!.dispose).toHaveBeenCalled();+ expect(mocks.softwareFrames).not.toHaveBeenCalled();});-test("cover extraction near the clip end seeks to the last available frame", async () => {- const { preloadVideoEncoderWeb, extractVideoFramesWeb } =- await import("../src/utils/video-encoding/web");- await preloadVideoEncoderWeb();- const worker = mocks.workers[0]!;- worker.readFile.mockImplementation((path: string) => {- if (path == "output.json") return JSON.stringify({ streams: [{}] });- if (path == "frame.json")- return JSON.stringify({- format: { start_time: "0" },- packets: [- { pts_time: "3.38" },- { pts_time: "3.413667" },- { pts_time: "3.430322" },- ],- });- return new Uint8Array([1, 2, 3]);+test("Firefox Android without VideoDecoder can prepare a cover and export an unmuted video", async () => {+ vi.stubGlobal("navigator", {+ userAgent:+ "Mozilla/5.0 (Android 16; Mobile; rv:144.0) Gecko/144.0 Firefox/144.0",});- const frames = await extractVideoFramesWeb(- new Blob(["video"]),- [3.4499],- 960,+ vi.stubGlobal("VideoDecoder", undefined);+ vi.stubGlobal("VideoEncoder", undefined);+ mocks.track.canDecode.mockImplementation(() =>+ Promise.resolve(typeof VideoDecoder !== "undefined"),);- expect(frames).toHaveLength(1);- expect(frames[0]!.type).toBe("image/png");- const command = worker.exec.mock.calls[0]![0] as string[];- expect(Number(command[command.indexOf("-ss") + 1])).toBeCloseTo(- 3.430321,- 6,+ const { spaceVideoInfo, spaceVideoCover, prepareSpaceVideo } =+ await import("../src/utils/post-video");+ const source = file();+ expect((await spaceVideoInfo(source)).duration).toBeGreaterThan(10);+ expect((await spaceVideoCover(source, 0)).file.size).toBeGreaterThan(0);+ const result = await prepareSpaceVideo(source, edit);+ expect(result.video.file.type).toBe("video/mp4");+ expect(new Uint8Array(await result.video.file.arrayBuffer())).toEqual(+ new Uint8Array([4, 5, 6]),);- expect(worker.unmount).toHaveBeenCalled();+ expect(mocks.softwareFrames).toHaveBeenCalledTimes(2);+ expect(mocks.softwareExport).toHaveBeenCalledWith(+ source,+ edit,+ false,+ expect.any(AbortSignal),+ );+ expect(mocks.init).not.toHaveBeenCalled();+ expect(+ mocks.inputs.every((input) => input.dispose.mock.calls.length > 0),+ ).toBe(true);+});++test("a browser without a native video encoder uses the software export", async () => {+ vi.stubGlobal("navigator", {+ userAgent:+ "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/144.0",+ });+ const cancel = vi.fn();+ mocks.init.mockResolvedValueOnce({+ isValid: false,+ discardedTracks: [{ reason: "no_encodable_target_codec" }],+ state: "ready",+ cancel,+ });+ expect(await (await exporter())(file(), edit)).toEqual(+ new Uint8Array([4, 5, 6]),+ );+ expect(cancel).toHaveBeenCalled();+ expect(mocks.softwareExport).toHaveBeenCalledOnce();+});++test.each([+ "Mozilla/5.0 (iPhone) AppleWebKit/605.1.15 Version/26.5 Mobile Safari/604.1",+ "Mozilla/5.0 (iPhone) AppleWebKit/605.1.15 CriOS/144.0 Mobile Safari/604.1",+ "Mozilla/5.0 (iPhone) AppleWebKit/605.1.15 FxiOS/144.0 Mobile Safari/605.1.15",+ "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 Version/26.5 Safari/605.1.15",+])(+ "WebKit never enters the full FFmpeg path when native decoding is unavailable: %s",+ async (userAgent) => {+ vi.stubGlobal("navigator", { userAgent });+ mocks.track.canDecode.mockResolvedValue(false);+ const { extractVideoFramesWeb, transcodeVideoWeb } =+ await import("../src/utils/video-encoding/web");+ await expect(extractVideoFramesWeb(file(), [0], 160)).rejects.toThrow(+ "can't preview",+ );+ await expect(transcodeVideoWeb(file(), edit)).rejects.toThrow(+ "can't decode",+ );+ expect(mocks.softwareFrames).not.toHaveBeenCalled();+ expect(mocks.softwareExport).not.toHaveBeenCalled();+ },+);++test("the supported iPhone path never imports either software fallback", async () => {+ vi.doMock("../src/utils/video-encoding/software", () => {+ throw new Error("Full FFmpeg must not load on iPhone");+ });+ vi.doMock("@mediabunny/aac-encoder", () => {+ throw new Error("AAC fallback must not load for passthrough audio");+ });+ try {+ const { extractVideoFramesWeb, transcodeVideoWeb } =+ await import("../src/utils/video-encoding/web");+ await expect(+ extractVideoFramesWeb(file(), [0], 160),+ ).resolves.toHaveLength(1);+ await expect(transcodeVideoWeb(file(), edit)).resolves.toEqual(+ new Uint8Array([1, 2, 3]),+ );+ } finally {+ vi.doMock("../src/utils/video-encoding/software", () => ({+ extractSoftwareVideoFrames: mocks.softwareFrames,+ transcodeSoftwareVideo: mocks.softwareExport,+ }));+ vi.doMock("@mediabunny/aac-encoder", () => ({+ registerAacEncoder: mocks.registerAacEncoder,+ }));+ }+});++test.each(["opus", "vorbis", "pcm-s16"])(+ "registers software AAC before converting unmuted %s when native AAC is unavailable",+ async (codec) => {+ vi.stubGlobal("navigator", {+ userAgent:+ "Mozilla/5.0 (X11; Linux x86_64) Gecko/20100101 Firefox/144.0",+ });+ mocks.audio.getCodec.mockResolvedValue(codec);+ mocks.canEncodeAudio.mockResolvedValue(false);+ await expect((await exporter())(file(), edit)).resolves.toBeDefined();+ expect(mocks.canEncodeAudio).toHaveBeenCalledWith(+ "aac",+ expect.objectContaining({+ numberOfChannels: 2,+ sampleRate: 48_000,+ }),+ );+ expect(mocks.registerAacEncoder).toHaveBeenCalledOnce();+ expect(+ mocks.registerAacEncoder.mock.invocationCallOrder[0],+ ).toBeLessThan(mocks.init.mock.invocationCallOrder[0]!);+ expect(mocks.init.mock.calls[0]![0]).toMatchObject({+ audio: { codec: "aac" },+ });+ expect(mocks.softwareExport).not.toHaveBeenCalled();+ },+);++test("uses native AAC when supported and preserves AAC input without requiring an encoder", async () => {+ mocks.audio.getCodec.mockResolvedValueOnce("opus");+ await (+ await exporter()+ )(file(), edit);+ expect(mocks.canEncodeAudio).toHaveBeenCalledOnce();+ expect(mocks.registerAacEncoder).not.toHaveBeenCalled();+ mocks.canEncodeAudio.mockClear().mockResolvedValue(false);+ await (+ await exporter()+ )(file(), edit);+ expect(mocks.canEncodeAudio).not.toHaveBeenCalled();+ expect(mocks.registerAacEncoder).not.toHaveBeenCalled();+});++test("muted and silent videos do not load an AAC encoder", async () => {+ mocks.audio.getCodec.mockResolvedValue("opus");+ mocks.canEncodeAudio.mockResolvedValue(false);+ await (+ await exporter()+ )(file(), { ...edit, muted: true });+ mocks.primaryAudio.mockResolvedValue(null);+ await (+ await exporter()+ )(file(), edit);+ expect(mocks.canEncodeAudio).not.toHaveBeenCalled();+ expect(mocks.registerAacEncoder).not.toHaveBeenCalled();});diff --git a/web/apps/space/tests/video-software.test.ts b/web/apps/space/tests/video-software.test.tsnew file mode 100644index 0000000000..fee689347e--- /dev/null+++ b/web/apps/space/tests/video-software.test.ts@@ -0,0 +1,162 @@+import { afterEach, beforeEach, expect, test, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ load: vi.fn(),+ mount: vi.fn(),+ exec: vi.fn(),+ terminate: vi.fn(),+ closeBitmap: vi.fn(),+ rejectPending: undefined as ((error: Error) => void) | undefined,+}));++vi.mock("ente-base/log-web", () => ({ logToDisk: vi.fn() }));+vi.mock("@ffmpeg/ffmpeg", () => ({+ FFFSType: { WORKERFS: "WORKERFS" },+ FFmpeg: class {+ load = mocks.load;+ createDir = vi.fn();+ mount = mocks.mount;+ exec = mocks.exec;+ ffprobe = vi.fn(() => Promise.resolve(0));+ readFile = vi.fn((path: string) =>+ Promise.resolve(+ path.endsWith("json")+ ? JSON.stringify({+ format: { start_time: 0 },+ packets: [{ pts_time: 0 }],+ })+ : new Uint8Array([1, 2, 3]),+ ),+ );+ deleteFile = vi.fn();+ terminate = () => {+ mocks.terminate();+ mocks.rejectPending?.(new Error("Worker terminated"));+ };+ },+}));++beforeEach(() => {+ vi.clearAllMocks();+ mocks.rejectPending = undefined;+ mocks.load.mockResolvedValue(true);+ mocks.exec.mockResolvedValue(0);+ vi.stubGlobal("document", {+ createElement: () => ({+ width: 0,+ height: 0,+ getContext: () => ({ drawImage: vi.fn() }),+ }),+ });+ vi.stubGlobal(+ "createImageBitmap",+ vi.fn(() =>+ Promise.resolve({+ width: 90,+ height: 160,+ close: mocks.closeBitmap,+ }),+ ),+ );+});+afterEach(() => vi.unstubAllGlobals());++const file = new File(["source"], "private.mov");+const edit = { start: 1.125, end: 9.875, muted: false, coverTime: 1.4 };++test("software covers use one worker for a batch and release bitmaps and the worker", async () => {+ const { extractSoftwareVideoFrames } =+ await import("../src/utils/video-encoding/software");+ const result = await extractSoftwareVideoFrames(+ file,+ [0, 1],+ 160,+ true,+ new AbortController().signal,+ );+ expect(result).toMatchObject([+ { width: 90, height: 160 },+ { width: 90, height: 160 },+ ]);+ expect(mocks.load).toHaveBeenCalledOnce();+ const mount = mocks.mount.mock.calls[0]![1] as { files: File[] };+ expect(mount.files[0]!.name).toBe("video");+ expect(mocks.closeBitmap).toHaveBeenCalledTimes(2);+ expect(mocks.terminate).toHaveBeenCalledOnce();+ const args = mocks.exec.mock.calls[0]![0] as string[];+ expect(args.join(" ")).toContain("tonemap=tonemap=hable");+});++test.each([false, true])(+ "software export preserves trimming, metadata removal, and mute=%s",+ async (muted) => {+ const { transcodeSoftwareVideo } =+ await import("../src/utils/video-encoding/software");+ await expect(+ transcodeSoftwareVideo(+ file,+ { ...edit, muted },+ false,+ new AbortController().signal,+ ),+ ).resolves.toEqual(new Uint8Array([1, 2, 3]));+ const args = mocks.exec.mock.calls[0]![0] as string[];+ expect(args.slice(0, 6)).toEqual([+ "-ss",+ "1.125",+ "-i",+ "/input/video",+ "-t",+ "8.75",+ ]);+ expect(args.join(" ")).toContain(+ "-map_metadata -1 -map_metadata:s -1 -map_chapters -1",+ );+ expect(args.includes("-an")).toBe(muted);+ expect(args.includes("aac")).toBe(!muted);+ expect(mocks.terminate).toHaveBeenCalledOnce();+ },+);++test("canceling software encoding terminates the worker and permits another export", async () => {+ mocks.exec.mockImplementationOnce(+ () =>+ new Promise((_, reject) => {+ mocks.rejectPending = reject;+ }),+ );+ const { transcodeSoftwareVideo } =+ await import("../src/utils/video-encoding/software");+ const controller = new AbortController();+ const pending = transcodeSoftwareVideo(+ file,+ edit,+ false,+ controller.signal,+ );+ const canceled = expect(pending).rejects.toMatchObject({+ name: "AbortError",+ });+ await vi.waitFor(() => expect(mocks.exec).toHaveBeenCalledOnce());+ controller.abort();+ await canceled;+ expect(mocks.terminate).toHaveBeenCalled();+ await expect(+ transcodeSoftwareVideo(file, edit, false, new AbortController().signal),+ ).resolves.toBeDefined();+});++test("a canceled request does not load FFmpeg, and a failed load releases the worker", async () => {+ const { transcodeSoftwareVideo } =+ await import("../src/utils/video-encoding/software");+ await expect(+ transcodeSoftwareVideo(file, edit, false, AbortSignal.abort()),+ ).rejects.toMatchObject({ name: "AbortError" });+ expect(mocks.load).not.toHaveBeenCalled();+ mocks.load.mockRejectedValueOnce(new Error("Unavailable"));+ await expect(+ transcodeSoftwareVideo(file, edit, false, new AbortController().signal),+ ).rejects.toThrow("Unavailable");+ expect(mocks.terminate).toHaveBeenCalledOnce();+ expect(mocks.mount).not.toHaveBeenCalled();+});diff --git a/web/apps/space/tests/video-webcodecs.test.ts b/web/apps/space/tests/video-webcodecs.test.tsdeleted file mode 100644index 22dc3797c3..0000000000--- a/web/apps/space/tests/video-webcodecs.test.ts+++ /dev/null@@ -1,329 +0,0 @@-import type { FFmpeg } from "@ffmpeg/ffmpeg";-import { afterEach, beforeEach, expect, test, vi } from "vitest";-import { encodeVideoWithWebCodecs } from "../src/utils/video-encoding/web-codecs";--let frames: Frame[];-let encodedSources: number[];-let encoderFailure: boolean;-let stallEncoder: (() => void) | undefined;-let supported: boolean;-let decoderConfig: VideoDecoderConfig;-let canvasOptions: CanvasRenderingContext2DSettings[];--class Canvas {- sourceTime = 0;- constructor(- public width: number,- public height: number,- ) {}- getContext(_type: string, options: CanvasRenderingContext2DSettings) {- canvasOptions.push(options);- return {- setTransform: vi.fn(),- rotate: vi.fn(),- drawImage: (frame: Frame) => {- this.sourceTime = frame.sourceTime;- },- };- }-}--class Frame {- timestamp: number;- duration: number;- sourceTime: number;- closed = false;- constructor(- source: { timestamp: number; duration: number; sourceTime?: number },- init?: VideoFrameInit,- ) {- this.timestamp = init?.timestamp ?? source.timestamp;- this.duration = init?.duration ?? source.duration;- this.sourceTime = source.sourceTime ?? source.timestamp;- frames.push(this);- }- close() {- this.closed = true;- }-}--class Decoder extends EventTarget {- static isConfigSupported = () => Promise.resolve({ supported });- state = "unconfigured";- decodeQueueSize = 0;- constructor(private init: VideoDecoderInit) {- super();- }- configure(config: VideoDecoderConfig) {- decoderConfig = config;- this.state = "configured";- }- decode(chunk: EncodedVideoChunk) {- this.init.output(- new Frame({- timestamp: chunk.timestamp,- duration: chunk.duration!,- }) as unknown as VideoFrame,- );- }- flush() {- return Promise.resolve();- }- close() {- this.state = "closed";- }-}--class Encoder extends EventTarget {- static isConfigSupported = () => Promise.resolve({ supported });- state = "unconfigured";- encodeQueueSize = 0;- constructor(private init: VideoEncoderInit) {- super();- }- configure() {- this.state = "configured";- }- encode(frame: Frame) {- if (encoderFailure) {- this.init.error(new DOMException("Encoder failed"));- return;- }- if (stallEncoder) {- this.encodeQueueSize++;- if (this.encodeQueueSize > 8) queueMicrotask(stallEncoder);- return;- }- encodedSources.push(frame.sourceTime);- this.init.output({- timestamp: frame.timestamp,- byteLength: 1,- copyTo: (bytes: Uint8Array) => bytes.fill(1),- } as unknown as EncodedVideoChunk);- }- flush() {- return Promise.resolve();- }- close() {- this.state = "closed";- }-}--beforeEach(() => {- frames = [];- encodedSources = [];- encoderFailure = false;- stallEncoder = undefined;- supported = true;- canvasOptions = [];- vi.stubGlobal("OffscreenCanvas", Canvas);- vi.stubGlobal("VideoFrame", Frame);- vi.stubGlobal("VideoEncoder", Encoder);- vi.stubGlobal("VideoDecoder", Decoder);- vi.stubGlobal(- "EncodedVideoChunk",- class {- timestamp: number;- duration?: number;- constructor(init: EncodedVideoChunkInit) {- this.timestamp = init.timestamp;- this.duration = init.duration;- }- },- );-});-afterEach(() => vi.unstubAllGlobals());--const fixture = (- fps = 30,- streamChanges = {},- description = new Uint8Array([1, 100, 0, 40, 255, 225, 0]),-) => {- const files = new Map<string, Uint8Array | string>();- const packets = Array.from({ length: Math.ceil(fps * 20) }, (_, index) => ({- pts_time: (index / fps).toFixed(6),- duration_time: (1 / fps).toFixed(6),- pos: index,- size: 1,- flags: index % 60 === 0 ? "K_" : "__",- }));- const ffmpeg = {- loaded: true,- createDir: vi.fn(),- deleteDir: vi.fn(),- deleteFile: vi.fn((path: string) => files.delete(path)),- listDir: (directory: string) =>- [...files.keys()]- .filter((path) => path.startsWith(directory + "/"))- .map((path) => ({- name: path.slice(directory.length + 1),- isDir: false,- })),- readFile: (path: string) => files.get(path),- writeFile: (path: string, bytes: Uint8Array) => files.set(path, bytes),- ffprobe: (args: string[]) => {- files.set(- args[args.indexOf("-o") + 1]!,- JSON.stringify({- format: { format_name: "mov,mp4,m4a,3gp,3g2,mj2" },- streams: [- {- codec_name: "h264",- codec_tag_string: "avc1",- width: 320,- height: 240,- pix_fmt: "yuv420p",- ...streamChanges,- },- ],- packets,- }),- );- return 0;- },- exec: vi.fn((args: string[]) => {- if (args[0] === "-dump_attachment:v:0")- files.set(args[1]!, description);- else files.set(args.at(-1)!, new Uint8Array([1, 2, 3]));- return 0;- }),- };- const encode = (start: number, end: number, signal?: AbortSignal) =>- encodeVideoWithWebCodecs(- ffmpeg as unknown as FFmpeg,- "/input",- new Blob([new Uint8Array(packets.length)]),- { start, end, muted: false },- signal,- );- return { encode, files, ffmpeg };-};--test("keeps every 30 fps frame without rounding duplicates", async () => {- const { encode, files } = fixture();- await encode(0, 10);- expect(encodedSources).toEqual(- Array.from({ length: 300 }, (_, index) =>- Math.round((index * 1_000_000) / 30),- ),- );- expect(frames.every((frame) => frame.closed)).toBe(true);- expect(files.size).toBe(0);-});--test("a trim between keyframes begins at the frame visible at the selected start", async () => {- const { encode } = fixture();- await encode(7.125, 17.125);- expect(encodedSources).toHaveLength(300);- expect(encodedSources[0]).toBe(7_100_000);- expect(encodedSources.at(-1)).toBe(17_066_667);- expect(new Set(encodedSources).size).toBe(300);-});--test("29.97 fps and fractional selections never export more than the selected duration", async () => {- const { encode } = fixture(30000 / 1001);- await encode(1.125, 11.125);- expect(encodedSources).toHaveLength(300);- encodedSources.forEach((time, index) => {- const target = 1_125_000 + Math.round((index * 1_000_000) / 30);- expect(time).toBeLessThanOrEqual(target);- expect(target - time).toBeLessThanOrEqual(33_367);- });- encodedSources = [];- await encode(1.125, 11.115);- expect(encodedSources).toHaveLength(299);-});--test("unsupported hardware is left to the existing exporter", async () => {- supported = false;- expect(await fixture().encode(0, 10)).toBeUndefined();- expect(encodedSources).toHaveLength(0);-});--test("cancelling a stalled encoder releases frames and temporary files", async () => {- const { encode, files, ffmpeg } = fixture();- const controller = new AbortController();- stallEncoder = () => controller.abort();- await expect(encode(0, 10, controller.signal)).rejects.toMatchObject({- name: "AbortError",- });- expect(frames.every((frame) => frame.closed)).toBe(true);- expect(files.size).toBe(0);- expect(ffmpeg.exec).toHaveBeenCalledTimes(1);-});--test("codec failures release resources and reject instead of leaving the export pending", async () => {- const { encode, files } = fixture();- encoderFailure = true;- await expect(encode(0, 10)).rejects.toThrow("Encoder failed");- expect(frames.every((frame) => frame.closed)).toBe(true);- expect(files.size).toBe(0);-});--test("full-range 8-bit H.264 uses WebCodecs", async () => {- const { encode, files } = fixture(30, { pix_fmt: "yuvj420p" });- expect(await encode(0, 10)).toBeDefined();- expect(encodedSources).toHaveLength(300);- expect(frames.every((frame) => frame.closed)).toBe(true);- expect(files.size).toBe(0);-});--test.each(["hvc1", "hev1"])(- "decodes HEVC %s with its configuration record",- async (tag) => {- const description = new Uint8Array([- 1, 1, 96, 0, 0, 0, 176, 0, 0, 0, 0, 0, 153, 240, 0, 252, 253, 248,- 248, 0, 0, 15, 0,- ]);- const { encode } = fixture(- 60,- { codec_name: "hevc", codec_tag_string: tag },- description,- );- expect(await encode(0, 10)).toBeDefined();- expect(decoderConfig.codec).toBe(`${tag}.1.6.L153.b0`);- expect(decoderConfig.description).toBe(description);- expect(encodedSources).toHaveLength(300);- expect(canvasOptions).toHaveLength(0);- },-);--test.each(["arib-std-b67", "smpte2084"])(- "converts HEVC %s frames to SDR before encoding",- async (transfer) => {- const description = new Uint8Array([- 1, 2, 32, 0, 0, 0, 144, 0, 0, 0, 0, 0, 153, 240, 0, 252, 253, 250,- 250, 0, 0, 15, 0,- ]);- const { encode, files } = fixture(- 30,- {- codec_name: "hevc",- codec_tag_string: "hvc1",- pix_fmt: "yuv420p10le",- color_primaries: "bt2020",- color_transfer: transfer,- },- description,- );- expect(await encode(0, 10)).toBeDefined();- expect(decoderConfig.codec).toBe("hvc1.2.4.L153.90");- expect(canvasOptions).toEqual([{ colorSpace: "srgb" }]);- expect(encodedSources).toEqual(- Array.from({ length: 300 }, (_, i) =>- Math.round((i * 1_000_000) / 30),- ),- );- expect(frames.every((frame) => frame.closed)).toBe(true);- expect(files.size).toBe(0);- },-);--test("a truncated HEVC configuration is left to the existing exporter", async () => {- const { encode } = fixture(30, {- codec_name: "hevc",- codec_tag_string: "hvc1",- });- expect(await encode(0, 10)).toBeUndefined();- expect(encodedSources).toHaveLength(0);-});diff --git a/web/package-lock.json b/web/package-lock.jsonindex 5566676ba3..601390d152 100644--- a/web/package-lock.json+++ b/web/package-lock.json@@ -353,12 +353,14 @@"@fontsource/nunito": "5.3.0","@hugeicons/core-free-icons": "4.3.5","@hugeicons/react": "1.1.10",+ "@mediabunny/aac-encoder": "1.61.6","@rive-app/canvas-lite": "2.43.1","ente-accounts": "*","ente-base": "*","ente-media": "*","ente-prelogin-wasm": "*","ente-space-wasm": "*",+ "mediabunny": "1.61.6","photoswipe": "5.4.4","react-easy-crop": "5.5.7","thumbhash": "0.1.1"@@ -1836,6 +1838,19 @@"dev": true,"license": "MIT"},+ "node_modules/@mediabunny/aac-encoder": {+ "version": "1.61.6",+ "resolved": "https://registry.npmjs.org/@mediabunny/aac-encoder/-/aac-encoder-1.61.6.tgz",+ "integrity": "sha512-rdeYDcjzi1VSpGcVlq9zBBCqrqualmh7cJjckl7lrKGpCrSIOTCdivdEHTFSCgHoH5G/aMxiCOb9M4fyAmzn6w==",+ "license": "MPL-2.0",+ "funding": {+ "type": "individual",+ "url": "https://github.com/sponsors/Vanilagy"+ },+ "peerDependencies": {+ "mediabunny": "^1.0.0"+ }+ },"node_modules/@mui/core-downloads-tracker": {"version": "9.4.0","resolved": "https://registry.npmjs.org/@mui/core-downloads-tracker/-/core-downloads-tracker-9.4.0.tgz",@@ -3675,6 +3690,21 @@"dev": true,"license": "MIT"},+ "node_modules/@types/dom-mediacapture-transform": {+ "version": "0.1.12",+ "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-transform/-/dom-mediacapture-transform-0.1.12.tgz",+ "integrity": "sha512-d7/QsLRwF864A5mgIM/YrfiglHoYn7zgCcAoJgW404r+2DwnNr7EBbLnCWpmOMgH8y0te73L1AV6H1bmauaWFw==",+ "license": "MIT",+ "dependencies": {+ "@types/dom-webcodecs": "*"+ }+ },+ "node_modules/@types/dom-webcodecs": {+ "version": "0.1.13",+ "resolved": "https://registry.npmjs.org/@types/dom-webcodecs/-/dom-webcodecs-0.1.13.tgz",+ "integrity": "sha512-O5hkiFIcjjszPIYyUSyvScyvrBoV3NOEEZx/pMlsu44TKzWNkLVBBxnxJz42in5n3QIolYOcBYFCPZZ0h8SkwQ==",+ "license": "MIT"+ },"node_modules/@types/esrecurse": {"version": "4.3.1","resolved": "https://registry.npmjs.org/@types/esrecurse/-/esrecurse-4.3.1.tgz",@@ -8132,6 +8162,25 @@"integrity": "sha512-l54rkKXlLBt3ob3zOLWHcnjvwUmX5bNEZ70igyapOZZC9imzqBmq1oz8p2roiV04KhjblFIi2hetLPF1oYVLRA==","license": "MIT"},+ "node_modules/mediabunny": {+ "version": "1.61.6",+ "resolved": "https://registry.npmjs.org/mediabunny/-/mediabunny-1.61.6.tgz",+ "integrity": "sha512-VHEZ53/7jGl4dQ+uOdndsZufFYwEUFWv9UXgvrGun3OapyzoqI9iaPOD7z3xxY/nAttLZBNwswgJzMtGJAquOQ==",+ "license": "MPL-2.0",+ "peer": true,+ "workspaces": [+ ".",+ "packages/*"+ ],+ "dependencies": {+ "@types/dom-mediacapture-transform": "^0.1.11",+ "@types/dom-webcodecs": "0.1.13"+ },+ "funding": {+ "type": "individual",+ "url": "https://github.com/sponsors/Vanilagy"+ }+ },"node_modules/memoize": {"version": "10.2.0","resolved": "https://registry.npmjs.org/memoize/-/memoize-10.2.0.tgz",