Open source CapCut alternative.
TypeScript
fix(site): read undecodable streams via compat
David Le0ff4b13307b7
diff --git a/site/src/cut/lib/codecSupport.test.ts b/site/src/cut/lib/codecSupport.test.tsindex e09dd5ca..aca5a031 100644--- a/site/src/cut/lib/codecSupport.test.ts+++ b/site/src/cut/lib/codecSupport.test.ts@@ -1,5 +1,5 @@import { describe, expect, test } from "bun:test";-import { readCodecs, type CodecProbes } from "./codecSupport";+import { compatNeedOf, noteUndecoded, readCodecs, type CodecProbes } from "./codecSupport";/** A storage that keeps what it is given, like a browser's localStorage. */function memoryStorage(): Pick<Storage, "getItem" | "setItem"> {@@ -45,3 +45,21 @@ describe("browser codecs", () => {expect(await readCodecs(io)).toEqual({ h264: false, aac: true });});});++describe("streams this page could not decode", () => {+ test("a stream that failed to decode asks for its compat cover", async () => {+ const asset = { id: "a1", type: "video" as const, url: "https://media.example/a.mp4" };+ expect(await compatNeedOf(asset)).toBeNull();++ expect(noteUndecoded("a1", "audio")).toBe(true);+ expect(await compatNeedOf(asset)).toBe("audio");++ // A second failure of the same stream is the file's own.+ expect(noteUndecoded("a1", "audio")).toBe(false);++ // Picture widens the cover, and a VP9 proxy carries the sound as well.+ expect(noteUndecoded("a1", "video")).toBe(true);+ expect(await compatNeedOf(asset)).toBe("video");+ expect(noteUndecoded("a1", "audio")).toBe(false);+ });+});diff --git a/site/src/cut/lib/codecSupport.ts b/site/src/cut/lib/codecSupport.tsindex 912b8cf9..8a5cba5e 100644--- a/site/src/cut/lib/codecSupport.ts+++ b/site/src/cut/lib/codecSupport.ts@@ -115,13 +115,30 @@ function codecsOf(asset: Pick<MediaAsset, "url">) {return read;}+/** Streams this page found it cannot decode by trying, by asset id, for the+ * session: a decoder that refused a track the codec checks passed, e.g. HEVC+ * in Firefox, or a malformed MP3 track in a screen recording. */+const refusedHere = new Map<string, CompatCover>();++/** Record that this page could not decode the asset's `cover` stream, so its+ * readers move to a compat proxy. False when that stream was already covered+ * this way: a second failure is the file's own. */+export function noteUndecoded(assetId: string, cover: CompatCover): boolean {+ const held = refusedHere.get(assetId);+ if (held === "video" || held === cover) {+ return false;+ }+ refusedHere.set(assetId, cover);+ return true;+}+/** The stream of the asset's master this browser cannot decode, as the compat* cover that carries it (proxyNames.ts): "video" for H.264 picture, which* covers the sound too, "audio" for AAC sound. Null when this browser decodes* the master — every browser that decodes both codecs answers without reading* the file. ProRes has its own proxy and WASM decoder, and takes none. */export async function compatNeedOf(- asset: Pick<MediaAsset, "type" | "url" | "block" | "color">+ asset: Pick<MediaAsset, "id" | "type" | "url" | "block" | "color">): Promise<CompatCover | null> {if ((asset.type !== "video" && asset.type !== "audio") || asset.block) {return null;@@ -129,6 +146,13 @@ export async function compatNeedOf(if (asset.color?.codec?.startsWith("ap")) {return null;}++ // A stream that already failed to decode here needs its cover whatever the+ // codec checks say.+ const refused = refusedHere.get(asset.id);+ if (refused) {+ return refused;+ }const codecs = await browserCodecs();if (codecs.h264 && codecs.aac) {return null;diff --git a/site/src/cut/lib/compatHandoff.test.ts b/site/src/cut/lib/compatHandoff.test.tsnew file mode 100644index 00000000..f29bf0ee--- /dev/null+++ b/site/src/cut/lib/compatHandoff.test.ts@@ -0,0 +1,85 @@+import { afterAll, describe, expect, spyOn, test } from "bun:test";++import { stubModule } from "@/lib/testing/stubModule";+import { bindFailureSink } from "./report";+import { useEditor } from "./store";+import type { MediaAsset } from "./types";+import { UserError } from "./userError";++// A stream this page turns out not to decode once it tries: HEVC in Firefox+// refuses the strip's track, a malformed MP3 track fails the waveform's+// decoder. The file is fine; the page hands the stream to a compat proxy.++await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, {+ withMedia: (async (_src: unknown, fn: (input: unknown) => Promise<unknown>) => fn({})) as never,+ videoTrackOf: (async () => null) as never,+ audioPeaks: (async () => {+ throw new DOMException("Decoding error. (malformed stream: mpa: invalid packet length)", "EncodingError");+ }) as never,+});+await stubModule<typeof import("./baseFrame")>("./baseFrame", import.meta.url, {+ // eslint-disable-next-line require-yield+ baseFramesAt: async function* () {+ throw new UserError("media_no_readable_video", "This file has no readable video.");+ },+});+// Each proxy ask, with the cover the asset needed when it was made.+const { compatNeedOf } = await import("./codecSupport");+const asks: string[] = [];+await stubModule<typeof import("./mediaProxy")>("./mediaProxy", import.meta.url, {+ ensureProxy: async (_projectId: string, asset: MediaAsset) => {+ asks.push(`${asset.id}:${await compatNeedOf(asset)}`);+ },+ awaitsCompat: async () => false,+});++const { enrichAsset, stripFailedFor } = await import("./media");++const reports: string[] = [];+bindFailureSink((error) => reports.push(error.message));+const warn = spyOn(console, "warn").mockImplementation(() => {});+const decoder = (globalThis as { AudioDecoder?: unknown }).AudioDecoder;+(globalThis as { AudioDecoder?: unknown }).AudioDecoder = class {};+afterAll(() => {+ bindFailureSink((error) => console.error(error));+ warn.mockRestore();+ (globalThis as { AudioDecoder?: unknown }).AudioDecoder = decoder;+});++const asset = (extra: Partial<MediaAsset>): MediaAsset => ({+ id: "a",+ fileName: "clip.mp4",+ name: "clip.mp4",+ type: "video",+ duration: 9,+ url: "stored:p1/clip.mp4",+ ...extra,+});++describe("a stream this page cannot decode", () => {+ test("a filmstrip that finds no readable video asks for a compat proxy", async () => {+ const tiktok = asset({ id: "v1", peaks: [] });+ useEditor.setState({ projectId: "p1", readOnly: false, playing: false, assets: [tiktok] });++ await enrichAsset(tiktok);+ await Promise.resolve();++ expect(asks.at(-1)).toBe("v1:video");+ expect(reports).toEqual([]);+ expect(stripFailedFor("v1")).toBe(false);+ });++ test("a waveform whose decoder fails asks for a compat proxy, once", async () => {+ const recording = asset({ id: "s1", type: "audio", fileName: "bandicam.mp3", name: "bandicam.mp3" });+ useEditor.setState({ projectId: "p1", assets: [recording] });++ await enrichAsset(recording);+ await Promise.resolve();+ expect(asks.at(-1)).toBe("s1:audio");+ expect(reports).toEqual([]);++ // The same stream failing again is the file's own failure.+ await enrichAsset(recording);+ expect(reports).toEqual(["[cut] enrich failed for bandicam.mp3 (audio, 9s)"]);+ });+});diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 3ada0e98..02b89919 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -18,12 +18,15 @@ import { FONT_RE, LUT_RE } from "./libraryFileType";import { markUploadStored, startUpload, uploadInFlight } from "./importQueue";import { convertProjectFile, dropProjectFile, mp4DisplayName } from "./mediaConvert";import { awaitsCompat, ensureProxy } from "./mediaProxy";+import { noteUndecoded } from "./codecSupport";+import { compatCovers, type CompatCover } from "./proxyNames";import {audioChunks,audioPeaks,audioTrackOf,decodeAudioSpan,frameSink,+ frameSinkIsCustom,openMedia,openMediaShared,probeMediaColor,@@ -2102,6 +2105,37 @@ function readFailed(asset: MediaAsset, what: string, err: unknown, subject = assreportSwallowed(what, err, subject);}+// The DOMException names WebCodecs gives a decoder that refused its input: a+// config it cannot open, or packets it failed on.+const DECODER_REFUSALS = new Set(["EncodingError", "NotSupportedError"]);++/** Whether `err` is this page's decoders refusing a stream: a track no decoder+ * here opens (HEVC in Firefox), or a decoder failing on the packets it was+ * handed (a malformed MP3 track in a screen recording). */+function decodeRefused(err: unknown): boolean {+ if (err instanceof DOMException) return DECODER_REFUSALS.has(err.name);+ if (err instanceof UnreadableMediaError) return err.undecodable;+ return err instanceof UserError && err.code === "media_no_readable_video";+}++/**+ * Hand a stream this page failed to decode to a compat proxy, which a+ * converter writes in codecs every browser decodes and whose landing enriches+ * the asset again. True when the failure went that way; false when it is the+ * file's own: the stream already reads from a compat proxy, or failed here+ * before.+ */+function coverRefused(asset: MediaAsset, cover: CompatCover, err: unknown): boolean {+ if (frameSinkIsCustom() || !decodeRefused(err)) return false;+ const live = liveAsset(asset.id) ?? asset;+ if (live.proxy && compatCovers(live.proxy.fileName, cover)) return false;+ if (!noteUndecoded(asset.id, cover)) return false;+ // An upload still running asks for its proxy when it lands (importQueue.ts).+ const projectId = useEditor.getState().projectId;+ if (projectId && !live.upload) void ensureProxy(projectId, live);+ return true;+}+// A failed enrich books itself more tries. Filmstrip and waveform reads drop// out together with the preview's when media reads fail, and the sweep that// would try again runs only on project open — so a blip at open used to leave@@ -2368,6 +2402,7 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {// next open reads the file again in place of keeping a// strip that stops partway.landDerived(asset.id, { peaks: undefined });+ if (coverRefused(asset, "audio", err)) return;readFailed(asset, "[cut] peaks failed", err);}): undefined;@@ -2429,6 +2464,9 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {enrichRetries.delete(asset.id);}} catch (err) {+ // A stream this page cannot decode reads from a compat proxy, whose+ // landing enriches the asset again.+ if ((asset.type === "video" || asset.type === "audio") && coverRefused(asset, asset.type, err)) return;// Thumbnails and waveforms are decorative; editing works without them.// The failure still gets recorded — clips stop their loading pulse — and// reported, which carries it into PostHog error tracking with the file's@@ -2508,6 +2546,7 @@ export async function ensurePeaks(asset: MediaAsset) {// Waveforms are decorative; editing works without them. A walk that// published partway comes off the asset, so the next read starts clean.landDerived(asset.id, { peaks: undefined });+ if (coverRefused(asset, "audio", err)) return;readFailed(asset, "[cut] peaks failed", err);}}diff --git a/site/src/cut/lib/mediaMissing.test.ts b/site/src/cut/lib/mediaMissing.test.tsindex 69ce1a3a..b942354a 100644--- a/site/src/cut/lib/mediaMissing.test.ts+++ b/site/src/cut/lib/mediaMissing.test.ts@@ -130,9 +130,10 @@ describe("other read failures", () => {expect(reported).toEqual([]);});- test("an undecodable file is reported against its stage", async () => {+ test("an undecodable file its compat proxy already covers is reported against its stage", async () => {failure = new DOMException("Decoding error.", "EncodingError");- const asset = audio();+ const asset = audio({ proxy: { fileName: "a.mp3.compat.mp4", sizeBytes: 1 } });+ useEditor.setState({ assets: [asset] });await ensurePeaks(asset);diff --git a/site/src/cut/lib/mediaProxy.test.ts b/site/src/cut/lib/mediaProxy.test.tsindex 7bf93c15..440192ca 100644--- a/site/src/cut/lib/mediaProxy.test.ts+++ b/site/src/cut/lib/mediaProxy.test.ts@@ -253,6 +253,30 @@ describe("compat proxy", () => {expect(coveredSrc("audio", asset.url)).toBe("stored:p/a.mp4.compat.webm");});+ test("an ask that arrives while one runs goes again once it lands", async () => {+ // The enrich asks at open and builds the preview proxy; the strip then+ // finds the picture undecodable here and asks for a cover mid-build.+ hevc = true;+ const asset = h264("a");+ useEditor.setState({ projectId: "p", assets: [asset] });+ let land!: () => void;+ const landed = new Promise<void>((r) => (land = r));+ const b = backend("cloud", ((_path: string, body: { compat?: string }) =>+ body.compat+ ? Response.json({ fileName: "a.mp4.compat.light.webm", sizeBytes: 9 })+ : landed.then(() => Response.json({ fileName: "a.mp4.light.mp4", sizeBytes: 9 }))) as never);+ const first = ensureProxy("p", asset, b.api);+ await new Promise((r) => setTimeout(r, 0));++ compatNeeds = { a: "video" };+ await ensureProxy("p", asset, b.api);+ land();+ await first;+ for (let i = 0; i < 20 && b.calls.length < 2; i++) await new Promise((r) => setTimeout(r, 0));++ expect(b.calls[1]?.body).toEqual({ file: "a.mp4", compat: "video", maxHeight: HEAVY_PROXY_MAX_HEIGHT });+ });+test("a heavy cloud video gets the light cover", async () => {compatNeeds = { a: "audio" };const asset = h264("a");diff --git a/site/src/cut/lib/mediaProxy.ts b/site/src/cut/lib/mediaProxy.tsindex d0751c4e..3671a617 100644--- a/site/src/cut/lib/mediaProxy.ts+++ b/site/src/cut/lib/mediaProxy.ts@@ -124,6 +124,10 @@ export const useProxyJobs = create<ProxyJobs>((set) => ({}));const inFlight = new Set<string>();+// Asks that arrived while one was running, by asset id. Each goes again once+// the running one lands: a stream found undecodable mid-build asks for a+// cover the running ask did not know it needed.+const askAgain = new Set<string>();/** The proxy an asset wants in this page: the compat proxy of a stream this* browser cannot decode (light for a heavy video the cloud streams), or the@@ -160,7 +164,11 @@ export async function ensureProxy(// project is next opened.if (frameSinkIsCustom()) return;watchCovers();- if (asset.upload || asset.block || inFlight.has(asset.id)) return;+ if (asset.upload || asset.block) return;+ if (inFlight.has(asset.id)) {+ askAgain.add(asset.id);+ return;+ }inFlight.add(asset.id);try {const ask = await proxyAsk(asset, backend);@@ -189,6 +197,9 @@ export async function ensureProxy(}} finally {inFlight.delete(asset.id);+ const s = useEditor.getState();+ const live = s.assets.find((a) => a.id === asset.id);+ if (askAgain.delete(asset.id) && live && s.projectId === projectId) void ensureProxy(projectId, live, backend);}}