Skip to content

Open source CapCut alternative.

TypeScript

fix(site): read undecodable streams via compat

David Le0ff4b13307b7
diff --git a/site/src/cut/lib/codecSupport.test.ts b/site/src/cut/lib/codecSupport.test.ts
index e09dd5ca..aca5a031 100644
--- a/site/src/cut/lib/codecSupport.test.ts
+++ b/site/src/cut/lib/codecSupport.test.ts
@@ -1,5 +1,5 @@
import { describe, expect, test } from "bun:test";
-import { readCodecs, type CodecProbes } from "./codecSupport";
+import { compatNeedOf, noteUndecoded, readCodecs, type CodecProbes } from "./codecSupport";
/** A storage that keeps what it is given, like a browser's localStorage. */
function memoryStorage(): Pick<Storage, "getItem" | "setItem"> {
@@ -45,3 +45,21 @@ describe("browser codecs", () => {
expect(await readCodecs(io)).toEqual({ h264: false, aac: true });
});
});
+
+describe("streams this page could not decode", () => {
+ test("a stream that failed to decode asks for its compat cover", async () => {
+ const asset = { id: "a1", type: "video" as const, url: "https://media.example/a.mp4" };
+ expect(await compatNeedOf(asset)).toBeNull();
+
+ expect(noteUndecoded("a1", "audio")).toBe(true);
+ expect(await compatNeedOf(asset)).toBe("audio");
+
+ // A second failure of the same stream is the file's own.
+ expect(noteUndecoded("a1", "audio")).toBe(false);
+
+ // Picture widens the cover, and a VP9 proxy carries the sound as well.
+ expect(noteUndecoded("a1", "video")).toBe(true);
+ expect(await compatNeedOf(asset)).toBe("video");
+ expect(noteUndecoded("a1", "audio")).toBe(false);
+ });
+});
diff --git a/site/src/cut/lib/codecSupport.ts b/site/src/cut/lib/codecSupport.ts
index 912b8cf9..8a5cba5e 100644
--- a/site/src/cut/lib/codecSupport.ts
+++ b/site/src/cut/lib/codecSupport.ts
@@ -115,13 +115,30 @@ function codecsOf(asset: Pick<MediaAsset, "url">) {
return read;
}
+/** Streams this page found it cannot decode by trying, by asset id, for the
+ * session: a decoder that refused a track the codec checks passed, e.g. HEVC
+ * in Firefox, or a malformed MP3 track in a screen recording. */
+const refusedHere = new Map<string, CompatCover>();
+
+/** Record that this page could not decode the asset's `cover` stream, so its
+ * readers move to a compat proxy. False when that stream was already covered
+ * this way: a second failure is the file's own. */
+export function noteUndecoded(assetId: string, cover: CompatCover): boolean {
+ const held = refusedHere.get(assetId);
+ if (held === "video" || held === cover) {
+ return false;
+ }
+ refusedHere.set(assetId, cover);
+ return true;
+}
+
/** The stream of the asset's master this browser cannot decode, as the compat
* cover that carries it (proxyNames.ts): "video" for H.264 picture, which
* covers the sound too, "audio" for AAC sound. Null when this browser decodes
* the master — every browser that decodes both codecs answers without reading
* the file. ProRes has its own proxy and WASM decoder, and takes none. */
export async function compatNeedOf(
- asset: Pick<MediaAsset, "type" | "url" | "block" | "color">
+ asset: Pick<MediaAsset, "id" | "type" | "url" | "block" | "color">
): Promise<CompatCover | null> {
if ((asset.type !== "video" && asset.type !== "audio") || asset.block) {
return null;
@@ -129,6 +146,13 @@ export async function compatNeedOf(
if (asset.color?.codec?.startsWith("ap")) {
return null;
}
+
+ // A stream that already failed to decode here needs its cover whatever the
+ // codec checks say.
+ const refused = refusedHere.get(asset.id);
+ if (refused) {
+ return refused;
+ }
const codecs = await browserCodecs();
if (codecs.h264 && codecs.aac) {
return null;
diff --git a/site/src/cut/lib/compatHandoff.test.ts b/site/src/cut/lib/compatHandoff.test.ts
new file mode 100644
index 00000000..f29bf0ee
--- /dev/null
+++ b/site/src/cut/lib/compatHandoff.test.ts
@@ -0,0 +1,85 @@
+import { afterAll, describe, expect, spyOn, test } from "bun:test";
+
+import { stubModule } from "@/lib/testing/stubModule";
+import { bindFailureSink } from "./report";
+import { useEditor } from "./store";
+import type { MediaAsset } from "./types";
+import { UserError } from "./userError";
+
+// A stream this page turns out not to decode once it tries: HEVC in Firefox
+// refuses the strip's track, a malformed MP3 track fails the waveform's
+// decoder. The file is fine; the page hands the stream to a compat proxy.
+
+await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, {
+ withMedia: (async (_src: unknown, fn: (input: unknown) => Promise<unknown>) => fn({})) as never,
+ videoTrackOf: (async () => null) as never,
+ audioPeaks: (async () => {
+ throw new DOMException("Decoding error. (malformed stream: mpa: invalid packet length)", "EncodingError");
+ }) as never,
+});
+await stubModule<typeof import("./baseFrame")>("./baseFrame", import.meta.url, {
+ // eslint-disable-next-line require-yield
+ baseFramesAt: async function* () {
+ throw new UserError("media_no_readable_video", "This file has no readable video.");
+ },
+});
+// Each proxy ask, with the cover the asset needed when it was made.
+const { compatNeedOf } = await import("./codecSupport");
+const asks: string[] = [];
+await stubModule<typeof import("./mediaProxy")>("./mediaProxy", import.meta.url, {
+ ensureProxy: async (_projectId: string, asset: MediaAsset) => {
+ asks.push(`${asset.id}:${await compatNeedOf(asset)}`);
+ },
+ awaitsCompat: async () => false,
+});
+
+const { enrichAsset, stripFailedFor } = await import("./media");
+
+const reports: string[] = [];
+bindFailureSink((error) => reports.push(error.message));
+const warn = spyOn(console, "warn").mockImplementation(() => {});
+const decoder = (globalThis as { AudioDecoder?: unknown }).AudioDecoder;
+(globalThis as { AudioDecoder?: unknown }).AudioDecoder = class {};
+afterAll(() => {
+ bindFailureSink((error) => console.error(error));
+ warn.mockRestore();
+ (globalThis as { AudioDecoder?: unknown }).AudioDecoder = decoder;
+});
+
+const asset = (extra: Partial<MediaAsset>): MediaAsset => ({
+ id: "a",
+ fileName: "clip.mp4",
+ name: "clip.mp4",
+ type: "video",
+ duration: 9,
+ url: "stored:p1/clip.mp4",
+ ...extra,
+});
+
+describe("a stream this page cannot decode", () => {
+ test("a filmstrip that finds no readable video asks for a compat proxy", async () => {
+ const tiktok = asset({ id: "v1", peaks: [] });
+ useEditor.setState({ projectId: "p1", readOnly: false, playing: false, assets: [tiktok] });
+
+ await enrichAsset(tiktok);
+ await Promise.resolve();
+
+ expect(asks.at(-1)).toBe("v1:video");
+ expect(reports).toEqual([]);
+ expect(stripFailedFor("v1")).toBe(false);
+ });
+
+ test("a waveform whose decoder fails asks for a compat proxy, once", async () => {
+ const recording = asset({ id: "s1", type: "audio", fileName: "bandicam.mp3", name: "bandicam.mp3" });
+ useEditor.setState({ projectId: "p1", assets: [recording] });
+
+ await enrichAsset(recording);
+ await Promise.resolve();
+ expect(asks.at(-1)).toBe("s1:audio");
+ expect(reports).toEqual([]);
+
+ // The same stream failing again is the file's own failure.
+ await enrichAsset(recording);
+ expect(reports).toEqual(["[cut] enrich failed for bandicam.mp3 (audio, 9s)"]);
+ });
+});
diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.ts
index 3ada0e98..02b89919 100644
--- a/site/src/cut/lib/media.ts
+++ b/site/src/cut/lib/media.ts
@@ -18,12 +18,15 @@ import { FONT_RE, LUT_RE } from "./libraryFileType";
import { markUploadStored, startUpload, uploadInFlight } from "./importQueue";
import { convertProjectFile, dropProjectFile, mp4DisplayName } from "./mediaConvert";
import { awaitsCompat, ensureProxy } from "./mediaProxy";
+import { noteUndecoded } from "./codecSupport";
+import { compatCovers, type CompatCover } from "./proxyNames";
import {
audioChunks,
audioPeaks,
audioTrackOf,
decodeAudioSpan,
frameSink,
+ frameSinkIsCustom,
openMedia,
openMediaShared,
probeMediaColor,
@@ -2102,6 +2105,37 @@ function readFailed(asset: MediaAsset, what: string, err: unknown, subject = ass
reportSwallowed(what, err, subject);
}
+// The DOMException names WebCodecs gives a decoder that refused its input: a
+// config it cannot open, or packets it failed on.
+const DECODER_REFUSALS = new Set(["EncodingError", "NotSupportedError"]);
+
+/** Whether `err` is this page's decoders refusing a stream: a track no decoder
+ * here opens (HEVC in Firefox), or a decoder failing on the packets it was
+ * handed (a malformed MP3 track in a screen recording). */
+function decodeRefused(err: unknown): boolean {
+ if (err instanceof DOMException) return DECODER_REFUSALS.has(err.name);
+ if (err instanceof UnreadableMediaError) return err.undecodable;
+ return err instanceof UserError && err.code === "media_no_readable_video";
+}
+
+/**
+ * Hand a stream this page failed to decode to a compat proxy, which a
+ * converter writes in codecs every browser decodes and whose landing enriches
+ * the asset again. True when the failure went that way; false when it is the
+ * file's own: the stream already reads from a compat proxy, or failed here
+ * before.
+ */
+function coverRefused(asset: MediaAsset, cover: CompatCover, err: unknown): boolean {
+ if (frameSinkIsCustom() || !decodeRefused(err)) return false;
+ const live = liveAsset(asset.id) ?? asset;
+ if (live.proxy && compatCovers(live.proxy.fileName, cover)) return false;
+ if (!noteUndecoded(asset.id, cover)) return false;
+ // An upload still running asks for its proxy when it lands (importQueue.ts).
+ const projectId = useEditor.getState().projectId;
+ if (projectId && !live.upload) void ensureProxy(projectId, live);
+ return true;
+}
+
// A failed enrich books itself more tries. Filmstrip and waveform reads drop
// out together with the preview's when media reads fail, and the sweep that
// would try again runs only on project open — so a blip at open used to leave
@@ -2368,6 +2402,7 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {
// next open reads the file again in place of keeping a
// strip that stops partway.
landDerived(asset.id, { peaks: undefined });
+ if (coverRefused(asset, "audio", err)) return;
readFailed(asset, "[cut] peaks failed", err);
})
: undefined;
@@ -2429,6 +2464,9 @@ export async function enrichAsset(asset: MediaAsset, src = asset.url) {
enrichRetries.delete(asset.id);
}
} catch (err) {
+ // A stream this page cannot decode reads from a compat proxy, whose
+ // landing enriches the asset again.
+ if ((asset.type === "video" || asset.type === "audio") && coverRefused(asset, asset.type, err)) return;
// Thumbnails and waveforms are decorative; editing works without them.
// The failure still gets recorded — clips stop their loading pulse — and
// reported, which carries it into PostHog error tracking with the file's
@@ -2508,6 +2546,7 @@ export async function ensurePeaks(asset: MediaAsset) {
// Waveforms are decorative; editing works without them. A walk that
// published partway comes off the asset, so the next read starts clean.
landDerived(asset.id, { peaks: undefined });
+ if (coverRefused(asset, "audio", err)) return;
readFailed(asset, "[cut] peaks failed", err);
}
}
diff --git a/site/src/cut/lib/mediaMissing.test.ts b/site/src/cut/lib/mediaMissing.test.ts
index 69ce1a3a..b942354a 100644
--- a/site/src/cut/lib/mediaMissing.test.ts
+++ b/site/src/cut/lib/mediaMissing.test.ts
@@ -130,9 +130,10 @@ describe("other read failures", () => {
expect(reported).toEqual([]);
});
- test("an undecodable file is reported against its stage", async () => {
+ test("an undecodable file its compat proxy already covers is reported against its stage", async () => {
failure = new DOMException("Decoding error.", "EncodingError");
- const asset = audio();
+ const asset = audio({ proxy: { fileName: "a.mp3.compat.mp4", sizeBytes: 1 } });
+ useEditor.setState({ assets: [asset] });
await ensurePeaks(asset);
diff --git a/site/src/cut/lib/mediaProxy.test.ts b/site/src/cut/lib/mediaProxy.test.ts
index 7bf93c15..440192ca 100644
--- a/site/src/cut/lib/mediaProxy.test.ts
+++ b/site/src/cut/lib/mediaProxy.test.ts
@@ -253,6 +253,30 @@ describe("compat proxy", () => {
expect(coveredSrc("audio", asset.url)).toBe("stored:p/a.mp4.compat.webm");
});
+ test("an ask that arrives while one runs goes again once it lands", async () => {
+ // The enrich asks at open and builds the preview proxy; the strip then
+ // finds the picture undecodable here and asks for a cover mid-build.
+ hevc = true;
+ const asset = h264("a");
+ useEditor.setState({ projectId: "p", assets: [asset] });
+ let land!: () => void;
+ const landed = new Promise<void>((r) => (land = r));
+ const b = backend("cloud", ((_path: string, body: { compat?: string }) =>
+ body.compat
+ ? Response.json({ fileName: "a.mp4.compat.light.webm", sizeBytes: 9 })
+ : landed.then(() => Response.json({ fileName: "a.mp4.light.mp4", sizeBytes: 9 }))) as never);
+ const first = ensureProxy("p", asset, b.api);
+ await new Promise((r) => setTimeout(r, 0));
+
+ compatNeeds = { a: "video" };
+ await ensureProxy("p", asset, b.api);
+ land();
+ await first;
+ for (let i = 0; i < 20 && b.calls.length < 2; i++) await new Promise((r) => setTimeout(r, 0));
+
+ expect(b.calls[1]?.body).toEqual({ file: "a.mp4", compat: "video", maxHeight: HEAVY_PROXY_MAX_HEIGHT });
+ });
+
test("a heavy cloud video gets the light cover", async () => {
compatNeeds = { a: "audio" };
const asset = h264("a");
diff --git a/site/src/cut/lib/mediaProxy.ts b/site/src/cut/lib/mediaProxy.ts
index d0751c4e..3671a617 100644
--- a/site/src/cut/lib/mediaProxy.ts
+++ b/site/src/cut/lib/mediaProxy.ts
@@ -124,6 +124,10 @@ export const useProxyJobs = create<ProxyJobs>((set) => ({
}));
const inFlight = new Set<string>();
+// Asks that arrived while one was running, by asset id. Each goes again once
+// the running one lands: a stream found undecodable mid-build asks for a
+// cover the running ask did not know it needed.
+const askAgain = new Set<string>();
/** The proxy an asset wants in this page: the compat proxy of a stream this
* browser cannot decode (light for a heavy video the cloud streams), or the
@@ -160,7 +164,11 @@ export async function ensureProxy(
// project is next opened.
if (frameSinkIsCustom()) return;
watchCovers();
- if (asset.upload || asset.block || inFlight.has(asset.id)) return;
+ if (asset.upload || asset.block) return;
+ if (inFlight.has(asset.id)) {
+ askAgain.add(asset.id);
+ return;
+ }
inFlight.add(asset.id);
try {
const ask = await proxyAsk(asset, backend);
@@ -189,6 +197,9 @@ export async function ensureProxy(
}
} finally {
inFlight.delete(asset.id);
+ const s = useEditor.getState();
+ const live = s.assets.find((a) => a.id === asset.id);
+ if (askAgain.delete(asset.id) && live && s.projectId === projectId) void ensureProxy(projectId, live, backend);
}
}