Skip to content

Open source CapCut alternative.

TypeScript

fix(cut): keep compare_to_source reading through a lost decoder [rebuild]

David Leafdd520af96f
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.ts
index 66982e0a..d41aabb3 100644
--- a/site/src/cut/components/AiPanel.tools.ts
+++ b/site/src/cut/components/AiPanel.tools.ts
@@ -188,6 +188,7 @@ export const AI_PANEL_TOOLS = [
"Check the cut against the thing it was built from, in numbers and in pictures. Every frame is scored by `diff`: the mean difference per pixel between your frame and the source's, 0 (the same) to 255, read at one small size so a score means the same thing in every call; under " +
`${SWEEP_TARGET} reads as the same picture. ` +
"mode sweep scores every step of the cut (or from/to) with no pictures and returns the median, p95 and max with the worst stretches, each with its peak frame and a 4×4 grid of where in the frame it differs. " +
+ "A decoder that fails mid-read is replaced and the read goes on: `lost` names source frames no decoder read, and a source that stops decoding ends the sweep at `coveredTo` with `stopped` saying where and why. " +
`mode moments (the default) takes up to ${MAX_COMPARE} \`times\` and hands back each as the source's frame and yours side by side, with its score and grid. ` +
"Either mode says when a frame matches the source better a few frames away: `shift` is that distance, and a positive one means move what is there later by that much. " +
"This is how a replica is verified. The loop: sweep, look at the worst stretches' peaks in moments mode, fix what differs, sweep again — until the median is under " +
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 023201e9..1ac28299 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -1497,9 +1497,15 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
...pictureReport(area),
source: sourceRef(asset),
note:
- (sweep.coveredTo !== undefined
- ? `The sweep ran out of time at ${sweep.coveredTo}s; sweep again from there for the rest. `
- : "") + SWEEP_NOTE,
+ (sweep.stopped
+ ? `The source stopped decoding, so the sweep ends at ${sweep.coveredTo}s: ${sweep.stopped} Sweep again from a little past it for the rest. `
+ : sweep.coveredTo !== undefined
+ ? `The sweep ran out of time at ${sweep.coveredTo}s; sweep again from there for the rest. `
+ : "") +
+ (sweep.lost
+ ? `Source frames at ${sweep.lost.map((l) => `${l.source}s`).join(", ")} did not decode (${sweep.lost[0].error}) and are left out of the scores. `
+ : "") +
+ SWEEP_NOTE,
};
}
diff --git a/site/src/cut/lib/compareToSource.test.ts b/site/src/cut/lib/compareToSource.test.ts
index 1f366241..fd137ae3 100644
--- a/site/src/cut/lib/compareToSource.test.ts
+++ b/site/src/cut/lib/compareToSource.test.ts
@@ -57,9 +57,19 @@ let sourceLevel: (t: number) => number = () => 0;
/** The source's own frame shape. */
let sourceShape = { w: 90, h: 160 };
-async function* framesAt(_src: string | Blob, times: number[], size?: unknown) {
+/** Whether the decoder dies on the frame at a second: the way Chrome's
+ * decoder fails ("Decoding error."), on a hardware read or on any. */
+let dies: (t: number, software: boolean) => boolean = () => false;
+/** Every read's decoder options, in order. */
+const readOpts: unknown[] = [];
+
+async function* framesAt(_src: string | Blob, times: number[], size?: unknown, opts?: { software?: boolean }) {
reads.push({ times: [...times], size });
- for (const t of times) yield { canvas: stubCanvas(sourceShape.w, sourceShape.h, sourceLevel(t)), timestamp: t, duration: 0 } as never;
+ readOpts.push(opts);
+ for (const t of times) {
+ if (dies(t, !!opts?.software)) throw new DOMException("Decoding error.", "EncodingError");
+ yield { canvas: stubCanvas(sourceShape.w, sourceShape.h, sourceLevel(t)), timestamp: t, duration: 0 } as never;
+ }
}
await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, { framesAt });
@@ -68,6 +78,8 @@ const { runAiTool } = await import("./aiTools");
beforeEach(() => {
reads.length = 0;
+ readOpts.length = 0;
+ dies = () => false;
sourceLevel = () => 0;
sourceShape = { w: 90, h: 160 };
useEditor.setState({
@@ -340,3 +352,48 @@ test("a sweep with nothing inside the source is turned away", async () => {
);
expect(error).toContain("offset");
});
+
+type Lost = { lost?: { source: number; error: string }[]; stopped?: string; note: string };
+
+test("a sweep whose decoder dies mid-read opens a fresh one where it died and finishes", async () => {
+ dies = (t, software) => t === 1 && !software;
+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 2, step: 0.5 })) as Sweep & Lost;
+ expect(out.sampled).toBe(4);
+ expect(out.coveredTo).toBeUndefined();
+ expect(out.lost).toBeUndefined();
+ // The second read picks up at the frame that failed, on software decoding.
+ expect(reads.map((r) => r.times)).toEqual([[0, 0.5, 1, 1.5], [1, 1.5]]);
+ expect(readOpts[1]).toMatchObject({ software: true });
+});
+
+test("a frame no fresh decoder reads is left out of the sweep and named, and the rest is scored", async () => {
+ dies = (t) => t === 1;
+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 2, step: 0.5 })) as Sweep & Lost;
+ expect(out.sampled).toBe(3);
+ expect(out.lost).toEqual([{ source: 1, error: "Decoding error." }]);
+ expect(out.note).toContain("1s");
+});
+
+test("a source that stops decoding ends the sweep where it stopped, saying where and why", async () => {
+ dies = (t) => t >= 1;
+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 3, step: 0.5 })) as Sweep & Lost;
+ expect(out.sampled).toBe(2);
+ expect(out.coveredTo).toBe(1);
+ expect(out.stopped).toContain("Decoding error.");
+ expect(out.stopped).toContain("1s");
+ expect(out.note).toContain("Decoding error.");
+});
+
+test("a picture call whose decoder dies reads on with a fresh one", async () => {
+ dies = (t, software) => t > 2 && !software;
+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1, 3] })) as { checked: unknown[] };
+ expect(out.checked).toHaveLength(2);
+ expect(readOpts.at(-1)).toMatchObject({ software: true });
+});
+
+test("a moment no decoder reads names the second and the decoder's own words", async () => {
+ dies = (t) => t === 3;
+ const error = await message(runAiTool("compare_to_source", { asset_id: "ref", times: [1, 3] }));
+ expect(error).toContain("3s");
+ expect(error).toContain("Decoding error.");
+});
diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.ts
index e4f74594..e20b3270 100644
--- a/site/src/cut/lib/mediaRead.ts
+++ b/site/src/cut/lib/mediaRead.ts
@@ -637,13 +637,14 @@ function capToTrack(track: InputVideoTrack, size?: FrameSize): FrameSize | undef
export async function* framesAt(
src: string | Blob,
times: number[],
- size?: FrameSize
+ size?: FrameSize,
+ opts?: FrameSinkOptions
): AsyncGenerator<WrappedCanvas | null> {
const input = openMedia(src);
try {
const track = await videoTrackOf(input);
if (!track) throw new UnreadableMediaError("This file has no readable video.");
- yield* frameSink(track, size).canvasesAtTimestamps(times);
+ yield* frameSink(track, size, opts).canvasesAtTimestamps(times);
} finally {
input.dispose();
}
diff --git a/site/src/cut/lib/sourceCompare.ts b/site/src/cut/lib/sourceCompare.ts
index 916f3145..68ce9d6f 100644
--- a/site/src/cut/lib/sourceCompare.ts
+++ b/site/src/cut/lib/sourceCompare.ts
@@ -3,7 +3,8 @@ import { FULL_FRAME, isFullFrame } from "./measure/picture";
import type { FrameBox } from "./measure/sourceMeasure";
import { bestShift, diffPixels, meanDiff, runShift, SCORE_LONG_SIDE, SWEEP_TARGET, sweepStats, worstStretches } from "./frameDiff";
import { composeComparison, makeStillFrame } from "./media";
-import { framesAt } from "./mediaRead";
+import type { WrappedCanvas } from "mediabunny";
+import { framesAt, type FrameSize } from "./mediaRead";
import { createRasterCanvas, decodeRasterImageUrl, rasterCanvasToDataUrl, type RasterSurface } from "./raster";
import type { ExportDoc } from "./renderSnapshot";
import { clipLen } from "./store";
@@ -38,6 +39,17 @@ const SWEEP_BUDGET_MS = 60_000;
/** How many of a sweep's worst stretches come back. */
const MAX_STRETCHES = 5;
+/** Readers a source frame is tried on — the one that failed, then fresh ones
+ * on software decoding — before the frame is given up. */
+const READ_TRIES = 3;
+
+/** Frames given up in a row before the source counts as unreadable from there. */
+const READ_LOST_RUN = 3;
+
+/** The breath before a fresh reader, per try: a decoder lost to contention
+ * comes back once the contention passes. */
+const READ_RETRY_MS = 100;
+
/** Aspect ratios this close draw edge to edge; anything further letterboxes. */
const ASPECT_SLACK = 0.01;
@@ -200,6 +212,7 @@ export async function compareMoments(
const centers = new Set(wanted.map((m) => key(m.sourceAt)));
const pixels = new Map<number, Uint8ClampedArray>();
const pictures = new Map<number, string>();
+ const lost: LostFrame[] = [];
if (asset.type === "image") {
const still = await stillPixels(asset, scorer, area);
for (const m of wanted) {
@@ -216,7 +229,7 @@ export async function compareMoments(
// height is its picture box's, so a picture inside a bigger frame is
// read at the size the cut's frame is.
let seen = 0;
- for await (const c of framesAt(asset.url, times, { height: readHeight(area, size.height) })) {
+ for await (const c of steadyFrames(asset, times, { height: readHeight(area, size.height) }, lost)) {
const k = key(times[seen++]);
if (!c) {
continue;
@@ -229,7 +242,11 @@ export async function compareMoments(
}
for (const m of wanted) {
if (!pictures.has(key(m.sourceAt))) {
- throw new Error(`Could not read "${asset.name}" at ${Math.round(m.sourceAt * 100) / 100}s.`);
+ const why = lost.find((l) => key(l.source) === key(m.sourceAt));
+ throw new Error(
+ `Could not read "${asset.name}" at ${Math.round(m.sourceAt * 100) / 100}s` +
+ (why ? `: ${why.error} ${READ_TRIES} fresh decoders, the last in software, failed on that frame.` : ".")
+ );
}
}
@@ -273,6 +290,85 @@ export async function compareMoments(
return { images, checked, ...(frame ? { frame } : {}) };
}
+/** A source frame the read gave up on: its second and what the decoder said. */
+type LostFrame = { source: number; error: string };
+
+/** The source stopped decoding: READ_LOST_RUN frames in a row failed fresh
+ * readers, the first of them at `source` seconds, the `index`th time asked. */
+class SourceReadError extends Error {
+ constructor(
+ readonly asset: MediaAsset,
+ readonly source: number,
+ readonly index: number,
+ readonly count: number,
+ readonly reason: string
+ ) {
+ super(
+ `"${asset.name}" stopped decoding at ${round3(source)}s (frame ${index + 1} of ${count} read, ${READ_LOST_RUN} in a row failing ` +
+ `${READ_TRIES} fresh decoders each, the last in software): ${reason}`
+ );
+ this.name = "SourceReadError";
+ }
+}
+
+/** What a failed read said, in the decoder's own words. */
+const reasonOf = (e: unknown) => (e instanceof Error ? e.message : String(e)) || "the decoder failed";
+
+/**
+ * The source's frames at `times`, one per time and in order, in one decode
+ * pass that outlives its decoder. A hardware decoder dies mid-read for reasons
+ * that pass — contention for the machine's decode sessions over a long
+ * session — and says only "Decoding error.", so a read that fails opens a
+ * fresh reader on software decoding at the frame it failed on and goes on
+ * from there. A frame that fails READ_TRIES readers is given up — null in its
+ * place and named in `lost` — and READ_LOST_RUN given up in a row end the
+ * read with a SourceReadError naming the first.
+ */
+async function* steadyFrames(
+ asset: MediaAsset,
+ times: number[],
+ size: FrameSize,
+ lost: LostFrame[]
+): AsyncGenerator<WrappedCanvas | null> {
+ let next = 0;
+ let software = false;
+ let tries = 0;
+ let run: { index: number; reason: string } | null = null;
+ while (next < times.length) {
+ const reader = framesAt(asset.url, times.slice(next), size, software ? { software: true } : undefined);
+ try {
+ for (;;) {
+ const step = await reader.next();
+ if (step.done) {
+ return;
+ }
+ next++;
+ tries = 0;
+ run = null;
+ yield step.value;
+ }
+ } catch (e) {
+ tries++;
+ software = true;
+ if (tries >= READ_TRIES) {
+ const reason = reasonOf(e);
+ run ??= { index: next, reason };
+ if (next - run.index + 1 >= READ_LOST_RUN) {
+ throw new SourceReadError(asset, times[run.index], run.index, times.length, run.reason);
+ }
+ lost.push({ source: round3(times[next]), error: reason });
+ next++;
+ tries = 0;
+ yield null;
+ } else {
+ await new Promise((r) => setTimeout(r, READ_RETRY_MS * tries));
+ }
+ } finally {
+ await reader.return(undefined).catch(() => {});
+ }
+ }
+}
+
/** Thrown inside the render loop when a sweep's time is up. */
class SweepOutOfTime extends Error {}
@@ -305,14 +401,20 @@ export type SweepResult = {
timing?: { shift: number; diff: number };
onBlocks?: number;
frame?: string;
+ /** Source frames no fresh decoder read, left out of the scores. */
+ lost?: LostFrame[];
+ /** Why the sweep ended at `coveredTo` short of its span, when the source
+ * stopped decoding there. */
+ stopped?: string;
};
/**
* The sweep: the cut scored against the source at every step of a span, with
* the source read at a constant offset, and no pictures. The source is read
- * in one pass in lockstep with the render, and each cut frame is also scored
- * against the source samples SHIFT_REACH steps either side, so a stretch that
- * is right but late says how late.
+ * first, in one pass down to the score's size, and its decoder let go before
+ * the cut renders, so no decoder sits open through the render; each cut frame
+ * is then scored against the source samples SHIFT_REACH steps either side, so
+ * a stretch that is right but late says how late.
*/
export async function sweepSource(
doc: ExportDoc,
@@ -346,40 +448,58 @@ export async function sweepSource(
const size = sizeOf(doc, SWEEP_LONG_SIDE);
const scorer = makeScorer(doc, area);
const width = SHIFT_REACH * 2 + 1;
- const near = new Float32Array(samples.length * width).fill(NaN);
+ const started = performance.now();
+ const outOfTime = (j: number) => j > 0 && performance.now() - started > SWEEP_BUDGET_MS;
+
+ // The source side, read whole before the render: a still answers every
+ // sample with its one frame; a video is one decode pass. A frame no decoder
+ // reads is null and drops out; a source that stops decoding ends the sweep
+ // at the first frame it lost.
+ const lost: LostFrame[] = [];
+ let stopped: SourceReadError | undefined;
+ const sources: (Uint8ClampedArray | null)[] = [];
+ const still = asset.type === "image" ? (await stillPixels(asset, scorer, area)).pixels : null;
+ if (still) {
+ sources.push(...samples.map(() => still));
+ } else {
+ try {
+ for await (const c of steadyFrames(asset, samples.map((s) => s.source), { height: readHeight(area, size.height) }, lost)) {
+ if (outOfTime(sources.length)) {
+ break;
+ }
+ sources.push(c ? scorer.pixels(c.canvas, c.canvas.width, c.canvas.height) : null);
+ }
+ } catch (e) {
+ if (!(e instanceof SourceReadError)) {
+ throw e;
+ }
+ stopped = e;
+ sources.length = Math.min(sources.length, e.index);
+ }
+ }
+
+ const near = new Float32Array(sources.length * width).fill(NaN);
const diffs: number[] = [];
const grids: number[][][] = [];
const cuts = new Map<number, Uint8ClampedArray>();
- const sources = new Map<number, Uint8ClampedArray | null>();
-
- // The source side: a still answers every sample with its one frame; a
- // video is pulled a frame at a time, in step with the render.
- const still = asset.type === "image" ? (await stillPixels(asset, scorer, area)).pixels : null;
- const reader = still ? null : framesAt(asset.url, samples.map((s) => s.source), { height: readHeight(area, size.height) });
- const started = performance.now();
try {
- await renderProjectFrames(doc, samples.map((s) => s.at), size, (a) => a.url, async (canvas) => {
+ await renderProjectFrames(doc, samples.slice(0, sources.length).map((s) => s.at), size, (a) => a.url, async (canvas) => {
const j = diffs.length;
- if (j > 0 && performance.now() - started > SWEEP_BUDGET_MS) {
+ if (outOfTime(j)) {
throw new SweepOutOfTime();
}
- let src = still;
- if (reader) {
- const next = await reader.next();
- const c = next.done ? null : next.value;
- src = c ? scorer.pixels(c.canvas, c.canvas.width, c.canvas.height) : null;
- }
+ const src = sources[j];
const cut = scorer.cut(canvas, canvas.width, canvas.height);
// The sample's own score, then the timing read: this cut frame against
// the source samples before it, and the cut frames before it against
- // this source sample. Only the last SHIFT_REACH of each are kept.
+ // this source sample. Only the last SHIFT_REACH cut frames are kept.
const own = src ? diffPixels(cut, src, scorer.size.width, scorer.size.height) : { diff: NaN, grid: [] };
diffs.push(own.diff);
grids.push(own.grid);
near[j * width + SHIFT_REACH] = own.diff;
for (let k = 1; k <= SHIFT_REACH && !still; k++) {
- const back = sources.get(j - k);
+ const back = j - k >= 0 ? sources[j - k] : null;
if (back) {
near[j * width + SHIFT_REACH - k] = meanDiff(cut, back);
}
@@ -389,22 +509,18 @@ export async function sweepSource(
}
}
cuts.set(j, cut);
- sources.set(j, src);
cuts.delete(j - SHIFT_REACH);
- sources.delete(j - SHIFT_REACH);
});
} catch (e) {
if (!(e instanceof SweepOutOfTime)) {
throw e;
}
- } finally {
- await reader?.return(undefined);
}
// What was scored: a sample the source had no frame for drops out.
const scored = diffs.flatMap((diff, i) => (Number.isNaN(diff) ? [] : [{ at: samples[i].at, diff, i }]));
if (scored.length === 0) {
- throw new Error(`Could not read "${asset.name}" over ${round3(from)}-${round3(to)}s.`);
+ throw new Error(`Could not read "${asset.name}" over ${round3(from)}-${round3(to)}s${stopped ? `: ${stopped.message}` : "."}`);
}
const stats = sweepStats(scored.map((s) => s.diff));
const worst: Stretch[] = worstStretches(scored, step, SWEEP_TARGET, MAX_STRETCHES).map((w) => {
@@ -441,5 +557,7 @@ export async function sweepSource(
...(timing ? { timing } : {}),
...(onBlocks > 0 ? { onBlocks: Math.round(onBlocks * 100) / 100 } : {}),
...(frame ? { frame } : {}),
+ ...(lost.length > 0 ? { lost } : {}),
+ ...(stopped ? { stopped: `${stopped.message} Cut ${samples[stopped.index].at}s.` } : {}),
};
}