Open source CapCut alternative.
TypeScript
fix(cut): keep compare_to_source reading through a lost decoder [rebuild]
David Leafdd520af96f
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.tsindex 66982e0a..d41aabb3 100644--- a/site/src/cut/components/AiPanel.tools.ts+++ b/site/src/cut/components/AiPanel.tools.ts@@ -188,6 +188,7 @@ export const AI_PANEL_TOOLS = ["Check the cut against the thing it was built from, in numbers and in pictures. Every frame is scored by `diff`: the mean difference per pixel between your frame and the source's, 0 (the same) to 255, read at one small size so a score means the same thing in every call; under " +`${SWEEP_TARGET} reads as the same picture. ` +"mode sweep scores every step of the cut (or from/to) with no pictures and returns the median, p95 and max with the worst stretches, each with its peak frame and a 4×4 grid of where in the frame it differs. " ++ "A decoder that fails mid-read is replaced and the read goes on: `lost` names source frames no decoder read, and a source that stops decoding ends the sweep at `coveredTo` with `stopped` saying where and why. " +`mode moments (the default) takes up to ${MAX_COMPARE} \`times\` and hands back each as the source's frame and yours side by side, with its score and grid. ` +"Either mode says when a frame matches the source better a few frames away: `shift` is that distance, and a positive one means move what is there later by that much. " +"This is how a replica is verified. The loop: sweep, look at the worst stretches' peaks in moments mode, fix what differs, sweep again — until the median is under " +diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 023201e9..1ac28299 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -1497,9 +1497,15 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {...pictureReport(area),source: sourceRef(asset),note:- (sweep.coveredTo !== undefined- ? `The sweep ran out of time at ${sweep.coveredTo}s; sweep again from there for the rest. `- : "") + SWEEP_NOTE,+ (sweep.stopped+ ? `The source stopped decoding, so the sweep ends at ${sweep.coveredTo}s: ${sweep.stopped} Sweep again from a little past it for the rest. `+ : sweep.coveredTo !== undefined+ ? `The sweep ran out of time at ${sweep.coveredTo}s; sweep again from there for the rest. `+ : "") ++ (sweep.lost+ ? `Source frames at ${sweep.lost.map((l) => `${l.source}s`).join(", ")} did not decode (${sweep.lost[0].error}) and are left out of the scores. `+ : "") ++ SWEEP_NOTE,};}diff --git a/site/src/cut/lib/compareToSource.test.ts b/site/src/cut/lib/compareToSource.test.tsindex 1f366241..fd137ae3 100644--- a/site/src/cut/lib/compareToSource.test.ts+++ b/site/src/cut/lib/compareToSource.test.ts@@ -57,9 +57,19 @@ let sourceLevel: (t: number) => number = () => 0;/** The source's own frame shape. */let sourceShape = { w: 90, h: 160 };-async function* framesAt(_src: string | Blob, times: number[], size?: unknown) {+/** Whether the decoder dies on the frame at a second: the way Chrome's+ * decoder fails ("Decoding error."), on a hardware read or on any. */+let dies: (t: number, software: boolean) => boolean = () => false;+/** Every read's decoder options, in order. */+const readOpts: unknown[] = [];++async function* framesAt(_src: string | Blob, times: number[], size?: unknown, opts?: { software?: boolean }) {reads.push({ times: [...times], size });- for (const t of times) yield { canvas: stubCanvas(sourceShape.w, sourceShape.h, sourceLevel(t)), timestamp: t, duration: 0 } as never;+ readOpts.push(opts);+ for (const t of times) {+ if (dies(t, !!opts?.software)) throw new DOMException("Decoding error.", "EncodingError");+ yield { canvas: stubCanvas(sourceShape.w, sourceShape.h, sourceLevel(t)), timestamp: t, duration: 0 } as never;+ }}await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, { framesAt });@@ -68,6 +78,8 @@ const { runAiTool } = await import("./aiTools");beforeEach(() => {reads.length = 0;+ readOpts.length = 0;+ dies = () => false;sourceLevel = () => 0;sourceShape = { w: 90, h: 160 };useEditor.setState({@@ -340,3 +352,48 @@ test("a sweep with nothing inside the source is turned away", async () => {);expect(error).toContain("offset");});++type Lost = { lost?: { source: number; error: string }[]; stopped?: string; note: string };++test("a sweep whose decoder dies mid-read opens a fresh one where it died and finishes", async () => {+ dies = (t, software) => t === 1 && !software;+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 2, step: 0.5 })) as Sweep & Lost;+ expect(out.sampled).toBe(4);+ expect(out.coveredTo).toBeUndefined();+ expect(out.lost).toBeUndefined();+ // The second read picks up at the frame that failed, on software decoding.+ expect(reads.map((r) => r.times)).toEqual([[0, 0.5, 1, 1.5], [1, 1.5]]);+ expect(readOpts[1]).toMatchObject({ software: true });+});++test("a frame no fresh decoder reads is left out of the sweep and named, and the rest is scored", async () => {+ dies = (t) => t === 1;+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 2, step: 0.5 })) as Sweep & Lost;+ expect(out.sampled).toBe(3);+ expect(out.lost).toEqual([{ source: 1, error: "Decoding error." }]);+ expect(out.note).toContain("1s");+});++test("a source that stops decoding ends the sweep where it stopped, saying where and why", async () => {+ dies = (t) => t >= 1;+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 3, step: 0.5 })) as Sweep & Lost;+ expect(out.sampled).toBe(2);+ expect(out.coveredTo).toBe(1);+ expect(out.stopped).toContain("Decoding error.");+ expect(out.stopped).toContain("1s");+ expect(out.note).toContain("Decoding error.");+});++test("a picture call whose decoder dies reads on with a fresh one", async () => {+ dies = (t, software) => t > 2 && !software;+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1, 3] })) as { checked: unknown[] };+ expect(out.checked).toHaveLength(2);+ expect(readOpts.at(-1)).toMatchObject({ software: true });+});++test("a moment no decoder reads names the second and the decoder's own words", async () => {+ dies = (t) => t === 3;+ const error = await message(runAiTool("compare_to_source", { asset_id: "ref", times: [1, 3] }));+ expect(error).toContain("3s");+ expect(error).toContain("Decoding error.");+});diff --git a/site/src/cut/lib/mediaRead.ts b/site/src/cut/lib/mediaRead.tsindex e4f74594..e20b3270 100644--- a/site/src/cut/lib/mediaRead.ts+++ b/site/src/cut/lib/mediaRead.ts@@ -637,13 +637,14 @@ function capToTrack(track: InputVideoTrack, size?: FrameSize): FrameSize | undefexport async function* framesAt(src: string | Blob,times: number[],- size?: FrameSize+ size?: FrameSize,+ opts?: FrameSinkOptions): AsyncGenerator<WrappedCanvas | null> {const input = openMedia(src);try {const track = await videoTrackOf(input);if (!track) throw new UnreadableMediaError("This file has no readable video.");- yield* frameSink(track, size).canvasesAtTimestamps(times);+ yield* frameSink(track, size, opts).canvasesAtTimestamps(times);} finally {input.dispose();}diff --git a/site/src/cut/lib/sourceCompare.ts b/site/src/cut/lib/sourceCompare.tsindex 916f3145..68ce9d6f 100644--- a/site/src/cut/lib/sourceCompare.ts+++ b/site/src/cut/lib/sourceCompare.ts@@ -3,7 +3,8 @@ import { FULL_FRAME, isFullFrame } from "./measure/picture";import type { FrameBox } from "./measure/sourceMeasure";import { bestShift, diffPixels, meanDiff, runShift, SCORE_LONG_SIDE, SWEEP_TARGET, sweepStats, worstStretches } from "./frameDiff";import { composeComparison, makeStillFrame } from "./media";-import { framesAt } from "./mediaRead";+import type { WrappedCanvas } from "mediabunny";+import { framesAt, type FrameSize } from "./mediaRead";import { createRasterCanvas, decodeRasterImageUrl, rasterCanvasToDataUrl, type RasterSurface } from "./raster";import type { ExportDoc } from "./renderSnapshot";import { clipLen } from "./store";@@ -38,6 +39,17 @@ const SWEEP_BUDGET_MS = 60_000;/** How many of a sweep's worst stretches come back. */const MAX_STRETCHES = 5;+/** Readers a source frame is tried on — the one that failed, then fresh ones+ * on software decoding — before the frame is given up. */+const READ_TRIES = 3;++/** Frames given up in a row before the source counts as unreadable from there. */+const READ_LOST_RUN = 3;++/** The breath before a fresh reader, per try: a decoder lost to contention+ * comes back once the contention passes. */+const READ_RETRY_MS = 100;+/** Aspect ratios this close draw edge to edge; anything further letterboxes. */const ASPECT_SLACK = 0.01;@@ -200,6 +212,7 @@ export async function compareMoments(const centers = new Set(wanted.map((m) => key(m.sourceAt)));const pixels = new Map<number, Uint8ClampedArray>();const pictures = new Map<number, string>();+ const lost: LostFrame[] = [];if (asset.type === "image") {const still = await stillPixels(asset, scorer, area);for (const m of wanted) {@@ -216,7 +229,7 @@ export async function compareMoments(// height is its picture box's, so a picture inside a bigger frame is// read at the size the cut's frame is.let seen = 0;- for await (const c of framesAt(asset.url, times, { height: readHeight(area, size.height) })) {+ for await (const c of steadyFrames(asset, times, { height: readHeight(area, size.height) }, lost)) {const k = key(times[seen++]);if (!c) {continue;@@ -229,7 +242,11 @@ export async function compareMoments(}for (const m of wanted) {if (!pictures.has(key(m.sourceAt))) {- throw new Error(`Could not read "${asset.name}" at ${Math.round(m.sourceAt * 100) / 100}s.`);+ const why = lost.find((l) => key(l.source) === key(m.sourceAt));+ throw new Error(+ `Could not read "${asset.name}" at ${Math.round(m.sourceAt * 100) / 100}s` ++ (why ? `: ${why.error} ${READ_TRIES} fresh decoders, the last in software, failed on that frame.` : ".")+ );}}@@ -273,6 +290,85 @@ export async function compareMoments(return { images, checked, ...(frame ? { frame } : {}) };}+/** A source frame the read gave up on: its second and what the decoder said. */+type LostFrame = { source: number; error: string };++/** The source stopped decoding: READ_LOST_RUN frames in a row failed fresh+ * readers, the first of them at `source` seconds, the `index`th time asked. */+class SourceReadError extends Error {+ constructor(+ readonly asset: MediaAsset,+ readonly source: number,+ readonly index: number,+ readonly count: number,+ readonly reason: string+ ) {+ super(+ `"${asset.name}" stopped decoding at ${round3(source)}s (frame ${index + 1} of ${count} read, ${READ_LOST_RUN} in a row failing ` ++ `${READ_TRIES} fresh decoders each, the last in software): ${reason}`+ );+ this.name = "SourceReadError";+ }+}++/** What a failed read said, in the decoder's own words. */+const reasonOf = (e: unknown) => (e instanceof Error ? e.message : String(e)) || "the decoder failed";++/**+ * The source's frames at `times`, one per time and in order, in one decode+ * pass that outlives its decoder. A hardware decoder dies mid-read for reasons+ * that pass — contention for the machine's decode sessions over a long+ * session — and says only "Decoding error.", so a read that fails opens a+ * fresh reader on software decoding at the frame it failed on and goes on+ * from there. A frame that fails READ_TRIES readers is given up — null in its+ * place and named in `lost` — and READ_LOST_RUN given up in a row end the+ * read with a SourceReadError naming the first.+ */+async function* steadyFrames(+ asset: MediaAsset,+ times: number[],+ size: FrameSize,+ lost: LostFrame[]+): AsyncGenerator<WrappedCanvas | null> {+ let next = 0;+ let software = false;+ let tries = 0;+ let run: { index: number; reason: string } | null = null;+ while (next < times.length) {+ const reader = framesAt(asset.url, times.slice(next), size, software ? { software: true } : undefined);+ try {+ for (;;) {+ const step = await reader.next();+ if (step.done) {+ return;+ }+ next++;+ tries = 0;+ run = null;+ yield step.value;+ }+ } catch (e) {+ tries++;+ software = true;+ if (tries >= READ_TRIES) {+ const reason = reasonOf(e);+ run ??= { index: next, reason };+ if (next - run.index + 1 >= READ_LOST_RUN) {+ throw new SourceReadError(asset, times[run.index], run.index, times.length, run.reason);+ }+ lost.push({ source: round3(times[next]), error: reason });+ next++;+ tries = 0;+ yield null;+ } else {+ await new Promise((r) => setTimeout(r, READ_RETRY_MS * tries));+ }+ } finally {+ await reader.return(undefined).catch(() => {});+ }+ }+}+/** Thrown inside the render loop when a sweep's time is up. */class SweepOutOfTime extends Error {}@@ -305,14 +401,20 @@ export type SweepResult = {timing?: { shift: number; diff: number };onBlocks?: number;frame?: string;+ /** Source frames no fresh decoder read, left out of the scores. */+ lost?: LostFrame[];+ /** Why the sweep ended at `coveredTo` short of its span, when the source+ * stopped decoding there. */+ stopped?: string;};/*** The sweep: the cut scored against the source at every step of a span, with* the source read at a constant offset, and no pictures. The source is read- * in one pass in lockstep with the render, and each cut frame is also scored- * against the source samples SHIFT_REACH steps either side, so a stretch that- * is right but late says how late.+ * first, in one pass down to the score's size, and its decoder let go before+ * the cut renders, so no decoder sits open through the render; each cut frame+ * is then scored against the source samples SHIFT_REACH steps either side, so+ * a stretch that is right but late says how late.*/export async function sweepSource(doc: ExportDoc,@@ -346,40 +448,58 @@ export async function sweepSource(const size = sizeOf(doc, SWEEP_LONG_SIDE);const scorer = makeScorer(doc, area);const width = SHIFT_REACH * 2 + 1;- const near = new Float32Array(samples.length * width).fill(NaN);+ const started = performance.now();+ const outOfTime = (j: number) => j > 0 && performance.now() - started > SWEEP_BUDGET_MS;++ // The source side, read whole before the render: a still answers every+ // sample with its one frame; a video is one decode pass. A frame no decoder+ // reads is null and drops out; a source that stops decoding ends the sweep+ // at the first frame it lost.+ const lost: LostFrame[] = [];+ let stopped: SourceReadError | undefined;+ const sources: (Uint8ClampedArray | null)[] = [];+ const still = asset.type === "image" ? (await stillPixels(asset, scorer, area)).pixels : null;+ if (still) {+ sources.push(...samples.map(() => still));+ } else {+ try {+ for await (const c of steadyFrames(asset, samples.map((s) => s.source), { height: readHeight(area, size.height) }, lost)) {+ if (outOfTime(sources.length)) {+ break;+ }+ sources.push(c ? scorer.pixels(c.canvas, c.canvas.width, c.canvas.height) : null);+ }+ } catch (e) {+ if (!(e instanceof SourceReadError)) {+ throw e;+ }+ stopped = e;+ sources.length = Math.min(sources.length, e.index);+ }+ }++ const near = new Float32Array(sources.length * width).fill(NaN);const diffs: number[] = [];const grids: number[][][] = [];const cuts = new Map<number, Uint8ClampedArray>();- const sources = new Map<number, Uint8ClampedArray | null>();-- // The source side: a still answers every sample with its one frame; a- // video is pulled a frame at a time, in step with the render.- const still = asset.type === "image" ? (await stillPixels(asset, scorer, area)).pixels : null;- const reader = still ? null : framesAt(asset.url, samples.map((s) => s.source), { height: readHeight(area, size.height) });- const started = performance.now();try {- await renderProjectFrames(doc, samples.map((s) => s.at), size, (a) => a.url, async (canvas) => {+ await renderProjectFrames(doc, samples.slice(0, sources.length).map((s) => s.at), size, (a) => a.url, async (canvas) => {const j = diffs.length;- if (j > 0 && performance.now() - started > SWEEP_BUDGET_MS) {+ if (outOfTime(j)) {throw new SweepOutOfTime();}- let src = still;- if (reader) {- const next = await reader.next();- const c = next.done ? null : next.value;- src = c ? scorer.pixels(c.canvas, c.canvas.width, c.canvas.height) : null;- }+ const src = sources[j];const cut = scorer.cut(canvas, canvas.width, canvas.height);// The sample's own score, then the timing read: this cut frame against// the source samples before it, and the cut frames before it against- // this source sample. Only the last SHIFT_REACH of each are kept.+ // this source sample. Only the last SHIFT_REACH cut frames are kept.const own = src ? diffPixels(cut, src, scorer.size.width, scorer.size.height) : { diff: NaN, grid: [] };diffs.push(own.diff);grids.push(own.grid);near[j * width + SHIFT_REACH] = own.diff;for (let k = 1; k <= SHIFT_REACH && !still; k++) {- const back = sources.get(j - k);+ const back = j - k >= 0 ? sources[j - k] : null;if (back) {near[j * width + SHIFT_REACH - k] = meanDiff(cut, back);}@@ -389,22 +509,18 @@ export async function sweepSource(}}cuts.set(j, cut);- sources.set(j, src);cuts.delete(j - SHIFT_REACH);- sources.delete(j - SHIFT_REACH);});} catch (e) {if (!(e instanceof SweepOutOfTime)) {throw e;}- } finally {- await reader?.return(undefined);}// What was scored: a sample the source had no frame for drops out.const scored = diffs.flatMap((diff, i) => (Number.isNaN(diff) ? [] : [{ at: samples[i].at, diff, i }]));if (scored.length === 0) {- throw new Error(`Could not read "${asset.name}" over ${round3(from)}-${round3(to)}s.`);+ throw new Error(`Could not read "${asset.name}" over ${round3(from)}-${round3(to)}s${stopped ? `: ${stopped.message}` : "."}`);}const stats = sweepStats(scored.map((s) => s.diff));const worst: Stretch[] = worstStretches(scored, step, SWEEP_TARGET, MAX_STRETCHES).map((w) => {@@ -441,5 +557,7 @@ export async function sweepSource(...(timing ? { timing } : {}),...(onBlocks > 0 ? { onBlocks: Math.round(onBlocks * 100) / 100 } : {}),...(frame ? { frame } : {}),+ ...(lost.length > 0 ? { lost } : {}),+ ...(stopped ? { stopped: `${stopped.message} Cut ${samples[stopped.index].at}s.` } : {}),};}