Open source CapCut alternative.
TypeScript
feat(cut): score compare_to_source and sweep the cut
David Lec421a46da13f
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.tsindex e6488525..0ff12de0 100644--- a/site/src/cut/components/AiPanel.tools.ts+++ b/site/src/cut/components/AiPanel.tools.ts@@ -12,6 +12,7 @@ import { WATCH_DETAIL_NOTES, WATCH_DETAILS } from "@/cut/lib/types";import { FACT_CLAIM_MAX_CHARS, FACT_CLAIMS_MAX } from "@/lib/inference/factCheck";import { URL_BATCH_MAX } from "@/cut/lib/urlBatch";import { FIND_MEDIA_DEFAULT, FIND_MEDIA_MAX, FIND_MEDIA_SNAPSHOT_CAP } from "@/cut/lib/findMedia";+import { COMPARE_MODES, MAX_COMPARE, SWEEP_TARGET } from "@/cut/lib/frameDiff";export const AI_PANEL_TOOLS = [{@@ -141,13 +142,22 @@ export const AI_PANEL_TOOLS = [{name: "compare_to_source",description:- "Check the cut against the thing it was built from: renders your frame at each moment and reads the source's frame at the moment it stands for, handing both back side by side in one picture. This is how a replica is verified — watching the source tells you what it does, and this tells you whether what you made does it too. Pass `times` as timeline seconds, or as {at, source} pairs when the cut sits at a different second than the source does. Up to 4 moments a call; check, fix what is off, call again.",+ "Check the cut against the thing it was built from, in numbers and in pictures. Every frame is scored by `diff`: the mean difference per pixel between your frame and the source's, 0 (the same) to 255, read at one small size so a score means the same thing in every call; under " ++ `${SWEEP_TARGET} reads as the same picture. ` ++ "mode sweep scores every step of the cut (or from/to) with no pictures and returns the median, p95 and max with the worst stretches, each with its peak frame and a 4×4 grid of where in the frame it differs. " ++ `mode moments (the default) takes up to ${MAX_COMPARE} \`times\` and hands back each as the source's frame and yours side by side, with its score and grid. ` ++ "Either mode says when a frame matches the source better a few frames away: `shift` is that distance, and a positive one means move what is there later by that much. " ++ "This is how a replica is verified. The loop: sweep, look at the worst stretches' peaks in moments mode, fix what differs, sweep again — until the median is under " ++ `${SWEEP_TARGET} and no stretch stands out. ` ++ "Blocks standing in for footage keep a difference no edit removes, so read those by the grid cells over your titles and shapes. " ++ "Pass `times` as timeline seconds, or {at, source} pairs when the cut sits at a different second than the source does; `offset` maps every bare second and the whole sweep (source second = timeline second + offset).",inputSchema: obj({asset_id: str("Project asset id of the source the cut was built from"),+ mode: { type: "string", enum: [...COMPARE_MODES], description: "moments: pictures and scores at `times` (default); sweep: scores over a span, no pictures" },times: {type: "array",- description: "Moments to check — a timeline second, or {at, source} when the two clocks differ",+ description: `moments: up to ${MAX_COMPARE} moments to check — a timeline second, or {at, source} when the two clocks differ`,items: {anyOf: [{ type: "number" },@@ -155,8 +165,12 @@ export const AI_PANEL_TOOLS = [],},},+ offset: num("Source second minus timeline second, for every moment without its own source and for the sweep (default 0)"),+ from: num("sweep: timeline second to start at (default 0)"),+ to: num("sweep: timeline second to stop at (default the cut's end)"),+ step: num("sweep: seconds between samples (default one frame, widened so a long span fits the call)"),},- ["asset_id", "times"]+ ["asset_id"]),},{diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 4b863d46..8a07b4b5 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -190,7 +190,6 @@ import {importStockAudio,importStockVideo,importUrlMedia,- composeComparison,composeSheets,makeStillFrame,renderAudioSpanWav,@@ -233,10 +232,12 @@ import { loadLibraryLut, lutIdOf, lutLabel } from "./linkedLibrary";import { type RawFrame, type ReadGrade, rawFrameAt, sampleClipRawFrame, sourceProfileOf, toBaseRendering, toGradedRendering } from "./baseFrame";import { applyOverlayPatchSettled, assetClipUses, assetIdsOnAnyTimeline, clipLen, footprints, nextFreeStart, track0Clips, laneGapAt, getClipSpans, overlayLaneOrder, overlayLayers, parkedTransitions, projectDuration, resolveTransitions, totalDuration, useEditor } from "./store";import { playheadAt } from "./playhead";-import { mixSpecFor, renderProjectFrame, renderProjectFrames } from "./exportRender";+import { mixSpecFor, renderProjectFrame } from "./exportRender";import { renderMix } from "./audioMix";import { measureLoudnessSliced, roundedLoudness, type LoudnessMeasure } from "./loudness";-import { framesAt, readMediaFileSize } from "./mediaRead";+import { readMediaFileSize } from "./mediaRead";+import { COMPARE_MODES, MAX_COMPARE, SWEEP_TARGET } from "./frameDiff";+import { compareMoments, sweepSource } from "./sourceCompare";import { renderStageFrame, storeStageStill } from "./stageFrame";import { createRasterCanvas, decodeRasterImageUrl, rasterCanvasToDataUrl } from "./raster";import { buildAiContext, describeCameraCard, describeDoc, hiddenFromChat, OTHER_CHATS_MEDIA_NOTE, roundT } from "./aiContext";@@ -573,9 +574,19 @@ const CAPTURE_LONG_SIDE = 640;* undo step, and a whole feature's worth of them in a single step is not* something anyone can step back through; a caller with more comes back. */const MAX_BLOCKS = 40;-// Each moment is two frames drawn and one picture composed; a handful at a-// time keeps the call inside the bridge's deadline and the reply readable.-const MAX_COMPARE = 4;+/** What a comparison's reply teaches about its own numbers. */+const COMPARE_NOTE =+ "Source on the left, your cut on the right, one picture per moment. diff is how far your frame is from the source's: the mean difference per pixel, 0 (the same) to 255, and under " ++ `${SWEEP_TARGET} reads as the same picture. grid is that difference over the frame in 4×4 cells, top row first, so the hot cells are where to look. ` ++ "shift means the source matches your frame better that many seconds away: positive, your cut shows it early, so move what is there later by that much; negative, earlier. " ++ "block names a block standing in for footage, which keeps a difference no edit removes, so judge those moments by the cells over your titles and shapes. " ++ "Name what differs — colour, the words and their face, where a thing sits, its size, what is moving and how far along it is — fix it with the tools and check again. A moment that matches needs no edit.";+const SWEEP_NOTE =+ "diff is the mean difference per pixel between your frame and the source's, 0 (the same) to 255; median is the typical frame, p95 the bad tail, max the worst. " ++ "worst lists the stretches most off, worst first, each with its peak frame (at on the cut, source on the source) and that frame's 4×4 grid, top row first. " ++ "Look at them: compare_to_source with times [{at, source}] at those peaks, fix what differs, then sweep again, until the median is under target. " ++ "shift on a stretch, or timing for the whole span, means the source matches better that many seconds away: positive, move it later by that much; negative, earlier. " ++ "onBlocks is the share of samples over blocks standing in for footage; those keep a difference no edit removes.";const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));const isNum = (v: unknown): v is number => typeof v === "number" && Number.isFinite(v);@@ -1274,9 +1285,10 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {compare_to_source: async (s, input) => {// Building from a reference and looking at the reference are separate- // acts, and until they are put in one picture nothing checks the first- // against the second. This renders the cut at a moment, reads the source- // at the moment it stands for, and hands back the two side by side.+ // acts, and until they are put side by side nothing checks the first+ // against the second. The picture call renders the cut at a few+ // moments beside the source's frames and scores each pair; the sweep+ // scores a whole stretch with no pictures, to find what to look at.const asset = requireItem(s.assets, input.asset_id, "project asset");if (asset.type === "audio")throw new ToolError(`"${asset.name}" is audio — listen_audio hears it, and a frame is a picture.`);@@ -1284,97 +1296,68 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {// back the same opening frame under four different stamps.if (!(asset.duration > 0))throw new ToolError(`"${asset.name}" has no length yet — its import is still being read. Try again in a moment.`);+ if (input.mode !== undefined && !(COMPARE_MODES as readonly unknown[]).includes(input.mode)) {+ throw new ToolError(`mode is one of ${COMPARE_MODES.join(", ")}.`);+ }+ const offset = isNum(input.offset) ? input.offset : 0;+ const total = projectDuration(s);+ const doc = {+ aspect: s.aspect,+ assets: s.assets,+ clips: s.clips,+ audioClips: s.audioClips,+ overlays: s.overlays,+ subtitles: s.subtitles,+ background: s.background,+ };+ const failed = (fallback: string) => (e: unknown) => {+ throw new ToolError(e instanceof Error ? e.message : fallback, { cause: e });+ };++ // The sweep: numbers over a span, at a step, under a time budget.+ if (input.mode === "sweep") {+ const sweep = await sweepSource(+ doc,+ asset,+ {+ from: isNum(input.from) ? input.from : 0,+ to: isNum(input.to) ? input.to : total,+ step: isNum(input.step) ? input.step : undefined,+ offset,+ },+ total+ ).catch(failed(`Could not sweep "${asset.name}".`));+ return {+ ...sweep,+ source: sourceRef(asset),+ note:+ (sweep.coveredTo !== undefined+ ? `The sweep ran out of time at ${sweep.coveredTo}s; sweep again from there for the rest. `+ : "") + SWEEP_NOTE,+ };+ }++ // The picture call: a few moments, each a picture and a score.const raw = Array.isArray(input.times) ? input.times : [];const moments = raw.map((x) => {const rec = (x ?? {}) as Record<string, unknown>;const at = isNum(x) ? (x as number) : isNum(rec.at) ? (rec.at as number) : NaN;- const sourceAt = isNum(rec.source) ? (rec.source as number) : at;+ const sourceAt = isNum(rec.source) ? (rec.source as number) : at + offset;return { at, sourceAt };}).filter((m) => Number.isFinite(m.at) && Number.isFinite(m.sourceAt));if (moments.length === 0)- throw new ToolError("Pass times: [seconds] — the moments of the cut to check, or [{at, source}] when the cut sits at a different second than the source does.");+ throw new ToolError("Pass times: [seconds] — the moments of the cut to check, or [{at, source}] when the cut sits at a different second than the source does. To score the whole cut first, pass mode: \"sweep\".");if (moments.length > MAX_COMPARE)- throw new ToolError(`That is ${moments.length} moments — ${MAX_COMPARE} is the most one call reads. Check these, fix what is off, then call again for the rest.`);- const total = projectDuration(s);- const frame = frameOf(s.aspect);- const k = CAPTURE_LONG_SIDE / Math.max(frame.w, frame.h);- const size = { width: Math.round(frame.w * k), height: Math.round(frame.h * k) };- // The source keeps its own aspect: a height alone lets a widescreen- // reference stay widescreen beside a vertical cut, which is half of what- // the comparison is for. Asking for both dimensions squashes it.- const sourceSize = { height: size.height };- const wanted = moments.map((m) => ({- at: clamp(m.at, 0, Math.max(0, total - 0.001)),- sourceAt: clamp(m.sourceAt, 0, Math.max(0, asset.duration - 0.001)),- }));- // One decode pass over the source, in play order, however the moments- // were asked for.- const order = wanted.map((_, i) => i).sort((a, b) => wanted[a].sourceAt - wanted[b].sourceAt);- const refs = new Map<number, string>();- if (asset.type === "image") {- const still = await makeStillFrame(asset.url, "read").catch((e) => {- throw new ToolError(e instanceof Error ? e.message : `Could not read "${asset.name}".`, { cause: e });- });- for (const i of order) refs.set(i, still.frames[0].image);- } else {- let seen = 0;- try {- for await (const c of framesAt(- asset.url,- order.map((i) => wanted[i].sourceAt),- sourceSize- )) {- const i = order[seen++];- if (c) refs.set(i, await rasterCanvasToDataUrl(c.canvas, "image/jpeg", 0.82));- }- } catch (e) {- throw new ToolError(e instanceof Error ? e.message : `Could not read "${asset.name}".`, { cause: e });- }- }- for (let i = 0; i < wanted.length; i++)- if (!refs.has(i)) throw new ToolError(`Could not read "${asset.name}" at ${round2(wanted[i].sourceAt)}s.`);- // One prepared painter for every moment: preparing stamps every title- // and may bring up the segmenter, and the document does not change- // between them.- const pairs: { t: number; sourceAt: number; source: string; cut: string }[] = [];- await renderProjectFrames(- {- aspect: s.aspect,- assets: s.assets,- clips: s.clips,- audioClips: s.audioClips,- overlays: s.overlays,- subtitles: s.subtitles,- background: s.background,- },- wanted.map((m) => m.at),- size,- (a) => a.url,- async (canvas, at) => {- const i = pairs.length;- pairs.push({- t: roundT(at),- sourceAt: round2(wanted[i].sourceAt),- source: refs.get(i)!,- cut: await rasterCanvasToDataUrl(canvas, "image/jpeg", 0.82),- });- }- ).catch((e) => {- throw new ToolError(e instanceof Error ? e.message : "Could not draw the cut's frame.", { cause: e });- });- const images = await composeComparison(pairs).catch((e) => {- throw new ToolError(e instanceof Error ? e.message : "Could not compose the comparison.", { cause: e });- });+ throw new ToolError(`That is ${moments.length} moments — ${MAX_COMPARE} is the most one call reads. Check these, fix what is off, then call again for the rest; mode: "sweep" scores every frame without pictures.`);+ const out = await compareMoments(doc, asset, moments, total).catch(failed("Could not compare the frames."));return {- images,- checked: pairs.map((p) => ({ at: p.t, source: p.sourceAt })),+ ...out,// The same shape every look returns, so a record opened by a// comparison knows what the other senses may ask of this source.source: sourceRef(asset),- note:- "Source on the left, your cut on the right, one picture per moment. Name what differs — colour, the words and their face, where a thing sits, its size, what is moving and how far along it is — then fix it with the tools and check again. A moment that matches needs no edit.",+ note: COMPARE_NOTE,};},diff --git a/site/src/cut/lib/compareToSource.test.ts b/site/src/cut/lib/compareToSource.test.tsindex 441712d8..762724a5 100644--- a/site/src/cut/lib/compareToSource.test.ts+++ b/site/src/cut/lib/compareToSource.test.ts@@ -3,29 +3,41 @@ import { stubModule } from "@/lib/testing/stubModule";import { setRasterFactory, type RasterSurface } from "./raster";import { useEditor } from "./store";-// A surface that records nothing and draws nothing: this pins the shape of the-// reads and the result, and the pixels themselves are the render path's own-// tests. Every call the draw code makes answers.-const stubContext = new Proxy(- {- canvas: null as unknown,- measureText: () => ({ width: 10, actualBoundingBoxAscent: 8, actualBoundingBoxDescent: 2 }),- createLinearGradient: () => ({ addColorStop() {} }),- getImageData: (_x: number, _y: number, w: number, h: number) => ({- data: new Uint8ClampedArray(Math.max(1, w * h * 4)),- width: w,- height: h,- }),- } as Record<string, unknown>,- {- get: (target, prop) =>- prop in target ? target[prop as string] : typeof prop === "string" ? () => undefined : undefined,- set: (target, prop, value) => ((target[prop as string] = value), true),- }-);+// A surface that draws nothing but a flat grey level: this pins the shape of+// the reads and the result, and the pixels themselves are the render path's+// own tests. Every call the draw code makes answers; drawing one stub canvas+// onto another carries its level across, and reading pixels hands that level+// back, so a test can say what the source shows at each second and read the+// score it comes to.+type StubCanvas = { width: number; height: number; level: number; getContext: () => unknown };++const contextOf = (canvas: StubCanvas) =>+ new Proxy(+ {+ canvas,+ measureText: () => ({ width: 10, actualBoundingBoxAscent: 8, actualBoundingBoxDescent: 2 }),+ createLinearGradient: () => ({ addColorStop() {} }),+ drawImage: (src: { level?: number }) => {+ if (typeof src?.level === "number") canvas.level = src.level;+ },+ getImageData: (_x: number, _y: number, w: number, h: number) => {+ const data = new Uint8ClampedArray(Math.max(1, w * h * 4));+ for (let i = 0; i < data.length; i += 4) data.set([canvas.level, canvas.level, canvas.level, 255], i);+ return { data, width: w, height: h };+ },+ } as Record<string, unknown>,+ {+ get: (target, prop) =>+ prop in target ? target[prop as string] : typeof prop === "string" ? () => undefined : undefined,+ set: (target, prop, value) => ((target[prop as string] = value), true),+ }+ );-const stubCanvas = (w: number, h: number): RasterSurface =>- ({ width: Math.max(1, w), height: Math.max(1, h), getContext: () => stubContext }) as unknown as RasterSurface;+const stubCanvas = (w: number, h: number, level = 0): RasterSurface => {+ const canvas: StubCanvas = { width: Math.max(1, w), height: Math.max(1, h), level, getContext: () => ctx };+ const ctx = contextOf(canvas);+ return canvas as unknown as RasterSurface;+};setRasterFactory({createCanvas: stubCanvas,@@ -40,10 +52,14 @@ setRasterFactory({/** Every time the source was asked for frames, and at what size. */const reads: { times: number[]; size: unknown }[] = [];+/** The grey level the source shows at a second; the cut always draws 0. */+let sourceLevel: (t: number) => number = () => 0;+/** The source's own frame shape. */+let sourceShape = { w: 90, h: 160 };async function* framesAt(_src: string | Blob, times: number[], size?: unknown) {reads.push({ times: [...times], size });- for (let i = 0; i < times.length; i++) yield { canvas: stubCanvas(90, 160), timestamp: 0, duration: 0 } as never;+ for (const t of times) yield { canvas: stubCanvas(sourceShape.w, sourceShape.h, sourceLevel(t)), timestamp: t, duration: 0 } as never;}await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, { framesAt });@@ -52,7 +68,11 @@ const { runAiTool } = await import("./aiTools");beforeEach(() => {reads.length = 0;+ sourceLevel = () => 0;+ sourceShape = { w: 90, h: 160 };useEditor.setState({+ aspect: "9:16",+ background: "#000000",clips: [{ id: "c1", assetId: "block", track: 0, start: 0, in: 0, out: 10, muted: false }],overlays: [],transitions: [],@@ -84,9 +104,9 @@ test("a moment is required, because there is nothing to compare without one", astest("more moments than one call reads are turned away with what to do instead", async () => {const error = await message(- runAiTool("compare_to_source", { asset_id: "ref", times: [1, 2, 3, 4, 5] })+ runAiTool("compare_to_source", { asset_id: "ref", times: [1, 2, 3, 4, 5, 6, 7, 8, 9] }));- expect(error).toContain("4 is the most one call reads");+ expect(error).toContain("8 is the most one call reads");});test("a sound has no frame to hold up", async () => {@@ -106,7 +126,7 @@ test("a comparison comes back as pictures, one per moment", async () => {asset_id: "ref",times: [1, { at: 5, source: 20 }],})) as { images: string[]; checked: { at: number; source: number }[]; source: { sound: string } };- expect(out.checked).toEqual([+ expect(out.checked).toMatchObject([{ at: 1, source: 1 },{ at: 5, source: 20 },]);@@ -120,7 +140,11 @@ test("a comparison comes back as pictures, one per moment", async () => {test("the source is read in one pass, in play order, at a height that keeps its own shape", async () => {await runAiTool("compare_to_source", { asset_id: "ref", times: [{ at: 1, source: 20 }, { at: 5, source: 3 }] });expect(reads).toHaveLength(1);- expect(reads[0].times).toEqual([3, 20]);+ // Each moment brings the few frames either side of it, for the timing read.+ const times = reads[0].times;+ expect(times).toEqual([...times].sort((a, b) => a - b));+ expect(times.includes(3) && times.includes(20)).toBe(true);+ expect(times.every((t) => Math.abs(t - 3) <= 0.101 || Math.abs(t - 20) <= 0.101)).toBe(true);// Both dimensions would squash a widescreen reference into a vertical cut's// frame — and mediabunny's sink refuses the pair outright.expect(Object.keys(reads[0].size as object)).toEqual(["height"]);@@ -145,3 +169,145 @@ test("a cut built only from elements is checked at the moments asked for", asyncglobalThis.createImageBitmap = was;}});++type Checked = { at: number; source: number; diff: number; grid: number[][]; shift?: number; block?: string };++test("every moment comes back with a score and a grid of where it differs", async () => {+ sourceLevel = () => 40;+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1, 2] })) as { checked: Checked[] };+ expect(out.checked).toHaveLength(2);+ for (const c of out.checked) {+ expect(c.diff).toBe(40);+ expect(c.grid).toHaveLength(4);+ expect(c.grid.flat()).toEqual(Array(16).fill(40));+ }+});++test("eight moments fit in one call", async () => {+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1, 2, 3, 4, 5, 6, 7, 8] })) as {+ images: string[];+ };+ expect(out.images).toHaveLength(8);+});++test("a moment that matches a neighbouring source frame says how far to move it", async () => {+ // The source shows what the cut shows two frames after the moment asked for.+ sourceLevel = (t) => (Math.abs(t - (5 + 2 / 30)) < 0.005 ? 0 : 30);+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [5] })) as { checked: Checked[] };+ expect(out.checked[0].diff).toBe(30);+ expect(out.checked[0].shift).toBeCloseTo(0.067, 3);+});++test("a moment with no better neighbour names no shift", async () => {+ sourceLevel = () => 30;+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [5] })) as { checked: Checked[] };+ expect(out.checked[0].shift).toBeUndefined();+});++test("an offset maps every bare moment onto the source", async () => {+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1, { at: 2, source: 9 }], offset: 2 })) as {+ checked: Checked[];+ };+ expect(out.checked.map((c) => c.source)).toEqual([3, 9]);+});++test("a moment over a block says so, because a block stands in for footage", async () => {+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1] })) as { checked: Checked[] };+ expect(out.checked[0].block).toBe("Shot 1");+});++test("a source of another shape is scored letterboxed into the cut's frame, and says so", async () => {+ sourceShape = { w: 160, h: 90 };+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1] })) as { frame?: string };+ expect(out.frame).toContain("letterboxed");+});++type Sweep = {+ images?: unknown;+ sampled: number;+ step: number;+ from: number;+ to: number;+ median: number;+ p95: number;+ max: number;+ worst: { from: number; to: number; diff: number; peak: number; at: number; grid: number[][]; block?: string }[];+ coveredTo?: number;+ source: { assetId: string };+};++test("a sweep scores the span at a step and hands back numbers, no pictures", async () => {+ const out = (await runAiTool("compare_to_source", {+ asset_id: "ref",+ mode: "sweep",+ from: 0,+ to: 2,+ step: 0.5,+ })) as Sweep;+ expect(out.images).toBeUndefined();+ expect(out.sampled).toBe(4);+ expect(reads).toHaveLength(1);+ expect(reads[0].times).toEqual([0, 0.5, 1, 1.5]);+ expect([out.median, out.p95, out.max]).toEqual([0, 0, 0]);+ expect(out.worst).toEqual([]);+ expect(out.source.assetId).toBe("ref");+});++test("a sweep reads the source at a constant offset and skips what falls outside it", async () => {+ const out = (await runAiTool("compare_to_source", {+ asset_id: "ref",+ mode: "sweep",+ from: 0,+ to: 2,+ step: 0.5,+ offset: 32,+ })) as Sweep;+ // The source runs to 33.17s, so only the first three samples land in it.+ expect(reads[0].times).toEqual([32, 32.5, 33]);+ expect(out.sampled).toBe(3);+});++test("a sweep points at the stretch that is off, with its grid and its block", async () => {+ sourceLevel = (t) => (t >= 1 && t < 2 ? 50 : 0);+ const out = (await runAiTool("compare_to_source", {+ asset_id: "ref",+ mode: "sweep",+ from: 0,+ to: 4,+ step: 0.5,+ })) as Sweep;+ expect(out.median).toBe(0);+ expect(out.max).toBe(50);+ expect(out.worst).toHaveLength(1);+ expect(out.worst[0]).toMatchObject({ from: 1, to: 2, diff: 50, peak: 50, block: "Shot 1" });+ expect(out.worst[0].grid.flat()).toEqual(Array(16).fill(50));+});++test("a sweep's step never drops under a frame and widens to fit the sample cap", async () => {+ const fine = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 1, step: 0.001 })) as Sweep;+ expect(fine.step).toBeCloseTo(1 / 30, 3);+ expect(fine.sampled).toBe(30);+ const long = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep" })) as Sweep;+ expect(long.sampled).toBeLessThanOrEqual(900);+ expect(long.to).toBeCloseTo(10, 3);+});++test("a sweep that runs out of time says how far it reached", async () => {+ const was = performance.now;+ let calls = 0;+ performance.now = () => (calls++) * 20_000;+ try {+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 4, step: 0.5 })) as Sweep;+ expect(out.sampled).toBeLessThan(8);+ expect(out.coveredTo).toBeLessThan(4);+ } finally {+ performance.now = was;+ }+});++test("a sweep with nothing inside the source is turned away", async () => {+ const error = await message(+ runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 2, offset: 40 })+ );+ expect(error).toContain("offset");+});diff --git a/site/src/cut/lib/frameDiff.test.ts b/site/src/cut/lib/frameDiff.test.tsnew file mode 100644index 00000000..940528ac--- /dev/null+++ b/site/src/cut/lib/frameDiff.test.ts@@ -0,0 +1,122 @@+import { expect, test } from "bun:test";+import { bestShift, diffPixels, meanDiff, runShift, sweepStats, worstStretches } from "./frameDiff";++// The numbers a replica is judged by. A score is only useful to the model if+// it means the same thing every call: 0 for identical frames, 255 for black+// against white, and a grid that puts the difference where it is.++/** A w×h RGBA frame, every pixel `rgb`. */+const solid = (w: number, h: number, rgb: [number, number, number]) => {+ const px = new Uint8ClampedArray(w * h * 4);+ for (let i = 0; i < w * h; i++) px.set([...rgb, 255], i * 4);+ return px;+};++test("identical frames score 0 everywhere", () => {+ const a = solid(8, 8, [40, 90, 200]);+ const out = diffPixels(a, a.slice(), 8, 8);+ expect(out.diff).toBe(0);+ expect(out.grid).toEqual([+ [0, 0, 0, 0],+ [0, 0, 0, 0],+ [0, 0, 0, 0],+ [0, 0, 0, 0],+ ]);+});++test("the score is the mean channel difference on a 0-255 scale", () => {+ expect(diffPixels(solid(8, 8, [0, 0, 0]), solid(8, 8, [255, 255, 255]), 8, 8).diff).toBe(255);+ // Alpha is not part of the picture, so it never counts.+ expect(diffPixels(solid(4, 4, [10, 20, 30]), solid(4, 4, [20, 10, 60]), 4, 4).diff).toBeCloseTo(50 / 3, 5);+});++test("a difference in one corner shows in that corner's cell", () => {+ const a = solid(8, 8, [0, 0, 0]);+ const b = a.slice();+ // The top-right 2×2 block of an 8×8 frame is grid cell [0][3].+ for (const [x, y] of [[6, 0], [7, 0], [6, 1], [7, 1]]) b.set([120, 120, 120], (y * 8 + x) * 4);+ const out = diffPixels(a, b, 8, 8);+ expect(out.grid[0][3]).toBe(120);+ expect(out.grid[3][0]).toBe(0);+ expect(out.diff).toBeCloseTo(120 / 16, 5);+});++test("a grid cell still reads on a frame smaller than the grid's split would like", () => {+ const out = diffPixels(solid(6, 10, [0, 0, 0]), solid(6, 10, [9, 9, 9]), 6, 10);+ expect(out.grid.flat().every((v) => v === 9)).toBe(true);+});++test("meanDiff is diffPixels' score without the grid", () => {+ const a = solid(5, 5, [1, 2, 3]);+ const b = solid(5, 5, [4, 8, 12]);+ expect(meanDiff(a, b)).toBeCloseTo(diffPixels(a, b, 5, 5).diff, 5);+});++test("a shift is named only when it matches clearly better than the moment asked for", () => {+ // A static frame matches every neighbour equally: no timing to fix.+ expect(bestShift(12, [{ shift: -0.033, diff: 12 }, { shift: 0.033, diff: 11.8 }])).toBeUndefined();+ // A move that lands two frames late in the cut: the source two frames on is+ // a near match.+ expect(bestShift(14, [{ shift: -0.033, diff: 20 }, { shift: 0.067, diff: 1.5 }])).toEqual({+ shift: 0.067,+ diff: 1.5,+ });+ // A frame that already matches has nothing to gain.+ expect(bestShift(0.6, [{ shift: 0.033, diff: 0.1 }])).toBeUndefined();+});++test("sweep stats are the median, the 95th percentile and the worst", () => {+ const scores = Array.from({ length: 100 }, (_, i) => i + 1);+ expect(sweepStats(scores)).toEqual({ median: 50, p95: 95, max: 100 });+ expect(sweepStats([7])).toEqual({ median: 7, p95: 7, max: 7 });+});++test("worst stretches group neighbouring bad samples and rank by how much is wrong", () => {+ const step = 0.5;+ const diffs = [1, 1, 30, 32, 1, 1, 1, 9, 1, 1, 1, 60];+ const samples = diffs.map((diff, i) => ({ at: i * step, diff }));+ const out = worstStretches(samples, step, 5, 5);+ // One frame far off outweighs two a long way off, and both outweigh one+ // barely past the target.+ expect(out.map((s) => [s.from, s.to])).toEqual([+ [5.5, 6],+ [1, 2],+ [3.5, 4],+ ]);+ expect(out[1].diff).toBe(31);+ expect(out[1].peak).toBe(32);+ expect(out[1].peakAt).toBe(1.5);+ expect(out[1].peakIndex).toBe(3);+ expect([out[1].first, out[1].last]).toEqual([2, 3]);+});++test("a single dip between bad samples does not split a stretch", () => {+ const samples = [1, 20, 2, 20, 1, 1].map((diff, i) => ({ at: i, diff }));+ expect(worstStretches(samples, 1, 5, 5).map((s) => [s.from, s.to])).toEqual([[1, 4]]);+});++test("a cut that matches everywhere has no worst stretch", () => {+ const samples = [1, 2, 3, 4, 2].map((diff, i) => ({ at: i, diff }));+ expect(worstStretches(samples, 1, 5, 5)).toEqual([]);+});++test("a cut off by the same amount everywhere is one stretch, not noise", () => {+ const samples = [30, 30, 30, 30].map((diff, i) => ({ at: i, diff }));+ expect(worstStretches(samples, 1, 5, 5).map((s) => [s.from, s.to])).toEqual([[0, 4]]);+});++test("worst stretches stop at the limit", () => {+ const samples = [9, 1, 1, 9, 1, 1, 9, 1, 1, 9].map((diff, i) => ({ at: i, diff }));+ expect(worstStretches(samples, 1, 5, 2)).toHaveLength(2);+});++test("a run of sweep samples that each match the source a step later reads as one shift", () => {+ // Three samples, each scored against the source one step either side:+ // [k=-1, k=0, k=+1]. The first has no source sample before it.+ const near = new Float32Array([NaN, 30, 2, 40, 28, 1, 35, 32, 3]);+ expect(runShift(near, 1, 0, 2, 0.5)).toEqual({ shift: 0.5, diff: 2 });+ // Over the middle sample alone it still reads late; with no neighbours+ // at all there is nothing to say.+ expect(runShift(near, 1, 1, 1, 0.5)?.shift).toBe(0.5);+ expect(runShift(new Float32Array([NaN, 9, NaN]), 1, 0, 0, 0.5)).toBeUndefined();+});diff --git a/site/src/cut/lib/frameDiff.ts b/site/src/cut/lib/frameDiff.tsnew file mode 100644index 00000000..5700597a--- /dev/null+++ b/site/src/cut/lib/frameDiff.ts@@ -0,0 +1,223 @@+// How far a replica's frame is from the frame it copies, as numbers a model+// can act on. A picture of the two side by side shows that something is off;+// a score says how much, a grid says where, and a sweep of scores says when.+// Every score reads frames at the same small size, so a 4 in one call and a+// 4 in the next mean the same thing.++/** How compare_to_source reads: a few moments as pictures with their scores,+ * or a sweep of scores over a span with no pictures. */+export const COMPARE_MODES = ["moments", "sweep"] as const;++/** The most moments one picture call reads: two frames drawn and one picture+ * composed each, small enough to keep the reply readable. */+export const MAX_COMPARE = 8;++/** The score a replica aims for: under it, two frames read as the same+ * picture to the eye. */+export const SWEEP_TARGET = 5;++/** Long side of the frame a score reads. Small enough to sweep a cut frame by+ * frame, large enough that a title or a shape still moves the number. */+export const SCORE_LONG_SIDE = 96;++/** The grid's split per side: 4×4 cells, row 0 at the top. */+const GRID = 4;++/** How much better a neighbouring source frame has to match before the+ * difference is called timing: under three quarters of the moment's own+ * score, and at least this many points under it. */+const SHIFT_RATIO = 0.75;+const SHIFT_MIN_GAIN = 1;++/** A run of bad samples survives one good sample in its middle. */+const STRETCH_GAP = 1;++const round1 = (v: number) => Math.round(v * 10) / 10;++/** Mean absolute difference over R, G and B, 0 (identical) to 255. */+export function meanDiff(cut: Uint8ClampedArray, source: Uint8ClampedArray): number {+ let sum = 0;+ for (let i = 0; i < cut.length; i += 4) {+ sum += Math.abs(cut[i] - source[i]) + Math.abs(cut[i + 1] - source[i + 1]) + Math.abs(cut[i + 2] - source[i + 2]);+ }+ return cut.length > 0 ? sum / ((cut.length / 4) * 3) : 0;+}++/** The score and where it comes from: `grid[row][col]` is each cell's own+ * mean difference, rounded, so a title in the wrong place reads as two hot+ * cells and a wrong background as a warm grid. */+export function diffPixels(+ cut: Uint8ClampedArray,+ source: Uint8ClampedArray,+ width: number,+ height: number+): { diff: number; grid: number[][] } {+ const sums = new Float64Array(GRID * GRID);+ const counts = new Float64Array(GRID * GRID);+ let total = 0;++ // One pass: every pixel adds to the total and to its cell.+ for (let y = 0; y < height; y++) {+ const row = Math.min(GRID - 1, Math.floor((y * GRID) / height));+ for (let x = 0; x < width; x++) {+ const i = (y * width + x) * 4;+ const d =+ Math.abs(cut[i] - source[i]) + Math.abs(cut[i + 1] - source[i + 1]) + Math.abs(cut[i + 2] - source[i + 2]);+ const cell = row * GRID + Math.min(GRID - 1, Math.floor((x * GRID) / width));+ sums[cell] += d;+ counts[cell] += 3;+ total += d;+ }+ }+ const grid = Array.from({ length: GRID }, (_, r) =>+ Array.from({ length: GRID }, (_, c) => {+ const cell = r * GRID + c;+ return counts[cell] > 0 ? Math.round(sums[cell] / counts[cell]) : 0;+ })+ );+ return { diff: width * height > 0 ? total / (width * height * 3) : 0, grid };+}++/** The neighbouring source frame that matches clearly better than the moment+ * itself, if one does. `shift` is how far from the moment that frame sits, so+ * a positive shift means the cut shows early what the source shows later. */+export function bestShift(+ base: number,+ candidates: { shift: number; diff: number }[]+): { shift: number; diff: number } | undefined {+ let best: { shift: number; diff: number } | undefined;+ for (const c of candidates) {+ if (!best || c.diff < best.diff) {+ best = c;+ }+ }+ if (!best || best.diff >= base * SHIFT_RATIO || base - best.diff < SHIFT_MIN_GAIN) {+ return undefined;+ }+ return best;+}++/**+ * Whether a run of sweep samples matches the source better a few steps away.+ * `near` holds, per sample, its cut frame scored against the source sample k+ * steps off for k in -reach..reach (NaN where there is none), so a stretch+ * whose move lands a frame late reads as one shift for the whole stretch.+ */+export function runShift(+ near: Float32Array,+ reach: number,+ first: number,+ last: number,+ step: number+): { shift: number; diff: number } | undefined {+ const width = reach * 2 + 1;++ // The run's mean score at each offset, over the samples that have one.+ const meanAt = (k: number) => {+ let sum = 0;+ let n = 0;+ for (let i = first; i <= last; i++) {+ const v = near[i * width + k + reach];+ if (Number.isNaN(v)) {+ continue;+ }+ sum += v;+ n++;+ }+ return n > 0 ? sum / n : NaN;+ };+ const base = meanAt(0);+ const candidates: { shift: number; diff: number }[] = [];+ for (let k = -reach; k <= reach; k++) {+ const diff = meanAt(k);+ if (k === 0 || Number.isNaN(diff)) {+ continue;+ }+ candidates.push({ shift: Math.round(k * step * 1000) / 1000, diff: round1(diff) });+ }+ return Number.isNaN(base) ? undefined : bestShift(base, candidates);+}++/** Nearest-rank quantile of an ascending list. */+const quantile = (sorted: number[], p: number) => sorted[Math.max(0, Math.min(sorted.length - 1, Math.ceil(p * sorted.length) - 1))];++/** The shape of a sweep's scores: the typical frame, the bad tail, the worst. */+export function sweepStats(scores: number[]): { median: number; p95: number; max: number } {+ const sorted = [...scores].sort((a, b) => a - b);+ return {+ median: round1(quantile(sorted, 0.5)),+ p95: round1(quantile(sorted, 0.95)),+ max: round1(sorted[sorted.length - 1] ?? 0),+ };+}++/**+ * The stretches of a sweep most worth a look, worst first.+ *+ * A sample is bad when it is over the target and at least as bad as the+ * median, so a cut that is off by the same amount everywhere reads as one+ * long stretch and a mostly good cut points at its outliers. Neighbouring bad+ * samples join into one stretch, and stretches rank by how far over the floor+ * they run in total: one frame far off and a long run a little off both+ * surface. `to` is the end of the last bad sample's step.+ */+export function worstStretches(+ samples: { at: number; diff: number }[],+ step: number,+ target: number,+ limit: number+): {+ from: number;+ to: number;+ diff: number;+ peak: number;+ peakAt: number;+ peakIndex: number;+ first: number;+ last: number;+ weight: number;+}[] {+ const median = quantile(samples.map((s) => s.diff).sort((a, b) => a - b), 0.5) ?? 0;+ const floor = Math.max(target, median);+ const bad = (d: number) => d > target && d >= median;++ // Runs of bad samples, bridging gaps of up to STRETCH_GAP good ones.+ const runs: { first: number; last: number }[] = [];+ for (let i = 0; i < samples.length; i++) {+ if (!bad(samples[i].diff)) {+ continue;+ }+ const open = runs[runs.length - 1];+ if (open && i - open.last <= STRETCH_GAP + 1) {+ open.last = i;+ continue;+ }+ runs.push({ first: i, last: i });+ }++ // Each run measured over every sample inside it, the bridged ones too.+ const stretches = runs.map(({ first, last }) => {+ let sum = 0;+ let weight = 0;+ let peakIndex = first;+ for (let i = first; i <= last; i++) {+ sum += samples[i].diff;+ weight += Math.max(0, samples[i].diff - floor);+ if (samples[i].diff > samples[peakIndex].diff) {+ peakIndex = i;+ }+ }+ return {+ from: samples[first].at,+ to: samples[last].at + step,+ diff: round1(sum / (last - first + 1)),+ peak: round1(samples[peakIndex].diff),+ peakAt: samples[peakIndex].at,+ peakIndex,+ first,+ last,+ weight,+ };+ });+ return stretches.sort((a, b) => b.weight - a.weight).slice(0, limit);+}diff --git a/site/src/cut/lib/media.ts b/site/src/cut/lib/media.tsindex 31c4bf7c..a79fbe58 100644--- a/site/src/cut/lib/media.ts+++ b/site/src/cut/lib/media.ts@@ -1880,11 +1880,8 @@ export async function composeSheets(** The pairing is the point. Two images handed over separately are two things* to hold in mind at once; side by side in one picture, a colour that drifted,- * a card sitting too low or a move that peaks early is simply visible. Nothing- * here measures the difference — a reference is often a recording of another- * app, where a pixel comparison reads the chrome around the picture as the- * picture — so the composite is made for an eye to judge and the judging is- * left to whoever is looking.+ * a card sitting too low or a move that peaks early is simply visible. The+ * composite is for the eye; the score beside it comes from frameDiff.ts.*/export async function composeComparison(pairs: { t: number; sourceAt: number; source: string; cut: string }[]diff --git a/site/src/cut/lib/sourceCompare.ts b/site/src/cut/lib/sourceCompare.tsnew file mode 100644index 00000000..de11338e--- /dev/null+++ b/site/src/cut/lib/sourceCompare.ts@@ -0,0 +1,397 @@+import { renderProjectFrames } from "./exportRender";+import { bestShift, diffPixels, meanDiff, runShift, SCORE_LONG_SIDE, SWEEP_TARGET, sweepStats, worstStretches } from "./frameDiff";+import { composeComparison, makeStillFrame } from "./media";+import { framesAt } from "./mediaRead";+import { createRasterCanvas, decodeRasterImageUrl, rasterCanvasToDataUrl } from "./raster";+import type { ExportDoc } from "./renderSnapshot";+import { clipLen } from "./store";+import { DEFAULT_BACKGROUND, frameOf, type MediaAsset } from "./types";++// Holding a replica up against the video it copies, as pictures and as+// numbers. The pictures are for the eye; the numbers say how far off each+// frame is, where in the frame, and whether the difference is the right thing+// a few frames early or late. A sweep is the same score run over a whole+// stretch of the cut, so the model can find what is off before it looks.++/** Long side of each frame in a comparison picture. */+const COMPARE_LONG_SIDE = 480;++/** Long side a sweep renders at, twice the score's: titles drawn at this size+ * and shrunk read the way the picture call's do. */+const SWEEP_LONG_SIDE = SCORE_LONG_SIDE * 2;++/** One frame of the cut, the step a sweep never goes under. */+const FRAME = 1 / 30;++/** How many frames either side of a moment the timing read looks. */+const SHIFT_REACH = 3;++/** The most samples one sweep scores; a longer span widens its step. */+const MAX_SWEEP_SAMPLES = 900;++/** How long a sweep scores before it hands back what it has, inside the+ * bridge's two-minute deadline with room for the reply. */+const SWEEP_BUDGET_MS = 60_000;++/** How many of a sweep's worst stretches come back. */+const MAX_STRETCHES = 5;++/** Aspect ratios this close draw edge to edge; anything further letterboxes. */+const ASPECT_SLACK = 0.01;++const round1 = (v: number) => Math.round(v * 10) / 10;+const round3 = (v: number) => Math.round(v * 1000) / 1000;+const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));++/** A frame size with the cut's shape and the given long side. */+function sizeOf(doc: ExportDoc, longSide: number): { width: number; height: number } {+ const frame = frameOf(doc.aspect);+ const k = longSide / Math.max(frame.w, frame.h);+ return { width: Math.max(1, Math.round(frame.w * k)), height: Math.max(1, Math.round(frame.h * k)) };+}++/** The label of the block standing in for footage at `t`, if one is. */+function blockAt(doc: ExportDoc, t: number): string | undefined {+ for (const c of doc.clips) {+ if (t < c.start || t >= c.start + clipLen(c)) {+ continue;+ }+ const asset = doc.assets.find((a) => a.id === c.assetId);+ if (asset?.origin === "block") {+ return asset.block?.label || asset.name;+ }+ }+ return undefined;+}++/**+ * Reads any frame down to the score's size, in the cut's shape. A source of+ * another shape is letterboxed into it over the cut's background, the way a+ * clip of that shape would sit on the timeline, and the scorer remembers it+ * did so the result can say so.+ */+function makeScorer(doc: ExportDoc) {+ const size = sizeOf(doc, SCORE_LONG_SIDE);+ const canvas = createRasterCanvas(size.width, size.height);+ const ctx = canvas.getContext("2d", { willReadFrequently: true }) as CanvasRenderingContext2D | null;+ if (!ctx) {+ throw new Error("No 2d context to score frames on.");+ }+ ctx.imageSmoothingQuality = "high";+ let letterboxed: { w: number; h: number } | undefined;+ return {+ // Held here so the surface lives as long as its context: a server+ // canvas's context keeps only a weak hold on it.+ canvas,+ size,+ get letterboxed() {+ return letterboxed;+ },+ pixels(source: CanvasImageSource, w: number, h: number): Uint8ClampedArray {+ ctx.fillStyle = doc.background ?? DEFAULT_BACKGROUND;+ ctx.fillRect(0, 0, size.width, size.height);+ if (Math.abs(w / h / (size.width / size.height) - 1) <= ASPECT_SLACK) {+ ctx.drawImage(source, 0, 0, size.width, size.height);+ } else {+ letterboxed = { w, h };+ const k = Math.min(size.width / w, size.height / h);+ const dw = w * k;+ const dh = h * k;+ ctx.drawImage(source, (size.width - dw) / 2, (size.height - dh) / 2, dw, dh);+ }+ return ctx.getImageData(0, 0, size.width, size.height).data;+ },+ };+}++/** The line a result carries when the source was letterboxed to score it. */+function frameNote(doc: ExportDoc, shape: { w: number; h: number } | undefined): string | undefined {+ if (!shape) {+ return undefined;+ }+ return (+ `The source's frame is ${round1(shape.w / shape.h)}:1 and the cut's is ${doc.aspect}, so scores read the source letterboxed into the cut's frame over its background. ` ++ "Bars the cut does not have count as difference; set_aspect to the source's shape when the copy should match it."+ );+}++/** A still source's one frame, as the scorer reads it. */+async function stillPixels(asset: MediaAsset, scorer: ReturnType<typeof makeScorer>) {+ const still = await makeStillFrame(asset.url, "read");+ const img = await decodeRasterImageUrl(still.frames[0].image);+ if (!img) {+ throw new Error(`Could not read "${asset.name}".`);+ }+ return { image: still.frames[0].image, pixels: scorer.pixels(img.source, img.width, img.height) };+}++/** One moment checked: where it was read on each clock, the score, the grid,+ * and the timing read when there is one. */+type Checked = { at: number; source: number; diff: number; grid: number[][]; shift?: number; block?: string };++/**+ * The picture call: each moment rendered, its source frame read, both put+ * side by side in one picture and scored. Every source frame within+ * SHIFT_REACH frames of a moment comes along in the same decode pass, so a+ * frame that is right but early or late says by how much.+ */+export async function compareMoments(+ doc: ExportDoc,+ asset: MediaAsset,+ moments: { at: number; sourceAt: number }[],+ total: number+): Promise<{ images: string[]; checked: Checked[]; frame?: string }> {+ const size = sizeOf(doc, COMPARE_LONG_SIDE);+ const scorer = makeScorer(doc);+ const wanted = moments.map((m) => ({+ at: clamp(m.at, 0, Math.max(0, total - 0.001)),+ sourceAt: clamp(m.sourceAt, 0, Math.max(0, asset.duration - 0.001)),+ }));++ // Every source second this call needs, keyed to the millisecond: each+ // moment's own and its neighbours', read in one pass in play order.+ const key = (t: number) => Math.round(t * 1000);+ const timeAt = (m: { sourceAt: number }, k: number) => clamp(m.sourceAt + k * FRAME, 0, Math.max(0, asset.duration - 0.001));+ const centers = new Set(wanted.map((m) => key(m.sourceAt)));+ const pixels = new Map<number, Uint8ClampedArray>();+ const pictures = new Map<number, string>();+ if (asset.type === "image") {+ const still = await stillPixels(asset, scorer);+ for (const m of wanted) {+ pixels.set(key(m.sourceAt), still.pixels);+ pictures.set(key(m.sourceAt), still.image);+ }+ } else {+ const times = [...new Set(wanted.flatMap((m) => Array.from({ length: SHIFT_REACH * 2 + 1 }, (_, i) => key(timeAt(m, i - SHIFT_REACH)))))]+ .sort((a, b) => a - b)+ .map((k) => k / 1000);+ // The source keeps its own aspect: a height alone lets a widescreen+ // reference stay widescreen beside a vertical cut, which is half of what+ // the comparison is for. Asking for both dimensions squashes it.+ let seen = 0;+ for await (const c of framesAt(asset.url, times, { height: size.height })) {+ const k = key(times[seen++]);+ if (!c) {+ continue;+ }+ pixels.set(k, scorer.pixels(c.canvas, c.canvas.width, c.canvas.height));+ if (centers.has(k)) {+ pictures.set(k, await rasterCanvasToDataUrl(c.canvas, "image/jpeg", 0.82));+ }+ }+ }+ for (const m of wanted) {+ if (!pictures.has(key(m.sourceAt))) {+ throw new Error(`Could not read "${asset.name}" at ${Math.round(m.sourceAt * 100) / 100}s.`);+ }+ }++ // One prepared painter for every moment: preparing stamps every title+ // and may bring up the segmenter, and the document does not change+ // between them. Each frame is scored before the next overwrites it.+ const pairs: { t: number; sourceAt: number; source: string; cut: string }[] = [];+ const checked: Checked[] = [];+ await renderProjectFrames(doc, wanted.map((m) => m.at), size, (a) => a.url, async (canvas, at) => {+ const m = wanted[pairs.length];+ const cut = scorer.pixels(canvas, canvas.width, canvas.height);+ const { diff, grid } = diffPixels(cut, pixels.get(key(m.sourceAt))!, scorer.size.width, scorer.size.height);+ const near: { shift: number; diff: number }[] = [];+ for (let k = -SHIFT_REACH; k <= SHIFT_REACH; k++) {+ const t = timeAt(m, k);+ const px = pixels.get(key(t));+ if (k === 0 || !px || key(t) === key(m.sourceAt)) {+ continue;+ }+ near.push({ shift: round3(t - m.sourceAt), diff: round1(meanDiff(cut, px)) });+ }+ const shift = bestShift(diff, near)?.shift;+ const block = blockAt(doc, at);+ checked.push({+ at: round3(at),+ source: Math.round(m.sourceAt * 100) / 100,+ diff: round1(diff),+ grid,+ ...(shift !== undefined ? { shift } : {}),+ ...(block ? { block } : {}),+ });+ pairs.push({+ t: round3(at),+ sourceAt: Math.round(m.sourceAt * 100) / 100,+ source: pictures.get(key(m.sourceAt))!,+ cut: await rasterCanvasToDataUrl(canvas, "image/jpeg", 0.82),+ });+ });+ const images = await composeComparison(pairs);+ const frame = frameNote(doc, scorer.letterboxed);+ return { images, checked, ...(frame ? { frame } : {}) };+}++/** Thrown inside the render loop when a sweep's time is up. */+class SweepOutOfTime extends Error {}++/** One stretch of a sweep worth a look. */+type Stretch = {+ from: number;+ to: number;+ diff: number;+ peak: number;+ at: number;+ source: number;+ grid: number[][];+ shift?: number;+ block?: string;+};++export type SweepResult = {+ mode: "sweep";+ from: number;+ to: number;+ step: number;+ offset: number;+ sampled: number;+ coveredTo?: number;+ median: number;+ p95: number;+ max: number;+ target: number;+ worst: Stretch[];+ timing?: { shift: number; diff: number };+ onBlocks?: number;+ frame?: string;+};++/**+ * The sweep: the cut scored against the source at every step of a span, with+ * the source read at a constant offset, and no pictures. The source is read+ * in one pass in lockstep with the render, and each cut frame is also scored+ * against the source samples SHIFT_REACH steps either side, so a stretch that+ * is right but late says how late.+ */+export async function sweepSource(+ doc: ExportDoc,+ asset: MediaAsset,+ span: { from: number; to: number; step?: number; offset: number },+ total: number+): Promise<SweepResult> {+ const from = clamp(span.from, 0, total);+ const to = clamp(span.to, from, total);+ const step = Math.max(FRAME, span.step ?? FRAME, (to - from) / MAX_SWEEP_SAMPLES);++ // The samples, kept to those whose source second is inside the source: a+ // constant offset makes them one unbroken run.+ const count = Math.ceil((to - from) / step - 1e-6);+ const samples: { at: number; source: number }[] = [];+ for (let i = 0; i < count; i++) {+ const at = round3(from + i * step);+ const source = round3(at + span.offset);+ if (source >= 0 && source < asset.duration) {+ samples.push({ at, source });+ }+ }+ if (samples.length === 0) {+ throw new Error(+ `No second of ${round3(from)}-${round3(to)}s lands inside "${asset.name}" (0-${round3(asset.duration)}s) at offset ${span.offset}s. ` ++ "The offset is the source second minus the timeline second; pass one that puts the span inside the source."+ );+ }++ const size = sizeOf(doc, SWEEP_LONG_SIDE);+ const scorer = makeScorer(doc);+ const width = SHIFT_REACH * 2 + 1;+ const near = new Float32Array(samples.length * width).fill(NaN);+ const diffs: number[] = [];+ const grids: number[][][] = [];+ const cuts = new Map<number, Uint8ClampedArray>();+ const sources = new Map<number, Uint8ClampedArray | null>();++ // The source side: a still answers every sample with its one frame; a+ // video is pulled a frame at a time, in step with the render.+ const still = asset.type === "image" ? (await stillPixels(asset, scorer)).pixels : null;+ const reader = still ? null : framesAt(asset.url, samples.map((s) => s.source), { height: size.height });+ const started = performance.now();+ try {+ await renderProjectFrames(doc, samples.map((s) => s.at), size, (a) => a.url, async (canvas) => {+ const j = diffs.length;+ if (j > 0 && performance.now() - started > SWEEP_BUDGET_MS) {+ throw new SweepOutOfTime();+ }+ let src = still;+ if (reader) {+ const next = await reader.next();+ const c = next.done ? null : next.value;+ src = c ? scorer.pixels(c.canvas, c.canvas.width, c.canvas.height) : null;+ }+ const cut = scorer.pixels(canvas, canvas.width, canvas.height);++ // The sample's own score, then the timing read: this cut frame against+ // the source samples before it, and the cut frames before it against+ // this source sample. Only the last SHIFT_REACH of each are kept.+ const own = src ? diffPixels(cut, src, scorer.size.width, scorer.size.height) : { diff: NaN, grid: [] };+ diffs.push(own.diff);+ grids.push(own.grid);+ near[j * width + SHIFT_REACH] = own.diff;+ for (let k = 1; k <= SHIFT_REACH && !still; k++) {+ const back = sources.get(j - k);+ if (back) {+ near[j * width + SHIFT_REACH - k] = meanDiff(cut, back);+ }+ const before = cuts.get(j - k);+ if (src && before) {+ near[(j - k) * width + SHIFT_REACH + k] = meanDiff(before, src);+ }+ }+ cuts.set(j, cut);+ sources.set(j, src);+ cuts.delete(j - SHIFT_REACH);+ sources.delete(j - SHIFT_REACH);+ });+ } catch (e) {+ if (!(e instanceof SweepOutOfTime)) {+ throw e;+ }+ } finally {+ await reader?.return(undefined);+ }++ // What was scored: a sample the source had no frame for drops out.+ const scored = diffs.flatMap((diff, i) => (Number.isNaN(diff) ? [] : [{ at: samples[i].at, diff, i }]));+ if (scored.length === 0) {+ throw new Error(`Could not read "${asset.name}" over ${round3(from)}-${round3(to)}s.`);+ }+ const stats = sweepStats(scored.map((s) => s.diff));+ const worst: Stretch[] = worstStretches(scored, step, SWEEP_TARGET, MAX_STRETCHES).map((w) => {+ const peak = scored[w.peakIndex];+ const shift = still ? undefined : runShift(near, SHIFT_REACH, scored[w.first].i, scored[w.last].i, step);+ const block = blockAt(doc, peak.at);+ return {+ from: w.from,+ to: round3(Math.min(w.to, to)),+ diff: w.diff,+ peak: w.peak,+ at: peak.at,+ source: samples[peak.i].source,+ grid: grids[peak.i],+ ...(shift ? { shift: shift.shift } : {}),+ ...(block ? { block } : {}),+ };+ });+ const timing = still ? undefined : runShift(near, SHIFT_REACH, 0, diffs.length - 1, step);+ const onBlocks = scored.filter((s) => blockAt(doc, s.at)).length / scored.length;+ const frame = frameNote(doc, scorer.letterboxed);+ const reached = diffs.length < samples.length ? samples[diffs.length].at : undefined;+ return {+ mode: "sweep",+ from: samples[0].at,+ to: round3(Math.min(samples[samples.length - 1].at + step, to)),+ step: round3(step),+ offset: span.offset,+ sampled: scored.length,+ ...(reached !== undefined ? { coveredTo: reached } : {}),+ ...stats,+ target: SWEEP_TARGET,+ worst,+ ...(timing ? { timing } : {}),+ ...(onBlocks > 0 ? { onBlocks: Math.round(onBlocks * 100) / 100 } : {}),+ ...(frame ? { frame } : {}),+ };+}