Open source CapCut alternative.
TypeScript
feat(site): let compare_to_source fill the cut's frame with a source of another shape
David Lebabe2b13a7bb
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.tsindex 7c03c2d3..eb35d4d2 100644--- a/site/src/cut/components/AiPanel.tools.ts+++ b/site/src/cut/components/AiPanel.tools.ts@@ -228,6 +228,12 @@ export const AI_PANEL_TOOLS = [from: num("sweep: timeline second to start at (default 0)"),to: num("sweep: timeline second to stop at (default the cut's end)"),step: num("sweep: seconds between samples (default one frame, widened so a long span fits the call)"),+ fit: {+ type: "string",+ enum: ["letterbox", "fill"],+ description:+ "How a source of another shape sits in the cut's frame to be scored. letterbox (the default): whole, over the background, the way a clip of that shape sits on the timeline. fill: scaled to cover the frame and centre-cropped, for a copy that fills its own frame edge to edge in another shape — what the crop cuts off is not checked, and type and shapes read at the fill's scale",+ },picture: pictureArg,},["asset_id"]diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.tsindex 562198ba..8ca3a0ba 100644--- a/site/src/cut/lib/aiTools.ts+++ b/site/src/cut/lib/aiTools.ts@@ -1743,6 +1743,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {};// Only the source's picture box is held against the cut's frame.const area = pictureArea(asset, input);+ // A source of another shape is letterboxed unless the call asks it to fill.+ const fit = input.fit === "fill" ? "fill" : "letterbox";// The sweep: numbers over a span, at a step, under a time budget.if (input.mode === "sweep") {@@ -1756,7 +1758,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {offset,},total,- area+ area,+ fit).catch(failed(`Could not sweep "${asset.name}".`));return {...sweep,@@ -1790,7 +1793,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {throw new ToolError("Pass times: [seconds] — the moments of the cut to check, or [{at, source}] when the cut sits at a different second than the source does. To score the whole cut first, pass mode: \"sweep\".");if (moments.length > MAX_COMPARE)throw new ToolError(`That is ${moments.length} moments — ${MAX_COMPARE} is the most one call reads. Check these, fix what is off, then call again for the rest; mode: "sweep" scores every frame without pictures.`);- const out = await compareMoments(doc, asset, moments, total, area).catch(failed("Could not compare the frames."));+ const out = await compareMoments(doc, asset, moments, total, area, fit).catch(failed("Could not compare the frames."));return {...out,...pictureReport(area),diff --git a/site/src/cut/lib/compareToSource.test.ts b/site/src/cut/lib/compareToSource.test.tsindex 0459cf0c..a6e9384f 100644--- a/site/src/cut/lib/compareToSource.test.ts+++ b/site/src/cut/lib/compareToSource.test.ts@@ -17,8 +17,10 @@ const contextOf = (canvas: StubCanvas) =>canvas,measureText: () => ({ width: 10, actualBoundingBoxAscent: 8, actualBoundingBoxDescent: 2 }),createLinearGradient: () => ({ addColorStop() {} }),- drawImage: (src: { level?: number }) => {+ drawImage: (src: { level?: number }, ...at: number[]) => {if (typeof src?.level === "number") canvas.level = src.level;+ // The destination box of a 9-argument draw, the one that places a frame.+ if (at.length === 8) draws.push({ into: { w: canvas.width, h: canvas.height }, x: at[4], y: at[5], w: at[6], h: at[7] });},getImageData: (_x: number, _y: number, w: number, h: number) => {const data = new Uint8ClampedArray(Math.max(1, w * h * 4));@@ -46,6 +48,9 @@ setRasterFactory({snapshot: async (canvas) => canvas as unknown as ImageBitmap,});+/** Where frames were drawn onto the score's surface, newest last. */+const draws: { into: { w: number; h: number }; x: number; y: number; w: number; h: number }[] = [];+// Checking a replica against its source reads two pictures per moment, so what// it accepts is worth pinning: the moments it is given, the ones it turns away// before drawing anything, and the shape of the read it makes of the source.@@ -79,6 +84,7 @@ const { runAiTool } = await import("./aiTools");beforeEach(() => {reads.length = 0;readOpts.length = 0;+ draws.length = 0;dies = () => false;sourceLevel = () => 0;sourceShape = { w: 90, h: 160 };@@ -234,6 +240,33 @@ test("a source of another shape is scored letterboxed into the cut's frame, andexpect(out.frame).toContain("letterboxed");});+test("a source of another shape fills the cut's frame when asked, centre-cropped, and says so", async () => {+ // A vertical reference rebuilt widescreen out of full-bleed colour: held+ // letterboxed, the bars score as difference on every frame and hide whether+ // the colour matches.+ useEditor.setState({ aspect: "16:9" });+ const boxed = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1] })) as { frame?: string };+ expect(boxed.frame).toContain('fit "fill"');++ draws.length = 0;+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1], fit: "fill" })) as { frame?: string };+ expect(out.frame).toContain("fill");+ expect(out.frame).not.toContain("letterboxed");+ // The source covers the surface it is scored on: as wide as it, taller,+ // and centred, so its top and bottom run off the edges.+ const placed = draws.filter((d) => d.h > d.into.h);+ expect(placed.length).toBeGreaterThan(0);+ for (const d of placed) {+ expect(d.w).toBeCloseTo(d.into.w, 0);+ expect(d.y).toBeCloseTo((d.into.h - d.h) / 2, 0);+ }++ const sweep = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 1, step: 0.5, fit: "fill" })) as {+ frame?: string;+ };+ expect(sweep.frame).toContain("fill");+});+test("a recording's picture box is what is held against the cut, read at the cut's size", async () => {// A 9:16 phone recording with a 16:9 picture across its middle, against a// 16:9 cut: the box is the cut's shape, so nothing is letterboxed.diff --git a/site/src/cut/lib/sourceCompare.ts b/site/src/cut/lib/sourceCompare.tsindex 35337299..a518b23b 100644--- a/site/src/cut/lib/sourceCompare.ts+++ b/site/src/cut/lib/sourceCompare.ts@@ -50,9 +50,14 @@ const READ_LOST_RUN = 3;* comes back once the contention passes. */const READ_RETRY_MS = 100;-/** Aspect ratios this close draw edge to edge; anything further letterboxes. */+/** Aspect ratios this close draw edge to edge; anything further letterboxes+ * or fills, as the call asks. */const ASPECT_SLACK = 0.01;+/** How a source of another shape sits in the cut's frame to be scored:+ * letterboxed whole over the background, or filling it, centre-cropped. */+export type SourceFit = "letterbox" | "fill";+const round1 = (v: number) => Math.round(v * 10) / 10;const round3 = (v: number) => Math.round(v * 1000) / 1000;const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));@@ -110,10 +115,12 @@ const readHeight = (area: FrameBox, height: number) => Math.round(height / area.* Reads any frame down to the score's size, in the cut's shape. Only the* source's picture box is read — the part of its frame the cut copies. A* picture of another shape is letterboxed into the cut's frame over its- * background, the way a clip of that shape would sit on the timeline, and the- * scorer remembers it did so the result can say so.+ * background, the way a clip of that shape would sit on the timeline — or,+ * with `fit` "fill", scaled to cover it and centre-cropped, the way a+ * full-bleed replica in another shape fills its frame — and the scorer+ * remembers it did so the result can say so.*/-function makeScorer(doc: ExportDoc, area: FrameBox) {+function makeScorer(doc: ExportDoc, area: FrameBox, fit: SourceFit) {const size = sizeOf(doc, SCORE_LONG_SIDE);const canvas = createRasterCanvas(size.width, size.height);const ctx = canvas.getContext("2d", { willReadFrequently: true }) as CanvasRenderingContext2D | null;@@ -121,14 +128,14 @@ function makeScorer(doc: ExportDoc, area: FrameBox) {throw new Error("No 2d context to score frames on.");}ctx.imageSmoothingQuality = "high";- let letterboxed: { w: number; h: number } | undefined;+ let reshaped: { w: number; h: number } | undefined;return {// Held here so the surface lives as long as its context: a server// canvas's context keeps only a weak hold on it.canvas,size,- get letterboxed() {- return letterboxed;+ get reshaped() {+ return reshaped;},/** A source frame, read inside its picture box. */pixels(source: CanvasImageSource, fw: number, fh: number): Uint8ClampedArray {@@ -146,8 +153,8 @@ function makeScorer(doc: ExportDoc, area: FrameBox) {if (Math.abs(r.w / r.h / (size.width / size.height) - 1) <= ASPECT_SLACK) {ctx!.drawImage(source, r.x, r.y, r.w, r.h, 0, 0, size.width, size.height);} else {- letterboxed = { w: r.w, h: r.h };- const k = Math.min(size.width / r.w, size.height / r.h);+ reshaped = { w: r.w, h: r.h };+ const k = (fit === "fill" ? Math.max : Math.min)(size.width / r.w, size.height / r.h);const dw = r.w * k;const dh = r.h * k;ctx!.drawImage(source, r.x, r.y, r.w, r.h, (size.width - dw) / 2, (size.height - dh) / 2, dw, dh);@@ -156,14 +163,28 @@ function makeScorer(doc: ExportDoc, area: FrameBox) {}}-/** The line a result carries when the source was letterboxed to score it. */-function frameNote(doc: ExportDoc, shape: { w: number; h: number } | undefined, area: FrameBox): string | undefined {+/** The line a result carries when the source was letterboxed or filled to+ * score it. */+function frameNote(+ doc: ExportDoc,+ shape: { w: number; h: number } | undefined,+ area: FrameBox,+ fit: SourceFit+): string | undefined {if (!shape) {return undefined;}+ const shapes = `The source's ${isFullFrame(area) ? "frame" : "picture box"} is ${round1(shape.w / shape.h)}:1 and the cut's is ${doc.aspect}`;+ if (fit === "fill") {+ return (+ `${shapes}, so scores read the source scaled to fill the cut's frame, centre-cropped. ` ++ "What the crop cuts off is not checked, and type and shapes read at the fill's scale; fit \"letterbox\" reads the whole source."+ );+ }return (- `The source's ${isFullFrame(area) ? "frame" : "picture box"} is ${round1(shape.w / shape.h)}:1 and the cut's is ${doc.aspect}, so scores read the source letterboxed into the cut's frame over its background. ` +- "Bars the cut does not have count as difference; set_aspect to the source's shape when the copy should match it."+ `${shapes}, so scores read the source letterboxed into the cut's frame over its background. ` ++ "Bars the cut does not have count as difference; set_aspect to the source's shape when the copy should match it, " ++ 'or pass fit "fill" when the copy fills its own frame edge to edge.');}@@ -196,10 +217,11 @@ export async function compareMoments(asset: MediaAsset,moments: { at: number; sourceAt: number }[],total: number,- area: FrameBox = FULL_FRAME+ area: FrameBox = FULL_FRAME,+ fit: SourceFit = "letterbox"): Promise<{ images: string[]; checked: Checked[]; frame?: string }> {const size = sizeOf(doc, COMPARE_LONG_SIDE);- const scorer = makeScorer(doc, area);+ const scorer = makeScorer(doc, area, fit);const wanted = moments.map((m) => ({at: clamp(m.at, 0, Math.max(0, total - 0.001)),sourceAt: clamp(m.sourceAt, 0, Math.max(0, asset.duration - 0.001)),@@ -286,7 +308,7 @@ export async function compareMoments(});});const images = await composeComparison(pairs);- const frame = frameNote(doc, scorer.letterboxed, area);+ const frame = frameNote(doc, scorer.reshaped, area, fit);return { images, checked, ...(frame ? { frame } : {}) };}@@ -424,7 +446,8 @@ export async function sweepSource(asset: MediaAsset,span: { from: number; to: number; step?: number; offset: number },total: number,- area: FrameBox = FULL_FRAME+ area: FrameBox = FULL_FRAME,+ fit: SourceFit = "letterbox"): Promise<SweepResult> {const from = clamp(span.from, 0, total);const to = clamp(span.to, from, total);@@ -449,7 +472,7 @@ export async function sweepSource(}const size = sizeOf(doc, SWEEP_LONG_SIDE);- const scorer = makeScorer(doc, area);+ const scorer = makeScorer(doc, area, fit);const width = SHIFT_REACH * 2 + 1;const started = performance.now();const outOfTime = (j: number) => j > 0 && performance.now() - started > SWEEP_BUDGET_MS;@@ -566,7 +589,7 @@ export async function sweepSource(});const timing = still ? undefined : runShift(near, SHIFT_REACH, 0, diffs.length - 1, step);const onBlocks = scored.filter((s) => blockAt(doc, s.at)).length / scored.length;- const frame = frameNote(doc, scorer.letterboxed, area);+ const frame = frameNote(doc, scorer.reshaped, area, fit);const reached = diffs.length < samples.length ? samples[diffs.length].at : undefined;return {mode: "sweep",