Skip to content

Open source CapCut alternative.

TypeScript

feat(site): let compare_to_source fill the cut's frame with a source of another shape

David Lebabe2b13a7bb
diff --git a/site/src/cut/components/AiPanel.tools.ts b/site/src/cut/components/AiPanel.tools.ts
index 7c03c2d3..eb35d4d2 100644
--- a/site/src/cut/components/AiPanel.tools.ts
+++ b/site/src/cut/components/AiPanel.tools.ts
@@ -228,6 +228,12 @@ export const AI_PANEL_TOOLS = [
from: num("sweep: timeline second to start at (default 0)"),
to: num("sweep: timeline second to stop at (default the cut's end)"),
step: num("sweep: seconds between samples (default one frame, widened so a long span fits the call)"),
+ fit: {
+ type: "string",
+ enum: ["letterbox", "fill"],
+ description:
+ "How a source of another shape sits in the cut's frame to be scored. letterbox (the default): whole, over the background, the way a clip of that shape sits on the timeline. fill: scaled to cover the frame and centre-cropped, for a copy that fills its own frame edge to edge in another shape — what the crop cuts off is not checked, and type and shapes read at the fill's scale",
+ },
picture: pictureArg,
},
["asset_id"]
diff --git a/site/src/cut/lib/aiTools.ts b/site/src/cut/lib/aiTools.ts
index 562198ba..8ca3a0ba 100644
--- a/site/src/cut/lib/aiTools.ts
+++ b/site/src/cut/lib/aiTools.ts
@@ -1743,6 +1743,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
};
// Only the source's picture box is held against the cut's frame.
const area = pictureArea(asset, input);
+ // A source of another shape is letterboxed unless the call asks it to fill.
+ const fit = input.fit === "fill" ? "fill" : "letterbox";
// The sweep: numbers over a span, at a step, under a time budget.
if (input.mode === "sweep") {
@@ -1756,7 +1758,8 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
offset,
},
total,
- area
+ area,
+ fit
).catch(failed(`Could not sweep "${asset.name}".`));
return {
...sweep,
@@ -1790,7 +1793,7 @@ const toolRuns: Record<BrowserToolName, ToolRun> = {
throw new ToolError("Pass times: [seconds] — the moments of the cut to check, or [{at, source}] when the cut sits at a different second than the source does. To score the whole cut first, pass mode: \"sweep\".");
if (moments.length > MAX_COMPARE)
throw new ToolError(`That is ${moments.length} moments — ${MAX_COMPARE} is the most one call reads. Check these, fix what is off, then call again for the rest; mode: "sweep" scores every frame without pictures.`);
- const out = await compareMoments(doc, asset, moments, total, area).catch(failed("Could not compare the frames."));
+ const out = await compareMoments(doc, asset, moments, total, area, fit).catch(failed("Could not compare the frames."));
return {
...out,
...pictureReport(area),
diff --git a/site/src/cut/lib/compareToSource.test.ts b/site/src/cut/lib/compareToSource.test.ts
index 0459cf0c..a6e9384f 100644
--- a/site/src/cut/lib/compareToSource.test.ts
+++ b/site/src/cut/lib/compareToSource.test.ts
@@ -17,8 +17,10 @@ const contextOf = (canvas: StubCanvas) =>
canvas,
measureText: () => ({ width: 10, actualBoundingBoxAscent: 8, actualBoundingBoxDescent: 2 }),
createLinearGradient: () => ({ addColorStop() {} }),
- drawImage: (src: { level?: number }) => {
+ drawImage: (src: { level?: number }, ...at: number[]) => {
if (typeof src?.level === "number") canvas.level = src.level;
+ // The destination box of a 9-argument draw, the one that places a frame.
+ if (at.length === 8) draws.push({ into: { w: canvas.width, h: canvas.height }, x: at[4], y: at[5], w: at[6], h: at[7] });
},
getImageData: (_x: number, _y: number, w: number, h: number) => {
const data = new Uint8ClampedArray(Math.max(1, w * h * 4));
@@ -46,6 +48,9 @@ setRasterFactory({
snapshot: async (canvas) => canvas as unknown as ImageBitmap,
});
+/** Where frames were drawn onto the score's surface, newest last. */
+const draws: { into: { w: number; h: number }; x: number; y: number; w: number; h: number }[] = [];
+
// Checking a replica against its source reads two pictures per moment, so what
// it accepts is worth pinning: the moments it is given, the ones it turns away
// before drawing anything, and the shape of the read it makes of the source.
@@ -79,6 +84,7 @@ const { runAiTool } = await import("./aiTools");
beforeEach(() => {
reads.length = 0;
readOpts.length = 0;
+ draws.length = 0;
dies = () => false;
sourceLevel = () => 0;
sourceShape = { w: 90, h: 160 };
@@ -234,6 +240,33 @@ test("a source of another shape is scored letterboxed into the cut's frame, and
expect(out.frame).toContain("letterboxed");
});
+test("a source of another shape fills the cut's frame when asked, centre-cropped, and says so", async () => {
+ // A vertical reference rebuilt widescreen out of full-bleed colour: held
+ // letterboxed, the bars score as difference on every frame and hide whether
+ // the colour matches.
+ useEditor.setState({ aspect: "16:9" });
+ const boxed = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1] })) as { frame?: string };
+ expect(boxed.frame).toContain('fit "fill"');
+
+ draws.length = 0;
+ const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1], fit: "fill" })) as { frame?: string };
+ expect(out.frame).toContain("fill");
+ expect(out.frame).not.toContain("letterboxed");
+ // The source covers the surface it is scored on: as wide as it, taller,
+ // and centred, so its top and bottom run off the edges.
+ const placed = draws.filter((d) => d.h > d.into.h);
+ expect(placed.length).toBeGreaterThan(0);
+ for (const d of placed) {
+ expect(d.w).toBeCloseTo(d.into.w, 0);
+ expect(d.y).toBeCloseTo((d.into.h - d.h) / 2, 0);
+ }
+
+ const sweep = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 1, step: 0.5, fit: "fill" })) as {
+ frame?: string;
+ };
+ expect(sweep.frame).toContain("fill");
+});
+
test("a recording's picture box is what is held against the cut, read at the cut's size", async () => {
// A 9:16 phone recording with a 16:9 picture across its middle, against a
// 16:9 cut: the box is the cut's shape, so nothing is letterboxed.
diff --git a/site/src/cut/lib/sourceCompare.ts b/site/src/cut/lib/sourceCompare.ts
index 35337299..a518b23b 100644
--- a/site/src/cut/lib/sourceCompare.ts
+++ b/site/src/cut/lib/sourceCompare.ts
@@ -50,9 +50,14 @@ const READ_LOST_RUN = 3;
* comes back once the contention passes. */
const READ_RETRY_MS = 100;
-/** Aspect ratios this close draw edge to edge; anything further letterboxes. */
+/** Aspect ratios this close draw edge to edge; anything further letterboxes
+ * or fills, as the call asks. */
const ASPECT_SLACK = 0.01;
+/** How a source of another shape sits in the cut's frame to be scored:
+ * letterboxed whole over the background, or filling it, centre-cropped. */
+export type SourceFit = "letterbox" | "fill";
+
const round1 = (v: number) => Math.round(v * 10) / 10;
const round3 = (v: number) => Math.round(v * 1000) / 1000;
const clamp = (v: number, lo: number, hi: number) => Math.max(lo, Math.min(hi, v));
@@ -110,10 +115,12 @@ const readHeight = (area: FrameBox, height: number) => Math.round(height / area.
* Reads any frame down to the score's size, in the cut's shape. Only the
* source's picture box is read — the part of its frame the cut copies. A
* picture of another shape is letterboxed into the cut's frame over its
- * background, the way a clip of that shape would sit on the timeline, and the
- * scorer remembers it did so the result can say so.
+ * background, the way a clip of that shape would sit on the timeline — or,
+ * with `fit` "fill", scaled to cover it and centre-cropped, the way a
+ * full-bleed replica in another shape fills its frame — and the scorer
+ * remembers it did so the result can say so.
*/
-function makeScorer(doc: ExportDoc, area: FrameBox) {
+function makeScorer(doc: ExportDoc, area: FrameBox, fit: SourceFit) {
const size = sizeOf(doc, SCORE_LONG_SIDE);
const canvas = createRasterCanvas(size.width, size.height);
const ctx = canvas.getContext("2d", { willReadFrequently: true }) as CanvasRenderingContext2D | null;
@@ -121,14 +128,14 @@ function makeScorer(doc: ExportDoc, area: FrameBox) {
throw new Error("No 2d context to score frames on.");
}
ctx.imageSmoothingQuality = "high";
- let letterboxed: { w: number; h: number } | undefined;
+ let reshaped: { w: number; h: number } | undefined;
return {
// Held here so the surface lives as long as its context: a server
// canvas's context keeps only a weak hold on it.
canvas,
size,
- get letterboxed() {
- return letterboxed;
+ get reshaped() {
+ return reshaped;
},
/** A source frame, read inside its picture box. */
pixels(source: CanvasImageSource, fw: number, fh: number): Uint8ClampedArray {
@@ -146,8 +153,8 @@ function makeScorer(doc: ExportDoc, area: FrameBox) {
if (Math.abs(r.w / r.h / (size.width / size.height) - 1) <= ASPECT_SLACK) {
ctx!.drawImage(source, r.x, r.y, r.w, r.h, 0, 0, size.width, size.height);
} else {
- letterboxed = { w: r.w, h: r.h };
- const k = Math.min(size.width / r.w, size.height / r.h);
+ reshaped = { w: r.w, h: r.h };
+ const k = (fit === "fill" ? Math.max : Math.min)(size.width / r.w, size.height / r.h);
const dw = r.w * k;
const dh = r.h * k;
ctx!.drawImage(source, r.x, r.y, r.w, r.h, (size.width - dw) / 2, (size.height - dh) / 2, dw, dh);
@@ -156,14 +163,28 @@ function makeScorer(doc: ExportDoc, area: FrameBox) {
}
}
-/** The line a result carries when the source was letterboxed to score it. */
-function frameNote(doc: ExportDoc, shape: { w: number; h: number } | undefined, area: FrameBox): string | undefined {
+/** The line a result carries when the source was letterboxed or filled to
+ * score it. */
+function frameNote(
+ doc: ExportDoc,
+ shape: { w: number; h: number } | undefined,
+ area: FrameBox,
+ fit: SourceFit
+): string | undefined {
if (!shape) {
return undefined;
}
+ const shapes = `The source's ${isFullFrame(area) ? "frame" : "picture box"} is ${round1(shape.w / shape.h)}:1 and the cut's is ${doc.aspect}`;
+ if (fit === "fill") {
+ return (
+ `${shapes}, so scores read the source scaled to fill the cut's frame, centre-cropped. ` +
+ "What the crop cuts off is not checked, and type and shapes read at the fill's scale; fit \"letterbox\" reads the whole source."
+ );
+ }
return (
- `The source's ${isFullFrame(area) ? "frame" : "picture box"} is ${round1(shape.w / shape.h)}:1 and the cut's is ${doc.aspect}, so scores read the source letterboxed into the cut's frame over its background. ` +
- "Bars the cut does not have count as difference; set_aspect to the source's shape when the copy should match it."
+ `${shapes}, so scores read the source letterboxed into the cut's frame over its background. ` +
+ "Bars the cut does not have count as difference; set_aspect to the source's shape when the copy should match it, " +
+ 'or pass fit "fill" when the copy fills its own frame edge to edge.'
);
}
@@ -196,10 +217,11 @@ export async function compareMoments(
asset: MediaAsset,
moments: { at: number; sourceAt: number }[],
total: number,
- area: FrameBox = FULL_FRAME
+ area: FrameBox = FULL_FRAME,
+ fit: SourceFit = "letterbox"
): Promise<{ images: string[]; checked: Checked[]; frame?: string }> {
const size = sizeOf(doc, COMPARE_LONG_SIDE);
- const scorer = makeScorer(doc, area);
+ const scorer = makeScorer(doc, area, fit);
const wanted = moments.map((m) => ({
at: clamp(m.at, 0, Math.max(0, total - 0.001)),
sourceAt: clamp(m.sourceAt, 0, Math.max(0, asset.duration - 0.001)),
@@ -286,7 +308,7 @@ export async function compareMoments(
});
});
const images = await composeComparison(pairs);
- const frame = frameNote(doc, scorer.letterboxed, area);
+ const frame = frameNote(doc, scorer.reshaped, area, fit);
return { images, checked, ...(frame ? { frame } : {}) };
}
@@ -424,7 +446,8 @@ export async function sweepSource(
asset: MediaAsset,
span: { from: number; to: number; step?: number; offset: number },
total: number,
- area: FrameBox = FULL_FRAME
+ area: FrameBox = FULL_FRAME,
+ fit: SourceFit = "letterbox"
): Promise<SweepResult> {
const from = clamp(span.from, 0, total);
const to = clamp(span.to, from, total);
@@ -449,7 +472,7 @@ export async function sweepSource(
}
const size = sizeOf(doc, SWEEP_LONG_SIDE);
- const scorer = makeScorer(doc, area);
+ const scorer = makeScorer(doc, area, fit);
const width = SHIFT_REACH * 2 + 1;
const started = performance.now();
const outOfTime = (j: number) => j > 0 && performance.now() - started > SWEEP_BUDGET_MS;
@@ -566,7 +589,7 @@ export async function sweepSource(
});
const timing = still ? undefined : runShift(near, SHIFT_REACH, 0, diffs.length - 1, step);
const onBlocks = scored.filter((s) => blockAt(doc, s.at)).length / scored.length;
- const frame = frameNote(doc, scorer.letterboxed, area);
+ const frame = frameNote(doc, scorer.reshaped, area, fit);
const reached = diffs.length < samples.length ? samples[diffs.length].at : undefined;
return {
mode: "sweep",