Open source CapCut alternative.
TypeScript
import { beforeEach, expect, test } from "bun:test";
import { stubModule } from "@/lib/testing/stubModule";
import { setRasterFactory, type RasterSurface } from "./raster";
import { useEditor } from "./store";
// A surface that draws nothing but a flat grey level: this pins the shape of
// the reads and the result, and the pixels themselves are the render path's
// own tests. Every call the draw code makes answers; drawing one stub canvas
// onto another carries its level across, and reading pixels hands that level
// back, so a test can say what the source shows at each second and read the
// score it comes to.
type StubCanvas = { width: number; height: number; level: number; getContext: () => unknown };
const contextOf = (canvas: StubCanvas) =>
new Proxy(
{
canvas,
measureText: () => ({ width: 10, actualBoundingBoxAscent: 8, actualBoundingBoxDescent: 2 }),
createLinearGradient: () => ({ addColorStop() {} }),
drawImage: (src: { level?: number }, ...at: number[]) => {
if (typeof src?.level === "number") canvas.level = src.level;
// The destination box of a 9-argument draw, the one that places a frame.
if (at.length === 8) draws.push({ into: { w: canvas.width, h: canvas.height }, x: at[4], y: at[5], w: at[6], h: at[7] });
},
getImageData: (_x: number, _y: number, w: number, h: number) => {
const data = new Uint8ClampedArray(Math.max(1, w * h * 4));
for (let i = 0; i < data.length; i += 4) data.set([canvas.level, canvas.level, canvas.level, 255], i);
return { data, width: w, height: h };
},
} as Record<string, unknown>,
{
get: (target, prop) =>
prop in target ? target[prop as string] : typeof prop === "string" ? () => undefined : undefined,
set: (target, prop, value) => ((target[prop as string] = value), true),
}
);
const stubCanvas = (w: number, h: number, level = 0): RasterSurface => {
const canvas: StubCanvas = { width: Math.max(1, w), height: Math.max(1, h), level, getContext: () => ctx };
const ctx = contextOf(canvas);
return canvas as unknown as RasterSurface;
};
setRasterFactory({
createCanvas: stubCanvas,
decodeImage: async () => ({ source: {} as CanvasImageSource, width: 90, height: 160 }),
canvasToBlob: async () => new Blob([new Uint8Array([1, 2, 3])], { type: "image/jpeg" }),
snapshot: async (canvas) => canvas as unknown as ImageBitmap,
});
/** Where frames were drawn onto the score's surface, newest last. */
const draws: { into: { w: number; h: number }; x: number; y: number; w: number; h: number }[] = [];
// Checking a replica against its source reads two pictures per moment, so what
// it accepts is worth pinning: the moments it is given, the ones it turns away
// before drawing anything, and the shape of the read it makes of the source.
/** Every time the source was asked for frames, and at what size. */
const reads: { times: number[]; size: unknown }[] = [];
/** The grey level the source shows at a second; the cut always draws 0. */
let sourceLevel: (t: number) => number = () => 0;
/** The source's own frame shape. */
let sourceShape = { w: 90, h: 160 };
/** Whether the decoder dies on the frame at a second: the way Chrome's
* decoder fails ("Decoding error."), on a hardware read or on any. */
let dies: (t: number, software: boolean) => boolean = () => false;
/** Every read's decoder options, in order. */
const readOpts: unknown[] = [];
async function* framesAt(_src: string | Blob, times: number[], size?: unknown, opts?: { software?: boolean }) {
reads.push({ times: [...times], size });
readOpts.push(opts);
for (const t of times) {
if (dies(t, !!opts?.software)) throw new DOMException("Decoding error.", "EncodingError");
yield { canvas: stubCanvas(sourceShape.w, sourceShape.h, sourceLevel(t)), timestamp: t, duration: 0 } as never;
}
}
await stubModule<typeof import("./mediaRead")>("./mediaRead", import.meta.url, { framesAt });
const { runAiTool } = await import("./aiTools");
beforeEach(() => {
reads.length = 0;
readOpts.length = 0;
draws.length = 0;
dies = () => false;
sourceLevel = () => 0;
sourceShape = { w: 90, h: 160 };
useEditor.setState({
aspect: "9:16",
background: "#000000",
clips: [{ id: "c1", assetId: "block", track: 0, start: 0, in: 0, out: 10, muted: false }],
overlays: [],
transitions: [],
audioClips: [],
assets: [
{ id: "ref", fileName: "ref.mp4", name: "Reference", type: "video", duration: 33.17, url: "" },
{ id: "unread", fileName: "unread.mp4", name: "Still importing", type: "video", duration: 0, url: "" },
{ id: "song", fileName: "song.mp3", name: "Song", type: "audio", duration: 33, url: "" },
{
id: "block",
fileName: "",
name: "Shot 1",
type: "image",
duration: 10,
url: "",
origin: "block",
block: { label: "Shot 1", color: "#FF5A00" },
},
],
});
});
const message = async (p: Promise<unknown>) =>
p.then(() => "", (e: unknown) => (e instanceof Error ? e.message : String(e)));
test("a moment is required, because there is nothing to compare without one", async () => {
expect(await message(runAiTool("compare_to_source", { asset_id: "ref", times: [] }))).toContain("Pass times");
});
test("more moments than one call reads are turned away with what to do instead", async () => {
const error = await message(
runAiTool("compare_to_source", { asset_id: "ref", times: [1, 2, 3, 4, 5, 6, 7, 8, 9] })
);
expect(error).toContain("8 is the most one call reads");
});
test("a sound has no frame to hold up", async () => {
expect(await message(runAiTool("compare_to_source", { asset_id: "song", times: [1] }))).toContain("listen_audio");
});
test("an unknown source says so before anything is drawn", async () => {
expect(await message(runAiTool("compare_to_source", { asset_id: "nope", times: [1] }))).toContain("nope");
});
test("a source with no length yet is turned away, because every moment would clamp to 0", async () => {
expect(await message(runAiTool("compare_to_source", { asset_id: "unread", times: [1, 8] }))).toContain("no length yet");
});
test("a comparison comes back as pictures, one per moment", async () => {
const out = (await runAiTool("compare_to_source", {
asset_id: "ref",
times: [1, { at: 5, source: 20 }],
})) as { images: string[]; checked: { at: number; source: number }[]; source: { sound: string } };
expect(out.checked).toMatchObject([
{ at: 1, source: 1 },
{ at: 5, source: 20 },
]);
expect(out.images).toHaveLength(2);
for (const sheet of out.images) expect(sheet.startsWith("data:image/")).toBe(true);
// The record every look returns, so a comparison does not erase what the
// ear already knows about this source.
expect(out.source.sound).toBeDefined();
});
test("the source is read in one pass, in play order, at a height that keeps its own shape", async () => {
await runAiTool("compare_to_source", { asset_id: "ref", times: [{ at: 1, source: 20 }, { at: 5, source: 3 }] });
expect(reads).toHaveLength(1);
// Each moment brings the few frames either side of it, for the timing read.
const times = reads[0].times;
expect(times).toEqual([...times].sort((a, b) => a - b));
expect(times.includes(3) && times.includes(20)).toBe(true);
expect(times.every((t) => Math.abs(t - 3) <= 0.101 || Math.abs(t - 20) <= 0.101)).toBe(true);
// Both dimensions would squash a widescreen reference into a vertical cut's
// frame — and mediabunny's sink refuses the pair outright.
expect(Object.keys(reads[0].size as object)).toEqual(["height"]);
});
test("a cut built only from elements is checked at the moments asked for", async () => {
// No track-0 clip: the cut's length is its elements' and its soundtrack's.
useEditor.setState({ clips: [] });
// A still element is drawn once and reused as a bitmap; the stub surface
// has no pixels to hand over. Another file's stub may be installed already.
const was = globalThis.createImageBitmap;
globalThis.createImageBitmap = (async () => ({ width: 1, height: 1, close() {} })) as never;
try {
await runAiTool("add_shape", { shape: "rect", start: 0, end: 3, w: 1, h: 1, fill: "#C53CD4" });
const frame = (await runAiTool("capture_frame", { t: 1.9 })) as { at: number };
expect(frame.at).toBeCloseTo(1.9, 3);
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [2.5] })) as {
checked: { at: number }[];
};
expect(out.checked[0].at).toBeCloseTo(2.5, 3);
} finally {
globalThis.createImageBitmap = was;
}
});
type Checked = { at: number; source: number; diff: number; grid: number[][]; shift?: number; block?: string };
test("every moment comes back with a score and a grid of where it differs", async () => {
sourceLevel = () => 40;
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1, 2] })) as { checked: Checked[] };
expect(out.checked).toHaveLength(2);
for (const c of out.checked) {
expect(c.diff).toBe(40);
expect(c.grid).toHaveLength(4);
expect(c.grid.flat()).toEqual(Array(16).fill(40));
}
});
test("eight moments fit in one call", async () => {
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1, 2, 3, 4, 5, 6, 7, 8] })) as {
images: string[];
};
expect(out.images).toHaveLength(8);
});
test("a moment that matches a neighbouring source frame says how far to move it", async () => {
// The source shows what the cut shows two frames after the moment asked for.
sourceLevel = (t) => (Math.abs(t - (5 + 2 / 30)) < 0.005 ? 0 : 30);
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [5] })) as { checked: Checked[] };
expect(out.checked[0].diff).toBe(30);
expect(out.checked[0].shift).toBeCloseTo(0.067, 3);
});
test("a moment with no better neighbour names no shift", async () => {
sourceLevel = () => 30;
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [5] })) as { checked: Checked[] };
expect(out.checked[0].shift).toBeUndefined();
});
test("an offset maps every bare moment onto the source", async () => {
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1, { at: 2, source: 9 }], offset: 2 })) as {
checked: Checked[];
};
expect(out.checked.map((c) => c.source)).toEqual([3, 9]);
});
test("a moment over a block says so, because a block stands in for footage", async () => {
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1] })) as { checked: Checked[] };
expect(out.checked[0].block).toBe("Shot 1");
});
test("a source of another shape is scored letterboxed into the cut's frame, and says so", async () => {
sourceShape = { w: 160, h: 90 };
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1] })) as { frame?: string };
expect(out.frame).toContain("letterboxed");
});
test("a source of another shape fills the cut's frame when asked, centre-cropped, and says so", async () => {
// A vertical reference rebuilt widescreen out of full-bleed colour: held
// letterboxed, the bars score as difference on every frame and hide whether
// the colour matches.
useEditor.setState({ aspect: "16:9" });
const boxed = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1] })) as { frame?: string };
expect(boxed.frame).toContain('fit "fill"');
// The fill crops a tall source into a wide frame and blows its type up
// with it, so type is sized from the letterboxed read.
expect(boxed.frame).toContain("size type and shapes from this read");
draws.length = 0;
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1], fit: "fill" })) as { frame?: string };
expect(out.frame).toContain("fill");
expect(out.frame).not.toContain("letterboxed");
expect(out.frame).toContain('size type and shapes from fit "letterbox"');
// The source covers the surface it is scored on: as wide as it, taller,
// and centred, so its top and bottom run off the edges.
const placed = draws.filter((d) => d.h > d.into.h);
expect(placed.length).toBeGreaterThan(0);
for (const d of placed) {
expect(d.w).toBeCloseTo(d.into.w, 0);
expect(d.y).toBeCloseTo((d.into.h - d.h) / 2, 0);
}
const sweep = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 1, step: 0.5, fit: "fill" })) as {
frame?: string;
};
expect(sweep.frame).toContain("fill");
});
test("a recording's picture box is what is held against the cut, read at the cut's size", async () => {
// A 9:16 phone recording with a 16:9 picture across its middle, against a
// 16:9 cut: the box is the cut's shape, so nothing is letterboxed.
useEditor.setState({ aspect: "16:9" });
useEditor.getState().updateAsset("ref", { picture: { x: 0.5, y: 0.5, w: 1, h: 0.3164 } });
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1] })) as {
frame?: string;
picture?: { h: number };
pictureNote?: string;
};
expect(out.frame).toBeUndefined();
expect(out.picture?.h).toBe(0.3164);
expect(out.pictureNote).toContain("fraction");
// The frame is read tall enough that its picture box is the cut's height.
const height = (reads[0].size as { height: number }).height;
expect(height * 0.3164).toBeCloseTo(270, -1);
const sweep = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 1, step: 0.5 })) as {
frame?: string;
picture?: unknown;
};
expect(sweep.frame).toBeUndefined();
expect(sweep.picture).toBeDefined();
// "full" scores the whole recorded frame, letterboxed.
const whole = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1], picture: "full" })) as { frame?: string };
expect(whole.frame).toContain("letterboxed");
});
type Sweep = {
images?: unknown;
sampled: number;
step: number;
from: number;
to: number;
median: number;
p95: number;
max: number;
worst: { from: number; to: number; diff: number; peak: number; at: number; grid: number[][]; block?: string }[];
coveredTo?: number;
source: { assetId: string };
};
test("a sweep scores the span at a step and hands back numbers, no pictures", async () => {
const out = (await runAiTool("compare_to_source", {
asset_id: "ref",
mode: "sweep",
from: 0,
to: 2,
step: 0.5,
})) as Sweep;
expect(out.images).toBeUndefined();
expect(out.sampled).toBe(4);
expect(reads).toHaveLength(1);
expect(reads[0].times).toEqual([0, 0.5, 1, 1.5]);
expect([out.median, out.p95, out.max]).toEqual([0, 0, 0]);
expect(out.worst).toEqual([]);
expect(out.source.assetId).toBe("ref");
});
test("a sweep reads the source at a constant offset and skips what falls outside it", async () => {
const out = (await runAiTool("compare_to_source", {
asset_id: "ref",
mode: "sweep",
from: 0,
to: 2,
step: 0.5,
offset: 32,
})) as Sweep;
// The source runs to 33.17s, so only the first three samples land in it.
expect(reads[0].times).toEqual([32, 32.5, 33]);
expect(out.sampled).toBe(3);
});
test("a sweep points at the stretch that is off, with its grid and its block", async () => {
sourceLevel = (t) => (t >= 1 && t < 2 ? 50 : 0);
const out = (await runAiTool("compare_to_source", {
asset_id: "ref",
mode: "sweep",
from: 0,
to: 4,
step: 0.5,
})) as Sweep;
expect(out.median).toBe(0);
expect(out.max).toBe(50);
expect(out.worst).toHaveLength(1);
expect(out.worst[0]).toMatchObject({ from: 1, to: 2, diff: 50, peak: 50, block: "Shot 1" });
expect(out.worst[0].grid.flat()).toEqual(Array(16).fill(50));
});
test("a sweep's step never drops under a frame and widens to fit the sample cap", async () => {
const fine = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 1, step: 0.001 })) as Sweep;
expect(fine.step).toBeCloseTo(1 / 30, 3);
expect(fine.sampled).toBe(30);
const long = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep" })) as Sweep;
expect(long.sampled).toBeLessThanOrEqual(900);
expect(long.to).toBeCloseTo(10, 3);
});
test("a sweep that runs out of time says how far it reached", async () => {
const was = performance.now;
let calls = 0;
performance.now = () => (calls++) * 20_000;
try {
const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 4, step: 0.5 })) as Sweep;
expect(out.sampled).toBeLessThan(8);
expect(out.coveredTo).toBeLessThan(4);
} finally {
performance.now = was;
}
});
test("a sweep with nothing inside the source is turned away", async () => {
const error = await message(
runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 2, offset: 40 })
);
expect(error).toContain("offset");
});
type Lost = { lost?: { source: number; error: string }[]; stopped?: string; note: string };
test("a sweep whose decoder dies mid-read opens a fresh one where it died and finishes", async () => {
dies = (t, software) => t === 1 && !software;
const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 2, step: 0.5 })) as Sweep & Lost;
expect(out.sampled).toBe(4);
expect(out.coveredTo).toBeUndefined();
expect(out.lost).toBeUndefined();
// The second read picks up at the frame that failed, on software decoding.
expect(reads.map((r) => r.times)).toEqual([[0, 0.5, 1, 1.5], [1, 1.5]]);
expect(readOpts[1]).toMatchObject({ software: true });
});
test("a frame no fresh decoder reads is left out of the sweep and named, and the rest is scored", async () => {
dies = (t) => t === 1;
const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 2, step: 0.5 })) as Sweep & Lost;
expect(out.sampled).toBe(3);
expect(out.lost).toEqual([{ source: 1, error: "Decoding error." }]);
expect(out.note).toContain("1s");
});
test("a source that stops decoding ends the sweep where it stopped, saying where and why", async () => {
dies = (t) => t >= 1;
const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 3, step: 0.5 })) as Sweep & Lost;
expect(out.sampled).toBe(2);
expect(out.coveredTo).toBe(1);
expect(out.stopped).toContain("Decoding error.");
expect(out.stopped).toContain("1s");
expect(out.note).toContain("Decoding error.");
});
test("a picture call whose decoder dies reads on with a fresh one", async () => {
dies = (t, software) => t > 2 && !software;
const out = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1, 3] })) as { checked: unknown[] };
expect(out.checked).toHaveLength(2);
expect(readOpts.at(-1)).toMatchObject({ software: true });
});
test("a moment no decoder reads names the second and the decoder's own words", async () => {
dies = (t) => t === 3;
const error = await message(runAiTool("compare_to_source", { asset_id: "ref", times: [1, 3] }));
expect(error).toContain("3s");
expect(error).toContain("Decoding error.");
});
test("a silent cut held up to a reference with sound says so", async () => {
// A blocked-out replica built from titles and shapes plays nothing; the
// reference's sound is what its cuts were timed to.
useEditor.getState().updateAsset("ref", { watch: { heard: [{ from: 0, to: 33 }] } as never });
const out = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 1, step: 0.5 })) as { note: string };
expect(out.note).toContain("plays no sound");
// A reference kept off the timelines still lends its sound, as a file of
// its own, so the reference can go once the build is done.
expect(out.note).toContain("audio_only");
expect(out.note).toContain("the reference stays off every timeline");
const look = (await runAiTool("compare_to_source", { asset_id: "ref", times: [1] })) as { note: string };
expect(look.note).toContain("plays no sound");
// With the reference's track on the soundtrack, nothing is said.
useEditor.setState({ audioClips: [{ id: "a1", assetId: "ref", start: 0, in: 0, out: 33.17, volume: 1, lane: 0 }] as never });
const heard = (await runAiTool("compare_to_source", { asset_id: "ref", mode: "sweep", from: 0, to: 1, step: 0.5 })) as { note: string };
expect(heard.note).not.toContain("plays no sound");
});