Skip to content

Open source CapCut alternative.

TypeScript
10.1 KB
Raw Blame
import { afterAll, beforeEach, expect, spyOn, test } from "bun:test";
import type { Input, InputVideoTrack, WrappedCanvas } from "mediabunny";
import * as mediaRead from "./mediaRead";
import { useEditor } from "./store";

// The measuring tools read a source the way the other senses do — through the
// frame sink — so a synthetic 60 fps video stands in for the file: every
// frame's pixels are painted from its index, and each tool's numbers are
// checked against what was painted.

const FPS = 60;
const FRAMES = 180; // 3 s
const W = 1080;
const H = 1920;

type Rgb = [number, number, number];
const RED: Rgb = [200, 30, 30];
const BLUE: Rgb = [30, 30, 200];
const GREEN: Rgb = [30, 160, 60];
const ORANGE: Rgb = [255, 90, 0];
const WHITE: Rgb = [255, 255, 255];

/** The picture at frame f, as a colour at each frame fraction. Red until 1s,
 * a five-frame blue shot, then green. On the green, an orange bar centred at
 * (0.5, 0.3), 0.1 tall, grows linearly from 0.1 to 0.5 wide over 1.5–2.0s; a
 * white "title" block spans x 0.2–0.8, y 0.7–0.75. From frame 165 to 174 a
 * corner square (x 0.8–0.95, y 0.05–0.2) strobes white and blue a frame each. */
function colourAt(f: number, fx: number, fy: number): Rgb {
  if (f < 60) return RED;
  if (f < 65) return BLUE;
  if (f >= 165 && f < 175 && fx >= 0.8 && fx < 0.95 && fy >= 0.05 && fy < 0.2) return f % 2 ? WHITE : BLUE;
  const t = f / FPS;
  const p = Math.max(0, Math.min(1, (t - 1.5) / 0.5));
  const bw = 0.1 + 0.4 * p;
  if (Math.abs(fx - 0.5) <= bw / 2 && Math.abs(fy - 0.3) <= 0.05) return ORANGE;
  if (fx >= 0.2 && fx < 0.8 && fy >= 0.7 && fy < 0.75) return WHITE;
  return GREEN;
}

function frameCanvas(f: number, w: number, h: number) {
  const data = new Uint8ClampedArray(w * h * 4);
  for (let y = 0; y < h; y++)
    for (let x = 0; x < w; x++) data.set([...colourAt(f, (x + 0.5) / w, (y + 0.5) / h), 255], (y * w + x) * 4);
  const ctx = { getImageData: () => ({ data, width: w, height: h }), drawImage() {}, putImageData() {} };
  return { width: w, height: h, getContext: () => ctx } as unknown as HTMLCanvasElement;
}

const wrap = (f: number, w: number, h: number) =>
  ({ canvas: frameCanvas(f, w, h), timestamp: f / FPS, duration: 1 / FPS }) as WrappedCanvas;
const frameShownAt = (t: number) => Math.max(0, Math.min(FRAMES - 1, Math.floor(t * FPS + 1e-6)));

const decoded: number[] = [];
const input = { dispose() {} } as unknown as Input;
const track = {
  getDisplayWidth: async () => W,
  getDisplayHeight: async () => H,
  getFirstTimestamp: async () => 0,
  displayWidth: W,
  displayHeight: H,
} as unknown as InputVideoTrack;
const spies = [
  spyOn(mediaRead, "openMedia").mockReturnValue(input),
  spyOn(mediaRead, "videoTrackOf").mockResolvedValue(track),
  spyOn(mediaRead, "frameSink").mockImplementation((_track, size) => {
    const w = size?.width ?? W;
    const h = size?.height ?? H;
    return {
      getCanvas: async (t: number) => wrap(frameShownAt(t), w, h),
      async *canvases(start = 0, end = Infinity) {
        for (let f = frameShownAt(start); f < FRAMES && f / FPS < end; f++) {
          decoded.push(f);
          yield wrap(f, w, h);
        }
      },
      async *canvasesAtTimestamps(times: Iterable<number> | AsyncIterable<number>) {
        for await (const t of times) yield wrap(frameShownAt(t), w, h);
      },
    };
  }),
];
afterAll(() => {
  for (const spy of spies) spy.mockRestore();
});

const { runAiTool } = await import("./aiTools");

beforeEach(() => {
  decoded.length = 0;
  useEditor.setState({
    assets: [
      { id: "ref", fileName: "ref.mp4", name: "Reference", type: "video", duration: FRAMES / FPS, url: "fixture.mp4", width: W, height: H },
      { id: "song", fileName: "song.mp3", name: "Song", type: "audio", duration: 30, url: "" },
    ],
  });
});

const message = async (p: Promise<unknown>) =>
  p.then(() => "", (e: unknown) => (e instanceof Error ? e.message : String(e)));

test("detect_cuts finds every hard cut to the frame, a five-frame shot included", async () => {
  const out = (await runAiTool("detect_cuts", { asset_id: "ref" })) as {
    fps: number;
    cuts: { t: number; frame: number; change: number }[];
    shots: { from: number; to: number; frames: number }[];
    coveredTo: number;
    truncated: boolean;
  };
  expect(out.fps).toBe(60);
  expect(out.cuts.map((c) => [c.t, c.frame])).toEqual([
    [1, 60],
    [1.083, 65],
  ]);
  expect(out.cuts[0].change).toBe(100);
  expect(out.shots.map((s) => s.frames)).toEqual([60, 5, 115]);
  expect(out.truncated).toBe(false);
  // Every frame of the span was looked at.
  expect(decoded).toHaveLength(FRAMES);
});

test("detect_cuts says when its first read of a source kept the picture box", async () => {
  const first = (await runAiTool("detect_cuts", { asset_id: "ref", from: 1.05, to: 2 })) as { pictureKept?: true };
  expect(first.pictureKept).toBe(true);
  const again = (await runAiTool("detect_cuts", { asset_id: "ref", from: 1.05, to: 2 })) as { pictureKept?: true };
  expect(again.pictureKept).toBeUndefined();
});

test("detect_cuts scans only the span asked for, and a sound has no cuts", async () => {
  const out = (await runAiTool("detect_cuts", { asset_id: "ref", from: 1.05, to: 2 })) as { cuts: { t: number }[] };
  expect(out.cuts.map((c) => c.t)).toEqual([1.083]);
  expect(await message(runAiTool("detect_cuts", { asset_id: "song" }))).toContain("listen_audio");
});

test("sample_color reads a point and a region at their own moments", async () => {
  const out = (await runAiTool("sample_color", {
    asset_id: "ref",
    samples: [
      { t: 0.5, x: 0.5, y: 0.5 },
      { t: 1.03, x: 0.5, y: 0.5 },
      // The bar at full width, over its own box and a strip of green each side.
      { t: 2.5, x: 0.5, y: 0.3, w: 0.6, h: 0.1 },
    ],
    colors: 2,
  })) as { samples: { t: number; hex: string; spread: number; dominant?: { hex: string; share: number }[] }[] };
  expect(out.samples.map((s) => s.hex)).toEqual(["#C81E1E", "#1E1EC8", "#FF5A00"]);
  expect(out.samples[0].spread).toBe(0);
  const bar = out.samples[2].dominant!;
  expect(bar[0].hex).toBe("#FF5A00");
  expect(bar[0].share).toBeCloseTo(0.83, 1);
  expect(bar[1].hex).toBe("#1EA03C");
});

test("sample_color turns away a call with nothing to read", async () => {
  expect(await message(runAiTool("sample_color", { asset_id: "ref", samples: [] }))).toContain("samples");
});

test("track_element follows a growing bar and names its ease", async () => {
  const out = (await runAiTool("track_element", {
    asset_id: "ref",
    from: 1.3,
    to: 2.3,
    seed: { x: 0.5, y: 0.3, w: 0.08, h: 0.08 },
  })) as {
    color: string;
    path: { t: number; x: number; y: number; w: number; h: number }[];
    moves: { prop: string; from: number; to: number; start: number; end: number; ease: string; error: number }[];
  };
  expect(out.color).toBe("#FF5A00");
  expect(out.path[0].w).toBeCloseTo(0.1, 2);
  expect(out.path[out.path.length - 1].w).toBeCloseTo(0.5, 2);
  expect(out.path[0].y).toBeCloseTo(0.3, 2);
  const grow = out.moves.find((m) => m.prop === "w")!;
  expect(grow.ease).toBe("linear");
  expect(grow.from).toBeCloseTo(1.5, 1);
  expect(grow.to).toBeCloseTo(2.0, 1);
  expect(grow.start).toBeCloseTo(0.1, 2);
  expect(grow.end).toBeCloseTo(0.5, 2);
  // The bar grows about its centre: its position never moves.
  expect(out.moves.find((m) => m.prop === "x")).toBeUndefined();
});

test("track_camera reads a held shot as one picture that never moves", async () => {
  const out = (await runAiTool("track_camera", { asset_id: "ref", from: 1.1, to: 1.4 })) as {
    still: boolean;
    path: { t: number; scale: number; dx: number; dy: number; rotation: number }[];
    moves: unknown[];
  };
  expect(out.still).toBe(true);
  expect(out.path.length).toBeGreaterThan(10);
  for (const p of out.path) expect([p.scale, p.dx, p.dy, p.rotation]).toEqual([1, 0, 0, 0]);
  expect(out.moves).toEqual([]);
});

test("detect_cuts marks a strobe as one flicker change", async () => {
  const out = (await runAiTool("detect_cuts", { asset_id: "ref" })) as {
    changes: { frame: number; frames: number; flicker?: true; box: { x: number; y: number } }[];
  };
  const flicker = out.changes.filter((c) => c.flicker);
  expect(flicker).toHaveLength(1);
  expect(flicker[0].frame).toBeGreaterThanOrEqual(165);
  expect(flicker[0].frame + flicker[0].frames).toBeLessThanOrEqual(176);
  expect(flicker[0].box.x).toBeCloseTo(0.875, 1);
});

test("track_camera from a cut detect_cuts gave starts on that shot's first frame", async () => {
  // The cut lands on frame 65 (1.0833 s); detect_cuts hands it back to the
  // millisecond, which can sit either side of the frame's own start.
  for (const from of [1.083, 1.084]) {
    const out = (await runAiTool("track_camera", { asset_id: "ref", from, to: 1.3 })) as { path: { t: number }[] };
    expect(out.path[0].t).toBeCloseTo(65 / FPS, 3);
  }
});

test("track_camera needs the stretch's end", async () => {
  expect(await message(runAiTool("track_camera", { asset_id: "ref", from: 1.1 }))).toContain("to");
});

test("track_element's ink box measures a title at one moment", async () => {
  const out = (await runAiTool("track_element", {
    asset_id: "ref",
    from: 2.5,
    match: "ink",
    color: "#FFFFFF",
    seed: { x: 0.5, y: 0.725, w: 0.9, h: 0.15 },
  })) as { path: { x: number; y: number; w: number; h: number }[] };
  expect(out.path).toHaveLength(1);
  const [box] = out.path;
  expect(box.x).toBeCloseTo(0.5, 2);
  expect(box.y).toBeCloseTo(0.725, 2);
  expect(box.w).toBeCloseTo(0.6, 2);
  expect(box.h).toBeCloseTo(0.05, 2);
});

test("detect_cuts says a pass ended at its span bound short of the source", async () => {
  // A long source: one pass covers DETECT_CUTS_SPAN_MAX seconds and hands
  // back where the next one starts.
  const { DETECT_CUTS_SPAN_MAX } = await import("./measure/limits");
  useEditor.setState({
    assets: [{ id: "long", fileName: "long.mp4", name: "Long", type: "video", duration: 500, url: "fixture.mp4", width: W, height: H }],
  });
  const out = (await runAiTool("detect_cuts", { asset_id: "long" })) as { truncated: boolean; note: string };
  expect(out.truncated).toBe(true);
  expect(out.note).toContain("call again");
  expect(DETECT_CUTS_SPAN_MAX).toBeLessThan(500);
});