Skip to content

Open source CapCut alternative.

TypeScript
12.6 KB
Raw Blame
import { beforeEach, describe, expect, test } from "bun:test";
import { runAiTool } from "./aiTools";
import { syncToSpeech } from "./cueSync";
import { useEditor } from "./store";
import { emptySubtitles, isTextOverlay, type SubtitleCue } from "./types";

/**
 * Re-timing what is already on the timeline onto the speech in the mix.
 *
 * The audio here is built the way cueAlign's own tests build it: noise bursts
 * where speech happens, quiet between them. What these check is the layer above
 * — that the move is reported honestly, that ids and words survive it, and that
 * a mix with nothing to read comes back saying so instead of claiming a fix.
 */

const RATE = 16000;

function mix(duration: number, spans: [number, number][]): Float32Array {
  const samples = new Float32Array(Math.round(duration * RATE));
  let seed = 7;
  const rnd = () => {
    seed = (seed * 1103515245 + 12345) % 2147483648;
    return seed / 2147483648 - 0.5;
  };
  for (let i = 0; i < samples.length; i++) samples[i] = rnd() * 0.002;
  for (const [from, to] of spans) {
    const start = Math.round(from * RATE);
    const stop = Math.min(samples.length, Math.round(to * RATE));
    for (let i = start; i < stop; i++) samples[i] += rnd() * 0.6;
  }
  return samples;
}

const SPEECH: [number, number][] = [
  [1, 2.5],
  [4, 5.5],
  [8, 9.5],
];
const audio = () => mix(12, SPEECH);

const cue = (id: string, start: number, end: number, text = "a line of speech"): SubtitleCue => ({
  id,
  start,
  end,
  text,
});

describe("syncToSpeech", () => {
  test("moves late times onto the speech and reports the move", () => {
    const late = [cue("a", 1.3, 2.8), cue("b", 4.3, 5.8), cue("c", 8.3, 9.8)];
    const report = syncToSpeech(late, audio(), RATE);
    expect(report.evidence).toBe(true);
    expect(report.moved).toHaveLength(3);
    expect(report.items[0].start).toBeCloseTo(1, 1);
    expect(report.items[1].start).toBeCloseTo(4, 1);
    expect(report.items[2].start).toBeCloseTo(8, 1);
    // The report is what the assistant repeats back, so it has to describe the
    // move that actually happened.
    expect(report.maxShift).toBeGreaterThanOrEqual(report.medianShift);
    expect(report.moved.every((m) => m.shift < 0)).toBe(true);
  });

  test("ids, text and word timings ride through", () => {
    const withWords = [
      { ...cue("a", 1.3, 2.8), words: [{ t0: 1.3, t1: 2, w: "a" }, { t0: 2, t1: 2.8, w: "line" }] },
      cue("b", 4.3, 5.8),
      cue("c", 8.3, 9.8),
    ];
    const report = syncToSpeech(withWords, audio(), RATE);
    expect(report.items[0].id).toBe("a");
    expect(report.items[0].text).toBe("a line of speech");
    expect(report.items[0].words?.[0].t0).toBeCloseTo(1, 1);
    expect(report.items[0].words).toHaveLength(2);
  });

  test("times already on the speech move nothing", () => {
    const report = syncToSpeech([cue("a", 1, 2.5), cue("b", 4, 5.5)], audio(), RATE);
    expect(report.moved).toHaveLength(0);
    expect(report.evidence).toBe(true);
  });

  test("audio with nothing to read says so rather than claiming a fix", () => {
    const silent = new Float32Array(RATE * 12);
    const wall = mix(12, [[0, 12]]);
    for (const samples of [silent, wall]) {
      const report = syncToSpeech([cue("a", 1.3, 2.8)], samples, RATE);
      expect(report.evidence).toBe(false);
      expect(report.moved).toHaveLength(0);
      expect(report.items[0].start).toBe(1.3);
    }
  });

  test("one cue out of a long recording is not enough to read the mix by", () => {
    // The fit is weighed over the whole transcript: a single line against
    // twelve seconds of speech describes something else, so nothing moves.
    // This is why the tool measures a whole track even when writing a subset.
    expect(syncToSpeech([cue("a", 1.3, 2.8)], audio(), RATE).evidence).toBe(false);
  });

  test("nothing to align is not an alignment", () => {
    expect(syncToSpeech([], audio(), RATE).evidence).toBe(false);
  });
});

describe("words at a time", () => {
  /** Six measured words a quarter-second apart, filling one speech span. */
  const spoken = (id: string, from: number): SubtitleCue => {
    const words = Array.from({ length: 6 }, (_, i) => ({
      w: `w${id}${i}`,
      t0: from + i * 0.25,
      t1: from + i * 0.25 + 0.25,
    }));
    return {
      id,
      start: from,
      end: from + 1.5,
      text: words.map((w) => w.w).join(" "),
      words,
    };
  };

  const track = () => [spoken("a", 1), spoken("b", 4), spoken("c", 8)];

  const openProject = (cues: SubtitleCue[]) =>
    useEditor.setState({
      projectId: "p1",
      aspect: "9:16",
      assets: [],
      clips: [],
      audioClips: [],
      overlays: [],
      subtitles: { ...emptySubtitles(), cues },
      cutMixSamples: async () => ({ samples: audio(), sampleRate: RATE }),
    });

  test("re-cuts every track and keeps the wording word for word", async () => {
    openProject(track());
    const said = useEditor
      .getState()
      .subtitles.cues.flatMap((c) => c.text.split(" "));
    await useEditor.getState().setSubtitleWordsPerCue(3);
    const cues = useEditor.getState().subtitles.cues;
    expect(cues).toHaveLength(6);
    expect(cues.flatMap((c) => c.text.split(" "))).toEqual(said);
    expect(useEditor.getState().subtitles.wordsPerCue).toBe(3);
  });

  test("the new captions land on the speech, and never on each other", async () => {
    openProject(track());
    await useEditor.getState().setSubtitleWordsPerCue(2);
    const cues = useEditor.getState().subtitles.cues;
    expect(cues[0].start).toBeCloseTo(1, 1);
    expect(cues[cues.length - 1].end).toBeCloseTo(9.5, 1);
    for (let i = 0; i + 1 < cues.length; i++) {
      expect(cues[i].end).toBeLessThanOrEqual(cues[i + 1].start + 1e-6);
      // Each caption still sits inside a run of speech in the mix.
      expect(SPEECH.some(([f, t]) => cues[i].start >= f - 0.2 && cues[i].end <= t + 0.2)).toBe(true);
    }
  });

  test("the re-cut and its alignment undo together", async () => {
    openProject(track());
    await useEditor.getState().setSubtitleWordsPerCue(3);
    expect(useEditor.getState().subtitles.cues).toHaveLength(6);
    useEditor.getState().undo();
    const cues = useEditor.getState().subtitles.cues;
    expect(cues).toHaveLength(3);
    expect(cues.map((c) => c.id)).toEqual(["a", "b", "c"]);
  });

  test("a mix with nothing to read still re-cuts the captions", async () => {
    openProject(track());
    useEditor.setState({ cutMixSamples: async () => null });
    await useEditor.getState().setSubtitleWordsPerCue(3);
    // The cut is made on the words the transcriber measured; only the
    // measurement against the mix is skipped.
    expect(useEditor.getState().subtitles.cues).toHaveLength(6);
  });
});

describe("align_to_audio", () => {
  const reset = (cues: SubtitleCue[], overlays: unknown[] = []) =>
    useEditor.setState({
      projectId: "p1",
      aspect: "16:9",
      assets: [],
      clips: [],
      audioClips: [],
      overlays: overlays as never,
      subtitles: { ...emptySubtitles(), cues },
      cutMixSamples: async () => ({ samples: audio(), sampleRate: RATE }),
    });

  const title = (id: string, start: number, end: number) => ({
    id,
    kind: "text" as const,
    text: `line ${id}`,
    start,
    end,
    x: 0.5,
    y: 0.5,
    size: 72,
    color: "#FFFFFF",
    font: "sf",
    weight: 700 as const,
  });

  beforeEach(() => reset([]));

  test("re-times a caption track in place, ids kept", async () => {
    reset([cue("a", 1.3, 2.8), cue("b", 4.3, 5.8), cue("c", 8.3, 9.8)]);
    const res = (await runAiTool("align_to_audio", {})) as {
      target: string;
      moved: number;
      evidence: boolean;
      items: { id: string; shift: number }[];
    };
    expect(res.target).toBe("captions");
    expect(res.evidence).toBe(true);
    expect(res.moved).toBe(3);
    const after = useEditor.getState().subtitles.cues;
    expect(after.map((c) => c.id)).toEqual(["a", "b", "c"]);
    expect(after[0].start).toBeCloseTo(1, 1);
    expect(after[2].start).toBeCloseTo(8, 1);
    // One undo step puts every cue back where it was.
    useEditor.getState().undo();
    expect(useEditor.getState().subtitles.cues[0].start).toBeCloseTo(1.3, 2);
  });

  test("writing part of a track never lands a cue on a neighbour", async () => {
    // The whole track is measured, only `b` is written. `b` wants to move back
    // onto the speech at 4, and `a` — left where it is — still ends at 4.4.
    // Captions on a track may not overlap, so the write is fitted to the room
    // it actually has.
    reset([cue("a", 2.9, 4.4), cue("b", 4.3, 5.8), cue("c", 8.3, 9.8)]);
    await runAiTool("align_to_audio", { ids: ["b"] });
    const after = useEditor.getState().subtitles.cues;
    const by = new Map(after.map((c) => [c.id, c]));
    expect(by.get("a")!.start).toBeCloseTo(2.9, 2);
    expect(by.get("a")!.end).toBeCloseTo(4.4, 2);
    expect(by.get("c")!.start).toBeCloseTo(8.3, 2);
    expect(by.get("b")!.start).toBeGreaterThanOrEqual(by.get("a")!.end - 1e-6);
    for (const [i, c] of after.entries()) {
      if (i > 0) expect(c.start).toBeGreaterThanOrEqual(after[i - 1].end - 1e-6);
    }
  });

  test("a card rides only with a word that actually moved", async () => {
    // `t2` is the only element written, so `c1` — the card behind `t1` — must
    // stay with the word it belongs to rather than following an alignment
    // nobody applied.
    const card = (id: string, start: number, end: number) => ({
      id,
      kind: "rect" as const,
      start,
      end,
      x: 0.5,
      y: 0.5,
      w: 1,
      h: 1,
      fill: "#FF5500",
    });
    reset(
      [],
      [
        { ...title("t1", 1.3, 2.8), lane: 0 },
        { ...title("t2", 4.3, 5.8), lane: 0 },
        { ...card("c1", 1.3, 2.8), lane: 1 },
      ]
    );
    await runAiTool("align_to_audio", { target: "elements", ids: ["t2"] });
    const by = new Map(useEditor.getState().overlays.map((o) => [o.id, o]));
    expect(by.get("t1")!.start).toBeCloseTo(1.3, 2);
    expect(by.get("c1")!.start).toBeCloseTo(1.3, 2);
    expect(by.get("t2")!.start).toBeCloseTo(4, 1);
  });

  test("re-times the elements of a text run", async () => {
    reset([], [title("t1", 1.3, 2.8), title("t2", 4.3, 5.8), title("t3", 8.3, 9.8)]);
    const res = (await runAiTool("align_to_audio", { target: "elements" })) as {
      target: string;
      lane: number;
      moved: number;
    };
    expect(res.target).toBe("elements");
    expect(res.moved).toBe(3);
    const after = useEditor.getState().overlays.filter(isTextOverlay);
    expect(after.map((o) => o.id)).toEqual(["t1", "t2", "t3"]);
    expect(after[0].start).toBeCloseTo(1, 1);
    expect(after[1].start).toBeCloseTo(4, 1);
    // Elements on one row must never overlap, and an alignment is no exception.
    for (let i = 0; i + 1 < after.length; i++)
      expect(after[i].end).toBeLessThanOrEqual(after[i + 1].start + 1e-6);
  });

  test("a mix it cannot read leaves the cut alone and says so", async () => {
    reset([cue("a", 1.3, 2.8)]);
    useEditor.setState({
      cutMixSamples: async () => ({ samples: new Float32Array(RATE * 12), sampleRate: RATE }),
    });
    const res = (await runAiTool("align_to_audio", {})) as { evidence: boolean; moved: number };
    expect(res.evidence).toBe(false);
    expect(res.moved).toBe(0);
    expect(useEditor.getState().subtitles.cues[0].start).toBeCloseTo(1.3, 2);
  });

  const failure = async (input: Record<string, unknown>) => {
    try {
      await runAiTool("align_to_audio", input);
      return "";
    } catch (err) {
      return err instanceof Error ? err.message : String(err);
    }
  };

  test("nothing to align, and nothing audible, are both refused", async () => {
    reset([]);
    expect(await failure({})).toContain("no captions");
    reset([cue("a", 1.3, 2.8)]);
    useEditor.setState({ cutMixSamples: async () => null });
    expect(await failure({})).toContain("audible");
  });

  test("a run's color cards ride with the words they sit behind", async () => {
    const card = (id: string, start: number, end: number) => ({
      id,
      kind: "rect" as const,
      start,
      end,
      x: 0.5,
      y: 0.5,
      w: 1,
      h: 1,
      fill: "#FF5500",
    });
    reset(
      [],
      [
        { ...title("t1", 1.3, 2.8), lane: 0 },
        { ...title("t2", 4.3, 5.8), lane: 0 },
        { ...card("c1", 1.3, 2.8), lane: 1 },
        { ...card("c2", 4.3, 5.8), lane: 1 },
      ]
    );
    await runAiTool("align_to_audio", { target: "elements" });
    const by = new Map(useEditor.getState().overlays.map((o) => [o.id, o]));
    expect(by.get("t1")!.start).toBeCloseTo(1, 1);
    expect(by.get("c1")!.start).toBeCloseTo(by.get("t1")!.start, 3);
    expect(by.get("c2")!.start).toBeCloseTo(by.get("t2")!.start, 3);
  });
});