Skip to content

Open source CapCut alternative.

TypeScript
14.6 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import { JUDGE_MAX_QUESTIONS } from "@/lib/inference/judge";
import { CUT_JUDGE } from "./chatRuntime";
import {
  availableActions,
  instantAction,
  instantQuestions,
  instantSnapshot,
  INSTANT_ACTIONS,
  scoreToNumber,
  type InstantAnswers,
  type TargetKind,
} from "./instantAction";

const settings = CUT_JUDGE;

/** An editor snapshot in the shape buildAiContext returns, with enough on it
 * for every tier of the registry to be offered. */
const context = {
  playhead: 3,
  playing: false,
  selection: { kind: "clip", id: "clip-b" },
  media: [{ id: "asset-a", name: "beach.mp4", type: "video", duration: 12 }],
  videoTrack: [
    { index: 0, id: "clip-a", asset: "intro.mp4", start: 0, len: 4, muted: false },
    { index: 1, id: "clip-b", asset: "beach.mp4", start: 4, len: 6, muted: false },
    { index: 2, id: "clip-c", asset: "sunset.mp4", start: 10, len: 5, muted: false },
  ],
  overlayVideo: [],
  soundtrack: [{ id: "audio-a", asset: "music.mp3", start: 0 }],
  overlays: [{ id: "ov-a", kind: "text", text: "Hello", start: 1, end: 3 }],
  transitions: [{ id: "tr-a", start: 4, seconds: 0.5, style: "crossfade" }],
  subtitles: {
    cues: [
      { id: "cue-a", start: 0, end: 1, text: "first line" },
      { id: "cue-b", start: 1, end: 2, text: "second line" },
    ],
    tracks: [{ track: 0, locale: "en-US", cues: 2 }],
  },
};

const snap = instantSnapshot(context);
const keyOf = (kind: TargetKind, id: string) => snap.by[kind].find((c) => c.id === id)!.key;

type Over = {
  action?: string;
  actionP?: number;
  targets?: Record<string, { key: string; p?: number }>;
  enums?: Record<string, { value: string; p?: number }>;
  levels?: Record<string, { score: number; confidence?: number }>;
  bools?: Record<string, number>;
  exact?: number;
  multi?: number;
  leftover?: number;
  remake?: number;
};

function answers(over: Over): InstantAnswers {
  const out: Record<string, unknown> = {
    action: {
      type: "choice",
      choice: over.action ?? "none",
      probabilities: { [over.action ?? "none"]: over.actionP ?? 0.8 },
      confidence: 0.5,
    },
    exact_value: { type: "noul", noul: over.exact ?? 0.05 },
    multi_target: { type: "noul", noul: over.multi ?? 0.05 },
    leftover: { type: "noul", noul: over.leftover ?? 0.05 },
    remake: { type: "noul", noul: over.remake ?? 0.05 },
  };
  for (const [kind, t] of Object.entries(over.targets ?? {}))
    out[`target::${kind}`] = { type: "choice", choice: t.key, probabilities: { [t.key]: t.p ?? 0.9 }, confidence: 0.5 };
  for (const [name, e] of Object.entries(over.enums ?? {}))
    out[`enum::${name}`] = { type: "choice", choice: e.value, probabilities: { [e.value]: e.p ?? 0.9 }, confidence: 0.5 };
  for (const [name, l] of Object.entries(over.levels ?? {}))
    out[`level::${name}`] = { type: "score", score: l.score, probabilities: {}, legend: {}, confidence: l.confidence ?? 0.8 };
  for (const [name, p] of Object.entries(over.bools ?? {})) out[`bool::${name}`] = { type: "noul", noul: p };
  return out as InstantAnswers;
}

describe("instantSnapshot", () => {
  test("names every item the timeline holds and marks the selected one", () => {
    expect(snap.by.clip.map((c) => c.id)).toEqual(["clip-a", "clip-b", "clip-c"]);
    expect(snap.by.clip.find((c) => c.id === "clip-b")!.label).toContain("currently selected");
    expect(snap.by.cue.map((c) => c.name)).toEqual(["first line", "second line"]);
    expect(snap.by.track.map((t) => t.track?.kind)).toEqual(["video", "soundtrack", "text", "subtitles"]);
  });

  test("an empty project offers only the actions that need nothing", () => {
    const empty = instantSnapshot({});
    const ids = availableActions(empty);
    expect(ids).toContain("undo");
    expect(ids).toContain("set_aspect");
    expect(ids).not.toContain("set_clip_muted");
    expect(ids).not.toContain("delete_cue");
  });

  // buildAiContext always reports at least one subtitle track, so a project
  // with no captions must still leave the caption tools out.
  test("a subtitle track with no cues is neither a track nor a caption tool", () => {
    const bare = instantSnapshot({
      ...context,
      subtitles: { count: 0, cues: [], tracks: [{ track: 0, locale: "en-US", cues: 0 }] },
    });
    expect(bare.captionTracks).toBe(0);
    expect(bare.by.track.map((t) => t.track?.kind)).not.toContain("subtitles");
    const ids = availableActions(bare);
    expect(ids).not.toContain("set_caption_look");
    expect(ids).not.toContain("subtitles_remove_track");
  });

  // The sound and title rows are numbered, so a project with music over
  // narration offers both and the track tools can tell them apart.
  test("every sound and title row is its own track candidate", () => {
    const layered = instantSnapshot({
      ...context,
      soundtrack: [
        { id: "audio-a", asset: "narration.mp3", start: 0 },
        { id: "audio-b", asset: "music.mp3", start: 0, lane: 1 },
      ],
      overlays: [
        { id: "ov-a", kind: "text", text: "Hello", start: 1, end: 3 },
        { id: "ov-b", kind: "shape", name: "Bar", start: 1, end: 3, lane: 2 },
      ],
    });
    const rows = layered.by.track.map((t) => `${t.track?.kind}-${t.track?.index}`);
    expect(rows).toContain("soundtrack-0");
    expect(rows).toContain("soundtrack-1");
    expect(rows).toContain("text-0");
    expect(rows).toContain("text-2");
    expect(layered.by.track.find((t) => t.track?.kind === "soundtrack" && t.track.index === 1)!.label).toContain(
      "music.mp3",
    );
  });
});

describe("instantQuestions", () => {
  test("one action question, the guards, and one question per family — under the cap", () => {
    const qs = instantQuestions(snap);
    expect(qs.action.type).toBe("choice");
    expect(qs.exact_value.type).toBe("noul");
    expect(qs.multi_target.type).toBe("noul");
    expect(Object.keys(qs).length).toBeLessThanOrEqual(JUDGE_MAX_QUESTIONS);
    // Targets are asked per item kind, never per action, so the count stays
    // flat as the registry grows.
    expect(Object.keys(qs).filter((k) => k.startsWith("target::")).length).toBeLessThanOrEqual(8);
  });

  test("the action options are the available actions plus the three ways out", () => {
    const qs = instantQuestions(snap);
    const options = Object.keys((qs.action as { criteria: Record<string, unknown> }).criteria);
    expect(new Set(options)).toEqual(new Set([...availableActions(snap), "none", "remake_it", "make_new"]));
  });

  test("a way out never resolves to a call", () => {
    for (const out of ["none", "remake_it", "make_new"])
      expect(instantAction(answers({ action: out }), snap, settings)).toBeNull();
  });

  test("delete_asset is never offered — undo does not bring the file back", () => {
    expect(Object.keys(INSTANT_ACTIONS)).not.toContain("delete_asset");
    expect(Object.values(INSTANT_ACTIONS).map((d) => d.tool)).not.toContain("delete_asset");
  });
});

describe("scoreToNumber", () => {
  const values = [0.25, 0.5, 1, 1.5, 2, 4];
  const ans = (score: number) => ({ type: "score" as const, score, probabilities: {}, legend: {}, confidence: 1 });

  test("a whole level reads back its own value", () => {
    expect(scoreToNumber(ans(0), values)).toBe(0.25);
    expect(scoreToNumber(ans(2), values)).toBe(1);
    expect(scoreToNumber(ans(5), values)).toBe(4);
  });

  test("between two levels it interpolates", () => {
    expect(scoreToNumber(ans(2.5), values)).toBe(1.25);
    expect(scoreToNumber(ans(3.25), values)).toBe(1.63);
  });

  test("outside the rubric it clamps", () => {
    expect(scoreToNumber(ans(-4), values)).toBe(0.25);
    expect(scoreToNumber(ans(99), values)).toBe(4);
  });

  test("places round to what the tool takes", () => {
    expect(scoreToNumber(ans(1.5), [-50, -25, 0, 25, 50], 0)).toBe(-12);
    expect(scoreToNumber(ans(3.2), [-50, -25, 0, 25, 50], 0)).toBe(30);
  });
});

describe("instantAction", () => {
  test("a resolved toggle builds its tool call and its line", () => {
    const r = instantAction(
      answers({ action: "set_clip_muted", targets: { clip: { key: keyOf("clip", "clip-a") } }, bools: { mute: 0.95 } }),
      snap,
      settings,
    );
    expect(r).toEqual({
      id: "set_clip_muted",
      tool: "set_clip_muted",
      args: { clipId: "clip-a", muted: true },
      say: 'Muted "intro.mp4".',
    });
  });

  test("a level becomes the number the tool takes", () => {
    const r = instantAction(
      answers({
        action: "set_speed",
        targets: { clip: { key: keyOf("clip", "clip-b") } },
        levels: { speed: { score: 1 } },
        bools: { smooth: 0.05 },
      }),
      snap,
      settings,
    );
    expect(r?.args).toEqual({ clipId: "clip-b", speed: 0.5 });
  });

  test("an enum comes from the shipped catalog", () => {
    const r = instantAction(
      answers({
        action: "set_transition",
        targets: { clip: { key: keyOf("clip", "clip-a") } },
        enums: { transition_style: { value: "crossfade" } },
        levels: { seconds_short: { score: 2 } },
      }),
      snap,
      settings,
    );
    expect(r?.args).toEqual({ clipId: "clip-a", style: "crossfade", seconds: 0.5 });
  });

  test("the action's own probability, not its confidence, decides", () => {
    const low = answers({
      action: "set_clip_muted",
      actionP: 0.2,
      targets: { clip: { key: keyOf("clip", "clip-a") } },
      bools: { mute: 0.95 },
    });
    expect(instantAction(low, snap, settings)).toBeNull();
  });

  test("an unsettled argument runs the loop instead", () => {
    const base = { action: "set_clip_muted", targets: { clip: { key: keyOf("clip", "clip-a") } }, bools: { mute: 0.95 } };
    expect(instantAction(answers({ ...base, targets: { clip: { key: keyOf("clip", "clip-a"), p: 0.2 } } }), snap, settings)).toBeNull();
    // A coin-flip direction is wrong half the time.
    expect(instantAction(answers({ ...base, bools: { mute: 0.5 } }), snap, settings)).toBeNull();
    expect(instantAction(answers({ ...base, targets: {} }), snap, settings)).toBeNull();
  });

  test("a modifier the ask never states takes its default, the direction does not", () => {
    // "select the sunset clip" says nothing about adding to the selection.
    const r = instantAction(
      answers({ action: "select_clip", targets: { clip: { key: keyOf("clip", "clip-c") } } }),
      snap,
      settings,
    );
    expect(r?.args).toEqual({ kind: "clip", id: "clip-c", additive: false });
    // "duck the music" says nothing about how far; the panel's own default does.
    const duck = instantAction(
      answers({ action: "set_audio_duck", targets: { audio: { key: keyOf("audio", "audio-a") } } }),
      snap,
      settings,
    );
    expect(duck?.args).toEqual({ id: "audio-a", duck: 0.5 });
    // Mute or unmute is the whole ask, so an undecided direction runs the loop.
    expect(
      instantAction(
        answers({ action: "set_clip_muted", targets: { clip: { key: keyOf("clip", "clip-a") } }, bools: { mute: 0.5 } }),
        snap,
        settings,
      ),
    ).toBeNull();
  });

  test("a named figure and a described set both take the model", () => {
    const base = {
      action: "set_speed",
      targets: { clip: { key: keyOf("clip", "clip-b") } },
      levels: { speed: { score: 4 } },
      bools: { smooth: 0.05 },
    };
    expect(instantAction(answers(base), snap, settings)).not.toBeNull();
    expect(instantAction(answers({ ...base, exact: 0.9 }), snap, settings)).toBeNull();
    expect(instantAction(answers({ ...base, multi: 0.9 }), snap, settings)).toBeNull();
    expect(instantAction(answers({ ...base, leftover: 0.9 }), snap, settings)).toBeNull();
    // A shot the person wants remade goes to the model however close a
    // colour setting reads to the ask.
    expect(instantAction(answers({ ...base, remake: 0.9 }), snap, settings)).toBeNull();
  });

  test("an exact figure still allows an action with no numeric argument", () => {
    const r = instantAction(
      answers({
        action: "set_clip_muted",
        targets: { clip: { key: keyOf("clip", "clip-a") } },
        bools: { mute: 0.95 },
        exact: 0.9,
      }),
      snap,
      settings,
    );
    expect(r?.tool).toBe("set_clip_muted");
  });

  test("no answers, no action, and the path switched off all run the loop", () => {
    expect(instantAction(null, snap, settings)).toBeNull();
    expect(instantAction(answers({ action: "none", actionP: 0.9 }), snap, settings)).toBeNull();
    expect(
      instantAction(
        answers({ action: "undo", actionP: 0.99 }),
        snap,
        { ...settings, instantAction: false },
      ),
    ).toBeNull();
  });

  test("an action this project cannot carry is refused even if it wins", () => {
    const empty = instantSnapshot({});
    expect(instantAction(answers({ action: "delete_cue", actionP: 0.99 }), empty, settings)).toBeNull();
  });

  test("a track reorder onto its own position writes nothing", () => {
    const row = snap.by.track.find((t) => t.track?.kind === "video")!;
    const to = snap.by.trackTo.find((t) => t.track?.kind === "video")!;
    expect(
      instantAction(
        answers({ action: "reorder_track", targets: { track: { key: row.key }, trackTo: { key: to.key } } }),
        snap,
        settings,
      ),
    ).toBeNull();
  });

  // Subtitle tracks have no stacking order and reorder_track's schema does
  // not take them, so the pair writes nothing.
  test("a track reorder across subtitle tracks writes nothing", () => {
    const subs = instantSnapshot({
      ...context,
      subtitles: {
        ...context.subtitles,
        tracks: [
          { track: 0, locale: "en-US", cues: 2 },
          { track: 1, locale: "ko-KR", cues: 2 },
        ],
      },
    });
    const rows = subs.by.track.filter((t) => t.track?.kind === "subtitles");
    const tos = subs.by.trackTo.filter((t) => t.track?.kind === "subtitles");
    expect(rows.length).toBe(2);
    expect(
      instantAction(
        answers({ action: "reorder_track", targets: { track: { key: rows[0].key }, trackTo: { key: tos[1].key } } }),
        subs,
        settings,
      ),
    ).toBeNull();
  });

  // set_color_hsl writes a band's hue, saturation and luminance together and
  // the snapshot carries no values, so a band already set is the model's.
  test("a colour band already set takes the model", () => {
    const picks = {
      action: "set_color_hsl",
      targets: { clip: { key: keyOf("clip", "clip-a") } },
      enums: { hsl_band: { value: "blue" } },
      levels: { hsl_sat: { score: 3 } },
    };
    expect(instantAction(answers(picks), snap, settings)?.tool).toBe("set_color_hsl");
    const graded = instantSnapshot({
      ...context,
      videoTrack: [{ ...context.videoTrack[0], hslBands: ["blue"] }, ...context.videoTrack.slice(1)],
    });
    expect(instantAction(answers(picks), graded, settings)).toBeNull();
  });
});