Skip to content

Open source CapCut alternative.

TypeScript
9.1 KB
Raw Blame
/**
 * The coverage invariant and the frame/second boundary.
 *
 * Shots must tile the whole audio with no gaps and no overlaps: the first
 * starts at frame 0, the last ends at `durationFrames`, and every shot ends
 * exactly where the next begins. Because frames are integers this holds
 * exactly — the orchestrator re-asserts it after every mutation, and any model
 * output is run through `repairCoverage` before it is trusted. `repairCoverage`
 * is total: it never throws for any input (a model routinely returns
 * overlapping, nested, or out-of-order ranges), and it treats a zero-length
 * timeline as covered by zero shots.
 *
 * Seconds cross this boundary only through `frameToSec`/`secToFrame`, computed
 * once from the project fps. Everything else in the plan is frames.
 */

import type { Shot } from "./types";

/** The timeline slice range a shot may occupy, in seconds. The floor is a
 * pacing choice, not the video model's: a narration names a new idea every
 * couple of seconds, and each idea deserves its own shot. The model picks its
 * own render length (up to ~10s) and always covers the slot, so the placement
 * trims the take to it (the dailies review picks which window survives). The
 * ceiling keeps one slot from swallowing a whole beat. */
export const MIN_SHOT_SEC = 2;
export const MAX_SHOT_SEC = 8;

export const secToFrame = (sec: number, fps: number): number =>
  Math.max(0, Math.round(sec * fps));

export const frameToSec = (frame: number, fps: number): number => frame / fps;

export const shotDurationFrames = (shot: Shot): number =>
  shot.endFrame - shot.startFrame;

/** Thrown when the invariant is violated — a bug, not a user-facing error. */
export class CoverageError extends Error {}

/**
 * Assert the invariant. Throws `CoverageError` with the exact break. Call this
 * after every mutation to the shot list.
 */
export function assertCoverage(shots: Shot[], durationFrames: number): void {
  if (shots.length === 0) {
    if (durationFrames <= 0) return;
    throw new CoverageError(`No shots cover ${durationFrames} frames.`);
  }
  if (shots[0].startFrame !== 0)
    throw new CoverageError(`First shot starts at ${shots[0].startFrame}, not 0.`);
  const last = shots[shots.length - 1];
  if (last.endFrame !== durationFrames)
    throw new CoverageError(
      `Last shot ends at ${last.endFrame}, not ${durationFrames}.`
    );
  for (let i = 0; i < shots.length; i++) {
    const s = shots[i];
    if (s.endFrame <= s.startFrame)
      throw new CoverageError(
        `Shot ${s.id} is empty or inverted (${s.startFrame}..${s.endFrame}).`
      );
    if (i > 0 && s.startFrame !== shots[i - 1].endFrame)
      throw new CoverageError(
        `Gap or overlap between shot ${shots[i - 1].id} (ends ${
          shots[i - 1].endFrame
        }) and ${s.id} (starts ${s.startFrame}).`
      );
  }
}

/** A raw shot boundary from a model, before it is trusted. */
export interface RawShot {
  startFrame: number;
  endFrame: number;
  audioText?: string;
  action?: string;
  characters?: string[];
  location?: string;
  framing?: string;
}

/**
 * Force a raw boundary list into a valid, fully-covering, correctly-ordered
 * shot list. Sorts, clamps into range, chains the boundaries into a monotonic
 * contiguous sequence (dropping nested/overlapped ranges rather than inverting
 * them), merges any slice shorter than the minimum clip into a neighbor, and
 * splits any slice longer than the maximum clip. The result always passes
 * `assertCoverage`; a zero-length timeline yields zero shots.
 */
export function repairCoverage(
  raw: RawShot[],
  durationFrames: number,
  fps: number,
  makeId: (i: number) => string
): Shot[] {
  if (durationFrames <= 0) return [];
  const minFrames = Math.round(MIN_SHOT_SEC * fps);
  const maxFrames = Math.round(MAX_SHOT_SEC * fps);

  // Sort by start, drop degenerate, clamp into range.
  const sorted = raw
    .map((r) => ({
      startFrame: clampFrame(r.startFrame, durationFrames),
      endFrame: clampFrame(r.endFrame, durationFrames),
      audioText: r.audioText ?? "",
      action: r.action ?? "",
      characters: r.characters ?? [],
      location: r.location ?? "",
      framing: r.framing ?? "",
    }))
    .filter((r) => r.endFrame > r.startFrame)
    .sort((a, b) => a.startFrame - b.startFrame || a.endFrame - b.endFrame);

  // Chain into a contiguous, monotonic sequence. Walk in start order behind a
  // cursor at the end of the last kept shot: each kept shot starts at the
  // cursor and ends at its own clamped end, and any shot ending at or before
  // the cursor — a nested or fully-overlapped range — is dropped. Non-monotonic
  // model output can never produce a gap or an inverted shot this way.
  const live: Slice[] = [];
  let cursor = 0;
  for (const r of sorted) {
    const end = Math.min(r.endFrame, durationFrames);
    if (end <= cursor) continue; // already covered by an earlier, longer shot
    live.push({ ...r, startFrame: cursor, endFrame: end });
    cursor = end;
    if (cursor >= durationFrames) break;
  }
  if (live.length === 0) {
    live.push({ startFrame: 0, endFrame: durationFrames, audioText: "", action: "", characters: [], location: "", framing: "" });
  }
  live[live.length - 1].endFrame = durationFrames;

  // Merge too-short slices, then split too-long ones (split has the last word —
  // the minimum is half the maximum, so an even division never re-slivers).
  const merged = mergeShort(live, minFrames, durationFrames);
  const normalized = resplitOverMax(merged, maxFrames);

  const shots: Shot[] = normalized.map((r, i) => ({
    id: makeId(i),
    startFrame: r.startFrame,
    endFrame: r.endFrame,
    audioText: r.audioText,
    action: r.action,
    characters: r.characters,
    location: r.location,
    framing: r.framing,
    status: "pending",
    attempts: 0,
  }));

  assertCoverage(shots, durationFrames);
  return shots;
}

type Slice = {
  startFrame: number;
  endFrame: number;
  audioText: string;
  action: string;
  characters: string[];
  location: string;
  framing: string;
};

const unionIds = (a: string[], b: string[]): string[] => Array.from(new Set([...a, ...b]));
const joinText = (a: string, b: string): string => [a, b].filter(Boolean).join(" ").trim();

function mergeShort(slices: Slice[], minFrames: number, durationFrames: number): Slice[] {
  // If the whole thing is shorter than one minimum shot, it is one shot.
  if (durationFrames <= minFrames) {
    const first = slices[0];
    return [{ ...first, startFrame: 0, endFrame: durationFrames }];
  }
  const out: Slice[] = [];
  for (const s of slices) {
    const prev = out[out.length - 1];
    if (s.endFrame - s.startFrame < minFrames && prev) {
      // Absorb the sliver forward, carrying its continuity metadata so a
      // character/location that appears only here isn't lost from the plan.
      prev.endFrame = s.endFrame;
      prev.characters = unionIds(prev.characters, s.characters);
      if (!prev.location) prev.location = s.location;
      if (!prev.action) prev.action = s.action;
      if (!prev.framing) prev.framing = s.framing;
      prev.audioText = joinText(prev.audioText, s.audioText);
    } else {
      out.push({ ...s });
    }
  }
  // A too-short head with no predecessor absorbs its successor instead.
  while (out.length > 1 && out[0].endFrame - out[0].startFrame < minFrames) {
    const head = out[0];
    const next = out[1];
    next.startFrame = head.startFrame;
    next.characters = unionIds(head.characters, next.characters);
    if (!next.location) next.location = head.location;
    if (!next.framing) next.framing = head.framing;
    next.audioText = joinText(head.audioText, next.audioText);
    out.shift();
  }
  out[0].startFrame = 0;
  out[out.length - 1].endFrame = durationFrames;
  return out;
}

function resplitOverMax(slices: Slice[], maxFrames: number): Slice[] {
  const out: Slice[] = [];
  for (const s of slices) {
    const len = s.endFrame - s.startFrame;
    if (len <= maxFrames) {
      out.push(s);
      continue;
    }
    const parts = Math.ceil(len / maxFrames);
    const base = Math.floor(len / parts);
    // Scope the transcript to each sub-part so a sub-clip carries only the words
    // spoken during its slice, not the whole original shot's span.
    const textParts = sliceWords(s.audioText, parts);
    let start = s.startFrame;
    for (let i = 0; i < parts; i++) {
      const end = i === parts - 1 ? s.endFrame : start + base;
      out.push({ ...s, startFrame: start, endFrame: end, audioText: textParts[i] });
      start = end;
    }
  }
  return out;
}

/** Split text into `parts` roughly-equal word chunks (in order). */
export function sliceWords(text: string, parts: number): string[] {
  if (parts <= 1) return [text];
  const words = text.split(/\s+/).filter(Boolean);
  const out: string[] = [];
  for (let i = 0; i < parts; i++) {
    const from = Math.floor((i * words.length) / parts);
    const to = Math.floor(((i + 1) * words.length) / parts);
    out.push(words.slice(from, to).join(" "));
  }
  return out;
}

const clampFrame = (v: unknown, durationFrames: number): number => {
  const n = typeof v === "number" && Number.isFinite(v) ? Math.round(v) : 0;
  return Math.max(0, Math.min(durationFrames, n));
};