Skip to content

Open source CapCut alternative.

TypeScript
3.9 KB
Raw Blame
/** Fused frame/transcript timeline: kept frame times woven into the caption
 * cues on one clock, so the model reads precomputed alignment — each speech
 * span lists the frames that fall inside it, and longer silences become
 * frame-only spans. Matching frames to lines by eye misaligns on long
 * ranges; this join is exact. All times are source seconds, the same numbers
 * stamped on the sheet cells. */

export interface FusedSpan {
  start: number;
  end: number;
  text?: string;
  frames: number[];
}

/** A silence between cues must reach this length to earn its own span. */
const MIN_SILENCE_SPAN = 1.5;

export function fuseTimeline(
  frameTimes: number[],
  cues: { start: number; end: number; text: string }[],
  range: { from: number; to: number }
): FusedSpan[] {
  const spans: FusedSpan[] = [];
  const inRange = cues
    .map((c) => ({
      start: Math.max(c.start, range.from),
      end: Math.min(c.end, range.to),
      text: c.text.trim(),
    }))
    .filter((c) => c.end > c.start && c.text.length > 0)
    .sort((a, b) => a.start - b.start || a.end - b.end);

  let cursor = range.from;
  for (const c of inRange) {
    if (c.start - cursor >= MIN_SILENCE_SPAN) spans.push({ start: cursor, end: c.start, frames: [] });
    spans.push({ start: c.start, end: Math.max(c.end, c.start), text: c.text, frames: [] });
    cursor = Math.max(cursor, c.end);
  }
  if (range.to - cursor >= MIN_SILENCE_SPAN || spans.length === 0)
    spans.push({ start: cursor, end: range.to, frames: [] });

  for (const t of [...frameTimes].sort((a, b) => a - b)) {
    const hit =
      spans.find((sp) => sp.start <= t && t < sp.end) ??
      // A frame in a crack between spans attaches to the closest one.
      spans.reduce((best, sp) =>
        Math.min(Math.abs(t - sp.start), Math.abs(t - sp.end)) <
        Math.min(Math.abs(t - best.start), Math.abs(t - best.end))
          ? sp
          : best
      );
    hit.frames.push(t);
  }
  return spans;
}

const sec = (t: number) => `${Math.round(t * 100) / 100}s`;

/** Render spans as the tool-result text block. Only spans that carry speech
 * or frames earn a line. */
export function renderFusedTimeline(spans: FusedSpan[]): string {
  const lines: string[] = [];
  for (const sp of spans) {
    if (!sp.text && sp.frames.length === 0) continue;
    const head = `[${sec(sp.start)}–${sec(sp.end)}]`;
    const body = sp.text ? ` "${sp.text}"` : " (no speech)";
    const frames = sp.frames.length > 0 ? `  frames: ${sp.frames.map(sec).join(", ")}` : "";
    lines.push(head + body + frames);
  }
  return lines.join("\n");
}

/** A cue lands and the picture under it changes with it: the caption swaps,
 * the speaker starts, the graphic cuts in. These are the moments a watch aims
 * its frames at, just after each cue starts so the new state is up and still.
 * A source nobody has transcribed offers none and sweeps on the steady floor
 * alone. */
export const CUE_LEAD_S = 0.2;

export function speechOnsets(
  cues: { start: number; end: number; text: string }[],
  from = 0,
  to = Infinity
): number[] {
  return cues
    .filter((c) => c.text.trim().length > 0)
    .map((c) => c.start + CUE_LEAD_S)
    .filter((t) => t >= from && t < to);
}

/** The speech that covers a stretch of a source: the project's own caption
 * cues where they reach it, the source's transcript everywhere else. A lane
 * holding cues for one clip leaves every other clip to its own transcript,
 * which is what a watch aims its frames at and fuses its timeline from. */
export function speechOver(
  cues: { start: number; end: number; text: string }[],
  transcript: { start: number; end: number; text: string }[],
  from: number,
  to: number
): { start: number; end: number; text: string }[] {
  const within = (list: { start: number; end: number; text: string }[]) =>
    list.filter((c) => c.end > from && c.start < to);
  const mapped = within(cues);
  return mapped.length > 0 ? mapped : within(transcript);
}