Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

13.0 KB
Raw Blame
/**
 * Music Video — intercut edit pass (#9290).
 *
 * The shot plan ties every cut to one generated clip, so cutting faster used
 * to mean paying for more clips. A reference analysis of a viral AI music
 * video (#9289) found ~2.6x our cut rate from ~the same per-frame motion: its
 * pace comes from the EDIT — shots of 1–4 beats in high-energy sections,
 * cuts on sung words, and A/B intercuts that reuse footage.
 *
 * This pass re-cuts the resolved clip list at render time without generating
 * anything:
 *
 *   - each section gets a target shot length from its energy rank (the
 *     loudest quarter of the song cuts every 2 beats, the quietest every 8);
 *   - a footage or still shot longer than ~1.3 targets is split; a cut prefers a sung
 *     word's onset (snapped to the half-beat grid when close), else the
 *     half-beat nearest the target;
 *   - pieces alternate between the host shot — which keeps its continuous
 *     time, so a lip-synced performance stays in sync — and a partner cutaway
 *     from the same section (else the nearest one), played from the part of
 *     its take the edit never showed (its tail after its own out-point);
 *   - performance clips are never partners: a sung take only matches the song
 *     at its own time. Stills and authored cards pass through untouched.
 *
 * Pure and deterministic. Every piece reads inside its source (a looping
 * source may wrap); the output tiles exactly the same timeline as the input.
 */

import { cueWordOnsets, hookKey, hookLines } from './hookTypography.js';

const EPS = 1e-6;
const MIN_PIECE_SEC = 0.4;
const WORD_SNAP_SEC = 0.08;
// Energy rank (0 = quietest section, 1 = loudest) → shot length in beats.
const TIERS = Object.freeze([[0.75, 2], [0.4, 3], [0.15, 4], [0, 8]]);

const round3 = (n) => Math.round(n * 1000) / 1000;
const finite = (n) => typeof n === 'number' && Number.isFinite(n);

/** Seconds per beat: the analysis BPM, else the median beat gap; null without either. */
function beatPeriod(bpm, beats) {
  if (finite(bpm) && bpm > 0) return 60 / bpm;
  const gaps = [];
  for (let i = 1; i < beats.length; i += 1) if (beats[i] > beats[i - 1]) gaps.push(beats[i] - beats[i - 1]);
  if (!gaps.length) return null;
  gaps.sort((a, b) => a - b);
  return gaps[Math.floor(gaps.length / 2)];
}

/** Each section's energy as a 0..1 rank among the song's sections. */
function sectionRanks(sections) {
  const valid = sections.filter((s) => finite(s?.startSec) && finite(s?.endSec) && s.endSec > s.startSec);
  const energies = valid.map((s) => (finite(s.energy) ? s.energy : 0));
  const sorted = [...energies].sort((a, b) => a - b);
  return valid.map((s, i) => {
    const e = energies[i];
    const below = sorted.filter((x) => x < e).length;
    return { startSec: s.startSec, endSec: s.endSec, rank: sorted.length > 1 ? below / (sorted.length - 1) : 0.5, index: i };
  });
}

const tierBeats = (rank) => TIERS.find(([min]) => rank >= min - EPS)[1];

function sectionAt(ranked, t) {
  return ranked.find((s) => t >= s.startSec - EPS && t < s.endSec - EPS) || ranked[ranked.length - 1] || null;
}

/**
 * Cut times strictly inside (t0, t1) for a target piece length: word onsets
 * (snapped to the half-beat grid when within WORD_SNAP_SEC) are preferred
 * within half a beat of the ideal cut; otherwise the half-beat nearest it.
 */
function cutTimes(t0, t1, target, halfGrid, onsets, period) {
  const cuts = [];
  const minPiece = Math.max(MIN_PIECE_SEC, period * 0.75);
  let from = t0;
  while (t1 - from > target * 1.3) {
    const ideal = from + target;
    const window = period / 2;
    const ok = (t) => t - from >= minPiece - EPS && t1 - t >= minPiece - EPS;
    let pick = null;
    let best = Infinity;
    for (const w of onsets) {
      if (w < ideal - window || w > ideal + window || !ok(w)) continue;
      const snapped = nearest(halfGrid, w);
      const at = snapped != null && Math.abs(snapped - w) <= WORD_SNAP_SEC ? snapped : w;
      if (ok(at) && Math.abs(at - ideal) < best) { best = Math.abs(at - ideal); pick = at; }
    }
    if (pick == null) {
      const g = nearest(halfGrid, ideal);
      pick = g != null && Math.abs(g - ideal) <= window && ok(g) ? g : ideal;
      if (!ok(pick)) break;
    }
    cuts.push(round3(pick));
    from = pick;
  }
  return cuts;
}

function nearest(sorted, t) {
  if (!sorted.length) return null;
  let lo = 0;
  let hi = sorted.length - 1;
  while (lo < hi) {
    const mid = (lo + hi) >> 1;
    if (sorted[mid] < t) lo = mid + 1; else hi = mid;
  }
  const a = sorted[lo];
  const b = lo > 0 ? sorted[lo - 1] : a;
  return Math.abs(a - t) <= Math.abs(b - t) ? a : b;
}

const STOP_WORDS = new Set(['a', 'an', 'and', 'are', 'as', 'at', 'be', 'but', 'by', 'for', 'from', 'i', 'in', 'is', 'it', 'me', 'my', 'of', 'on', 'or', 'our', 'so', 'the', 'to', 'we', 'you', 'your']);
const wordKey = (word) => hookKey(word).replace(/\s+/g, '');
const isNumber = (word) => /^\p{N}+(?:[.,]\p{N}+)*$/u.test(String(word).replace(/^[^\p{L}\p{N}]+|[^\p{L}\p{N}]+$/gu, ''));

/** The most frequent content word of each repeated hook line, plus numbers. */
function graphicWords(cues) {
  const hooks = hookLines(cues);
  const candidates = [];
  for (const cue of Array.isArray(cues) ? cues : []) {
    const aligned = cueWordOnsets(cue, cue.endSec);
    const counts = new Map();
    if (hooks.has(hookKey(cue.text))) {
      for (const { w } of aligned) {
        const key = wordKey(w);
        if (key && !STOP_WORDS.has(key) && !/^\p{N}+$/u.test(key)) counts.set(key, (counts.get(key) || 0) + 1);
      }
    }
    const keyword = [...counts].sort((a, b) => b[1] - a[1])[0]?.[0];
    for (const { w, atSec } of aligned) if (isNumber(w) || (keyword && wordKey(w) === keyword)) candidates.push({ atSec, text: w });
  }
  return candidates.sort((a, b) => a.atSec - b.atSec);
}

/** Replace one beat within a single source piece; keep footage read heads intact. */
function insertGraphicCards(clips, ranked, period, lyricCues, accentColor, locked) {
  const candidates = graphicWords(lyricCues);
  if (!candidates.length) return clips;
  const accent = /^#[0-9a-f]{6}$/i.test(accentColor || '') ? accentColor : '#ff5a1f';
  const out = [];
  let cursor = 0;
  let previousCardAt = -Infinity;
  let count = 0;
  let timeline = 0;
  const authoredCards = clips.flatMap((clip) => {
    const start = timeline;
    timeline += clip.outSec - clip.inSec;
    return clip.layer === 'card' ? [{ start, end: timeline }] : [];
  });
  for (const clip of clips) {
    const span = clip.outSec - clip.inSec;
    const start = cursor;
    const end = start + span;
    cursor = end;
    if (clip.layer === 'card') {
      previousCardAt = start;
      out.push(clip);
      continue;
    }
    if (locked.has(clip.sceneId)) { out.push(clip); continue; }
    const eligible = candidates.filter(({ atSec }) => {
      const section = sectionAt(ranked, atSec);
      return section && tierBeats(section.rank) <= 3 && atSec >= section.startSec - EPS && atSec >= start - EPS
        && atSec + period <= Math.min(end, section.endSec) + EPS
        && atSec - previousCardAt >= 4 * period - EPS
        && authoredCards.every((card) => Math.abs(atSec - card.start) >= 4 * period - EPS
          && (atSec + period < card.start - EPS || atSec > card.end + EPS));
    });
    let from = start;
    for (const { atSec, text } of eligible) {
      if (atSec < from - EPS || atSec - previousCardAt < 4 * period - EPS) continue;
      if (atSec > from + EPS) out.push(sliceClip(clip, from - start, atSec - from));
      const duration = round3(period);
      out.push({ sceneId: `graphic-card-${count}`, layer: 'card', inSec: 0, outSec: duration, duration,
        cardText: text, cardColor: count % 2 === 0 ? accent : '#000000', cardOf: clip.sceneId });
      count += 1;
      previousCardAt = atSec;
      from = atSec + period;
    }
    if (end > from + EPS) out.push(from === start ? clip : sliceClip(clip, from - start, end - from));
  }
  return out;
}

function sliceClip(clip, offset, length) {
  const duration = round3(length);
  // A still's image effect picks up where the piece before it left off.
  if (clip.layer === 'still') return { ...clip, inSec: 0, outSec: duration, duration, sourceSec: duration, ...(clip.effect ? { effectAtSec: round3((clip.effectAtSec || 0) + offset) } : {}) };
  const inSec = round3(clip.inSec + offset);
  return { ...clip, inSec, outSec: round3(inSec + duration), duration,
    ...(clip.loop === false ? { sourceSec: round3(clip.inSec + (clip.sourceSec ?? clip.duration) - inSec) } : {}) };
}

/**
 * Re-cut `clips` (the snapped, coverage-checked render list, in timeline
 * order). `scenes` supply each clip's shot mode; `sections` (with `energy`),
 * `beats`/`bpm` and sung `words` (`[{ startSec }]`) drive the pacing.
 * Returns a new clip list; the input is returned as-is when there is no beat
 * grid to cut on.
 */
export function intercutClips(clips, { scenes = [], sections = [], beats = [], bpm = null, words = [], lyricCues = [], graphicCards = false, accentColor = null, lockedSceneIds = [] } = {}) {
  const locked = new Set(lockedSceneIds);
  const list = Array.isArray(clips) ? clips : [];
  const grid = (Array.isArray(beats) ? beats : []).filter(finite).sort((a, b) => a - b);
  const period = beatPeriod(bpm, grid);
  const ranked = sectionRanks(Array.isArray(sections) ? sections : []);
  if (!period || !grid.length || !ranked.length || !list.length) return list;
  const halfGrid = [];
  for (let i = 0; i < grid.length; i += 1) {
    halfGrid.push(grid[i]);
    halfGrid.push(grid[i] + (i + 1 < grid.length ? (grid[i + 1] - grid[i]) / 2 : period / 2));
  }
  const onsets = (Array.isArray(words) ? words : []).map((w) => w?.startSec).filter(finite).sort((a, b) => a - b);
  const modes = new Map((Array.isArray(scenes) ? scenes : []).map((s) => [s?.sceneId, s?.shotMode]));

  // Place every clip on the timeline.
  let cursor = 0;
  const placed = list.map((clip) => {
    const span = clip.outSec - clip.inSec;
    const entry = { clip, t0: cursor, t1: cursor + span, section: sectionAt(ranked, cursor + span / 2) };
    cursor += span;
    return entry;
  });

  // Partner cutaways: footage that is not a sung performance, each with a
  // read head starting after the part the edit already shows.
  const partners = placed
    .filter(({ clip }) => !clip.layer && clip.videoPath && modes.get(clip.sceneId) !== 'performance' && !locked.has(clip.sceneId) && finite(clip.sourceSec) && clip.sourceSec > MIN_PIECE_SEC)
    .map((p) => ({ ...p, head: p.clip.outSec, start: p.clip.inSec, end: p.clip.inSec + p.clip.sourceSec }));

  // A piece ending where its partner's own shot begins would show that take's
  // tail and then its start, which reads as the clip looping, so that shot is
  // never the partner.
  function takePartner(host, len, endT) {
    const pool = partners.filter((p) => p.clip.sceneId !== host.clip.sceneId && p.clip.videoPath !== host.clip.videoPath && p.end - p.start >= len - EPS
      && Math.abs(p.t0 - endT) > EPS);
    if (!pool.length) return null;
    const sameSection = pool.filter((p) => p.section?.index === host.section?.index);
    const candidates = sameSection.length ? sameSection : pool;
    // Nearest in time, then the one with the most unshown footage left.
    candidates.sort((a, b) => Math.abs(a.t0 - host.t0) - Math.abs(b.t0 - host.t0) || (b.end - b.head) - (a.end - a.head));
    const p = candidates[0];
    if (p.head + len > p.end + EPS) p.head = p.start; // wrap: reuse from the take's start
    const inSec = round3(p.head);
    p.head = inSec + len;
    return { ...p.clip, inSec, outSec: round3(inSec + len), duration: round3(len), loop: false, sourceSec: round3(p.end - inSec), intercutOf: host.clip.sceneId };
  }

  const out = [];
  for (const host of placed) {
    const { clip, t0, t1, section } = host;
    const still = clip.layer === 'still';
    // A locked shot is approved as it plays, so the final render keeps it whole
    // and never lends its footage to another shot.
    if ((clip.layer && !still) || (!still && !clip.videoPath) || !section || locked.has(clip.sceneId)) { out.push(clip); continue; }
    const target = tierBeats(section.rank) * period;
    const cuts = cutTimes(t0, t1, target, halfGrid, onsets, period);
    if (!cuts.length) { out.push(clip); continue; }
    const bounds = [t0, ...cuts, t1];
    for (let k = 0; k < bounds.length - 1; k += 1) {
      const a = bounds[k];
      const len = round3(bounds[k + 1] - a);
      const partner = k % 2 === 1 ? takePartner(host, len, bounds[k + 1]) : null;
      if (partner) { out.push(partner); continue; }
      if (still) { out.push({ ...clip, inSec: 0, outSec: len, duration: len, sourceSec: len, ...(clip.effect ? { effectAtSec: round3(a - t0) } : {}) }); continue; }
      // The host keeps its own continuous time (a performance stays in sync).
      const inSec = round3(clip.inSec + (a - t0));
      out.push({ ...clip, inSec, outSec: round3(inSec + len), duration: len, ...(clip.loop === false ? { sourceSec: round3(clip.inSec + (clip.sourceSec ?? clip.duration) - inSec) } : {}) });
    }
  }
  return graphicCards ? insertGraphicCards(out, ranked, period, lyricCues, accentColor, locked) : out;
}