Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

23.4 KB
Raw Blame
"""Signal math behind multicam editing: sync, drift, who-speaks-when, and the cut plan.

Everything here works on numpy arrays so it can be tested with synthetic audio.
The orchestration (ffmpeg extraction, sessions, rendering) lives in multicam.py.

Sync runs in two passes. A coarse pass cross-correlates 10 ms onset envelopes
over the full length of both files, which tolerates different mics, gain and EQ.
A fine pass runs GCC-PHAT on raw audio at several checkpoints; fitting a line
through those checkpoints yields both the offset and the clock drift between
devices (cameras and recorders commonly drift a frame every 10-20 minutes).
"""

from __future__ import annotations

import heapq
from dataclasses import dataclass
from typing import Optional

import numpy as np

FRAME_SECONDS = 0.01


def onset_envelope(samples: np.ndarray, sample_rate: int) -> np.ndarray:
    """Half-wave rectified log-energy rise per 10 ms frame, z-scored."""
    hop = int(round(sample_rate * FRAME_SECONDS))
    frames = len(samples) // hop
    if frames < 2:
        return np.zeros(0, dtype=np.float32)
    x = samples[: frames * hop].astype(np.float32).reshape(frames, hop)
    log_energy = np.log10(np.mean(x * x, axis=1) + 1e-9)
    onset = np.maximum(0.0, np.diff(log_energy, prepend=log_energy[0]))
    std = float(onset.std())
    if std < 1e-9:
        return np.zeros(frames, dtype=np.float32)
    return ((onset - onset.mean()) / std).astype(np.float32)


@dataclass
class CoarseMatch:
    lag_seconds: float
    score: float
    peak_ratio: float

    @property
    def ok(self) -> bool:
        return self.score >= 6.0 and self.peak_ratio <= 0.85


def coarse_lag(
    ref_env: np.ndarray,
    src_env: np.ndarray,
    *,
    min_overlap_seconds: float = 20.0,
) -> Optional[CoarseMatch]:
    """Lag (seconds) such that src time 0 sits at that time on the reference.

    score is the peak's robust z-score over all valid lags; peak_ratio is the
    runner-up peak (at least 1 s away) divided by the best one.
    """
    n, m = len(ref_env), len(src_env)
    if n == 0 or m == 0:
        return None
    size = 1 << int(np.ceil(np.log2(n + m)))
    corr = np.fft.irfft(np.fft.rfft(ref_env, size) * np.conj(np.fft.rfft(src_env, size)), size)
    lags = np.concatenate([np.arange(0, n), np.arange(-(m - 1), 0)])
    values = np.concatenate([corr[:n], corr[size - (m - 1):]]) if m > 1 else corr[:n]
    overlap = np.minimum(n, lags + m) - np.maximum(0, lags)
    min_overlap = min(int(min_overlap_seconds / FRAME_SECONDS), int(0.5 * min(n, m)))
    valid = overlap >= max(1, min_overlap)
    if not valid.any():
        return None
    lags, values = lags[valid], values[valid]

    best = int(np.argmax(values))
    peak = float(values[best])
    median = float(np.median(values))
    mad = float(np.median(np.abs(values - median))) * 1.4826 + 1e-9
    guard = int(1.0 / FRAME_SECONDS)
    away = np.abs(lags - lags[best]) > guard
    runner_up = float(values[away].max()) if away.any() else median
    ratio = (runner_up - median) / (peak - median) if peak > median else 1.0
    return CoarseMatch(
        lag_seconds=float(lags[best]) * FRAME_SECONDS,
        score=(peak - median) / mad,
        peak_ratio=max(0.0, float(ratio)),
    )


def gcc_phat(ref: np.ndarray, src: np.ndarray, sample_rate: int, max_shift_seconds: float) -> tuple[float, float]:
    """Shift (seconds) of src inside ref, and the peak's prominence.

    ref must be the src window padded by max_shift on both sides; a return of 0
    means src lines up with the center of ref.
    """
    size = 1 << int(np.ceil(np.log2(len(ref) + len(src))))
    cross = np.fft.rfft(ref.astype(np.float64), size) * np.conj(np.fft.rfft(src.astype(np.float64), size))
    cross /= np.abs(cross) + 1e-12
    corr = np.fft.irfft(cross, size)
    pad = int(round(max_shift_seconds * sample_rate))
    window = corr[: 2 * pad + 1]
    best = int(np.argmax(window))
    peak = float(window[best])
    spread = float(np.std(window)) + 1e-12
    shift = best
    # Parabolic interpolation gets sub-sample precision.
    if 0 < best < len(window) - 1:
        a, b, c = window[best - 1], window[best], window[best + 1]
        denom = a - 2 * b + c
        if abs(denom) > 1e-12:
            shift = best + 0.5 * (a - c) / denom
    return (shift - pad) / sample_rate, peak / spread


@dataclass
class ClockFit:
    offset: float
    speed: float
    residual_ms: float  # worst residual among the surviving (inlier) checkpoints
    checkpoints: int  # surviving checkpoint count
    total_checkpoints: int = 0  # checkpoints offered to the fit, before outliers were dropped
    residual_all_ms: float = 0.0  # worst residual against every checkpoint, outliers included
    speed_fallback: bool = False  # drift looked implausible, so speed was forced back to 1.0


# Real clock drift between recorders stays far under this (|speed - 1|, so 1000 ppm).
MAX_DRIFT = 1e-3


def fit_clock(points: list[tuple[float, float, float]], *, max_residual: float = 0.02) -> Optional[ClockFit]:
    """Fit timeline = offset + speed * source from (source_s, timeline_s, weight) points.

    Checkpoints further than max_residual seconds from a Theil-Sen line are
    dropped before the weighted least-squares fit.
    A single point yields speed 1. Implausible speeds (over MAX_DRIFT) fall back
    to the median offset at speed 1, since real clock drift is far smaller.
    """
    if not points:
        return None
    all_pts = np.array(points, dtype=np.float64)
    total = len(all_pts)

    def solve(p: np.ndarray) -> tuple[float, float]:
        if len(p) < 2 or np.ptp(p[:, 0]) < 1.0:
            return float(np.median(p[:, 1] - p[:, 0])), 1.0
        w = p[:, 2]
        slope, intercept = np.polyfit(p[:, 0], p[:, 1], 1, w=w)
        return float(intercept), float(slope)

    # Theil-Sen seeds the outlier test; least squares alone lets one bad
    # checkpoint drag the line far enough that good points look bad too.
    slopes = [
        (all_pts[j, 1] - all_pts[i, 1]) / (all_pts[j, 0] - all_pts[i, 0])
        for i in range(total) for j in range(i + 1, total)
        if all_pts[j, 0] - all_pts[i, 0] >= 1.0
    ]
    speed = float(np.median(slopes)) if slopes else 1.0
    offset = float(np.median(all_pts[:, 1] - speed * all_pts[:, 0]))
    keep = np.abs(all_pts[:, 1] - (offset + speed * all_pts[:, 0])) <= max_residual
    pts = all_pts[keep] if keep.any() else all_pts
    offset, speed = solve(pts)
    residual = np.abs(pts[:, 1] - (offset + speed * pts[:, 0]))
    speed_fallback = abs(speed - 1.0) > MAX_DRIFT
    if speed_fallback:
        offset, speed = float(np.median(pts[:, 1] - pts[:, 0])), 1.0
        residual = np.abs(pts[:, 1] - (offset + pts[:, 0]))
    # Residual over every checkpoint offered to the fit, outliers included: a
    # fit that dropped its way to a clean-looking inlier residual can still be
    # wrong if most of the checkpoints it threw out disagreed with the line.
    residual_all = np.abs(all_pts[:, 1] - (offset + speed * all_pts[:, 0]))
    return ClockFit(
        offset=offset,
        speed=speed,
        residual_ms=float(residual.max()) * 1000.0,
        checkpoints=len(pts),
        total_checkpoints=total,
        residual_all_ms=float(residual_all.max()) * 1000.0,
        speed_fallback=speed_fallback,
    )


def level_db(samples: np.ndarray, sample_rate: int) -> np.ndarray:
    """RMS level in dBFS per 10 ms frame for int16 or float samples."""
    hop = int(round(sample_rate * FRAME_SECONDS))
    frames = len(samples) // hop
    if frames == 0:
        return np.zeros(0, dtype=np.float32)
    x = samples[: frames * hop].astype(np.float32)
    if samples.dtype == np.int16:
        x /= 32768.0
    power = np.mean(x.reshape(frames, hop) ** 2, axis=1)
    return (10.0 * np.log10(power + 1e-10)).astype(np.float32)


def _smooth_power_db(db: np.ndarray, frames: int) -> np.ndarray:
    if frames <= 1 or len(db) == 0:
        return db
    power = 10.0 ** (db / 10.0)
    kernel = np.ones(frames, dtype=np.float32) / frames
    return (10.0 * np.log10(np.convolve(power, kernel, mode="same") + 1e-10)).astype(np.float32)


SILENT = -1
BOTH = -2


def speaker_frames(
    levels: list[np.ndarray],
    *,
    smooth_seconds: float = 0.3,
    speech_window_db: float = 20.0,
    floor_margin_db: float = 10.0,
    both_margin_db: float = 4.0,
) -> np.ndarray:
    """Per-frame dominant speaker index from one timeline-aligned level track per person.

    Each track is normalized to its own typical speech level (95th percentile)
    so a hot guest mic doesn't win every frame. Bleed from the other person
    lands far below that level and loses the argmax. Returns SILENT when nobody
    clears their noise floor and BOTH when two people are within both_margin_db.
    """
    if not levels:
        return np.zeros(0, dtype=np.int32)
    length = min(len(level) for level in levels)
    frames = int(round(smooth_seconds / FRAME_SECONDS))
    stacked = np.stack([_smooth_power_db(level[:length], frames) for level in levels])
    floors = np.percentile(stacked, 20, axis=1, keepdims=True)
    speech = np.percentile(stacked, 95, axis=1, keepdims=True)
    normalized = stacked - speech
    active = (normalized > -speech_window_db) & (stacked > floors + floor_margin_db)
    scores = np.where(active, normalized, -np.inf)

    order = np.argsort(-scores, axis=0)
    best = order[0]
    labels = best.astype(np.int32)
    best_score = np.take_along_axis(scores, order[:1], axis=0)[0]
    labels[~np.isfinite(best_score)] = SILENT
    if len(levels) > 1:
        second_score = np.take_along_axis(scores, order[1:2], axis=0)[0]
        gap = np.where(np.isfinite(second_score), best_score - np.nan_to_num(second_score, neginf=0.0), np.inf)
        both = gap < both_margin_db
        labels[both] = BOTH
    return labels


def segments_to_frames(segments: list[dict], people: list[str], length: int) -> np.ndarray:
    """Rasterize diarized {start, end, person} turns into the speaker_frames label format."""
    labels = np.full(length, SILENT, dtype=np.int32)
    index = {person: i for i, person in enumerate(people)}
    for seg in segments:
        who = index.get(seg.get("person"))
        if who is None:
            continue
        a = max(0, int(float(seg["start"]) / FRAME_SECONDS))
        b = min(length, int(float(seg["end"]) / FRAME_SECONDS))
        if b <= a:
            continue
        current = labels[a:b]
        labels[a:b] = np.where((current != SILENT) & (current != who), BOTH, who)
    return labels


def _runs(values: np.ndarray) -> list[list[int]]:
    """Run-length encode into [value, start_frame, end_frame] lists."""
    if len(values) == 0:
        return []
    change = np.flatnonzero(np.diff(values)) + 1
    starts = np.concatenate([[0], change])
    ends = np.concatenate([change, [len(values)]])
    return [[int(values[s]), int(s), int(e)] for s, e in zip(starts, ends)]


@dataclass
class Camera:
    source_id: str
    person: str
    start: float
    end: float

    def covers(self, a: float, b: float) -> bool:
        return self.start <= a + 1e-6 and self.end >= b - 1e-6


def plan_cuts(
    labels: np.ndarray,
    people: list[str],
    cameras: list[Camera],
    *,
    range_start: float,
    range_end: float,
    min_shot: float = 2.0,
    max_shot: float = 30.0,
    wide_insert: float = 4.0,
    backchannel: float = 1.2,
    guests: frozenset[str] = frozenset(),
    hold_guest: bool = True,
    host_solo: bool = True,
    guest_min: float = 0.0,
    guest_delay: float = 0.0,
) -> list[dict]:
    """Turn per-frame speaker labels into [{start, end, source_id}] on the timeline.

    Silence holds the previous shot; crosstalk goes wide. A backchannel (an
    "mm-hm", a laugh, a one-word interjection under backchannel seconds) inside
    someone else's turn never cuts away from them. Shots shorter than min_shot
    fold into a neighbour whose camera also covers them. A single shot longer
    than max_shot gets a wide_insert-second wide shot in the middle when a wide
    camera exists, except on a guest when hold_guest is set: a guest's answer
    stays on the guest from start to finish. With host_solo off a host speaks
    on the wide shot. A guest turn shorter than guest_min stays wide, and a
    longer one opens wide for guest_delay seconds before going to the guest,
    as a call recording cuts between the split screen and the guest. Split
    camera files (part 1, part 2) are handled because every decision checks
    which camera actually covers the moment. Once the shots are settled, a
    cut made where someone starts to speak moves LEAD_IN seconds earlier.
    """
    if not cameras or range_end <= range_start:
        return []
    wide = [c for c in cameras if c.person == "wide"]
    by_person = {p: [c for c in cameras if c.person == p] for p in people}

    def pick(target: str, a: float, b: float) -> Optional[Camera]:
        pools = []
        if target == "wide":
            pools = [wide]
        else:
            pools = [by_person.get(target, []), wide]
        pools.append(cameras)
        for pool in pools:
            for cam in pool:
                if cam.covers(a, b):
                    return cam
        return None

    first = int(range_start / FRAME_SECONDS)
    last = int(np.ceil(range_end / FRAME_SECONDS))
    window = np.full(last - first, SILENT, dtype=np.int32)
    avail = labels[max(0, first):min(len(labels), last)]
    lead = max(0, -first)
    window[lead:lead + len(avail)] = avail
    window = suppress_backchannels(window, backchannel)

    targets: list[str] = []
    speech: list[str] = []
    previous = "wide" if wide else (people[0] if people else "wide")
    for value in window:
        if value == SILENT:
            targets.append(previous)
            speech.append("")
            continue
        current = "wide" if value == BOTH else people[value]
        targets.append(current)
        speech.append(current)
        previous = current
    if wide and (not host_solo or guest_min > 0 or guest_delay > 0):
        targets = _call_layout(targets, speech, guests, host_solo, guest_min, guest_delay)
    codes = {name: i for i, name in enumerate(dict.fromkeys(targets))}
    names = list(codes)
    shot_runs = _runs(np.array([codes[t] for t in targets], dtype=np.int32))

    # Camera coverage can change inside a run (a camera file ends); split there.
    edges = sorted({c.start for c in cameras} | {c.end for c in cameras})
    shots: list[dict] = []
    for code, a, b in shot_runs:
        start = range_start + a * FRAME_SECONDS
        end = min(range_end, range_start + b * FRAME_SECONDS)
        cuts = [start] + [e for e in edges if start < e < end] + [end]
        for s, e in zip(cuts, cuts[1:]):
            cam = pick(names[code], s, e)
            if cam is None:
                continue
            if shots and shots[-1]["source_id"] == cam.source_id and abs(shots[-1]["end"] - s) < 1e-6:
                shots[-1]["end"] = e
            else:
                shots.append({"start": s, "end": e, "source_id": cam.source_id})

    by_id = {c.source_id: c for c in cameras}
    shots = _fold_short_shots(shots, min_shot, by_id)

    if wide and max_shot > 0 and wide_insert > 0:
        broken: list[dict] = []
        for shot in shots:
            length = shot["end"] - shot["start"]
            person = by_id[shot["source_id"]].person
            held = hold_guest and person in guests
            if person == "wide" or held or length <= max_shot:
                broken.append(shot)
                continue
            # Spacing inserts at least a shortest shot apart keeps them from
            # overlapping when the wide cutaway is longer than max_shot.
            pieces = int(length // max(max_shot, wide_insert + min_shot))
            step = length / (pieces + 1)
            cursor = shot["start"]
            for k in range(1, pieces + 1):
                mid = shot["start"] + step * k
                a, b = mid - wide_insert / 2, mid + wide_insert / 2
                if a - cursor < min_shot or shot["end"] - b < min_shot:
                    continue
                cam = pick("wide", a, b)
                if cam is None or cam.person != "wide":
                    continue
                broken.append({**shot, "start": cursor, "end": a})
                broken.append({"start": a, "end": b, "source_id": cam.source_id})
                cursor = b
            broken.append({**shot, "start": cursor, "end": shot["end"]})
        shots = broken

    _lead_in(shots, labels, by_id, range_start, min_shot)
    return [
        {"start": round(s["start"], 3), "end": round(s["end"], 3), "source_id": s["source_id"]}
        for s in shots
        if s["end"] - s["start"] > 1e-3
    ]


# A cut made where someone starts to speak lands this much before their first
# word, so the picture is already on them when the sound arrives.
LEAD_IN = 0.12


def _lead_in(shots: list[dict], labels: np.ndarray, by_id: dict[str, Camera], range_start: float,
             min_shot: float) -> None:
    """Move each cut that a speech onset caused up to LEAD_IN seconds earlier, in place.

    The cut only moves back through silence (or the incoming speaker's own
    sound), never across anyone else's words. It never leaves the outgoing
    shot shorter than min_shot, and never reaches before the incoming camera
    started rolling.
    """
    first = int(range_start / FRAME_SECONDS)
    steps = int(round(LEAD_IN / FRAME_SECONDS))
    for prev, shot in zip(shots, shots[1:]):
        f = first + int(round((shot["start"] - range_start) / FRAME_SECONDS))
        if not 0 < f < len(labels) or labels[f] == SILENT or labels[f] == labels[f - 1]:
            continue
        earliest = max(prev["start"] + min_shot, by_id[shot["source_id"]].start)
        k = f
        while (f - k < steps and k > 0 and labels[k - 1] in (SILENT, labels[f])
               and range_start + (k - 1 - first) * FRAME_SECONDS >= earliest - 1e-9):
            k -= 1
        if k < f:
            prev["end"] = shot["start"] = range_start + (k - first) * FRAME_SECONDS


# A guest's full-frame shot runs this far past their last word, then the
# split returns before the next speaker, as the measured remote edits do.
SOLO_TAIL = 1.0


def _call_layout(targets: list[str], speech: list[str], guests: frozenset[str], host_solo: bool,
                 guest_min: float, guest_delay: float) -> list[str]:
    """Send host turns and short guest turns to the wide shot; open long guest turns on it.

    `targets` has pauses filled with the last speaker; `speech` keeps them
    empty, so a guest turn is measured from their first word to their last,
    and silence after an answer never counts toward it.
    """
    out = list(targets)
    delay = int(guest_delay / FRAME_SECONDS)
    tail = int(SOLO_TAIL / FRAME_SECONDS)
    start = 0
    for i in range(1, len(targets) + 1):
        if i < len(targets) and targets[i] == targets[start]:
            continue
        who = targets[start]
        if who in guests:
            spoken = [k for k in range(start, i) if speech[k] == who]
            first, last = (spoken[0], spoken[-1] + 1) if spoken else (start, start)
            if (last - first) * FRAME_SECONDS < guest_min:
                out[start:i] = ["wide"] * (i - start)
            else:
                solo_from, solo_to = min(i, first + delay), min(i, last + tail)
                out[start:i] = ["wide"] * (i - start)
                out[solo_from:solo_to] = [who] * max(0, solo_to - solo_from)
        elif who != "wide" and not host_solo:
            out[start:i] = ["wide"] * (i - start)
        start = i
    return out


def suppress_backchannels(labels: np.ndarray, backchannel: float) -> np.ndarray:
    """Relabel short interjections inside one person's turn as that person.

    A run shorter than `backchannel` seconds that sits between two stretches of
    the same speaker (silence in between doesn't count) belongs to that
    speaker's turn: a listener's "yeah" or a shared laugh shouldn't pull the
    camera off someone mid-answer. Both stretches must be at least that long
    themselves, so a blip inside real crosstalk doesn't swallow it.
    """
    limit = int(backchannel / FRAME_SECONDS)
    if limit <= 0 or len(labels) == 0:
        return labels
    out = labels.copy()
    runs = [r for r in _runs(labels) if r[0] != SILENT]
    for k in range(1, len(runs) - 1):
        value, a, b = runs[k]
        (before, a0, b0), (after, a1, b1) = runs[k - 1], runs[k + 1]
        if (b - a < limit and before == after and before >= 0 and value != before
                and b0 - a0 >= limit and b1 - a1 >= limit):
            out[a:b] = before
    return out


def _merge_neighbours(shots: list[dict]) -> list[dict]:
    merged: list[dict] = []
    for s in shots:
        if merged and merged[-1]["source_id"] == s["source_id"] and abs(merged[-1]["end"] - s["start"]) < 1e-6:
            merged[-1]["end"] = s["end"]
        else:
            merged.append(s)
    return merged


def _fold_short_shots(shots: list[dict], min_shot: float, by_id: dict[str, Camera]) -> list[dict]:
    """Fold shots under min_shot into their longer neighbour whose camera covers them.

    Always folds the globally shortest shot next, so flickery crosstalk
    consolidates into one wide shot before the long shots around it grow into
    it. A heap over a linked list keeps that O(n log n) on 3-hour episodes.
    """
    shots = _merge_neighbours(shots)
    n = len(shots)
    prev = list(range(-1, n - 1))
    nxt = list(range(1, n + 1))
    nxt[-1:] = [-1] if n else []
    alive = [True] * n
    version = [0] * n
    heap: list[tuple[float, int, int]] = []

    def length(i: int) -> float:
        return shots[i]["end"] - shots[i]["start"]

    def push(i: int) -> None:
        if 0 <= i and alive[i] and length(i) < min_shot:
            heapq.heappush(heap, (length(i), i, version[i]))

    def unlink(i: int) -> None:
        alive[i] = False
        if prev[i] >= 0:
            nxt[prev[i]] = nxt[i]
        if nxt[i] >= 0:
            prev[nxt[i]] = prev[i]

    for i in range(n):
        push(i)
    while heap:
        _, i, v = heapq.heappop(heap)
        if not alive[i] or v != version[i]:
            continue
        shot = shots[i]
        candidates = [
            j for j in (prev[i], nxt[i])
            if j >= 0
            and abs((shots[j]["end"] if j == prev[i] else shots[j]["start"]) - (shot["start"] if j == prev[i] else shot["end"])) < 1e-6
            and by_id[shots[j]["source_id"]].covers(shot["start"], shot["end"])
        ]
        if not candidates:
            continue
        j = max(candidates, key=length)
        if j == prev[i]:
            shots[j]["end"] = shot["end"]
        else:
            shots[j]["start"] = shot["start"]
        unlink(i)
        # The fold can leave j touching a shot on the same camera; join them.
        for k in (prev[j], nxt[j]):
            if k >= 0 and shots[k]["source_id"] == shots[j]["source_id"]:
                shots[j]["start"] = min(shots[j]["start"], shots[k]["start"])
                shots[j]["end"] = max(shots[j]["end"], shots[k]["end"])
                unlink(k)
        # j grew, so its old heap entry is stale; its neighbours may now have a
        # camera to fold into that they lacked before.
        version[j] += 1
        for k in (j, prev[j], nxt[j]):
            push(k)

    out, i = [], next((k for k in range(n) if alive[k] and prev[k] == -1), -1)
    while i >= 0:
        out.append(shots[i])
        i = nxt[i]
    return out


def speech_bounds(labels: np.ndarray) -> Optional[tuple[float, float]]:
    """First and last frame where anyone speaks, in seconds."""
    spoken = np.flatnonzero(labels != SILENT)
    if len(spoken) == 0:
        return None
    return float(spoken[0]) * FRAME_SECONDS, float(spoken[-1] + 1) * FRAME_SECONDS