Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

13.5 KB
Raw Blame
"""Pure helper functions for the face-tracking / camera-movement
pipeline.

Extracted from video_processor.py. These are all side-effect-free:
they take dicts / scalars / lists and return decisions (a camera
center, a speaker id, a clamped crop_x). No ffmpeg, no cv2, no
numpy. Trivially unit-testable in isolation, but they govern the
camera's behavior during every clip render.
"""

from __future__ import annotations


def update_tripod_camera(
    current_center_x: float,
    target_center_x: float | None,
    crop_w: int,
    video_width: int,
    dt: float,
    force_snap: bool = False,
) -> float:
    """Heavy-tripod camera movement inspired by OpenShorts.

    The camera stays still while the subject remains inside a safe
    zone, then moves at a bounded speed instead of constantly chasing
    every face twitch. Always clamps the result so the crop window
    never falls off the source frame.
    """
    half_crop = crop_w / 2.0
    min_center = half_crop
    max_center = max(half_crop, video_width - half_crop)

    if target_center_x is None:
        return min(max(current_center_x, min_center), max_center)

    if force_snap:
        current_center_x = target_center_x
    else:
        diff = target_center_x - current_center_x
        # Wide enough to absorb natural head movement and detector
        # noise, narrow enough to keep the speaker composed near center.
        safe_zone_radius = crop_w * 0.22
        if abs(diff) > safe_zone_radius:
            slow_speed = 72.0
            fast_speed = 360.0
            speed = fast_speed if abs(diff) > crop_w * 0.5 else slow_speed
            step = min(abs(diff), speed * max(dt, 0.01))
            current_center_x += step if diff > 0 else -step

    return min(max(current_center_x, min_center), max_center)


def choose_camera_speaker(
    transcript_speaker: str | None,
    transcript_duration: float,
    active_speaker: str | None,
    pending_speaker: str | None,
    pending_count: int,
    min_turn_duration: float = 2.6,
    confirmation_frames: int = 3,
) -> tuple[str | None, str | None, int, bool]:
    """Stabilize diarization before the camera switches.

    Brief interjections don't move the camera; sustained turns must
    be confirmed for `confirmation_frames` samples before the switch
    is committed. Returns:
        (camera_speaker, pending_speaker, pending_count, switched_now)
    """
    if transcript_speaker is None:
        return active_speaker, pending_speaker, pending_count, False

    if active_speaker is None:
        return transcript_speaker, None, 0, True

    if transcript_speaker == active_speaker:
        return active_speaker, None, 0, False

    # Brief interjections should not move the camera.
    if transcript_duration < min_turn_duration:
        return active_speaker, None, 0, False

    if pending_speaker != transcript_speaker:
        return active_speaker, transcript_speaker, 1, False

    pending_count += 1
    if pending_count < confirmation_frames:
        return active_speaker, pending_speaker, pending_count, False

    return transcript_speaker, None, 0, True


def safe_default_center(
    width: int,
    crop_w: int,
    face_map: dict | None,
    has_any_split: bool,
    first_speaker: str | None,
    speaker_anchor_x: dict,
) -> float:
    """Pick a safe initial camera center.

    On split-screen, width/2 is the dead zone between two feeds.  Use
    the face_map or speaker_anchor to start on an actual speaker
    instead. Fallback order:
      1. first_speaker's learned anchor
      2. largest face_map cluster
      3. plain center (non-split)
      4. left quarter (split with no map)
    """
    if first_speaker and first_speaker in speaker_anchor_x:
        return float(speaker_anchor_x[first_speaker])

    if face_map and face_map.get("clusters"):
        clusters = face_map["clusters"]
        best = max(clusters, key=lambda c: c.get("count", 0))
        return float(best["center_x"])

    if not has_any_split:
        return float(width) / 2

    return float(width) / 4


def clamp_away_from_dead_zone(
    crop_x: int,
    crop_w: int,
    width: int,
    face_map: dict | None,
    has_any_split: bool,
) -> int:
    """Snap a crop that centers on the split-screen seam to the nearest
    cluster instead.

    Only triggers when the crop CENTER is close to the midline of the
    source frame: a crop that merely straddles the midline is fine on
    fullscreen layouts where the face genuinely sits near center.
    """
    if not has_any_split or not face_map:
        return crop_x

    mid_x = width // 2
    crop_center = crop_x + crop_w // 2

    seam_margin = crop_w // 8  # ~75px on a 607-wide crop
    if abs(crop_center - mid_x) > seam_margin:
        return crop_x

    clusters = face_map.get("clusters", [])
    if not clusters:
        return max(0, width // 4 - crop_w // 2)

    best_cluster = min(
        clusters, key=lambda c: abs(c["center_x"] - crop_center)
    )
    snapped = max(0, min(best_cluster["center_x"] - crop_w // 2, width - crop_w))
    return snapped


def upgrade_speaker_mappings(face_map: dict) -> dict:
    """Invalidate stale speaker-to-cluster mappings from old caches.

    v1 caches used "first-to-speak = left" which breaks when the host
    speaks first but sits on the right. This function clears the
    mappings entirely and stamps `_mappings_v2 = True` so the clear
    only runs once per cache. Cluster positions are kept because they're
    still useful for dead-zone clamping and safe defaults.

    Note: face_analysis.py writes its own v2-quality mappings and
    stamps them with `_mappings_v2=True`, so newly-built face_maps
    skip this function entirely.
    """
    face_map = dict(face_map)
    face_map["speaker_mappings"] = {}
    face_map["_mappings_v2"] = True
    return face_map


def crop_center_keeping_faces_visible(
    face_xs: list[float],
    crop_w: int,
    video_width: int,
    margin_ratio: float = 0.15,
) -> float:
    """Pick the crop center that leaves the most sampled faces in frame.

    The median of a run of face positions is only the right answer when
    those positions are unimodal. When a run mixes two layouts, a
    fullscreen shot at 960 and a split-screen tile at 1440, the median lands
    at 1200, which on a 607-wide crop is the wall between them, and the
    render holds on nothing for the length of the run.

    So score candidate centers by how many faces they actually keep inside
    the crop, and only fall back to closeness as a tiebreak. On a unimodal
    run every candidate keeps every face and this returns the median, which
    is what the caller wanted in the first place.
    """
    if not face_xs:
        return float(video_width) / 2

    from statistics import median

    max_crop_x = max(0, video_width - crop_w)
    margin = crop_w * margin_ratio

    def _center_of(candidate: float) -> float:
        crop_x = min(max(candidate - crop_w / 2.0, 0.0), float(max_crop_x))
        return crop_x + crop_w / 2.0

    def _score(candidate: float) -> tuple[int, float]:
        center = _center_of(candidate)
        low = center - crop_w / 2.0 + margin
        high = center + crop_w / 2.0 - margin
        inside = [x for x in face_xs if low <= x <= high]
        if not inside:
            return 0, -float("inf")
        spread = sum(abs(x - center) for x in inside) / len(inside)
        return len(inside), -spread

    candidates = list(face_xs) + [float(median(face_xs))]
    best = max(candidates, key=_score)
    return _center_of(best)


def seats_from_frames(
    frame_positions: list[list[float]],
    video_width: int,
    min_paired_frames: int = 3,
    min_separation_ratio: float = 0.20,
    min_paired_share: float = 0.10,
) -> tuple[int, int] | None:
    """Find the two fixed positions people occupy, or None if there is one.

    Takes the x of every face in every sampled frame, grouped by frame. Two
    seats are only real if some frame held two faces at once and they sat far
    enough apart to be two people rather than two heads sharing one camera.

    Counting a flat list of positions either side of the midline cannot make
    that distinction, and mistaking one person for two is expensive: the
    invented seat is empty wall, and every speaker mapped to it renders as a
    hold on nothing.
    """
    from statistics import median

    paired = [sorted(xs) for xs in frame_positions if len(xs) >= 2]
    with_faces = sum(1 for xs in frame_positions if xs)
    # A share as well as a count, so a handful of false positives, a face in a
    # photo on the shelf behind someone, cannot seat a second person.
    if len(paired) < max(min_paired_frames, with_faces * min_paired_share):
        return None

    left = int(median([xs[0] for xs in paired]))
    right = int(median([xs[-1] for xs in paired]))
    if (right - left) <= video_width * min_separation_ratio:
        return None
    return left, right


def longest_layout_run(detections: list, split: bool) -> float:
    best = 0.0
    start = None
    for t, faces in detections:
        if not faces:
            continue
        if (len(faces) >= 2) == split:
            if start is None:
                start = t
            best = max(best, t - start)
        else:
            start = None
    return best


def follow_within_runs(
    runs: list,
    face_points: list,
    crop_w: int,
    video_width: int,
    fast: float = 480.0,
    slow: float = 140.0,
) -> list:
    half = crop_w / 2.0
    lowest, highest = half, max(half, video_width - half)
    keys = []
    for start, end, center in runs:
        cam = min(max(center, lowest), highest)
        if keys:
            keys.append((max(keys[-1][0], start - 0.001), keys[-1][1]))
        keys.append((start, cam))
        last = start
        for t, cx, fw in face_points:
            if not start <= t < end:
                continue
            room = max(24.0, (crop_w - fw) / 2.0 - crop_w * 0.04)
            calm = min(crop_w * 0.22, room)
            diff = cx - cam
            if abs(diff) > calm:
                speed = fast if abs(diff) > room else slow
                cam += min(abs(diff), speed * max(t - last, 0.01)) * (1 if diff > 0 else -1)
                cam = min(max(cam, lowest), highest)
            last = t
            keys.append((t, cam))
    return keys


def clip_layout_is_mixed(
    detections: list,
    face_map: dict | None = None,
    min_share: float = 0.15,
    min_frames: int = 3,
    min_run: float = 1.0,
) -> bool:
    """Does this clip switch between a split screen and a fullscreen shot?

    Judged on the clip's own frames first. The layout used to be read only
    from the episode-wide face_map, and a face_map cached by a podcli older
    than the mixed-layout work carries no is_mixed_layout key at all, so the
    default turned those into "not mixed". That sent Riverside recordings down
    a path that holds one camera position across a layout change, and every
    fullscreen stretch rendered as the wall beside the speaker.

    Both layouts must hold a real share of the frames. A count alone would
    call a few frames with a missed second face a layout change and route
    almost every clip through the mixed path.
    """
    split_frames = sum(1 for _, faces in detections if len(faces) >= 2)
    single_frames = sum(1 for _, faces in detections if len(faces) == 1)
    layout_frames = split_frames + single_frames

    if layout_frames and (
        split_frames >= max(min_frames, layout_frames * min_share)
        and single_frames >= max(min_frames, layout_frames * min_share)
    ):
        return True

    if (
        longest_layout_run(detections, split=True) >= min_run
        and longest_layout_run(detections, split=False) >= min_run
    ):
        return True

    # A clip sitting entirely inside one layout still belongs to a mixed
    # episode, so the episode-wide flag stays a valid hint.
    return bool(face_map and face_map.get("is_mixed_layout", False))


def followed_face_cx_at(
    t_target: float,
    tracked_detections: list,
    segment_tracks: list,
    fallback_track_id: int | None = None,
    window: float = 1.5,
) -> float | None:
    """Where the speaker the camera is following actually is at a given time.

    Used to check a planned crop still holds its subject. It has to ask about
    the followed track, not the biggest face in the frame: on a split screen
    the biggest face is whoever sits closest to their own camera, and on the
    recording this was written for one speaker measures 845px across against
    the other's 518. Taking the larger one made every correct crop onto the
    quieter side look like a crop onto nobody, so each keyframe was pulled
    back to the same person and the camera stopped switching.

    Falls back to the most prominent face only when the followed speaker is
    not on screen anywhere near this moment, where any face beats none.
    """
    track_id = None
    for start_t, end_t, _speaker, tid, *_rest in segment_tracks:
        if start_t <= t_target <= end_t:
            track_id = tid
            break
    if track_id is None:
        track_id = fallback_track_id

    best_cx = best_dt = None
    fallback_cx, fallback_dt = None, None
    for t, faces in tracked_detections:
        dt = abs(t - t_target)
        if dt > window or not faces:
            continue
        if fallback_dt is None or dt < fallback_dt:
            fallback_cx = float(max(faces, key=lambda f: f["fw"])["cx"])
            fallback_dt = dt
        if track_id is None:
            continue
        face = next((f for f in faces if f["track_id"] == track_id), None)
        if face is not None and (best_dt is None or dt < best_dt):
            best_cx, best_dt = float(face["cx"]), dt

    return best_cx if best_cx is not None else fallback_cx