Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

12.0 KB
Raw Blame
"""Per-clip mouth-motion reframe planner."""

from __future__ import annotations

import logging
import os
import re
import tempfile
from pathlib import Path
from typing import Optional

from utils.proc import run as proc_run, ProcError

log = logging.getLogger("podcli.local_reframe")


_PTS_RE = re.compile(r"pts_time:([0-9.]+)")
_YAVG_RE = re.compile(r"lavfi\.signalstats\.YAVG=([0-9.]+)")


def count_scene_cuts(video_path: str, threshold: float = 0.35) -> int:
    """Count scene cuts in the clip via FFmpeg's `select='gt(scene,...)'` filter.

    Returns the number of detected cuts. Returns 0 if FFmpeg fails - callers
    should treat 0 as "no info" and proceed with their default plan.
    """
    try:
        result = proc_run(
            [
                "ffmpeg",
                "-i", str(video_path),
                # Audio and subtitles are irrelevant to scene detection, and the
                # scene score is a whole-frame statistic, so computing it on a
                # 320px-wide copy gives the same cuts for a fraction of the
                # decode. Downscale before select so the filter sees small frames.
                "-an", "-sn",
                "-filter:v", f"scale=320:-2,select='gt(scene,{threshold})',showinfo",
                "-f", "null", "-",
            ],
            timeout=180,
            check=False,
        )
    except ProcError as exc:
        log.warning("scene-cut count failed: %s", exc)
        return 0
    if result.returncode != 0:
        return 0
    return len(_PTS_RE.findall(result.stderr or ""))


def parse_motion_metadata(path: Path) -> tuple[list[float], list[float]]:
    """Parse FFmpeg metadata-print output into (times, YAVG values) lists."""
    times: list[float] = []
    values: list[float] = []
    current_time: Optional[float] = None
    try:
        raw = path.read_text(errors="ignore")
    except OSError:
        return [], []
    for line in raw.splitlines():
        pts_match = _PTS_RE.search(line)
        if pts_match:
            current_time = float(pts_match.group(1))
            continue
        val_match = _YAVG_RE.search(line)
        if val_match and current_time is not None:
            times.append(current_time)
            values.append(float(val_match.group(1)))
            current_time = None
    return times, values


def smooth_values(values: list[float], window: int = 15) -> list[float]:
    """Box-filter smoothing, no scipy dependency."""
    if not values:
        return []
    half = window // 2
    smoothed: list[float] = []
    n = len(values)
    for i in range(n):
        lo = max(0, i - half)
        hi = min(n, i + half + 1)
        smoothed.append(sum(values[lo:hi]) / (hi - lo))
    return smoothed


def build_speaker_timeline_from_motion(
    times: list[float],
    left_values: list[float],
    right_values: list[float],
    min_duration: float = 1.0,
    margin: float = 1.15,
) -> list[dict]:
    """Build a list of {start, end, speaker: "left"|"right"} segments.

    Inputs are per-frame YAVG motion values for the two mouth ROIs. We
    normalize each side by its own mean (so we compare *relative* activity,
    not absolute brightness), smooth with a box filter, then walk the
    timeline with hysteresis: only flip the active speaker when the other
    side's smoothed motion exceeds the current side by `margin`x. Finally
    merge segments shorter than `min_duration` into their neighbors so a
    half-second laugh from the listener doesn't yank the camera.
    """
    if not times or len(left_values) != len(right_values) or not left_values:
        return []
    n = min(len(times), len(left_values), len(right_values))
    times = times[:n]
    left_values = left_values[:n]
    right_values = right_values[:n]

    def _normalize(values: list[float]) -> list[float]:
        mean_value = sum(values) / max(len(values), 1)
        if mean_value <= 0:
            return [0.0] * len(values)
        return [v / mean_value for v in values]

    left = smooth_values(_normalize(left_values))
    right = smooth_values(_normalize(right_values))
    if not left or not right:
        return []

    current = 0 if left[0] >= right[0] else 1
    states: list[int] = []
    for lv, rv in zip(left, right):
        if current == 0 and rv > lv * margin:
            current = 1
        elif current == 1 and lv > rv * margin:
            current = 0
        states.append(current)

    segments: list[dict] = []
    i = 0
    while i < len(states):
        j = i
        while j + 1 < len(states) and states[j + 1] == states[i]:
            j += 1
        seg_start = times[i]
        seg_end = times[min(j + 1, len(times) - 1)]
        if seg_end <= seg_start:
            seg_end = seg_start + 0.05
        segments.append({
            "start": seg_start,
            "end": seg_end,
            "speaker": "left" if states[i] == 0 else "right",
        })
        i = j + 1

    merged: list[dict] = []
    for seg in segments:
        too_short = (seg["end"] - seg["start"]) < min_duration
        if merged and (too_short or merged[-1]["speaker"] == seg["speaker"]):
            merged[-1]["end"] = seg["end"]
            continue
        merged.append(dict(seg))
    return merged


def build_pan_x_expression(timeline: list[dict], left_x: int, right_x: int) -> str:
    """Compile the speaker timeline into one nested if(lt(t,...)) expression.

    The expression is evaluated by FFmpeg's crop filter per-frame, so the
    crop window snaps to the active speaker without keyframe interpolation.
    No animation by design. Smooth pans on speaker switches produce a
    "drunken cameraman" effect on split-screen footage; hard cuts read as
    intentional editing.
    """
    if not timeline:
        return str(left_x)

    def x_for(speaker: str) -> int:
        return left_x if speaker == "left" else right_x

    expr = str(x_for(timeline[-1]["speaker"]))
    for seg in reversed(timeline[:-1]):
        # Escape commas so the expression is safe inside a filtergraph.
        expr = (
            f"if(lt(t\\,{seg['end']:.4f})\\,{x_for(seg['speaker'])}\\,{expr})"
        )
    return expr


def _safe_clip(value: int, lo: int, hi: int) -> int:
    return max(lo, min(int(value), hi))


def _even(value: int) -> int:
    return value - (value % 2)


def detect_local_speaker_reframe_plan(
    clip_path: str,
    face_map: dict,
    target_ratio: float = 9 / 16,
    max_scene_cuts: int = 2,
    motion_timeout: float = 120.0,
) -> Optional[dict]:
    """Plan a per-clip pan based on local mouth motion.

    Returns:
      {
        "mode": "pan",
        "width": int, "height": int,
        "crop_w": int, "crop_h": int,
        "left_x": int, "right_x": int,
        "x_expression": str,        # ready to drop into ffmpeg crop=...:x='...'
        "timeline": [...],
        "scene_cuts": int,
      }
    or None when the plan doesn't apply (not split-screen, too many cuts,
    motion analysis failed, single-side timeline).

    The caller is responsible for falling back to the existing reframe
    pipeline when this returns None.
    """
    if not face_map:
        return None
    clusters = face_map.get("clusters") or []
    if len(clusters) < 2 or not face_map.get("is_split_screen"):
        return None

    width = int(face_map.get("video_width") or 0)
    height = int(face_map.get("video_height") or 0)
    if width <= 0 or height <= 0:
        return None

    if width / max(height, 1) <= 1.2:
        return None

    cuts = count_scene_cuts(clip_path)
    if cuts > max_scene_cuts:
        log.info("local-reframe skip: %d scene cuts in clip", cuts)
        return None

    left_cluster, right_cluster = clusters[0], clusters[1]
    left_cx = int(left_cluster.get("center_x", width // 4))
    right_cx = int(right_cluster.get("center_x", 3 * width // 4))
    if abs(right_cx - left_cx) < width * 0.15:
        return None

    # PIP guard: a face ratio over 3x usually means inset, not split-screen.
    obs = face_map.get("observations") or []
    def _mean_face_w_for(side_cx: int) -> float:
        widths = [
            float(o.get("face_width") or 0)
            for o in obs
            if abs(int(o.get("face_center_x", 0)) - side_cx) < width * 0.15
            and o.get("face_width")
        ]
        return sum(widths) / len(widths) if widths else 0.0

    left_w = _mean_face_w_for(left_cx)
    right_w = _mean_face_w_for(right_cx)
    if left_w > 0 and right_w > 0:
        ratio = max(left_w, right_w) / max(1.0, min(left_w, right_w))
        if ratio > 3.0:
            log.info(
                "local-reframe skip: face-size ratio %.1f looks like PIP",
                ratio,
            )
            return None

    crop_w = _even(min(width, int(height * target_ratio)))
    crop_h = _even(height)
    max_x = max(0, width - crop_w)
    left_x = _safe_clip(left_cx - crop_w // 2, 0, max_x)
    right_x = _safe_clip(right_cx - crop_w // 2, 0, max_x)

    def _mouth_roi(cluster: dict, side_cx: int) -> tuple[int, int, int, int]:
        approx_face_w = 0
        observations = face_map.get("observations") or []
        side_obs = [
            o for o in observations
            if abs(int(o.get("face_center_x", 0)) - side_cx) < width * 0.15
        ]
        if side_obs:
            approx_face_w = int(
                sum(o.get("face_width", 0) for o in side_obs) / len(side_obs)
            )
        if approx_face_w < 60:
            approx_face_w = max(80, width // 8)
        cy_values = [o.get("face_center_y", height // 2) for o in side_obs]
        cy = int(sum(cy_values) / len(cy_values)) if cy_values else height // 2
        roi_w = _even(max(80, int(approx_face_w * 1.2)))
        roi_h = _even(max(70, int(approx_face_w * 0.85)))
        roi_x = _safe_clip(side_cx - roi_w // 2, 0, max(0, width - roi_w))
        roi_y = _safe_clip(cy, 0, max(0, height - roi_h))
        return roi_x, roi_y, roi_w, roi_h

    lx, ly, lw, lh = _mouth_roi(left_cluster, left_cx)
    rx, ry, rw, rh = _mouth_roi(right_cluster, right_cx)

    with tempfile.TemporaryDirectory(prefix="podcli_reframe_") as tmp:
        left_motion = Path(tmp) / "left.txt"
        right_motion = Path(tmp) / "right.txt"
        filter_complex = (
            f"[0:v]split=2[l][r];"
            f"[l]crop={lw}:{lh}:{lx}:{ly},format=gray,"
            f"tblend=all_mode=difference,signalstats,"
            f"metadata=mode=print:key=lavfi.signalstats.YAVG"
            f":file={_escape_path_for_filter(left_motion)}[lo];"
            f"[r]crop={rw}:{rh}:{rx}:{ry},format=gray,"
            f"tblend=all_mode=difference,signalstats,"
            f"metadata=mode=print:key=lavfi.signalstats.YAVG"
            f":file={_escape_path_for_filter(right_motion)}[ro]"
        )
        try:
            proc_run(
                [
                    "ffmpeg", "-y",
                    "-i", str(clip_path),
                    "-filter_complex", filter_complex,
                    "-map", "[lo]", "-f", "null", "-",
                    "-map", "[ro]", "-f", "null", "-",
                ],
                timeout=motion_timeout,
                check=False,
            )
        except ProcError as exc:
            log.warning("local-reframe motion ffmpeg failed: %s", exc)
            return None

        if not left_motion.exists() or not right_motion.exists():
            return None
        times, left_values = parse_motion_metadata(left_motion)
        _, right_values = parse_motion_metadata(right_motion)

    timeline = build_speaker_timeline_from_motion(times, left_values, right_values)
    if len(timeline) < 2:
        return None

    x_expression = build_pan_x_expression(timeline, left_x, right_x)

    return {
        "mode": "pan",
        "width": width,
        "height": height,
        "crop_w": crop_w,
        "crop_h": crop_h,
        "left_x": left_x,
        "right_x": right_x,
        "x_expression": x_expression,
        "timeline": timeline,
        "scene_cuts": cuts,
    }


def _escape_path_for_filter(path: Path) -> str:
    """Escape a filesystem path for use inside an FFmpeg filter argument."""
    return (
        str(path)
        .replace("\\", "\\\\")
        .replace(":", "\\:")
        .replace("'", "\\'")
        .replace(" ", "\\ ")
    )