Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

Point the camera at whoever is speaking (2.7.4) (#174)

Nika Siradzec3460898b8b9
diff --git a/backend/services/face_analysis.py b/backend/services/face_analysis.py
index f9501b5..e855a50 100644
--- a/backend/services/face_analysis.py
+++ b/backend/services/face_analysis.py
@@ -44,6 +44,7 @@ def analyze_faces(
import cv2
import numpy as np
from services.face_detector import create_detector, detect_faces
+ from services.face_track_helpers import seats_from_frames
except ImportError:
print("Warning: OpenCV not available, skipping face analysis", file=sys.stderr)
return None
@@ -74,12 +75,19 @@ def analyze_faces(
if progress_callback:
progress_callback(10, f"Scanning {sample_count} frames for faces...")
- # We compare each face's mouth ROI against the SAME face's mouth ROI
- # from the previous sample. High frame-to-frame difference = the
- # mouth is moving = this person is speaking. Listeners stay static.
- # Matching is done by nearest cx — good enough for a split-screen
- # where faces don't move between samples.
- prev_mouth_gray: dict[int, "np.ndarray"] = {}
+ # We compare each face's mouth ROI against the same face's mouth ROI one
+ # frame later. High difference = the mouth is moving = this person is
+ # speaking. Listeners stay static.
+ #
+ # The pair has to be adjacent frames. This used to diff against the
+ # previous *sample*, and sample_count is capped at 300, so on a 60-minute
+ # episode the two frames were 12 seconds apart and the difference measured
+ # nothing but the passage of time. Every split-screen episode over about
+ # two and a half minutes was choosing its speaker from noise.
+
+ # Per sampled frame, the x of every face in it. Clustering needs to know
+ # which positions were simultaneous, which a flat observation list cannot say.
+ frame_positions: list[list[int]] = []
def _mouth_roi_gray(frame, cx: int, cy: int, fw: int, fh: int):
"""Return a small downsampled grayscale of the mouth area,
@@ -123,8 +131,19 @@ def analyze_faces(
pos_ms = cap.get(cv2.CAP_PROP_POS_MSEC)
t = pos_ms / 1000.0 if pos_ms and pos_ms > 0 else frame_idx / fps
+ # The very next frame, decoded but never detected on: the face boxes
+ # from this frame are still accurate one frame later, so the second
+ # decode buys the mouth diff and costs no inference. Skipped when the
+ # sampler is already taking every frame, where borrowing one would
+ # halve the sample count.
+ next_frame = None
+ if step > 1 and cap.grab():
+ frame_idx += 1
+ got_next, candidate = cap.retrieve()
+ if got_next:
+ next_frame = candidate
+
faces = detect_faces(detector, frame, width, height)
- current_mouth_gray: dict[int, "np.ndarray"] = {}
frame_faces = 0
for f in faces:
@@ -132,14 +151,10 @@ def analyze_faces(
mouth = _mouth_roi_gray(frame, f["cx"], f["cy"], f["fw"], fh)
activity = 0.0
- if mouth is not None:
- # Bucket faces by cluster-ish column (left/right half here).
- # We'll refine to proper clusters after they're built.
- key = 0 if f["cx"] < width // 2 else 1
- prev = prev_mouth_gray.get(key)
- if prev is not None and prev.shape == mouth.shape:
- activity = float(np.mean(np.abs(mouth - prev)))
- current_mouth_gray[key] = mouth
+ if mouth is not None and next_frame is not None:
+ later = _mouth_roi_gray(next_frame, f["cx"], f["cy"], f["fw"], fh)
+ if later is not None and later.shape == mouth.shape:
+ activity = float(np.mean(np.abs(later - mouth)))
observations.append({
"time": round(t, 3),
@@ -151,8 +166,8 @@ def analyze_faces(
})
frame_faces += 1
- prev_mouth_gray = current_mouth_gray
faces_per_frame.append(frame_faces)
+ frame_positions.append([f["cx"] for f in faces])
if progress_callback and sampled % 20 == 0:
pct = 10 + int(60 * sampled / sample_count)
@@ -167,35 +182,45 @@ def analyze_faces(
if progress_callback:
progress_callback(75, "Clustering face positions...")
- # Cluster faces by left/right half split (consistent with _build_speaker_aware_crop)
+ # Cluster faces into seats — the fixed positions a person occupies.
+ #
+ # Two seats mean two people on screen at once, and the only evidence for
+ # that is a frame holding both. Splitting a flat list of positions at the
+ # midline cannot tell two people apart from one person who leaned across
+ # it: three sampled frames on the far side were enough to mint a second
+ # seat. Every speaker mapped to that invented seat then rendered as a hold
+ # on empty wall for the length of their turn.
target_ratio = 1080 / 1920 # 9:16
crop_w = int(height * target_ratio)
positions = np.array([o["face_center_x"] for o in observations])
mid_x = width // 2
seam_margin = 20
- left_pos = positions[positions < mid_x]
- right_pos = positions[positions >= mid_x]
+ seats = seats_from_frames(frame_positions, width)
clusters_list = []
- if len(left_pos) >= 3:
- cx = int(np.median(left_pos))
+ if seats:
+ left_seat, right_seat = seats
clusters_list.append({
- "center_x": cx,
- "count": len(left_pos),
- "crop_x": max(0, min(cx - crop_w // 2, mid_x - crop_w - seam_margin)),
+ "center_x": left_seat,
+ "count": sum(1 for x in positions if x < mid_x),
+ "crop_x": max(0, min(left_seat - crop_w // 2, mid_x - crop_w - seam_margin)),
})
- if len(right_pos) >= 3:
- cx = int(np.median(right_pos))
+ clusters_list.append({
+ "center_x": right_seat,
+ "count": sum(1 for x in positions if x >= mid_x),
+ "crop_x": max(mid_x + seam_margin, min(right_seat - crop_w // 2, width - crop_w)),
+ })
+ else:
+ # One seat. Not "one speaker" — the guest may be off camera entirely,
+ # and the render should stay on the person it can actually see.
+ cx = int(np.median(positions))
clusters_list.append({
"center_x": cx,
- "count": len(right_pos),
- "crop_x": max(mid_x + seam_margin, min(cx - crop_w // 2, width - crop_w)),
+ "count": len(positions),
+ "crop_x": max(0, min(cx - crop_w // 2, width - crop_w)),
})
- if not clusters_list:
- return None
-
clusters_list.sort(key=lambda c: c["center_x"])
# Detect split-screen and mixed layouts (Riverside-style recordings
diff --git a/backend/services/face_track_helpers.py b/backend/services/face_track_helpers.py
index 930b39b..3aa5ecc 100644
--- a/backend/services/face_track_helpers.py
+++ b/backend/services/face_track_helpers.py
@@ -173,3 +173,163 @@ def upgrade_speaker_mappings(face_map: dict) -> dict:
face_map["speaker_mappings"] = {}
face_map["_mappings_v2"] = True
return face_map
+
+
+def crop_center_keeping_faces_visible(
+ face_xs: list[float],
+ crop_w: int,
+ video_width: int,
+ margin_ratio: float = 0.15,
+) -> float:
+ """Pick the crop center that leaves the most sampled faces in frame.
+
+ The median of a run of face positions is only the right answer when
+ those positions are unimodal. When a run mixes two layouts — a
+ fullscreen shot at 960 and a split-screen tile at 1440 — the median
+ lands at 1200, which on a 607-wide crop is the wall between them, and
+ the render holds on nothing for the length of the run.
+
+ So score candidate centers by how many faces they actually keep inside
+ the crop, and only fall back to closeness as a tiebreak. On a unimodal
+ run every candidate keeps every face and this returns the median, which
+ is what the caller wanted in the first place.
+ """
+ if not face_xs:
+ return float(video_width) / 2
+
+ from statistics import median
+
+ max_crop_x = max(0, video_width - crop_w)
+ margin = crop_w * margin_ratio
+
+ def _center_of(candidate: float) -> float:
+ crop_x = min(max(candidate - crop_w / 2.0, 0.0), float(max_crop_x))
+ return crop_x + crop_w / 2.0
+
+ def _score(candidate: float) -> tuple[int, float]:
+ center = _center_of(candidate)
+ low = center - crop_w / 2.0 + margin
+ high = center + crop_w / 2.0 - margin
+ inside = [x for x in face_xs if low <= x <= high]
+ if not inside:
+ return 0, -float("inf")
+ spread = sum(abs(x - center) for x in inside) / len(inside)
+ return len(inside), -spread
+
+ candidates = list(face_xs) + [float(median(face_xs))]
+ best = max(candidates, key=_score)
+ return _center_of(best)
+
+
+def seats_from_frames(
+ frame_positions: list[list[float]],
+ video_width: int,
+ min_paired_frames: int = 3,
+ min_separation_ratio: float = 0.20,
+ min_paired_share: float = 0.10,
+) -> tuple[int, int] | None:
+ """Find the two fixed positions people occupy, or None if there is one.
+
+ Takes the x of every face in every sampled frame, grouped by frame. Two
+ seats are only real if some frame held two faces at once and they sat far
+ enough apart to be two people rather than two heads sharing one camera.
+
+ Counting a flat list of positions either side of the midline cannot make
+ that distinction, and mistaking one person for two is expensive: the
+ invented seat is empty wall, and every speaker mapped to it renders as a
+ hold on nothing.
+ """
+ from statistics import median
+
+ paired = [sorted(xs) for xs in frame_positions if len(xs) >= 2]
+ with_faces = sum(1 for xs in frame_positions if xs)
+ # A share as well as a count, so a handful of false positives — a face in a
+ # photo on the shelf behind someone — cannot seat a second person.
+ if len(paired) < max(min_paired_frames, with_faces * min_paired_share):
+ return None
+
+ left = int(median([xs[0] for xs in paired]))
+ right = int(median([xs[-1] for xs in paired]))
+ if (right - left) <= video_width * min_separation_ratio:
+ return None
+ return left, right
+
+
+def clip_layout_is_mixed(
+ detections: list,
+ face_map: dict | None = None,
+ min_share: float = 0.15,
+ min_frames: int = 3,
+) -> bool:
+ """Does this clip switch between a split screen and a fullscreen shot?
+
+ Judged on the clip's own frames first. The layout used to be read only
+ from the episode-wide face_map, and a face_map cached by a podcli older
+ than the mixed-layout work carries no is_mixed_layout key at all, so the
+ default turned those into "not mixed". That sent Riverside recordings down
+ a path that holds one camera position across a layout change, and every
+ fullscreen stretch rendered as the wall beside the speaker.
+
+ Both layouts must hold a real share of the frames. A count alone would
+ call a few frames with a missed second face a layout change and route
+ almost every clip through the mixed path.
+ """
+ split_frames = sum(1 for _, faces in detections if len(faces) >= 2)
+ single_frames = sum(1 for _, faces in detections if len(faces) == 1)
+ layout_frames = split_frames + single_frames
+
+ if layout_frames and (
+ split_frames >= max(min_frames, layout_frames * min_share)
+ and single_frames >= max(min_frames, layout_frames * min_share)
+ ):
+ return True
+
+ # A clip sitting entirely inside one layout still belongs to a mixed
+ # episode, so the episode-wide flag stays a valid hint.
+ return bool(face_map and face_map.get("is_mixed_layout", False))
+
+
+def followed_face_cx_at(
+ t_target: float,
+ tracked_detections: list,
+ segment_tracks: list,
+ fallback_track_id: int | None = None,
+ window: float = 1.5,
+) -> float | None:
+ """Where the speaker the camera is following actually is at a given time.
+
+ Used to check a planned crop still holds its subject. It has to ask about
+ the followed track, not the biggest face in the frame: on a split screen
+ the biggest face is whoever sits closest to their own camera, and on the
+ recording this was written for one speaker measures 845px across against
+ the other's 518. Taking the larger one made every correct crop onto the
+ quieter side look like a crop onto nobody, so each keyframe was pulled
+ back to the same person and the camera stopped switching.
+
+ Falls back to the most prominent face only when the followed speaker is
+ not on screen anywhere near this moment, where any face beats none.
+ """
+ track_id = None
+ for start_t, end_t, _speaker, tid, *_rest in segment_tracks:
+ if start_t <= t_target <= end_t:
+ track_id = tid
+ break
+ if track_id is None:
+ track_id = fallback_track_id
+
+ best_cx = best_dt = None
+ fallback_cx, fallback_dt = None, None
+ for t, faces in tracked_detections:
+ dt = abs(t - t_target)
+ if dt > window or not faces:
+ continue
+ if fallback_dt is None or dt < fallback_dt:
+ fallback_cx = float(max(faces, key=lambda f: f["fw"])["cx"])
+ fallback_dt = dt
+ if track_id is None:
+ continue
+ face = next((f for f in faces if f["track_id"] == track_id), None)
+ if face is not None and (best_dt is None or dt < best_dt):
+ best_cx, best_dt = float(face["cx"]), dt
+
+ return best_cx if best_cx is not None else fallback_cx
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index 79bd37e..1b64f3c 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -41,6 +41,9 @@ from services.motion_filters import (
from services.face_track_helpers import (
choose_camera_speaker as _choose_camera_speaker,
clamp_away_from_dead_zone as _clamp_away_from_dead_zone,
+ clip_layout_is_mixed as _clip_layout_is_mixed,
+ crop_center_keeping_faces_visible as _crop_center_keeping_faces_visible,
+ followed_face_cx_at as _followed_face_cx_at,
safe_default_center as _safe_default_center,
update_tripod_camera as _update_tripod_camera,
upgrade_speaker_mappings as _upgrade_speaker_mappings,
@@ -809,6 +812,35 @@ def _choose_track_segment_targets(
return trimmed
+ def _anchored_center_with_a_face(
+ anchor_x: float, start_t: float, end_t: float,
+ ) -> float:
+ """Keep the anchor if a detected face sits inside the crop it implies,
+ otherwise return a center that holds the faces this clip really has.
+
+ Searches the segment first, then widens to the whole clip, because a
+ speaker turn with no detections at all still has to point somewhere,
+ and anywhere a face was seen beats a position from another render.
+ """
+ in_segment = [
+ float(face["cx"])
+ for t, faces in tracked_detections
+ if start_t <= t <= end_t
+ for face in faces
+ ]
+ nearby = in_segment or [
+ float(face["cx"]) for _t, faces in tracked_detections for face in faces
+ ]
+ if not nearby:
+ return anchor_x
+
+ anchor_crop_x = _crop_x_for_center(anchor_x)
+ low = anchor_crop_x + crop_w * 0.15
+ high = anchor_crop_x + crop_w * 0.85
+ if any(low <= cx <= high for cx in nearby):
+ return anchor_x
+ return _crop_center_keeping_faces_visible(nearby, crop_w, width)
+
local_confirmation_seen = set()
max_anchor_only_segment = 2.5
is_first_segment = True
@@ -862,7 +894,15 @@ def _choose_track_segment_targets(
)
and (is_first_segment or (end_t - start_t) <= max_anchor_only_segment)
):
- center_x = float(speaker_anchor_x[speaker])
+ # The anchor is an episode-wide position, and nothing so far has
+ # checked that anybody is standing on it in this clip. Holding an
+ # unchecked anchor is how a turn renders as seconds of empty wall:
+ # the first segment used to take one at any length. Only keep it
+ # when a face the clip actually saw lands inside the crop it asks
+ # for; otherwise go where the faces are.
+ center_x = _anchored_center_with_a_face(
+ float(speaker_anchor_x[speaker]), start_t, end_t,
+ )
segment_targets.append((start_t, end_t, _crop_x_for_center(center_x), speaker))
is_first_segment = False
else:
@@ -1352,7 +1392,7 @@ def _track_and_crop(
# pipeline assumes stable face positions within a turn, which
# breaks on every layout transition. For mixed layouts, use
# simple per-frame largest-face following instead.
- is_mixed = face_map.get("is_mixed_layout", False) if face_map else False
+ is_mixed = _clip_layout_is_mixed(detections, face_map)
if is_mixed:
# Build a time→speaker lookup from segments
def _speaker_at(t_sec: float) -> str | None:
@@ -1396,21 +1436,35 @@ def _track_and_crop(
if face_points:
jump_thresh = max(180.0, crop_w * 0.35)
- runs = [] # [(start_t, end_t, median_cx), ...]
- run_start = face_points[0][0]
+ # A run's camera position must keep that run's faces in frame. The
+ # median only does when the run is unimodal, and a run that spans a
+ # layout change is not: the median of a fullscreen shot and a
+ # split-screen tile is the wall between them.
+ def _run_center(cxs: list[float]) -> float:
+ return _crop_center_keeping_faces_visible(cxs, crop_w, width)
+
+ runs = [] # [(start_t, end_t, center_cx), ...]
+ # From 0, not from the first detection. Runs bound the segments
+ # this path cuts and cross-dissolves back together, so a first run
+ # that starts late and a last run that ends early drop that video
+ # from the render: the clip came out short and lost its last word.
+ run_start = 0.0
run_cxs = [face_points[0][1]]
for i in range(1, len(face_points)):
t, cx = face_points[i]
run_median = float(median(run_cxs))
if abs(cx - run_median) > jump_thresh:
- # Layout changed — close current run, start new one
- runs.append((run_start, face_points[i-1][0], float(median(run_cxs))))
+ # Layout changed — close current run, start new one.
+ # Close it where the next one opens, not at the previous
+ # sample: the sampler runs at about 12 Hz, so ending a run
+ # one sample early threw away 80ms of video at every cut.
+ runs.append((run_start, t, _run_center(run_cxs)))
run_start = t
run_cxs = [cx]
else:
run_cxs.append(cx)
- runs.append((run_start, face_points[-1][0], float(median(run_cxs))))
+ runs.append((run_start, duration, _run_center(run_cxs)))
# Merge very short runs (<0.8s) into their neighbors
if len(runs) > 1:
@@ -1438,10 +1492,16 @@ def _track_and_crop(
# Multiple layouts — crop each segment separately, join
# with cross-dissolve so transitions feel editorial.
- import tempfile
work_dir = os.path.dirname(output_path) or "."
xfade_dur = 0.18 # Short dissolve
part_paths = []
+ cut_paths = []
+ # (path, measured duration, padding it carries). Offsets are built
+ # from what ffmpeg actually wrote, not from what was asked for: a
+ # skipped short run or a seek landing a few frames off used to
+ # shift every later transition, and the drift showed up as a clip
+ # whose video ran shorter than its own audio.
+ part_specs = []
try:
for ri, (r_start, r_end, r_cx) in enumerate(runs):
@@ -1460,8 +1520,8 @@ def _track_and_crop(
"-ss", str(seg_start), "-t", str(seg_end - seg_start),
"-i", input_path,
"-vf", seg_vf,
+ "-an",
"-c:v", "libx264", "-crf", "18", "-preset", "fast",
- "-c:a", "aac", "-b:a", "192k",
"-avoid_negative_ts", "make_zero",
part_path,
]
@@ -1469,88 +1529,129 @@ def _track_and_crop(
if r.returncode != 0:
continue
part_paths.append(part_path)
+ measured = _get_media_duration_seconds(part_path) or (seg_end - seg_start)
+ part_specs.append((part_path, measured, pad))
+
+ # The last part carries no padding whichever run it came from,
+ # so a dropped final run does not leave a dangling overlap.
+ if part_specs:
+ last_path, last_len, _ = part_specs[-1]
+ part_specs[-1] = (last_path, last_len, 0.0)
if len(part_paths) < 2:
- # Fallback to static crop of longest run
- best_run = max(runs, key=lambda r: r[1] - r[0])
- crop_x = _crop_for(best_run[2])
- vf = f"crop={crop_w}:{crop_h}:{crop_x}:{crop_y},scale={target_w}:{target_h}"
- return _run_ffmpeg_with_fallback(
- cmd_parts_before_enc=["ffmpeg", "-y", "-i", input_path, "-vf", vf],
- cmd_parts_after_enc=["-c:a", "aac", "-b:a", "192k", "-ar", "44100", "-movflags", "+faststart"],
- output_path=output_path, label="crop_mixed_fallback",
- )
-
- # Chain xfade filters between all segments
- if len(part_paths) == 2:
- # Simple 2-segment xfade
- offset = max(0.01, (runs[0][1] - runs[0][0]) - xfade_dur)
- cmd = [
- "ffmpeg", "-y",
- "-i", part_paths[0], "-i", part_paths[1],
- "-filter_complex",
- f"[0:v][1:v]xfade=transition=fade:duration={xfade_dur}:offset={offset:.3f}[v];"
- f"[0:a][1:a]acrossfade=d={xfade_dur}[a]",
- "-map", "[v]", "-map", "[a]",
- "-c:v", "libx264", "-crf", "18", "-preset", "fast",
- "-c:a", "aac", "-b:a", "192k",
- "-movflags", "+faststart",
- output_path,
- ]
- r = proc_run(cmd, timeout=_FFMPEG_TIMEOUT, check=False)
- if r.returncode == 0:
- return output_path
+ log_event("crop", "fallback", reason="mixed_too_few_parts",
+ parts=len(part_paths), to="hard-cuts")
else:
- # 3+ segments: chain xfades
+ # Where each part's own content begins in the output: the
+ # running total of the parts before it, each counted without
+ # the padding that only exists to give the dissolve
+ # somewhere to happen. Counting the padding, or subtracting
+ # the dissolve a second time, is what made the video end
+ # early and drift against its own audio.
+ offsets = []
+ running = 0.0
+ for _path, part_len, part_pad in part_specs[:-1]:
+ running += max(0.01, part_len - part_pad)
+ offsets.append(running)
+
+ # Dissolve the picture, and take the sound straight from the
+ # source. Cross-fading each part's re-encoded audio slurred
+ # about a quarter of a second of speech across the joins for
+ # no benefit: both sides of every dissolve are the same
+ # moment of the same recording.
inputs = []
- for p in part_paths:
- inputs.extend(["-i", p])
+ for path in part_paths:
+ inputs.extend(["-i", path])
+ inputs.extend(["-i", input_path])
+ source_index = len(part_paths)
- # Build filter chain: xfade each pair
filter_parts = []
- audio_parts = []
prev_label = "[0:v]"
- prev_audio = "[0:a]"
- cumulative_offset = 0.0
-
for i in range(1, len(part_paths)):
- seg_dur = runs[i-1][1] - runs[i-1][0]
- cumulative_offset += max(0.01, seg_dur - xfade_dur)
out_label = f"[v{i}]" if i < len(part_paths) - 1 else "[v]"
- out_audio = f"[a{i}]" if i < len(part_paths) - 1 else "[a]"
filter_parts.append(
- f"{prev_label}[{i}:v]xfade=transition=fade:duration={xfade_dur}:offset={cumulative_offset:.3f}{out_label}"
- )
- audio_parts.append(
- f"{prev_audio}[{i}:a]acrossfade=d={xfade_dur}{out_audio}"
+ f"{prev_label}[{i}:v]xfade=transition=fade"
+ f":duration={xfade_dur}:offset={offsets[i-1]:.3f}{out_label}"
)
prev_label = out_label
- prev_audio = out_audio
- filter_complex = ";".join(filter_parts + audio_parts)
cmd = ["ffmpeg", "-y"] + inputs + [
- "-filter_complex", filter_complex,
- "-map", "[v]", "-map", "[a]",
+ "-filter_complex", ";".join(filter_parts),
+ "-map", "[v]",
+ ]
+ if _has_audio_stream(input_path):
+ cmd += ["-map", f"{source_index}:a", "-c:a", "aac", "-b:a", "192k"]
+ cmd += [
"-c:v", "libx264", "-crf", "18", "-preset", "fast",
- "-c:a", "aac", "-b:a", "192k",
"-movflags", "+faststart",
+ "-shortest",
output_path,
]
r = proc_run(cmd, timeout=_FFMPEG_TIMEOUT, check=False)
if r.returncode == 0:
return output_path
- # If xfade failed, fall back to static
+ log_event("crop", "fallback", reason="mixed_xfade_failed",
+ parts=len(part_paths), to="hard-cuts")
+ # The dissolve is the only thing that failed, so drop the
+ # dissolve and keep the framing: one cut per run, joined end to
+ # end. This used to fall back to a static crop of the longest
+ # run, which is right for that run and wrong for every other
+ # layout in the clip — on a recording that cuts to a fullscreen
+ # shot, a hold on the wall for a third of its length.
+ for ri, (r_start, r_end, r_cx) in enumerate(runs):
+ seg_start = max(0.0, r_start)
+ seg_end = min(duration, r_end)
+ if seg_end - seg_start < 0.05:
+ continue
+ cut_path = os.path.join(work_dir, f"_mixed_cut_{ri}.mp4")
+ cut_cmd = [
+ "ffmpeg", "-y",
+ "-ss", str(seg_start), "-t", str(seg_end - seg_start),
+ "-i", input_path,
+ "-vf", (f"crop={crop_w}:{crop_h}:{_crop_for(r_cx)}:{crop_y},"
+ f"scale={target_w}:{target_h}"),
+ "-c:v", "libx264", "-crf", "18", "-preset", "fast",
+ "-c:a", "aac", "-b:a", "192k", "-ar", "44100",
+ "-avoid_negative_ts", "make_zero",
+ cut_path,
+ ]
+ if proc_run(cut_cmd, timeout=_FFMPEG_TIMEOUT, check=False).returncode == 0:
+ cut_paths.append(cut_path)
+
+ if len(cut_paths) >= 2:
+ list_path = os.path.join(work_dir, "_mixed_cuts.txt")
+ with open(list_path, "w", encoding="utf-8") as fh:
+ for cut in cut_paths:
+ fh.write(f"file '{os.path.abspath(cut)}'\n")
+ concat_cmd = [
+ "ffmpeg", "-y", "-f", "concat", "-safe", "0",
+ "-i", list_path, "-c", "copy",
+ "-movflags", "+faststart", output_path,
+ ]
+ concat_ok = proc_run(
+ concat_cmd, timeout=_FFMPEG_TIMEOUT, check=False,
+ ).returncode == 0
+ if os.path.exists(list_path):
+ os.remove(list_path)
+ if concat_ok:
+ log_event("crop", "chose=mixed-hard-cuts", runs=len(cut_paths))
+ return output_path
+
+ # Every joining strategy failed. Hold the run that keeps the
+ # most faces in frame rather than the longest one.
best_run = max(runs, key=lambda r: r[1] - r[0])
crop_x = _crop_for(best_run[2])
vf = f"crop={crop_w}:{crop_h}:{crop_x}:{crop_y},scale={target_w}:{target_h}"
+ log_event("crop", "fallback", reason="mixed_join_failed", to="static")
return _run_ffmpeg_with_fallback(
cmd_parts_before_enc=["ffmpeg", "-y", "-i", input_path, "-vf", vf],
- cmd_parts_after_enc=["-c:a", "aac", "-b:a", "192k", "-ar", "44100", "-movflags", "+faststart"],
- output_path=output_path, label="crop_mixed_xfade_fallback",
+ cmd_parts_after_enc=["-c:a", "aac", "-b:a", "192k", "-ar", "44100",
+ "-movflags", "+faststart"],
+ output_path=output_path, label="crop_mixed_static_last_resort",
)
finally:
- for p in part_paths:
+ for p in part_paths + cut_paths:
if os.path.exists(p):
os.remove(p)
@@ -1688,21 +1789,14 @@ def _track_and_crop(
if keyframes_x and len(keyframes_x) >= 2:
max_crop_x = max(0, width - crop_w)
- # Build a time→nearest-face-cx lookup from detections
- def _nearest_face_cx(t_target: float, window: float = 1.5) -> float | None:
- best_cx, best_dt = None, window + 1
- for t, faces in detections:
- dt = abs(t - t_target)
- if dt > window or not faces:
- continue
- if dt < best_dt:
- best_cx = float(max(faces, key=lambda f: f["fw"])["cx"])
- best_dt = dt
- return best_cx
+ def _face_cx_for(kf_t: float) -> float | None:
+ return _followed_face_cx_at(
+ kf_t, tracked_detections, segment_tracks, fallback_track_id,
+ )
validated = []
for kf_t, kf_x in keyframes_x:
- face_cx = _nearest_face_cx(kf_t)
+ face_cx = _face_cx_for(kf_t)
if face_cx is not None:
# Check if face is inside the crop window
crop_left = kf_x
diff --git a/cli/VERSION b/cli/VERSION
index 2c9b4ef..a4dd9db 100644
--- a/cli/VERSION
+++ b/cli/VERSION
@@ -1 +1 @@
-2.7.3
+2.7.4
diff --git a/package.json b/package.json
index f52f431..c1d5892 100644
--- a/package.json
+++ b/package.json
@@ -1,6 +1,6 @@
{
"name": "podcli",
- "version": "2.7.3",
+ "version": "2.7.4",
"private": true,
"description": "AI-powered podcast clip generator for TikTok/YouTube Shorts. Transcribe, find viral moments, export vertical clips with burned captions.",
"type": "module",
diff --git a/tests/test_choose_segment_tracks.py b/tests/test_choose_segment_tracks.py
index f627ffe..65c5235 100644
--- a/tests/test_choose_segment_tracks.py
+++ b/tests/test_choose_segment_tracks.py
@@ -166,3 +166,84 @@ class ChooseSegmentTracksTests(unittest.TestCase):
if __name__ == "__main__":
unittest.main()
+
+
+class AnchorOnlySegmentTests(unittest.TestCase):
+ """_choose_track_segment_targets must not aim the camera at a face_map
+ anchor that nothing in this clip stands on.
+
+ Segments the chosen track never appears in fall through to the speaker's
+ episode-wide anchor. That anchor was taken outright, so an invented or
+ stale seat pointed the camera at the wall next to the only person on
+ screen and held it there for the length of the turn.
+
+ Segments longer than 1.5s recover through the gap-points path first, so
+ these use short turns to reach the anchor branch itself.
+ """
+
+ WIDTH = 1920
+ CROP_W = 607
+
+ def _targets(self, segment_tracks, detections, anchors):
+ return vp._choose_track_segment_targets(
+ segment_tracks=segment_tracks,
+ tracked_detections=detections,
+ speaker_anchor_x=anchors,
+ width=self.WIDTH,
+ crop_w=self.CROP_W,
+ )
+
+ def _one_person(self, cx, end, fps=10):
+ return [(i / fps, [_face(cx, track_id=0)]) for i in range(int(end * fps) + 1)]
+
+ def _holds(self, crop_x, cx):
+ return crop_x <= cx <= crop_x + self.CROP_W
+
+ def test_unstood_anchor_is_replaced_by_a_real_face(self):
+ # One person at 1400 all clip. The turn's speaker has no track of
+ # their own and an anchor at 400 — the invented seat.
+ targets = self._targets(
+ segment_tracks=[(0.0, 1.2, "SPEAKER_01", 7, 400.0)],
+ detections=self._one_person(1400, 8.0),
+ anchors={"SPEAKER_01": 400.0},
+ )
+ self.assertEqual(len(targets), 1)
+ self.assertTrue(
+ self._holds(targets[0][2], 1400),
+ f"crop at {targets[0][2]} holds nobody; the only face is at 1400",
+ )
+
+ def test_a_stood_anchor_is_left_alone(self):
+ # Two real tiles, anchor on the right one. Somebody is standing there,
+ # so the anchor survives untouched.
+ targets = self._targets(
+ segment_tracks=[(0.0, 1.2, "SPEAKER_01", 7, 1440.0)],
+ detections=_build_split_detections(480, 1440, 8.0),
+ anchors={"SPEAKER_01": 1440.0},
+ )
+ self.assertEqual(len(targets), 1)
+ self.assertTrue(self._holds(targets[0][2], 1440))
+
+ def test_anchor_survives_when_the_face_is_merely_off_centre(self):
+ # 1250 is inside the crop the 1400 anchor asks for. Nudging the camera
+ # for that would fight the tripod, so the anchor stands.
+ targets = self._targets(
+ segment_tracks=[(0.0, 1.2, "SPEAKER_00", 7, 1400.0)],
+ detections=self._one_person(1250, 8.0),
+ anchors={"SPEAKER_00": 1400.0},
+ )
+ self.assertEqual(targets[0][2], vp_crop_x(1400.0, self.CROP_W, self.WIDTH))
+
+ def test_no_faces_at_all_keeps_the_anchor(self):
+ # Nothing to correct towards, so the anchor is still the best guess.
+ targets = self._targets(
+ segment_tracks=[(0.0, 1.2, "SPEAKER_00", 3, 800.0)],
+ detections=[(i / 10, []) for i in range(30)],
+ anchors={"SPEAKER_00": 800.0},
+ )
+ self.assertEqual(targets[0][2], vp_crop_x(800.0, self.CROP_W, self.WIDTH))
+
+
+def vp_crop_x(center_x: float, crop_w: int, width: int) -> int:
+ """The clamp _choose_track_segment_targets applies to a chosen centre."""
+ return max(0, min(int(round(center_x - crop_w / 2.0)), width - crop_w))
diff --git a/tests/test_crop_path_golden.py b/tests/test_crop_path_golden.py
index 2897fb5..18ba2bc 100644
--- a/tests/test_crop_path_golden.py
+++ b/tests/test_crop_path_golden.py
@@ -354,3 +354,50 @@ class SaneSpeakerMappingsTests(unittest.TestCase):
def test_passes_none_through(self):
self.assertIsNone(vp._sane_speaker_mappings(None))
+
+
+class MixedLayoutDetectionTests(unittest.TestCase):
+ """A clip that switches between a split screen and a fullscreen shot must
+ be recognised from its own frames.
+
+ The layout used to be read only from the episode-wide face_map, and
+ face_maps cached by a podcli older than the mixed-layout work carry no
+ is_mixed_layout key at all. `.get(..., False)` turned those into "not
+ mixed", which sent the clip down the keyframe path. That path holds one
+ camera position across a layout change, so on a Riverside recording every
+ fullscreen stretch rendered as the wall beside the speaker.
+ """
+
+ def _decide(self, detections, face_map=None):
+ return vp._clip_layout_is_mixed(detections, face_map)
+
+ def _frames(self, n_split, n_single):
+ det = [(i * 0.1, [{"cx": 1181}, {"cx": 2909}]) for i in range(n_split)]
+ det += [((n_split + i) * 0.1, [{"cx": 2050}]) for i in range(n_single)]
+ return det
+
+ def test_riverside_shape_is_mixed_without_any_face_map(self):
+ # The real proportions from the reported episode: 55 split-screen
+ # frames and 28 fullscreen frames in one 41s clip.
+ self.assertTrue(self._decide(self._frames(55, 28)))
+
+ def test_stale_face_map_does_not_override_the_clip(self):
+ stale = {"clusters": [{"center_x": 1006}, {"center_x": 2964}]} # no key
+ self.assertTrue(self._decide(self._frames(55, 28), stale))
+
+ def test_pure_split_screen_is_not_mixed(self):
+ self.assertFalse(self._decide(self._frames(80, 0)))
+
+ def test_pure_fullscreen_is_not_mixed(self):
+ self.assertFalse(self._decide(self._frames(0, 80)))
+
+ def test_a_few_missed_second_faces_are_not_a_layout(self):
+ # 4 dropped detections in 80 frames is detector noise, not a switch to
+ # a fullscreen shot. Counting them would route almost every clip
+ # through the mixed path.
+ self.assertFalse(self._decide(self._frames(76, 4)))
+
+ def test_face_map_flag_still_wins_when_the_clip_looks_uniform(self):
+ # A clip that sits entirely inside one layout still belongs to a mixed
+ # episode; the episode-wide flag remains a valid hint.
+ self.assertTrue(self._decide(self._frames(80, 0), {"is_mixed_layout": True}))
diff --git a/tests/test_face_track_helpers.py b/tests/test_face_track_helpers.py
index 99b27ba..4dda790 100644
--- a/tests/test_face_track_helpers.py
+++ b/tests/test_face_track_helpers.py
@@ -298,6 +298,75 @@ class UpgradeSpeakerMappingsTests(unittest.TestCase):
self.assertEqual(original["speaker_mappings"], {"A": 0})
+class CropCenterKeepingFacesVisibleTests(unittest.TestCase):
+ CROP_W = 607 # 9:16 out of a 1920x1080 source
+ WIDTH = 1920
+
+ def _center(self, xs):
+ return fth.crop_center_keeping_faces_visible(xs, self.CROP_W, self.WIDTH)
+
+ def test_unimodal_run_lands_on_the_group(self):
+ xs = [940, 950, 960, 970, 980]
+ center = self._center(xs)
+ self.assertLess(abs(center - 960), 30)
+
+ def test_bimodal_run_picks_the_bigger_side_not_the_gap(self):
+ # A run that spans a layout change: four samples fullscreen at 960,
+ # two on a split-screen tile at 1500. The median is 1230, which on a
+ # 607-wide crop holds on the wall between them and shows no one.
+ xs = [960, 955, 965, 950, 1500, 1505]
+ center = self._center(xs)
+ self.assertGreaterEqual(
+ sum(1 for x in xs if abs(x - center) < self.CROP_W / 2), 4
+ )
+ self.assertLess(abs(center - 960), 60)
+
+ def test_never_returns_a_center_with_nobody_in_frame(self):
+ # The exact shape that rendered as a hold on empty wall: two tiles far
+ # apart, nothing between them.
+ xs = [480, 470, 490, 1440, 1450, 1430]
+ center = self._center(xs)
+ self.assertTrue(any(abs(x - center) < self.CROP_W / 2 for x in xs))
+
+ def test_clamps_into_the_frame(self):
+ center = self._center([20, 25, 30])
+ self.assertGreaterEqual(center - self.CROP_W / 2, 0)
+
+ def test_no_faces_falls_back_to_center(self):
+ self.assertEqual(self._center([]), self.WIDTH / 2)
+
+
+class SeatsFromFramesTests(unittest.TestCase):
+ WIDTH = 1920
+
+ def test_two_tiles_are_two_seats(self):
+ frames = [[480, 1440]] * 20
+ self.assertEqual(fth.seats_from_frames(frames, self.WIDTH), (480, 1440))
+
+ def test_one_person_wandering_across_the_midline_is_one_seat(self):
+ # The bug this replaced: 40 solo frames, some left of centre and some
+ # right of it, were counted as two people and rendered as a cut
+ # between a face and a wall.
+ frames = [[900]] * 20 + [[1020]] * 20
+ self.assertIsNone(fth.seats_from_frames(frames, self.WIDTH))
+
+ def test_two_heads_sharing_one_camera_are_one_seat(self):
+ frames = [[900, 1050]] * 20
+ self.assertIsNone(fth.seats_from_frames(frames, self.WIDTH))
+
+ def test_a_few_false_positives_cannot_seat_a_second_person(self):
+ frames = [[960]] * 100 + [[300, 1600]] * 3
+ self.assertIsNone(fth.seats_from_frames(frames, self.WIDTH))
+
+ def test_mixed_layout_still_finds_both_seats(self):
+ # Riverside-style: sometimes both tiles, sometimes one fullscreen.
+ frames = ([[480, 1440]] * 15) + ([[960]] * 25)
+ self.assertEqual(fth.seats_from_frames(frames, self.WIDTH), (480, 1440))
+
+ def test_no_frame_ever_held_two_faces(self):
+ self.assertIsNone(fth.seats_from_frames([[960]] * 50, self.WIDTH))
+
+
class ReExportTests(unittest.TestCase):
def test_video_processor_reexports_all_helpers(self):
from services import video_processor as vp
@@ -306,7 +375,66 @@ class ReExportTests(unittest.TestCase):
self.assertIs(vp._safe_default_center, fth.safe_default_center)
self.assertIs(vp._clamp_away_from_dead_zone, fth.clamp_away_from_dead_zone)
self.assertIs(vp._upgrade_speaker_mappings, fth.upgrade_speaker_mappings)
+ self.assertIs(
+ vp._crop_center_keeping_faces_visible,
+ fth.crop_center_keeping_faces_visible,
+ )
+ self.assertIs(vp._clip_layout_is_mixed, fth.clip_layout_is_mixed)
+ self.assertIs(vp._followed_face_cx_at, fth.followed_face_cx_at)
if __name__ == "__main__":
unittest.main()
+
+
+class FollowedFaceCxAtTests(unittest.TestCase):
+ """The crop validator must ask where the *followed speaker* is, not where
+ the biggest face is.
+
+ On the recording this was written for, the two speakers sit at very
+ different distances from their own cameras: 845px of face against 518px.
+ Taking the larger one made a correct crop onto the quieter speaker look
+ like a crop onto nobody, so the validator pulled every keyframe back onto
+ the same person and the camera stopped switching for a 29-second turn.
+ """
+
+ def _split(self, duration=10.0, fps=10):
+ # Left speaker has the much larger face, as the real recording does.
+ return [
+ (i / fps, [
+ {"cx": 950, "fw": 845, "track_id": 0},
+ {"cx": 2880, "fw": 518, "track_id": 1},
+ ])
+ for i in range(int(duration * fps) + 1)
+ ]
+
+ def test_follows_the_smaller_face_when_it_is_the_speaker(self):
+ tracks = [(0.0, 10.0, "SPEAKER_R", 1, 2880.0)]
+ cx = fth.followed_face_cx_at(5.0, self._split(), tracks)
+ self.assertEqual(cx, 2880.0)
+
+ def test_follows_the_larger_face_when_it_is_the_speaker(self):
+ tracks = [(0.0, 10.0, "SPEAKER_L", 0, 950.0)]
+ cx = fth.followed_face_cx_at(5.0, self._split(), tracks)
+ self.assertEqual(cx, 950.0)
+
+ def test_picks_the_track_for_the_turn_that_covers_the_time(self):
+ tracks = [(0.0, 4.0, "SPEAKER_L", 0, 950.0), (4.0, 10.0, "SPEAKER_R", 1, 2880.0)]
+ self.assertEqual(fth.followed_face_cx_at(2.0, self._split(), tracks), 950.0)
+ self.assertEqual(fth.followed_face_cx_at(8.0, self._split(), tracks), 2880.0)
+
+ def test_falls_back_to_the_prominent_face_when_the_speaker_is_absent(self):
+ # Followed track 1 is never on screen; any face beats none.
+ detections = [(i / 10, [{"cx": 950, "fw": 845, "track_id": 0}]) for i in range(50)]
+ tracks = [(0.0, 5.0, "SPEAKER_R", 1, 2880.0)]
+ self.assertEqual(fth.followed_face_cx_at(2.0, detections, tracks), 950.0)
+
+ def test_uses_the_fallback_track_outside_every_turn(self):
+ tracks = [(0.0, 2.0, "SPEAKER_L", 0, 950.0)]
+ cx = fth.followed_face_cx_at(6.0, self._split(), tracks, fallback_track_id=1)
+ self.assertEqual(cx, 2880.0)
+
+ def test_no_detections_nearby_gives_nothing(self):
+ detections = [(0.0, [{"cx": 950, "fw": 845, "track_id": 0}])]
+ tracks = [(0.0, 10.0, "SPEAKER_L", 0, 950.0)]
+ self.assertIsNone(fth.followed_face_cx_at(9.0, detections, tracks))