Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
backend: extract pure camera helpers into face_track_helpers
Nika Siradzedba17546dc11
diff --git a/backend/services/face_track_helpers.py b/backend/services/face_track_helpers.pynew file mode 100644index 0000000..930b39b--- /dev/null+++ b/backend/services/face_track_helpers.py@@ -0,0 +1,175 @@+"""Pure helper functions for the face-tracking / camera-movement+pipeline.++Extracted from video_processor.py. These are all side-effect-free:+they take dicts / scalars / lists and return decisions (a camera+center, a speaker id, a clamped crop_x). No ffmpeg, no cv2, no+numpy. Trivially unit-testable in isolation, but they govern the+camera's behavior during every clip render.+"""++from __future__ import annotations+++def update_tripod_camera(+ current_center_x: float,+ target_center_x: float | None,+ crop_w: int,+ video_width: int,+ dt: float,+ force_snap: bool = False,+) -> float:+ """Heavy-tripod camera movement inspired by OpenShorts.++ The camera stays still while the subject remains inside a safe+ zone, then moves at a bounded speed instead of constantly chasing+ every face twitch. Always clamps the result so the crop window+ never falls off the source frame.+ """+ half_crop = crop_w / 2.0+ min_center = half_crop+ max_center = max(half_crop, video_width - half_crop)++ if target_center_x is None:+ return min(max(current_center_x, min_center), max_center)++ if force_snap:+ current_center_x = target_center_x+ else:+ diff = target_center_x - current_center_x+ # Wide enough to absorb natural head movement and detector+ # noise, narrow enough to keep the speaker composed near center.+ safe_zone_radius = crop_w * 0.22+ if abs(diff) > safe_zone_radius:+ slow_speed = 72.0+ fast_speed = 360.0+ speed = fast_speed if abs(diff) > crop_w * 0.5 else slow_speed+ step = min(abs(diff), speed * max(dt, 0.01))+ current_center_x += step if diff > 0 else -step++ return min(max(current_center_x, min_center), max_center)+++def choose_camera_speaker(+ transcript_speaker: str | None,+ transcript_duration: float,+ active_speaker: str | None,+ pending_speaker: str | None,+ pending_count: int,+ min_turn_duration: float = 2.6,+ confirmation_frames: int = 3,+) -> tuple[str | None, str | None, int, bool]:+ """Stabilize diarization before the camera switches.++ Brief interjections don't move the camera; sustained turns must+ be confirmed for `confirmation_frames` samples before the switch+ is committed. Returns:+ (camera_speaker, pending_speaker, pending_count, switched_now)+ """+ if transcript_speaker is None:+ return active_speaker, pending_speaker, pending_count, False++ if active_speaker is None:+ return transcript_speaker, None, 0, True++ if transcript_speaker == active_speaker:+ return active_speaker, None, 0, False++ # Brief interjections should not move the camera.+ if transcript_duration < min_turn_duration:+ return active_speaker, None, 0, False++ if pending_speaker != transcript_speaker:+ return active_speaker, transcript_speaker, 1, False++ pending_count += 1+ if pending_count < confirmation_frames:+ return active_speaker, pending_speaker, pending_count, False++ return transcript_speaker, None, 0, True+++def safe_default_center(+ width: int,+ crop_w: int,+ face_map: dict | None,+ has_any_split: bool,+ first_speaker: str | None,+ speaker_anchor_x: dict,+) -> float:+ """Pick a safe initial camera center.++ On split-screen, width/2 is the dead zone between two feeds. Use+ the face_map or speaker_anchor to start on an actual speaker+ instead. Fallback order:+ 1. first_speaker's learned anchor+ 2. largest face_map cluster+ 3. plain center (non-split)+ 4. left quarter (split with no map)+ """+ if first_speaker and first_speaker in speaker_anchor_x:+ return float(speaker_anchor_x[first_speaker])++ if face_map and face_map.get("clusters"):+ clusters = face_map["clusters"]+ best = max(clusters, key=lambda c: c.get("count", 0))+ return float(best["center_x"])++ if not has_any_split:+ return float(width) / 2++ return float(width) / 4+++def clamp_away_from_dead_zone(+ crop_x: int,+ crop_w: int,+ width: int,+ face_map: dict | None,+ has_any_split: bool,+) -> int:+ """Snap a crop that centers on the split-screen seam to the nearest+ cluster instead.++ Only triggers when the crop CENTER is close to the midline of the+ source frame — a crop that merely straddles the midline is fine on+ fullscreen layouts where the face genuinely sits near center.+ """+ if not has_any_split or not face_map:+ return crop_x++ mid_x = width // 2+ crop_center = crop_x + crop_w // 2++ seam_margin = crop_w // 8 # ~75px on a 607-wide crop+ if abs(crop_center - mid_x) > seam_margin:+ return crop_x++ clusters = face_map.get("clusters", [])+ if not clusters:+ return max(0, width // 4 - crop_w // 2)++ best_cluster = min(+ clusters, key=lambda c: abs(c["center_x"] - crop_center)+ )+ snapped = max(0, min(best_cluster["center_x"] - crop_w // 2, width - crop_w))+ return snapped+++def upgrade_speaker_mappings(face_map: dict) -> dict:+ """Invalidate stale speaker-to-cluster mappings from old caches.++ v1 caches used "first-to-speak = left" which breaks when the host+ speaks first but sits on the right. This function clears the+ mappings entirely and stamps `_mappings_v2 = True` so the clear+ only runs once per cache. Cluster positions are kept — they're+ still useful for dead-zone clamping and safe defaults.++ Note: face_analysis.py writes its own v2-quality mappings and+ stamps them with `_mappings_v2=True`, so newly-built face_maps+ skip this function entirely.+ """+ face_map = dict(face_map)+ face_map["speaker_mappings"] = {}+ face_map["_mappings_v2"] = True+ return face_mapdiff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex 384e729..d7cd6c8 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -35,6 +35,13 @@ from services.motion_filters import (motion_windows_from_keyframes as _motion_windows_from_keyframes,simplify_keyframes as _simplify_keyframes,)+from services.face_track_helpers import (+ choose_camera_speaker as _choose_camera_speaker,+ clamp_away_from_dead_zone as _clamp_away_from_dead_zone,+ safe_default_center as _safe_default_center,+ update_tripod_camera as _update_tripod_camera,+ upgrade_speaker_mappings as _upgrade_speaker_mappings,+)import sys@@ -469,44 +476,6 @@ def _choose_segment_tracks(return segment_tracks, learned_anchor_x, learned_side-def _update_tripod_camera(- current_center_x: float,- target_center_x: float | None,- crop_w: int,- video_width: int,- dt: float,- force_snap: bool = False,-) -> float:- """- Heavy-tripod camera movement inspired by OpenShorts.-- The camera stays still while the subject remains inside a safe zone, then- moves at a bounded speed instead of constantly chasing every face twitch.- """- half_crop = crop_w / 2.0- min_center = half_crop- max_center = max(half_crop, video_width - half_crop)-- if target_center_x is None:- return min(max(current_center_x, min_center), max_center)-- if force_snap:- current_center_x = target_center_x- else:- diff = target_center_x - current_center_x- # Wide enough to absorb natural head movement and detector noise,- # narrow enough to keep the speaker composed near center.- safe_zone_radius = crop_w * 0.22- if abs(diff) > safe_zone_radius:- slow_speed = 72.0- fast_speed = 360.0- speed = fast_speed if abs(diff) > crop_w * 0.5 else slow_speed- step = min(abs(diff), speed * max(dt, 0.01))- current_center_x += step if diff > 0 else -step-- return min(max(current_center_x, min_center), max_center)--def _choose_track_segment_targets(segment_tracks: list,tracked_detections: list,@@ -831,132 +800,6 @@ def _pick_tracking_face(return face, side_ok or near_anchor-def _choose_camera_speaker(- transcript_speaker: str | None,- transcript_duration: float,- active_speaker: str | None,- pending_speaker: str | None,- pending_count: int,- min_turn_duration: float = 2.6,- confirmation_frames: int = 3,-) -> tuple[str | None, str | None, int, bool]:- """- Stabilize diarization before the camera switches.-- Returns (camera_speaker, pending_speaker, pending_count, switched_now).- """- if transcript_speaker is None:- return active_speaker, pending_speaker, pending_count, False-- if active_speaker is None:- return transcript_speaker, None, 0, True-- if transcript_speaker == active_speaker:- return active_speaker, None, 0, False-- # Brief interjections should not move the camera.- if transcript_duration < min_turn_duration:- return active_speaker, None, 0, False-- if pending_speaker != transcript_speaker:- return active_speaker, transcript_speaker, 1, False-- pending_count += 1- if pending_count < confirmation_frames:- return active_speaker, pending_speaker, pending_count, False-- return transcript_speaker, None, 0, True---def _safe_default_center(- width: int,- crop_w: int,- face_map: dict | None,- has_any_split: bool,- first_speaker: str | None,- speaker_anchor_x: dict,-) -> float:- """- Pick a safe initial camera center.-- On split-screen, width/2 is the dead zone between two feeds. Use the- face_map or speaker_anchor to start on an actual speaker instead.- """- # Best case: we know where the first speaker sits- if first_speaker and first_speaker in speaker_anchor_x:- return float(speaker_anchor_x[first_speaker])-- # Face-map cluster fallback: pick the cluster with more observations- if face_map and face_map.get("clusters"):- clusters = face_map["clusters"]- best = max(clusters, key=lambda c: c.get("count", 0))- return float(best["center_x"])-- # Non-split: center is fine- if not has_any_split:- return float(width) / 2-- # Split with no map: pick the left quarter as a safer guess than center- return float(width) / 4---def _clamp_away_from_dead_zone(- crop_x: int,- crop_w: int,- width: int,- face_map: dict | None,- has_any_split: bool,-) -> int:- """- If crop_x centers the window on the split-screen seam (where no face- lives), snap to the nearest cluster position instead.-- Only triggers when the crop CENTER is close to mid_x. A crop that- merely straddles mid_x is fine — in fullscreen mode the face genuinely- sits near center.- """- if not has_any_split or not face_map:- return crop_x-- mid_x = width // 2- crop_center = crop_x + crop_w // 2-- # Only clamp when the crop is centered near the seam itself.- # A tight margin avoids false-positives on fullscreen layouts where- # the face legitimately sits near center.- seam_margin = crop_w // 8 # ~75px on a 607-wide crop- if abs(crop_center - mid_x) > seam_margin:- return crop_x-- # Snap to nearest cluster- clusters = face_map.get("clusters", [])- if not clusters:- return max(0, width // 4 - crop_w // 2)-- best_cluster = min(clusters, key=lambda c: abs(c["center_x"] - crop_center))- snapped = max(0, min(best_cluster["center_x"] - crop_w // 2, width - crop_w))- return snapped---def _upgrade_speaker_mappings(face_map: dict) -> dict:- """- Invalidate stale speaker-to-cluster mappings from old caches.-- Old caches used "first-to-speak = left" which breaks when the host- speaks first but sits on the right. Rather than trying to re-compute- mappings from clip-scoped data, we clear the mappings entirely so the- clip-local tracking decides which face to follow based on visual- evidence (which is more reliable for any single clip).-- Cluster positions are kept — they're still useful for dead-zone- clamping and safe defaults.- """- face_map = dict(face_map)- face_map["speaker_mappings"] = {}- face_map["_mappings_v2"] = True- return face_map--def _track_and_crop(input_path: str,output_path: str,diff --git a/tests/test_face_track_helpers.py b/tests/test_face_track_helpers.pynew file mode 100644index 0000000..99b27ba--- /dev/null+++ b/tests/test_face_track_helpers.py@@ -0,0 +1,312 @@+"""Tests for backend.services.face_track_helpers — pure camera /+speaker-switch decision helpers.+"""++import os+import sys+import unittest++ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))+BACKEND_ROOT = os.path.join(ROOT, "backend")+if BACKEND_ROOT not in sys.path:+ sys.path.insert(0, BACKEND_ROOT)++from services import face_track_helpers as fth+++class UpdateTripodCameraTests(unittest.TestCase):+ def test_no_target_clamps_within_frame(self):+ # No target → camera sits at current_center but clamped+ result = fth.update_tripod_camera(+ current_center_x=500, target_center_x=None,+ crop_w=600, video_width=1920, dt=0.1,+ )+ self.assertEqual(result, 500)++ def test_clamps_below_half_crop(self):+ # current 100 with crop_w=600 → min center = 300+ result = fth.update_tripod_camera(+ current_center_x=100, target_center_x=None,+ crop_w=600, video_width=1920, dt=0.1,+ )+ self.assertEqual(result, 300)++ def test_clamps_above_width_minus_half(self):+ # video_width - half_crop = 1920 - 300 = 1620+ result = fth.update_tripod_camera(+ current_center_x=1800, target_center_x=None,+ crop_w=600, video_width=1920, dt=0.1,+ )+ self.assertEqual(result, 1620)++ def test_force_snap_jumps_to_target(self):+ result = fth.update_tripod_camera(+ current_center_x=500, target_center_x=1200,+ crop_w=600, video_width=1920, dt=0.1, force_snap=True,+ )+ self.assertEqual(result, 1200)++ def test_small_drift_inside_safe_zone_holds(self):+ # diff = 50, safe_zone_radius = 600 * 0.22 = 132 → hold+ result = fth.update_tripod_camera(+ current_center_x=600, target_center_x=650,+ crop_w=600, video_width=1920, dt=0.1,+ )+ self.assertEqual(result, 600)++ def test_large_drift_moves_toward_target(self):+ # diff = 400 → outside safe zone → move toward it+ result = fth.update_tripod_camera(+ current_center_x=500, target_center_x=900,+ crop_w=600, video_width=1920, dt=0.1,+ )+ # Should have moved right but not snapped all the way+ self.assertGreater(result, 500)+ self.assertLess(result, 900)++ def test_movement_is_bounded_by_speed(self):+ # Very fast diff, small dt → step limited by max speed × dt+ result = fth.update_tripod_camera(+ current_center_x=500, target_center_x=1800, # huge diff → fast_speed+ crop_w=600, video_width=1920, dt=0.1,+ )+ # fast_speed=360, dt=0.1 → max step 36 → 500 + 36 = 536+ self.assertEqual(result, 536)+++class ChooseCameraSpeakerTests(unittest.TestCase):+ def test_no_transcript_speaker_holds(self):+ self.assertEqual(+ fth.choose_camera_speaker(+ transcript_speaker=None,+ transcript_duration=5.0,+ active_speaker="A",+ pending_speaker=None,+ pending_count=0,+ ),+ ("A", None, 0, False),+ )++ def test_first_speaker_becomes_active(self):+ self.assertEqual(+ fth.choose_camera_speaker(+ transcript_speaker="A",+ transcript_duration=5.0,+ active_speaker=None,+ pending_speaker=None,+ pending_count=0,+ ),+ ("A", None, 0, True),+ )++ def test_same_speaker_continues(self):+ self.assertEqual(+ fth.choose_camera_speaker(+ transcript_speaker="A",+ transcript_duration=5.0,+ active_speaker="A",+ pending_speaker="B",+ pending_count=2,+ ),+ ("A", None, 0, False),+ )++ def test_brief_interjection_ignored(self):+ # transcript_duration < min_turn_duration → hold+ self.assertEqual(+ fth.choose_camera_speaker(+ transcript_speaker="B",+ transcript_duration=1.0, # brief+ active_speaker="A",+ pending_speaker=None,+ pending_count=0,+ ),+ ("A", None, 0, False),+ )++ def test_new_speaker_enters_pending(self):+ cam, pending, count, switched = fth.choose_camera_speaker(+ transcript_speaker="B",+ transcript_duration=5.0,+ active_speaker="A",+ pending_speaker=None,+ pending_count=0,+ )+ self.assertEqual(cam, "A")+ self.assertEqual(pending, "B")+ self.assertEqual(count, 1)+ self.assertFalse(switched)++ def test_pending_confirmed_after_confirmation_frames(self):+ # Previous pending=B, count=2; with confirmation_frames=3 the+ # third sighting commits the switch.+ cam, pending, count, switched = fth.choose_camera_speaker(+ transcript_speaker="B",+ transcript_duration=5.0,+ active_speaker="A",+ pending_speaker="B",+ pending_count=2,+ confirmation_frames=3,+ )+ self.assertEqual(cam, "B")+ self.assertEqual(pending, None)+ self.assertTrue(switched)++ def test_different_pending_resets_count(self):+ # Pending was B=1, now transcript speaker is C → reset to C=1+ cam, pending, count, switched = fth.choose_camera_speaker(+ transcript_speaker="C",+ transcript_duration=5.0,+ active_speaker="A",+ pending_speaker="B",+ pending_count=1,+ )+ self.assertEqual(pending, "C")+ self.assertEqual(count, 1)+ self.assertFalse(switched)+++class SafeDefaultCenterTests(unittest.TestCase):+ def test_uses_first_speaker_anchor_when_available(self):+ self.assertEqual(+ fth.safe_default_center(+ width=1920, crop_w=600,+ face_map=None, has_any_split=False,+ first_speaker="A",+ speaker_anchor_x={"A": 800.0},+ ),+ 800.0,+ )++ def test_uses_largest_face_map_cluster(self):+ face_map = {+ "clusters": [+ {"center_x": 400, "count": 10},+ {"center_x": 1500, "count": 50}, # largest+ ],+ }+ self.assertEqual(+ fth.safe_default_center(+ width=1920, crop_w=600,+ face_map=face_map, has_any_split=True,+ first_speaker=None,+ speaker_anchor_x={},+ ),+ 1500.0,+ )++ def test_non_split_returns_center(self):+ self.assertEqual(+ fth.safe_default_center(+ width=1920, crop_w=600,+ face_map=None, has_any_split=False,+ first_speaker=None, speaker_anchor_x={},+ ),+ 960.0,+ )++ def test_split_with_no_map_returns_left_quarter(self):+ # Safer than the seam at the exact center.+ self.assertEqual(+ fth.safe_default_center(+ width=1920, crop_w=600,+ face_map=None, has_any_split=True,+ first_speaker=None, speaker_anchor_x={},+ ),+ 480.0,+ )+++class ClampAwayFromDeadZoneTests(unittest.TestCase):+ def test_non_split_is_noop(self):+ self.assertEqual(+ fth.clamp_away_from_dead_zone(+ crop_x=100, crop_w=600, width=1920,+ face_map={"clusters": []}, has_any_split=False,+ ),+ 100,+ )++ def test_no_face_map_is_noop(self):+ self.assertEqual(+ fth.clamp_away_from_dead_zone(+ crop_x=660, crop_w=600, width=1920,+ face_map=None, has_any_split=True,+ ),+ 660,+ )++ def test_crop_far_from_seam_passes_through(self):+ # crop_center = 100 + 300 = 400; mid_x = 960; distance = 560 > margin+ face_map = {"clusters": [{"center_x": 400}, {"center_x": 1500}]}+ self.assertEqual(+ fth.clamp_away_from_dead_zone(+ crop_x=100, crop_w=600, width=1920,+ face_map=face_map, has_any_split=True,+ ),+ 100,+ )++ def test_crop_near_seam_snaps_to_nearest_cluster(self):+ # crop_x=660 → crop_center=960 (exactly on seam); nearest cluster = 1500+ face_map = {"clusters": [{"center_x": 400}, {"center_x": 1500}]}+ snapped = fth.clamp_away_from_dead_zone(+ crop_x=660, crop_w=600, width=1920,+ face_map=face_map, has_any_split=True,+ )+ # Should have moved off the seam+ self.assertNotEqual(snapped, 660)+ # crop_x still keeps the crop window on-frame+ self.assertLessEqual(snapped + 600, 1920)+ self.assertGreaterEqual(snapped, 0)++ def test_near_seam_with_no_clusters_returns_left_quarter(self):+ face_map = {"clusters": []}+ result = fth.clamp_away_from_dead_zone(+ crop_x=660, crop_w=600, width=1920,+ face_map=face_map, has_any_split=True,+ )+ # width // 4 - crop_w // 2 = 480 - 300 = 180+ self.assertEqual(result, 180)+++class UpgradeSpeakerMappingsTests(unittest.TestCase):+ def test_clears_old_mappings(self):+ original = {+ "clusters": [{"center_x": 400}],+ "speaker_mappings": {"A": 0, "B": 1},+ }+ result = fth.upgrade_speaker_mappings(original)+ self.assertEqual(result["speaker_mappings"], {})+ self.assertTrue(result["_mappings_v2"])++ def test_preserves_clusters(self):+ original = {+ "clusters": [{"center_x": 400}, {"center_x": 1500}],+ "speaker_mappings": {"A": 0},+ }+ result = fth.upgrade_speaker_mappings(original)+ self.assertEqual(len(result["clusters"]), 2)++ def test_does_not_mutate_input(self):+ original = {+ "clusters": [],+ "speaker_mappings": {"A": 0},+ }+ fth.upgrade_speaker_mappings(original)+ # Original should still have the old mapping+ self.assertEqual(original["speaker_mappings"], {"A": 0})+++class ReExportTests(unittest.TestCase):+ def test_video_processor_reexports_all_helpers(self):+ from services import video_processor as vp+ self.assertIs(vp._update_tripod_camera, fth.update_tripod_camera)+ self.assertIs(vp._choose_camera_speaker, fth.choose_camera_speaker)+ self.assertIs(vp._safe_default_center, fth.safe_default_center)+ self.assertIs(vp._clamp_away_from_dead_zone, fth.clamp_away_from_dead_zone)+ self.assertIs(vp._upgrade_speaker_mappings, fth.upgrade_speaker_mappings)+++if __name__ == "__main__":+ unittest.main()