Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

backend: extract pure camera helpers into face_track_helpers

Nika Siradzedba17546dc11
diff --git a/backend/services/face_track_helpers.py b/backend/services/face_track_helpers.py
new file mode 100644
index 0000000..930b39b
--- /dev/null
+++ b/backend/services/face_track_helpers.py
@@ -0,0 +1,175 @@
+"""Pure helper functions for the face-tracking / camera-movement
+pipeline.
+
+Extracted from video_processor.py. These are all side-effect-free:
+they take dicts / scalars / lists and return decisions (a camera
+center, a speaker id, a clamped crop_x). No ffmpeg, no cv2, no
+numpy. Trivially unit-testable in isolation, but they govern the
+camera's behavior during every clip render.
+"""
+
+from __future__ import annotations
+
+
+def update_tripod_camera(
+ current_center_x: float,
+ target_center_x: float | None,
+ crop_w: int,
+ video_width: int,
+ dt: float,
+ force_snap: bool = False,
+) -> float:
+ """Heavy-tripod camera movement inspired by OpenShorts.
+
+ The camera stays still while the subject remains inside a safe
+ zone, then moves at a bounded speed instead of constantly chasing
+ every face twitch. Always clamps the result so the crop window
+ never falls off the source frame.
+ """
+ half_crop = crop_w / 2.0
+ min_center = half_crop
+ max_center = max(half_crop, video_width - half_crop)
+
+ if target_center_x is None:
+ return min(max(current_center_x, min_center), max_center)
+
+ if force_snap:
+ current_center_x = target_center_x
+ else:
+ diff = target_center_x - current_center_x
+ # Wide enough to absorb natural head movement and detector
+ # noise, narrow enough to keep the speaker composed near center.
+ safe_zone_radius = crop_w * 0.22
+ if abs(diff) > safe_zone_radius:
+ slow_speed = 72.0
+ fast_speed = 360.0
+ speed = fast_speed if abs(diff) > crop_w * 0.5 else slow_speed
+ step = min(abs(diff), speed * max(dt, 0.01))
+ current_center_x += step if diff > 0 else -step
+
+ return min(max(current_center_x, min_center), max_center)
+
+
+def choose_camera_speaker(
+ transcript_speaker: str | None,
+ transcript_duration: float,
+ active_speaker: str | None,
+ pending_speaker: str | None,
+ pending_count: int,
+ min_turn_duration: float = 2.6,
+ confirmation_frames: int = 3,
+) -> tuple[str | None, str | None, int, bool]:
+ """Stabilize diarization before the camera switches.
+
+ Brief interjections don't move the camera; sustained turns must
+ be confirmed for `confirmation_frames` samples before the switch
+ is committed. Returns:
+ (camera_speaker, pending_speaker, pending_count, switched_now)
+ """
+ if transcript_speaker is None:
+ return active_speaker, pending_speaker, pending_count, False
+
+ if active_speaker is None:
+ return transcript_speaker, None, 0, True
+
+ if transcript_speaker == active_speaker:
+ return active_speaker, None, 0, False
+
+ # Brief interjections should not move the camera.
+ if transcript_duration < min_turn_duration:
+ return active_speaker, None, 0, False
+
+ if pending_speaker != transcript_speaker:
+ return active_speaker, transcript_speaker, 1, False
+
+ pending_count += 1
+ if pending_count < confirmation_frames:
+ return active_speaker, pending_speaker, pending_count, False
+
+ return transcript_speaker, None, 0, True
+
+
+def safe_default_center(
+ width: int,
+ crop_w: int,
+ face_map: dict | None,
+ has_any_split: bool,
+ first_speaker: str | None,
+ speaker_anchor_x: dict,
+) -> float:
+ """Pick a safe initial camera center.
+
+ On split-screen, width/2 is the dead zone between two feeds. Use
+ the face_map or speaker_anchor to start on an actual speaker
+ instead. Fallback order:
+ 1. first_speaker's learned anchor
+ 2. largest face_map cluster
+ 3. plain center (non-split)
+ 4. left quarter (split with no map)
+ """
+ if first_speaker and first_speaker in speaker_anchor_x:
+ return float(speaker_anchor_x[first_speaker])
+
+ if face_map and face_map.get("clusters"):
+ clusters = face_map["clusters"]
+ best = max(clusters, key=lambda c: c.get("count", 0))
+ return float(best["center_x"])
+
+ if not has_any_split:
+ return float(width) / 2
+
+ return float(width) / 4
+
+
+def clamp_away_from_dead_zone(
+ crop_x: int,
+ crop_w: int,
+ width: int,
+ face_map: dict | None,
+ has_any_split: bool,
+) -> int:
+ """Snap a crop that centers on the split-screen seam to the nearest
+ cluster instead.
+
+ Only triggers when the crop CENTER is close to the midline of the
+ source frame — a crop that merely straddles the midline is fine on
+ fullscreen layouts where the face genuinely sits near center.
+ """
+ if not has_any_split or not face_map:
+ return crop_x
+
+ mid_x = width // 2
+ crop_center = crop_x + crop_w // 2
+
+ seam_margin = crop_w // 8 # ~75px on a 607-wide crop
+ if abs(crop_center - mid_x) > seam_margin:
+ return crop_x
+
+ clusters = face_map.get("clusters", [])
+ if not clusters:
+ return max(0, width // 4 - crop_w // 2)
+
+ best_cluster = min(
+ clusters, key=lambda c: abs(c["center_x"] - crop_center)
+ )
+ snapped = max(0, min(best_cluster["center_x"] - crop_w // 2, width - crop_w))
+ return snapped
+
+
+def upgrade_speaker_mappings(face_map: dict) -> dict:
+ """Invalidate stale speaker-to-cluster mappings from old caches.
+
+ v1 caches used "first-to-speak = left" which breaks when the host
+ speaks first but sits on the right. This function clears the
+ mappings entirely and stamps `_mappings_v2 = True` so the clear
+ only runs once per cache. Cluster positions are kept — they're
+ still useful for dead-zone clamping and safe defaults.
+
+ Note: face_analysis.py writes its own v2-quality mappings and
+ stamps them with `_mappings_v2=True`, so newly-built face_maps
+ skip this function entirely.
+ """
+ face_map = dict(face_map)
+ face_map["speaker_mappings"] = {}
+ face_map["_mappings_v2"] = True
+ return face_map
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index 384e729..d7cd6c8 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -35,6 +35,13 @@ from services.motion_filters import (
motion_windows_from_keyframes as _motion_windows_from_keyframes,
simplify_keyframes as _simplify_keyframes,
)
+from services.face_track_helpers import (
+ choose_camera_speaker as _choose_camera_speaker,
+ clamp_away_from_dead_zone as _clamp_away_from_dead_zone,
+ safe_default_center as _safe_default_center,
+ update_tripod_camera as _update_tripod_camera,
+ upgrade_speaker_mappings as _upgrade_speaker_mappings,
+)
import sys
@@ -469,44 +476,6 @@ def _choose_segment_tracks(
return segment_tracks, learned_anchor_x, learned_side
-def _update_tripod_camera(
- current_center_x: float,
- target_center_x: float | None,
- crop_w: int,
- video_width: int,
- dt: float,
- force_snap: bool = False,
-) -> float:
- """
- Heavy-tripod camera movement inspired by OpenShorts.
-
- The camera stays still while the subject remains inside a safe zone, then
- moves at a bounded speed instead of constantly chasing every face twitch.
- """
- half_crop = crop_w / 2.0
- min_center = half_crop
- max_center = max(half_crop, video_width - half_crop)
-
- if target_center_x is None:
- return min(max(current_center_x, min_center), max_center)
-
- if force_snap:
- current_center_x = target_center_x
- else:
- diff = target_center_x - current_center_x
- # Wide enough to absorb natural head movement and detector noise,
- # narrow enough to keep the speaker composed near center.
- safe_zone_radius = crop_w * 0.22
- if abs(diff) > safe_zone_radius:
- slow_speed = 72.0
- fast_speed = 360.0
- speed = fast_speed if abs(diff) > crop_w * 0.5 else slow_speed
- step = min(abs(diff), speed * max(dt, 0.01))
- current_center_x += step if diff > 0 else -step
-
- return min(max(current_center_x, min_center), max_center)
-
-
def _choose_track_segment_targets(
segment_tracks: list,
tracked_detections: list,
@@ -831,132 +800,6 @@ def _pick_tracking_face(
return face, side_ok or near_anchor
-def _choose_camera_speaker(
- transcript_speaker: str | None,
- transcript_duration: float,
- active_speaker: str | None,
- pending_speaker: str | None,
- pending_count: int,
- min_turn_duration: float = 2.6,
- confirmation_frames: int = 3,
-) -> tuple[str | None, str | None, int, bool]:
- """
- Stabilize diarization before the camera switches.
-
- Returns (camera_speaker, pending_speaker, pending_count, switched_now).
- """
- if transcript_speaker is None:
- return active_speaker, pending_speaker, pending_count, False
-
- if active_speaker is None:
- return transcript_speaker, None, 0, True
-
- if transcript_speaker == active_speaker:
- return active_speaker, None, 0, False
-
- # Brief interjections should not move the camera.
- if transcript_duration < min_turn_duration:
- return active_speaker, None, 0, False
-
- if pending_speaker != transcript_speaker:
- return active_speaker, transcript_speaker, 1, False
-
- pending_count += 1
- if pending_count < confirmation_frames:
- return active_speaker, pending_speaker, pending_count, False
-
- return transcript_speaker, None, 0, True
-
-
-def _safe_default_center(
- width: int,
- crop_w: int,
- face_map: dict | None,
- has_any_split: bool,
- first_speaker: str | None,
- speaker_anchor_x: dict,
-) -> float:
- """
- Pick a safe initial camera center.
-
- On split-screen, width/2 is the dead zone between two feeds. Use the
- face_map or speaker_anchor to start on an actual speaker instead.
- """
- # Best case: we know where the first speaker sits
- if first_speaker and first_speaker in speaker_anchor_x:
- return float(speaker_anchor_x[first_speaker])
-
- # Face-map cluster fallback: pick the cluster with more observations
- if face_map and face_map.get("clusters"):
- clusters = face_map["clusters"]
- best = max(clusters, key=lambda c: c.get("count", 0))
- return float(best["center_x"])
-
- # Non-split: center is fine
- if not has_any_split:
- return float(width) / 2
-
- # Split with no map: pick the left quarter as a safer guess than center
- return float(width) / 4
-
-
-def _clamp_away_from_dead_zone(
- crop_x: int,
- crop_w: int,
- width: int,
- face_map: dict | None,
- has_any_split: bool,
-) -> int:
- """
- If crop_x centers the window on the split-screen seam (where no face
- lives), snap to the nearest cluster position instead.
-
- Only triggers when the crop CENTER is close to mid_x. A crop that
- merely straddles mid_x is fine — in fullscreen mode the face genuinely
- sits near center.
- """
- if not has_any_split or not face_map:
- return crop_x
-
- mid_x = width // 2
- crop_center = crop_x + crop_w // 2
-
- # Only clamp when the crop is centered near the seam itself.
- # A tight margin avoids false-positives on fullscreen layouts where
- # the face legitimately sits near center.
- seam_margin = crop_w // 8 # ~75px on a 607-wide crop
- if abs(crop_center - mid_x) > seam_margin:
- return crop_x
-
- # Snap to nearest cluster
- clusters = face_map.get("clusters", [])
- if not clusters:
- return max(0, width // 4 - crop_w // 2)
-
- best_cluster = min(clusters, key=lambda c: abs(c["center_x"] - crop_center))
- snapped = max(0, min(best_cluster["center_x"] - crop_w // 2, width - crop_w))
- return snapped
-
-
-def _upgrade_speaker_mappings(face_map: dict) -> dict:
- """
- Invalidate stale speaker-to-cluster mappings from old caches.
-
- Old caches used "first-to-speak = left" which breaks when the host
- speaks first but sits on the right. Rather than trying to re-compute
- mappings from clip-scoped data, we clear the mappings entirely so the
- clip-local tracking decides which face to follow based on visual
- evidence (which is more reliable for any single clip).
-
- Cluster positions are kept — they're still useful for dead-zone
- clamping and safe defaults.
- """
- face_map = dict(face_map)
- face_map["speaker_mappings"] = {}
- face_map["_mappings_v2"] = True
- return face_map
-
-
def _track_and_crop(
input_path: str,
output_path: str,
diff --git a/tests/test_face_track_helpers.py b/tests/test_face_track_helpers.py
new file mode 100644
index 0000000..99b27ba
--- /dev/null
+++ b/tests/test_face_track_helpers.py
@@ -0,0 +1,312 @@
+"""Tests for backend.services.face_track_helpers — pure camera /
+speaker-switch decision helpers.
+"""
+
+import os
+import sys
+import unittest
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+BACKEND_ROOT = os.path.join(ROOT, "backend")
+if BACKEND_ROOT not in sys.path:
+ sys.path.insert(0, BACKEND_ROOT)
+
+from services import face_track_helpers as fth
+
+
+class UpdateTripodCameraTests(unittest.TestCase):
+ def test_no_target_clamps_within_frame(self):
+ # No target → camera sits at current_center but clamped
+ result = fth.update_tripod_camera(
+ current_center_x=500, target_center_x=None,
+ crop_w=600, video_width=1920, dt=0.1,
+ )
+ self.assertEqual(result, 500)
+
+ def test_clamps_below_half_crop(self):
+ # current 100 with crop_w=600 → min center = 300
+ result = fth.update_tripod_camera(
+ current_center_x=100, target_center_x=None,
+ crop_w=600, video_width=1920, dt=0.1,
+ )
+ self.assertEqual(result, 300)
+
+ def test_clamps_above_width_minus_half(self):
+ # video_width - half_crop = 1920 - 300 = 1620
+ result = fth.update_tripod_camera(
+ current_center_x=1800, target_center_x=None,
+ crop_w=600, video_width=1920, dt=0.1,
+ )
+ self.assertEqual(result, 1620)
+
+ def test_force_snap_jumps_to_target(self):
+ result = fth.update_tripod_camera(
+ current_center_x=500, target_center_x=1200,
+ crop_w=600, video_width=1920, dt=0.1, force_snap=True,
+ )
+ self.assertEqual(result, 1200)
+
+ def test_small_drift_inside_safe_zone_holds(self):
+ # diff = 50, safe_zone_radius = 600 * 0.22 = 132 → hold
+ result = fth.update_tripod_camera(
+ current_center_x=600, target_center_x=650,
+ crop_w=600, video_width=1920, dt=0.1,
+ )
+ self.assertEqual(result, 600)
+
+ def test_large_drift_moves_toward_target(self):
+ # diff = 400 → outside safe zone → move toward it
+ result = fth.update_tripod_camera(
+ current_center_x=500, target_center_x=900,
+ crop_w=600, video_width=1920, dt=0.1,
+ )
+ # Should have moved right but not snapped all the way
+ self.assertGreater(result, 500)
+ self.assertLess(result, 900)
+
+ def test_movement_is_bounded_by_speed(self):
+ # Very fast diff, small dt → step limited by max speed × dt
+ result = fth.update_tripod_camera(
+ current_center_x=500, target_center_x=1800, # huge diff → fast_speed
+ crop_w=600, video_width=1920, dt=0.1,
+ )
+ # fast_speed=360, dt=0.1 → max step 36 → 500 + 36 = 536
+ self.assertEqual(result, 536)
+
+
+class ChooseCameraSpeakerTests(unittest.TestCase):
+ def test_no_transcript_speaker_holds(self):
+ self.assertEqual(
+ fth.choose_camera_speaker(
+ transcript_speaker=None,
+ transcript_duration=5.0,
+ active_speaker="A",
+ pending_speaker=None,
+ pending_count=0,
+ ),
+ ("A", None, 0, False),
+ )
+
+ def test_first_speaker_becomes_active(self):
+ self.assertEqual(
+ fth.choose_camera_speaker(
+ transcript_speaker="A",
+ transcript_duration=5.0,
+ active_speaker=None,
+ pending_speaker=None,
+ pending_count=0,
+ ),
+ ("A", None, 0, True),
+ )
+
+ def test_same_speaker_continues(self):
+ self.assertEqual(
+ fth.choose_camera_speaker(
+ transcript_speaker="A",
+ transcript_duration=5.0,
+ active_speaker="A",
+ pending_speaker="B",
+ pending_count=2,
+ ),
+ ("A", None, 0, False),
+ )
+
+ def test_brief_interjection_ignored(self):
+ # transcript_duration < min_turn_duration → hold
+ self.assertEqual(
+ fth.choose_camera_speaker(
+ transcript_speaker="B",
+ transcript_duration=1.0, # brief
+ active_speaker="A",
+ pending_speaker=None,
+ pending_count=0,
+ ),
+ ("A", None, 0, False),
+ )
+
+ def test_new_speaker_enters_pending(self):
+ cam, pending, count, switched = fth.choose_camera_speaker(
+ transcript_speaker="B",
+ transcript_duration=5.0,
+ active_speaker="A",
+ pending_speaker=None,
+ pending_count=0,
+ )
+ self.assertEqual(cam, "A")
+ self.assertEqual(pending, "B")
+ self.assertEqual(count, 1)
+ self.assertFalse(switched)
+
+ def test_pending_confirmed_after_confirmation_frames(self):
+ # Previous pending=B, count=2; with confirmation_frames=3 the
+ # third sighting commits the switch.
+ cam, pending, count, switched = fth.choose_camera_speaker(
+ transcript_speaker="B",
+ transcript_duration=5.0,
+ active_speaker="A",
+ pending_speaker="B",
+ pending_count=2,
+ confirmation_frames=3,
+ )
+ self.assertEqual(cam, "B")
+ self.assertEqual(pending, None)
+ self.assertTrue(switched)
+
+ def test_different_pending_resets_count(self):
+ # Pending was B=1, now transcript speaker is C → reset to C=1
+ cam, pending, count, switched = fth.choose_camera_speaker(
+ transcript_speaker="C",
+ transcript_duration=5.0,
+ active_speaker="A",
+ pending_speaker="B",
+ pending_count=1,
+ )
+ self.assertEqual(pending, "C")
+ self.assertEqual(count, 1)
+ self.assertFalse(switched)
+
+
+class SafeDefaultCenterTests(unittest.TestCase):
+ def test_uses_first_speaker_anchor_when_available(self):
+ self.assertEqual(
+ fth.safe_default_center(
+ width=1920, crop_w=600,
+ face_map=None, has_any_split=False,
+ first_speaker="A",
+ speaker_anchor_x={"A": 800.0},
+ ),
+ 800.0,
+ )
+
+ def test_uses_largest_face_map_cluster(self):
+ face_map = {
+ "clusters": [
+ {"center_x": 400, "count": 10},
+ {"center_x": 1500, "count": 50}, # largest
+ ],
+ }
+ self.assertEqual(
+ fth.safe_default_center(
+ width=1920, crop_w=600,
+ face_map=face_map, has_any_split=True,
+ first_speaker=None,
+ speaker_anchor_x={},
+ ),
+ 1500.0,
+ )
+
+ def test_non_split_returns_center(self):
+ self.assertEqual(
+ fth.safe_default_center(
+ width=1920, crop_w=600,
+ face_map=None, has_any_split=False,
+ first_speaker=None, speaker_anchor_x={},
+ ),
+ 960.0,
+ )
+
+ def test_split_with_no_map_returns_left_quarter(self):
+ # Safer than the seam at the exact center.
+ self.assertEqual(
+ fth.safe_default_center(
+ width=1920, crop_w=600,
+ face_map=None, has_any_split=True,
+ first_speaker=None, speaker_anchor_x={},
+ ),
+ 480.0,
+ )
+
+
+class ClampAwayFromDeadZoneTests(unittest.TestCase):
+ def test_non_split_is_noop(self):
+ self.assertEqual(
+ fth.clamp_away_from_dead_zone(
+ crop_x=100, crop_w=600, width=1920,
+ face_map={"clusters": []}, has_any_split=False,
+ ),
+ 100,
+ )
+
+ def test_no_face_map_is_noop(self):
+ self.assertEqual(
+ fth.clamp_away_from_dead_zone(
+ crop_x=660, crop_w=600, width=1920,
+ face_map=None, has_any_split=True,
+ ),
+ 660,
+ )
+
+ def test_crop_far_from_seam_passes_through(self):
+ # crop_center = 100 + 300 = 400; mid_x = 960; distance = 560 > margin
+ face_map = {"clusters": [{"center_x": 400}, {"center_x": 1500}]}
+ self.assertEqual(
+ fth.clamp_away_from_dead_zone(
+ crop_x=100, crop_w=600, width=1920,
+ face_map=face_map, has_any_split=True,
+ ),
+ 100,
+ )
+
+ def test_crop_near_seam_snaps_to_nearest_cluster(self):
+ # crop_x=660 → crop_center=960 (exactly on seam); nearest cluster = 1500
+ face_map = {"clusters": [{"center_x": 400}, {"center_x": 1500}]}
+ snapped = fth.clamp_away_from_dead_zone(
+ crop_x=660, crop_w=600, width=1920,
+ face_map=face_map, has_any_split=True,
+ )
+ # Should have moved off the seam
+ self.assertNotEqual(snapped, 660)
+ # crop_x still keeps the crop window on-frame
+ self.assertLessEqual(snapped + 600, 1920)
+ self.assertGreaterEqual(snapped, 0)
+
+ def test_near_seam_with_no_clusters_returns_left_quarter(self):
+ face_map = {"clusters": []}
+ result = fth.clamp_away_from_dead_zone(
+ crop_x=660, crop_w=600, width=1920,
+ face_map=face_map, has_any_split=True,
+ )
+ # width // 4 - crop_w // 2 = 480 - 300 = 180
+ self.assertEqual(result, 180)
+
+
+class UpgradeSpeakerMappingsTests(unittest.TestCase):
+ def test_clears_old_mappings(self):
+ original = {
+ "clusters": [{"center_x": 400}],
+ "speaker_mappings": {"A": 0, "B": 1},
+ }
+ result = fth.upgrade_speaker_mappings(original)
+ self.assertEqual(result["speaker_mappings"], {})
+ self.assertTrue(result["_mappings_v2"])
+
+ def test_preserves_clusters(self):
+ original = {
+ "clusters": [{"center_x": 400}, {"center_x": 1500}],
+ "speaker_mappings": {"A": 0},
+ }
+ result = fth.upgrade_speaker_mappings(original)
+ self.assertEqual(len(result["clusters"]), 2)
+
+ def test_does_not_mutate_input(self):
+ original = {
+ "clusters": [],
+ "speaker_mappings": {"A": 0},
+ }
+ fth.upgrade_speaker_mappings(original)
+ # Original should still have the old mapping
+ self.assertEqual(original["speaker_mappings"], {"A": 0})
+
+
+class ReExportTests(unittest.TestCase):
+ def test_video_processor_reexports_all_helpers(self):
+ from services import video_processor as vp
+ self.assertIs(vp._update_tripod_camera, fth.update_tripod_camera)
+ self.assertIs(vp._choose_camera_speaker, fth.choose_camera_speaker)
+ self.assertIs(vp._safe_default_center, fth.safe_default_center)
+ self.assertIs(vp._clamp_away_from_dead_zone, fth.clamp_away_from_dead_zone)
+ self.assertIs(vp._upgrade_speaker_mappings, fth.upgrade_speaker_mappings)
+
+
+if __name__ == "__main__":
+ unittest.main()