Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

fix: face detection audit — pass face_map from CLI, fix fast path split-screen

Nika Siradze7629f3af5e6c
diff --git a/backend/cli.py b/backend/cli.py
index 6820f4f..632106c 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -296,6 +296,9 @@ def cmd_process(args):
# Save to cache for next run
_save_cache(video_path, result)
+ # Extract face_map before result gets overwritten in clip loop
+ face_map = result.get("face_map")
+
# Check speaker data availability (needed for smart cropping)
speakers_in_words = set(w.get("speaker") for w in words if w.get("speaker"))
diarization_warning = result.get("diarization_warning")
@@ -429,6 +432,7 @@ def cmd_process(args):
logo_path=config.get("logo_path") or None,
outro_path=config.get("outro_path") or None,
keep_segments=clip.get("segments"),
+ face_map=face_map,
)
results.append(result)
print(f" ✓ {result['file_size_mb']}MB")
diff --git a/backend/services/face_analysis.py b/backend/services/face_analysis.py
index bf88821..c09893c 100644
--- a/backend/services/face_analysis.py
+++ b/backend/services/face_analysis.py
@@ -124,29 +124,31 @@ def analyze_faces(
if progress_callback:
progress_callback(75, "Clustering face positions...")
- # Cluster faces by position
+ # Cluster faces by left/right half split (consistent with _build_speaker_aware_crop)
target_ratio = 1080 / 1920 # 9:16
crop_w = int(height * target_ratio)
positions = np.array([o["face_center_x"] for o in observations])
- cluster_radius = width * 0.20
+ mid_x = width // 2
+ seam_margin = 20
- clusters_list = []
- used = np.zeros(len(positions), dtype=bool)
- sorted_idx = np.argsort(positions)
+ left_pos = positions[positions < mid_x]
+ right_pos = positions[positions >= mid_x]
- for idx in sorted_idx:
- if used[idx]:
- continue
- mask = (np.abs(positions - positions[idx]) < cluster_radius) & ~used
- if np.any(mask):
- cluster_center = int(np.median(positions[mask]))
- crop_x = max(0, min(cluster_center - crop_w // 2, width - crop_w))
- clusters_list.append({
- "center_x": cluster_center,
- "count": int(np.sum(mask)),
- "crop_x": crop_x,
- })
- used |= mask
+ clusters_list = []
+ if len(left_pos) >= 3:
+ cx = int(np.median(left_pos))
+ clusters_list.append({
+ "center_x": cx,
+ "count": len(left_pos),
+ "crop_x": max(0, min(cx - crop_w // 2, mid_x - crop_w - seam_margin)),
+ })
+ if len(right_pos) >= 3:
+ cx = int(np.median(right_pos))
+ clusters_list.append({
+ "center_x": cx,
+ "count": len(right_pos),
+ "crop_x": max(mid_x + seam_margin, min(cx - crop_w // 2, width - crop_w)),
+ })
if not clusters_list:
return None
@@ -164,16 +166,27 @@ def analyze_faces(
if progress_callback:
progress_callback(85, "Mapping speakers to face positions...")
- if is_split_screen and len(clusters_list) >= 2 and len(speakers) == 2:
- # Split-screen: map by who speaks first → left face
+ if is_split_screen and len(clusters_list) >= 2:
+ # Split-screen: map top-2 speakers by talk time, first-to-speak = left
+ speaker_talk_time = {}
speaker_first_time = {}
for seg in speaker_segments:
sp = seg.get("speaker")
- if sp and sp not in speaker_first_time:
- speaker_first_time[sp] = seg["start"]
- speakers_sorted = sorted(speakers, key=lambda s: speaker_first_time.get(s, float("inf")))
- speaker_mappings[speakers_sorted[0]] = 0 # left cluster
- speaker_mappings[speakers_sorted[1]] = 1 # right cluster
+ if sp:
+ speaker_talk_time[sp] = speaker_talk_time.get(sp, 0) + (seg["end"] - seg["start"])
+ if sp not in speaker_first_time:
+ speaker_first_time[sp] = seg["start"]
+
+ speakers_by_talk = sorted(speakers, key=lambda s: speaker_talk_time.get(s, 0), reverse=True)
+ top_2 = speakers_by_talk[:2]
+ top_2_by_first = sorted(top_2, key=lambda s: speaker_first_time.get(s, float("inf")))
+ speaker_mappings[top_2_by_first[0]] = 0 # left cluster
+ if len(top_2_by_first) > 1:
+ speaker_mappings[top_2_by_first[1]] = 1 # right cluster
+ # Extra speakers → dominant speaker's cluster
+ dominant_idx = speaker_mappings.get(speakers_by_talk[0], 0)
+ for sp in speakers_by_talk[2:]:
+ speaker_mappings[sp] = dominant_idx
elif len(clusters_list) >= 2 and len(speakers) >= 2:
# Non-split-screen: vote by which face is visible when speaker talks
obs_arr = [(o["time"], o["face_center_x"]) for o in observations]
@@ -184,7 +197,7 @@ def analyze_faces(
for obs_t, obs_cx in obs_arr:
if abs(obs_t - t) < 1.5:
for ci, cl in enumerate(clusters_list):
- if abs(obs_cx - cl["center_x"]) < cluster_radius:
+ if abs(obs_cx - cl["center_x"]) < width * 0.15:
votes[ci] += 1
if any(votes):
speaker_mappings[speaker] = int(np.argmax(votes))
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index 8c0b9d4..c4ee751 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -275,7 +275,10 @@ def _use_face_map(
clusters = face_map.get("clusters", [])
speaker_mappings = face_map.get("speaker_mappings", {})
dominant = face_map.get("dominant_speaker")
+ is_split_screen = face_map.get("is_split_screen", False)
crop_w = int(height * target_ratio)
+ mid_x = width // 2
+ seam_margin = 20
if not clusters:
return None
@@ -283,9 +286,17 @@ def _use_face_map(
# Verify face_map was computed at the same resolution
map_w = face_map.get("video_width", width)
if map_w != width:
- # Resolution mismatch — face_map positions don't apply
return None
+ # Apply seam margin to crop_x values for split-screen (copy to avoid mutating face_map)
+ if is_split_screen and len(clusters) >= 2:
+ clusters = [dict(cl) for cl in clusters]
+ for cl in clusters:
+ if cl["center_x"] < mid_x:
+ cl["crop_x"] = max(0, min(cl["crop_x"], mid_x - crop_w - seam_margin))
+ else:
+ cl["crop_x"] = max(mid_x + seam_margin, min(cl["crop_x"], width - crop_w))
+
# Check if clip has multiple speakers
speakers_in_clip = set()
if transcript_words:
@@ -297,7 +308,6 @@ def _use_face_map(
ci = speaker_mappings.get(sp)
if ci is not None and ci < len(clusters):
return str(clusters[ci]["crop_x"])
- # Fallback to dominant speaker's position
if dominant and dominant in speaker_mappings:
ci = speaker_mappings[dominant]
if ci < len(clusters):
@@ -341,8 +351,8 @@ def _use_face_map(
default_ci = speaker_mappings.get(dominant, 0)
default_x = clusters[min(default_ci, len(clusters) - 1)]["crop_x"]
- # Build keyframes
- pan_duration = 0.4
+ # Build keyframes — instant cut for split-screen, smooth pan otherwise
+ pan_duration = 0.0 if is_split_screen else 0.4
duration = segments[-1][1] if segments else 1.0
keyframes = []
prev_x = default_x
@@ -352,8 +362,12 @@ def _use_face_map(
target_x = clusters[ci]["crop_x"] if ci is not None and ci < len(clusters) else default_x
if target_x != prev_x:
- keyframes.append((start_t, prev_x))
- keyframes.append((start_t + pan_duration, target_x))
+ if pan_duration > 0:
+ keyframes.append((start_t, prev_x))
+ keyframes.append((start_t + pan_duration, target_x))
+ else:
+ keyframes.append((start_t, prev_x))
+ keyframes.append((start_t + 0.01, target_x))
prev_x = target_x
if not keyframes: