Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

fix: reliable camera tracking for mixed-layout Riverside recordings

Nika Siradzefbb5ef74d597
diff --git a/backend/presets.py b/backend/presets.py
index cb2d58a..9e6c646 100644
--- a/backend/presets.py
+++ b/backend/presets.py
@@ -21,7 +21,7 @@ PRESETS_DIR = os.path.join(
DEFAULT_PRESET = {
"caption_style": "branded",
- "crop_strategy": "face",
+ "crop_strategy": "speaker",
"time_adjust": -1.0,
"logo_path": "",
"outro_path": "",
@@ -30,8 +30,8 @@ DEFAULT_PRESET = {
"output_dir": "",
"whisper_model": "base",
"top_clips": 5,
- "max_clip_duration": 90,
- "min_clip_duration": 20,
+ "max_clip_duration": 40,
+ "min_clip_duration": 15,
"target_lufs": -14.0,
"energy_boost": True,
"quality": "max",
diff --git a/backend/services/claude_suggest.py b/backend/services/claude_suggest.py
index 8c249ae..71da193 100644
--- a/backend/services/claude_suggest.py
+++ b/backend/services/claude_suggest.py
@@ -72,6 +72,15 @@ def _engine_label(engine: str) -> str:
return "AI"
+def _format_timeout_label(timeout: int) -> str:
+ """Render a human-readable timeout label for progress messages."""
+ if timeout % 60 == 0 and timeout >= 60:
+ minutes = timeout // 60
+ unit = "minute" if minutes == 1 else "minutes"
+ return f"{minutes} {unit}"
+ return f"{timeout}s"
+
+
def _run_ai_command(
cli_path: str,
engine: str,
@@ -294,11 +303,68 @@ Transcript ({segment_count} segments, ~{duration_min:.0f} min):
{transcript_text}"""
+def _build_transcript_text(segments: list[dict]) -> str:
+ """Serialize transcript segments into the prompt-friendly text format."""
+ lines = []
+ for seg in segments:
+ speaker = seg.get("speaker", "")
+ speaker_label = f"[{speaker}] " if speaker else ""
+ start = seg.get("start", 0)
+ text = seg.get("text", "").strip()
+ if text:
+ lines.append(f"[{start:.1f}s] {speaker_label}{text}")
+ return "\n".join(lines)
+
+
+def _segments_duration_seconds(segments: list[dict]) -> float:
+ """Estimate total covered duration from transcript segments."""
+ if not segments:
+ return 0.0
+ return max(0.0, float(segments[-1].get("end", segments[-1].get("start", 0))) - float(segments[0].get("start", 0)))
+
+
+def _should_bucket_initial_selection(segments: list[dict]) -> bool:
+ """
+ Use bucketed AI search for long or dense transcripts.
+
+ This keeps initial clip discovery from spending many minutes on a single
+ whole-episode prompt for long podcasts.
+ """
+ if not segments:
+ return False
+
+ duration_seconds = _segments_duration_seconds(segments)
+ transcript_chars = sum(len(str(seg.get("text", ""))) for seg in segments)
+
+ return bool(
+ duration_seconds >= 45 * 60
+ or len(segments) >= 180
+ or transcript_chars >= 18000
+ )
+
+
+def _dedupe_clips_by_range(clips: list[dict]) -> list[dict]:
+ """Drop duplicate clip suggestions that share the same rounded range."""
+ deduped = []
+ seen_ranges = set()
+ for clip in sorted(clips, key=lambda c: c.get("start_second", 0)):
+ key = (
+ round(float(clip.get("start_second", 0)), 1),
+ round(float(clip.get("end_second", 0)), 1),
+ )
+ if key in seen_ranges:
+ continue
+ seen_ranges.add(key)
+ deduped.append(clip)
+ return deduped
+
+
def suggest_with_claude(
segments: list[dict],
top_n: int = 5,
exclude_clips: list[dict] | None = None,
progress_callback: Optional[Callable[[int, str], None]] = None,
+ timeout: int = 300,
) -> Optional[list[dict]]:
"""
Use an AI CLI (Claude Code or Codex) to extract the best clip moments.
@@ -314,18 +380,7 @@ def suggest_with_claude(
label = _engine_label(candidates[0][1])
progress_callback(0, f"Preparing transcript for {label}...")
- # Build transcript text from segments
- lines = []
- for seg in segments:
- speaker = seg.get("speaker", "")
- speaker_label = f"[{speaker}] " if speaker else ""
- start = seg.get("start", 0)
- text = seg.get("text", "").strip()
- if text:
- # Use absolute seconds (not M:SS) so Claude returns seconds too
- lines.append(f"[{start:.1f}s] {speaker_label}{text}")
-
- transcript_text = "\n".join(lines)
+ transcript_text = _build_transcript_text(segments)
# Estimate duration
duration_min = 0
@@ -381,11 +436,11 @@ def suggest_with_claude(
prompt=prompt,
prompt_file=prompt_file,
project_dir=project_dir,
- timeout=300,
+ timeout=timeout,
)
except subprocess.TimeoutExpired:
if progress_callback:
- progress_callback(0, f"{label} timed out (5 min limit)")
+ progress_callback(0, f"{label} timed out ({_format_timeout_label(timeout)} limit)")
continue
except Exception as e:
if progress_callback:
@@ -446,7 +501,7 @@ def suggest_with_claude(
keep_segments = [{"start": start_sec, "end": end_sec}]
kept_duration = sum(seg["end"] - seg["start"] for seg in keep_segments)
- if kept_duration < 15 or kept_duration > 55:
+ if kept_duration < 15 or kept_duration > 45:
continue
normalized.append({
@@ -486,6 +541,111 @@ def suggest_with_claude(
pass
+def suggest_initial_with_claude(
+ segments: list[dict],
+ top_n: int = 5,
+ progress_callback: Optional[Callable[[int, str], None]] = None,
+) -> Optional[list[dict]]:
+ """
+ Initial clip discovery entry point.
+
+ For long podcasts, search transcript buckets first so the AI can return
+ quickly and cover the full timeline instead of timing out on one giant
+ prompt.
+ """
+ if not _should_bucket_initial_selection(segments):
+ return suggest_with_claude(
+ segments=segments,
+ top_n=top_n,
+ progress_callback=progress_callback,
+ timeout=180,
+ )
+
+ start_bound = float(segments[0].get("start", 0))
+ end_bound = float(segments[-1].get("end", segments[-1].get("start", 0)))
+ duration = max(0.0, end_bound - start_bound)
+ if duration <= 0:
+ return None
+
+ bucket_count = min(6, max(3, math.ceil((duration / 60.0) / 25.0)))
+ bucket_size = duration / bucket_count
+ buckets = []
+ for idx in range(bucket_count):
+ bucket_start = start_bound + idx * bucket_size
+ bucket_end = end_bound if idx == bucket_count - 1 else start_bound + (idx + 1) * bucket_size
+ bucket_segments = _slice_segments_for_range(segments, bucket_start, bucket_end)
+ if len(bucket_segments) < 3:
+ continue
+ buckets.append({
+ "start": bucket_start,
+ "end": bucket_end,
+ "segments": bucket_segments,
+ })
+
+ if not buckets:
+ return suggest_with_claude(
+ segments=segments,
+ top_n=top_n,
+ progress_callback=progress_callback,
+ timeout=180,
+ )
+
+ if progress_callback:
+ progress_callback(0, f"Long episode detected — searching {len(buckets)} timeline buckets...")
+
+ aggregated: list[dict] = []
+ per_bucket_top_n = max(2, math.ceil(top_n / max(1, len(buckets))))
+
+ for idx, bucket in enumerate(buckets):
+ bucket_label = (
+ f"bucket {idx + 1}/{len(buckets)} "
+ f"[{int(bucket['start'] // 60)}:{int(bucket['start'] % 60):02d}-"
+ f"{int(bucket['end'] // 60)}:{int(bucket['end'] % 60):02d}]"
+ )
+ if progress_callback:
+ progress_callback(0, f"Searching {bucket_label}...")
+
+ bucket_clips = suggest_with_claude(
+ segments=bucket["segments"],
+ top_n=per_bucket_top_n,
+ exclude_clips=aggregated,
+ progress_callback=(
+ None if progress_callback is None
+ else lambda pct, msg, bucket_label=bucket_label: progress_callback(
+ pct,
+ f"{bucket_label}: {msg}" if msg else msg,
+ )
+ ),
+ timeout=90,
+ )
+ if not bucket_clips:
+ continue
+
+ aggregated.extend(bucket_clips)
+
+ deduped = _dedupe_clips_by_range(aggregated)
+ if len(deduped) >= top_n:
+ return deduped[:top_n]
+
+ fallback_clips = suggest_with_claude(
+ segments=segments,
+ top_n=top_n,
+ exclude_clips=deduped,
+ progress_callback=(
+ None if progress_callback is None
+ else lambda pct, msg: progress_callback(
+ pct,
+ f"global pass: {msg}" if msg else msg,
+ )
+ ),
+ timeout=120,
+ )
+ if fallback_clips:
+ deduped = _dedupe_clips_by_range(deduped + fallback_clips)
+
+ return deduped[:top_n] if deduped else None
+
+
def _bucket_coverage_seconds(existing_clips: list[dict], start: float, end: float) -> float:
"""Total selected-clip overlap inside a time bucket."""
covered = 0.0
diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.py
index f373543..2e3fad8 100644
--- a/backend/services/clip_generator.py
+++ b/backend/services/clip_generator.py
@@ -9,6 +9,8 @@ import os
import sys
import tempfile
import shutil
+import subprocess
+import re
from typing import Optional, Callable
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@@ -30,6 +32,232 @@ _FILLER_WORDS = frozenset([
"ah", "er", "erm", "eh",
])
+# Weak lead-in markers that often add setup without adding viral value.
+# These are only used for opening-trim decisions (not caption deletion).
+_WEAK_OPENING_WORDS = frozenset([
+ "so", "well", "okay", "ok", "like", "you", "know",
+ "right", "yeah", "yes", "actually", "basically",
+]) | _FILLER_WORDS
+
+_SCENE_TIME_RE = re.compile(r"pts_time:([0-9]+(?:\.[0-9]+)?)")
+
+
+def _trim_weak_opening(
+ words: list[dict],
+ start_second: float,
+ end_second: float,
+ max_trim: float = 3.0,
+ min_gain: float = 0.25,
+) -> float:
+ """
+ Tighten clip opening toward the first substantive sentence.
+
+ Rules:
+ - Remove obvious dead air at the head.
+ - Optionally skip a short run of weak opener words.
+ - Never trim past max_trim seconds or into a likely hook ("?" / "!").
+ """
+ if not words or end_second <= start_second + 0.6:
+ return start_second
+
+ clip_words = sorted(
+ [w for w in words if w["end"] > start_second and w["start"] < end_second],
+ key=lambda w: w["start"],
+ )
+ if len(clip_words) < 2:
+ return start_second
+
+ candidate_start = start_second
+ first_word = clip_words[0]
+
+ # Dead air before first spoken word.
+ if first_word["start"] - start_second >= 0.45:
+ candidate_start = first_word["start"]
+
+ weak_window_end = min(end_second, start_second + max_trim)
+ weak_run_end = None
+ saw_real_word_after_weak = False
+
+ for w in clip_words:
+ if w["start"] > weak_window_end:
+ break
+ raw = str(w.get("word", "")).strip()
+ if not raw:
+ continue
+ # If the opening already has question/exclamation energy, keep it.
+ if "?" in raw or "!" in raw:
+ break
+ bare = raw.lower().strip(".,!?;:-–—'\"")
+ if bare in _WEAK_OPENING_WORDS:
+ weak_run_end = max(float(w["end"]), weak_run_end or 0.0)
+ continue
+ saw_real_word_after_weak = True
+ break
+
+ if weak_run_end is not None and saw_real_word_after_weak:
+ candidate_start = max(candidate_start, weak_run_end)
+
+ candidate_start = min(candidate_start, start_second + max_trim, end_second - 0.5)
+ if candidate_start - start_second >= min_gain:
+ return round(candidate_start, 2)
+ return start_second
+
+
+def _get_media_duration(path: str) -> float:
+ """Read media duration in seconds (best effort)."""
+ try:
+ cmd = [
+ "ffprobe", "-v", "error",
+ "-show_entries", "format=duration",
+ "-of", "default=nk=1:nw=1",
+ path,
+ ]
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=10)
+ if result.returncode != 0:
+ return 0.0
+ return float((result.stdout or "0").strip() or 0.0)
+ except Exception:
+ return 0.0
+
+
+def _detect_scene_cuts(path: str, threshold: float = 0.22, max_cuts: int = 32) -> list[float]:
+ """
+ Detect hard visual changes likely to feel like jump cuts.
+ Returns scene-change timestamps in seconds.
+ """
+ try:
+ cmd = [
+ "ffmpeg", "-hide_banner",
+ "-i", path,
+ "-filter:v", f"select='gt(scene,{threshold})',showinfo",
+ "-an",
+ "-f", "null", "-",
+ ]
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=90)
+ text = (result.stderr or "") + "\n" + (result.stdout or "")
+ raw = [float(m.group(1)) for m in _SCENE_TIME_RE.finditer(text)]
+ if not raw:
+ return []
+ cuts: list[float] = []
+ for t in raw:
+ if not cuts or abs(t - cuts[-1]) > 0.08:
+ cuts.append(round(t, 3))
+ if len(cuts) >= max_cuts:
+ break
+ return cuts
+ except Exception:
+ return []
+
+
+def _select_problematic_scene_cuts(cuts: list[float], duration: float) -> list[float]:
+ """
+ Pick cuts that are most likely to feel bad:
+ - clustered cuts (rapid switches)
+ - tail-end cuts right before clip ends
+ """
+ if not cuts or duration <= 0:
+ return []
+
+ flagged: list[float] = []
+ for i, t in enumerate(cuts):
+ prev_t = cuts[i - 1] if i > 0 else None
+ next_t = cuts[i + 1] if i + 1 < len(cuts) else None
+ is_clustered = (
+ (prev_t is not None and (t - prev_t) <= 0.9)
+ or (next_t is not None and (next_t - t) <= 0.9)
+ )
+ is_tail = (duration - t) <= 2.0
+ if is_clustered or is_tail:
+ flagged.append(t)
+
+ # Keep order and de-dup (safety)
+ out: list[float] = []
+ for t in flagged:
+ if not out or abs(t - out[-1]) > 0.08:
+ out.append(t)
+ return out
+
+
+def _apply_local_transition_smoothing(
+ input_path: str,
+ output_path: str,
+ cut_times: list[float],
+ pass_index: int = 0,
+) -> bool:
+ """Apply short blur masks around selected cut times (adaptive by pass)."""
+ if not cut_times:
+ return False
+
+ # Pass 0: clean/light smoothing. Pass 1+: stronger smoothing.
+ if pass_index <= 0:
+ outer_pad = 0.15
+ core_pad = 0.08
+ outer_sigma = 1.8
+ core_sigma = 3.2
+ else:
+ outer_pad = 0.20
+ core_pad = 0.11
+ outer_sigma = 2.2
+ core_sigma = 4.0
+
+ outer_windows = [(max(0.0, t - outer_pad), t + outer_pad) for t in cut_times]
+ core_windows = [(max(0.0, t - core_pad), t + core_pad) for t in cut_times]
+
+ outer_expr = "+".join(f"between(t,{a:.3f},{b:.3f})" for a, b in outer_windows)
+ core_expr = "+".join(f"between(t,{a:.3f},{b:.3f})" for a, b in core_windows)
+
+ vf = (
+ f"gblur=sigma={outer_sigma:.1f}:steps=1:enable='{outer_expr}',"
+ f"gblur=sigma={core_sigma:.1f}:steps=1:enable='{core_expr}'"
+ )
+ cmd = [
+ "ffmpeg", "-y",
+ "-i", input_path,
+ "-vf", vf,
+ "-c:v", "libx264", "-crf", "18", "-preset", "medium",
+ "-c:a", "copy",
+ "-movflags", "+faststart",
+ output_path,
+ ]
+ try:
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
+ return result.returncode == 0 and os.path.exists(output_path)
+ except Exception:
+ return False
+
+
+def _auto_fix_transition_jumps(video_path: str, max_passes: int = 1) -> bool:
+ """
+ Bounded auto-fix for jumpy transitions. Never loops indefinitely.
+ Returns True if at least one fix pass succeeded.
+ """
+ fixed_any = False
+ if max_passes <= 0:
+ return False
+
+ current = video_path
+ for i in range(max_passes):
+ duration = _get_media_duration(current)
+ scene_threshold = 0.22 if i == 0 else 0.18
+ cuts = _detect_scene_cuts(current, threshold=scene_threshold)
+ problematic = _select_problematic_scene_cuts(cuts, duration)
+ if not problematic:
+ break
+
+ temp_out = current + f".autofix{i+1}.mp4"
+ ok = _apply_local_transition_smoothing(
+ current,
+ temp_out,
+ problematic,
+ pass_index=i,
+ )
+ if not ok:
+ break
+ os.replace(temp_out, current)
+ fixed_any = True
+
+ return fixed_any
+
def _snap_to_sentence_end(
transcript_words: list[dict], start_second: float, end_second: float,
@@ -332,6 +560,7 @@ def generate_clip(
outro_path: Optional[str] = None,
clean_fillers: bool = True,
keep_segments: list[dict] = None,
+ allow_ass_fallback: bool = False,
progress_callback: Optional[Callable[[int, str], None]] = None,
) -> dict:
"""
@@ -342,7 +571,7 @@ def generate_clip(
start_second: Clip start time
end_second: Clip end time
caption_style: "hormozi", "karaoke", "subtle", or "branded"
- crop_strategy: "center" or "face"
+ crop_strategy: "center", "face", or "speaker"
transcript_words: Word-level timestamps from transcription
title: Clip title (used in filename)
output_dir: Where to save the final clip (defaults to temp)
@@ -369,6 +598,17 @@ def generate_clip(
# Validate segments from Claude
keep_segments = [s for s in keep_segments if s["end"] > s["start"]]
keep_segments.sort(key=lambda s: s["start"])
+
+ # Tighten the opening of the first kept segment to reduce setup/preamble.
+ if transcript_words:
+ trimmed_start = _trim_weak_opening(
+ transcript_words,
+ keep_segments[0]["start"],
+ keep_segments[0]["end"],
+ )
+ if trimmed_start < keep_segments[0]["end"] - 0.5:
+ keep_segments[0]["start"] = trimmed_start
+
start_second = keep_segments[0]["start"]
end_second = keep_segments[-1]["end"]
@@ -383,6 +623,10 @@ def generate_clip(
duration = sum(s["end"] - s["start"] for s in keep_segments)
else:
+ # Tighten weak setup at the head before sentence/end trimming.
+ if transcript_words:
+ start_second = _trim_weak_opening(transcript_words, start_second, end_second)
+
# Snap clip boundaries to sentence endings
if transcript_words:
end_second = _snap_to_sentence_end(transcript_words, start_second, end_second)
@@ -402,8 +646,8 @@ def generate_clip(
keep_segments = None
duration = end_second - start_second
- if duration > 60:
- raise ValueError(f"Clip too long ({duration:.0f}s). Max 60 seconds for shorts.")
+ if duration > 45:
+ raise ValueError(f"Clip too long ({duration:.0f}s). Max 45 seconds for shorts.")
# Load style config for branded-specific settings
style_config = get_style(caption_style)
@@ -474,7 +718,7 @@ def generate_clip(
face_map=face_map,
)
- # Step 3: Render captions (Remotion or ASS fallback)
+ # Step 3: Render captions (Remotion-first; ASS fallback optional)
if transcript_words:
if progress_callback:
progress_callback(50, f"Adding {caption_style} captions (3/{total_steps})")
@@ -497,7 +741,6 @@ def generate_clip(
captioned_path = os.path.join(work_dir, "captioned.mp4")
- # Try Remotion first, fall back to ASS
remotion_ok = _render_with_remotion(
video_path=cropped_path,
words=clip_words,
@@ -507,8 +750,14 @@ def generate_clip(
logo_path=logo_path if (style_config.get("logo_support", False) and logo_path) else None,
)
- if not remotion_ok:
- # Fallback: ASS subtitle burn-in
+ if not remotion_ok and not allow_ass_fallback:
+ raise RuntimeError(
+ "Remotion caption render failed. ASS fallback is disabled "
+ "(set allow_ass_fallback=true to permit fallback)."
+ )
+
+ if not remotion_ok and allow_ass_fallback:
+ # Optional fallback: ASS subtitle burn-in
ass_path = os.path.join(work_dir, "captions.ass")
render_captions(
words=clip_words,
@@ -572,6 +821,15 @@ def generate_clip(
shutil.copy2(final_video_path, final_path)
+ # Optional bounded QA/autofix pass for transition jumps.
+ # Hard-capped to avoid any infinite rerender loop.
+ max_autofix_passes = int(os.environ.get("PODCLI_TRANSITION_AUTOFIX_PASSES", "2") or "2")
+ max_autofix_passes = max(0, min(max_autofix_passes, 2))
+ if max_autofix_passes > 0:
+ if progress_callback:
+ progress_callback(97, "Quality gate: checking transitions...")
+ _auto_fix_transition_jumps(final_path, max_passes=max_autofix_passes)
+
# Get file size
file_size = os.path.getsize(final_path)
file_size_mb = round(file_size / (1024 * 1024), 2)
diff --git a/backend/services/face_analysis.py b/backend/services/face_analysis.py
index 78a4237..2e238f9 100644
--- a/backend/services/face_analysis.py
+++ b/backend/services/face_analysis.py
@@ -154,7 +154,11 @@ def analyze_faces(
progress_callback(85, "Mapping speakers to face positions...")
if is_split_screen and len(clusters_list) >= 2:
- # Split-screen: map top-2 speakers by talk time, first-to-speak = left
+ # Split-screen: map speakers to clusters using observation evidence.
+ # For each speaker, count which cluster has more face observations
+ # during that speaker's segments (±1.5s window). This replaces the
+ # old "first-to-speak = left" heuristic which broke when the host
+ # spoke first but sat on the right.
speaker_talk_time = {}
speaker_first_time = {}
for seg in speaker_segments:
@@ -166,10 +170,53 @@ def analyze_faces(
speakers_by_talk = sorted(speakers, key=lambda s: speaker_talk_time.get(s, 0), reverse=True)
top_2 = speakers_by_talk[:2]
- top_2_by_first = sorted(top_2, key=lambda s: speaker_first_time.get(s, float("inf")))
- speaker_mappings[top_2_by_first[0]] = 0 # left cluster
- if len(top_2_by_first) > 1:
- speaker_mappings[top_2_by_first[1]] = 1 # right cluster
+
+ obs_arr = [(o["time"], o["face_center_x"]) for o in observations]
+ for speaker in top_2:
+ sp_times = [s["start"] for s in speaker_segments if s.get("speaker") == speaker]
+ votes = [0] * len(clusters_list)
+ for t in sp_times:
+ for obs_t, obs_cx in obs_arr:
+ if abs(obs_t - t) < 1.5:
+ for ci, cl in enumerate(clusters_list):
+ if abs(obs_cx - cl["center_x"]) < width * 0.15:
+ votes[ci] += 1
+ if any(votes):
+ speaker_mappings[speaker] = int(np.argmax(votes))
+
+ # Resolve conflicts: if both speakers mapped to the same cluster,
+ # assign the weaker one to the other cluster.
+ if len(top_2) == 2 and top_2[0] in speaker_mappings and top_2[1] in speaker_mappings:
+ if speaker_mappings[top_2[0]] == speaker_mappings[top_2[1]]:
+ other_ci = 1 - speaker_mappings[top_2[0]]
+ # Give the less-confident speaker the other cluster
+ sp0_votes = sum(
+ 1 for t in [s["start"] for s in speaker_segments if s.get("speaker") == top_2[0]]
+ for obs_t, obs_cx in obs_arr
+ if abs(obs_t - t) < 1.5 and abs(obs_cx - clusters_list[speaker_mappings[top_2[0]]]["center_x"]) < width * 0.15
+ )
+ sp1_votes = sum(
+ 1 for t in [s["start"] for s in speaker_segments if s.get("speaker") == top_2[1]]
+ for obs_t, obs_cx in obs_arr
+ if abs(obs_t - t) < 1.5 and abs(obs_cx - clusters_list[speaker_mappings[top_2[1]]]["center_x"]) < width * 0.15
+ )
+ if sp0_votes >= sp1_votes:
+ speaker_mappings[top_2[1]] = other_ci
+ else:
+ speaker_mappings[top_2[0]] = other_ci
+
+ # If observation-based mapping failed for either speaker, fall back
+ # to temporal ordering (first-to-speak = left).
+ if len(top_2) == 2:
+ top_2_by_first = sorted(top_2, key=lambda s: speaker_first_time.get(s, float("inf")))
+ if top_2[0] not in speaker_mappings and top_2[1] not in speaker_mappings:
+ speaker_mappings[top_2_by_first[0]] = 0
+ speaker_mappings[top_2_by_first[1]] = 1
+ elif top_2[0] not in speaker_mappings:
+ speaker_mappings[top_2[0]] = 1 - speaker_mappings[top_2[1]]
+ elif top_2[1] not in speaker_mappings:
+ speaker_mappings[top_2[1]] = 1 - speaker_mappings[top_2[0]]
+
# Extra speakers → dominant speaker's cluster
dominant_idx = speaker_mappings.get(speakers_by_talk[0], 0)
for sp in speakers_by_talk[2:]:
@@ -211,4 +258,5 @@ def analyze_faces(
"dominant_speaker": dominant_speaker,
"video_width": width,
"video_height": height,
+ "_mappings_v2": True,
}
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index e2b69fc..15706c6 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -398,20 +398,22 @@ def _build_cam_expr(keyframes: list, duration: float, is_split: bool, max_parts:
t1, x1 = keyframes[i + 1]
dt = max(0.01, t1 - t0)
jump = abs(x1 - x0)
- blurred_cut_jump = 90 if is_split else 180
- quick_reframe_jump = 40 if is_split else 90
+ # In split/mixed layouts, avoid both long pans and hard snaps.
+ # Force very short eased handoffs for virtually all jumps.
+ blurred_cut_jump = 100000 if is_split else 180
+ quick_reframe_jump = 24 if is_split else 90
if jump < 2:
# Negligible movement — hold
parts.append(f"if(between(t\\,{t0:.3f}\\,{t1:.3f})\\,{x0}\\,")
elif jump >= blurred_cut_jump:
- # Very large jumps look worse as animated crop slides than as a
- # short blurred cut. The blur/zoom filters carry the transition.
+ # Very large jumps in non-split layouts still read better as short
+ # blurred cuts than as long animated slides.
parts.append(f"if(between(t\\,{t0:.3f}\\,{t1:.3f})\\,{x1}\\,")
elif jump >= quick_reframe_jump:
# Moderate jump: use a short bounded reframe, but keep it much
# tighter than a literal camera pan.
- pan_t = min(0.14 if is_split else 0.18, dt)
+ pan_t = min(0.08 if is_split else 0.18, dt)
pan_end_t = round(t0 + pan_t, 3)
parts.append(
f"if(between(t\\,{t0:.3f}\\,{pan_end_t:.3f})\\,"
@@ -854,9 +856,9 @@ def _update_tripod_camera(
current_center_x = target_center_x
else:
diff = target_center_x - current_center_x
- # Keep the hold zone fairly tight so the speaker stays composed near
- # center, but still ignore ordinary head movement and detector noise.
- safe_zone_radius = crop_w * 0.14
+ # Wide enough to absorb natural head movement and detector noise,
+ # narrow enough to keep the speaker composed near center.
+ safe_zone_radius = crop_w * 0.22
if abs(diff) > safe_zone_radius:
slow_speed = 72.0
fast_speed = 360.0
@@ -893,7 +895,16 @@ def _choose_track_segment_targets(
first_x = points[0][1]
median_x = float(median([cx for _, cx in points]))
seed_x = first_x * 0.72 + median_x * 0.28
- return min(points, key=lambda p: abs(p[1] - seed_x))[1]
+ preferred_margin = max(90.0, crop_w * 0.16)
+
+ def _score(point: tuple[float, float]) -> float:
+ _, cx = point
+ crop_x = _crop_x_for_center(cx)
+ edge_margin = min(crop_x, max_crop_x - crop_x) if max_crop_x > 0 else preferred_margin
+ edge_penalty = max(0.0, preferred_margin - edge_margin) * 3.0
+ return abs(cx - seed_x) + edge_penalty
+
+ return min(points, key=_score)[1]
def _split_points_into_stable_runs(
points: list[tuple[float, float]],
@@ -953,6 +964,56 @@ def _choose_track_segment_targets(
runs.append((run_start_t, end_t, run_points))
return runs
+ def _trim_bad_boundary_runs(
+ runs: list[tuple[float, float, list[tuple[float, float]]]],
+ ) -> list[tuple[float, float, list[tuple[float, float]]]]:
+ if len(runs) < 2:
+ return runs
+
+ preferred_margin = max(90.0, crop_w * 0.16)
+
+ def _run_center(run_points: list[tuple[float, float]]) -> float:
+ return float(median([cx for _, cx in run_points]))
+
+ def _edge_margin(center_x: float) -> float:
+ crop_x = _crop_x_for_center(center_x)
+ return min(crop_x, max_crop_x - crop_x) if max_crop_x > 0 else preferred_margin
+
+ trimmed = list(runs)
+ changed = True
+ while len(trimmed) >= 2 and changed:
+ changed = False
+
+ start_t, end_t, run_points = trimmed[0]
+ next_start_t, next_end_t, next_points = trimmed[1]
+ center_x = _run_center(run_points)
+ next_center_x = _run_center(next_points)
+ is_short_boundary = (end_t - start_t) <= 1.4 or len(run_points) <= 2
+ is_edge_hugging = _edge_margin(center_x) < preferred_margin
+ next_is_more_central = _edge_margin(next_center_x) > _edge_margin(center_x) + preferred_margin * 0.6
+ if is_short_boundary and is_edge_hugging and next_is_more_central:
+ trimmed[1] = (start_t, next_end_t, next_points)
+ trimmed.pop(0)
+ changed = True
+ continue
+
+ prev_start_t, prev_end_t, prev_points = trimmed[-2]
+ start_t, end_t, run_points = trimmed[-1]
+ center_x = _run_center(run_points)
+ prev_center_x = _run_center(prev_points)
+ is_short_boundary = (end_t - start_t) <= 1.4 or len(run_points) <= 2
+ is_edge_hugging = _edge_margin(center_x) < preferred_margin
+ prev_is_more_central = _edge_margin(prev_center_x) > _edge_margin(center_x) + preferred_margin * 0.6
+ if is_short_boundary and is_edge_hugging and prev_is_more_central:
+ trimmed[-2] = (prev_start_t, end_t, prev_points)
+ trimmed.pop()
+ changed = True
+
+ return trimmed
+
+ local_confirmation_seen = set()
+ max_anchor_only_segment = 2.5
+ is_first_segment = True
segment_targets = []
for start_t, end_t, speaker, track_id, _ in segment_tracks:
points = []
@@ -963,15 +1024,49 @@ def _choose_track_segment_targets(
if face is not None:
points.append((t, float(face["cx"])))
+ # Layout-change recovery: if the chosen track covers less than half
+ # the segment (e.g. Riverside split→fullscreen switch mid-clip),
+ # supplement with the most visible face from ANY track in the gap.
+ seg_duration = max(0.01, end_t - start_t)
if points:
- for idx, (sub_start_t, sub_end_t, run_points) in enumerate(_split_points_into_stable_runs(points, end_t)):
+ point_coverage = (points[-1][0] - points[0][0]) / seg_duration
+ else:
+ point_coverage = 0.0
+
+ if point_coverage < 0.5 and seg_duration > 1.5:
+ last_point_t = points[-1][0] if points else start_t
+ gap_points = []
+ for t, faces in tracked_detections:
+ if t <= last_point_t or t > end_t or not faces:
+ continue
+ # Pick the most prominent face (largest)
+ best = max(faces, key=lambda f: f["fw"])
+ gap_points.append((t, float(best["cx"])))
+ if gap_points:
+ points = points + gap_points
+
+ if points:
+ if speaker is not None:
+ local_confirmation_seen.add(speaker)
+ stable_runs = _trim_bad_boundary_runs(_split_points_into_stable_runs(points, end_t))
+ for idx, (sub_start_t, sub_end_t, run_points) in enumerate(stable_runs):
if idx == 0:
sub_start_t = start_t
center_x = _pick_representative_center(run_points)
segment_targets.append((sub_start_t, sub_end_t, _crop_x_for_center(center_x), speaker))
- elif speaker is not None and speaker in speaker_anchor_x:
+ is_first_segment = False
+ elif (
+ speaker is not None
+ and speaker in speaker_anchor_x
+ and (
+ speaker in local_confirmation_seen
+ or is_first_segment # Trust face_map for the first segment
+ )
+ and (is_first_segment or (end_t - start_t) <= max_anchor_only_segment)
+ ):
center_x = float(speaker_anchor_x[speaker])
segment_targets.append((start_t, end_t, _crop_x_for_center(center_x), speaker))
+ is_first_segment = False
else:
continue
@@ -981,6 +1076,10 @@ def _choose_track_segment_targets(
def _build_track_turn_keyframes(
segment_targets: list,
default_x: int,
+ crop_w: int = 0,
+ width: int = 0,
+ face_map: dict = None,
+ has_any_split: bool = False,
) -> list[tuple[float, int]]:
"""
Build short bounded reframes between stable speaker-turn targets.
@@ -988,7 +1087,14 @@ def _build_track_turn_keyframes(
The camera should hold through the turn, then move quickly onto the new
speaker instead of slowly chasing the face for seconds.
"""
- keyframes = [(0.0, default_x)]
+ # Only clamp the default (no-face-data) position away from the dead
+ # zone. Segment targets come from real face detections and may
+ # legitimately sit near center in fullscreen layouts.
+ safe_default = _clamp_away_from_dead_zone(
+ default_x, crop_w, width, face_map, has_any_split,
+ ) if crop_w > 0 else default_x
+
+ keyframes = [(0.0, safe_default)]
prev_x = default_x
prev_speaker = None
left_edge_margin = 90
@@ -1124,6 +1230,95 @@ def _choose_camera_speaker(
return transcript_speaker, None, 0, True
+def _safe_default_center(
+ width: int,
+ crop_w: int,
+ face_map: dict | None,
+ has_any_split: bool,
+ first_speaker: str | None,
+ speaker_anchor_x: dict,
+) -> float:
+ """
+ Pick a safe initial camera center.
+
+ On split-screen, width/2 is the dead zone between two feeds. Use the
+ face_map or speaker_anchor to start on an actual speaker instead.
+ """
+ # Best case: we know where the first speaker sits
+ if first_speaker and first_speaker in speaker_anchor_x:
+ return float(speaker_anchor_x[first_speaker])
+
+ # Face-map cluster fallback: pick the cluster with more observations
+ if face_map and face_map.get("clusters"):
+ clusters = face_map["clusters"]
+ best = max(clusters, key=lambda c: c.get("count", 0))
+ return float(best["center_x"])
+
+ # Non-split: center is fine
+ if not has_any_split:
+ return float(width) / 2
+
+ # Split with no map: pick the left quarter as a safer guess than center
+ return float(width) / 4
+
+
+def _clamp_away_from_dead_zone(
+ crop_x: int,
+ crop_w: int,
+ width: int,
+ face_map: dict | None,
+ has_any_split: bool,
+) -> int:
+ """
+ If crop_x centers the window on the split-screen seam (where no face
+ lives), snap to the nearest cluster position instead.
+
+ Only triggers when the crop CENTER is close to mid_x. A crop that
+ merely straddles mid_x is fine — in fullscreen mode the face genuinely
+ sits near center.
+ """
+ if not has_any_split or not face_map:
+ return crop_x
+
+ mid_x = width // 2
+ crop_center = crop_x + crop_w // 2
+
+ # Only clamp when the crop is centered near the seam itself.
+ # A tight margin avoids false-positives on fullscreen layouts where
+ # the face legitimately sits near center.
+ seam_margin = crop_w // 8 # ~75px on a 607-wide crop
+ if abs(crop_center - mid_x) > seam_margin:
+ return crop_x
+
+ # Snap to nearest cluster
+ clusters = face_map.get("clusters", [])
+ if not clusters:
+ return max(0, width // 4 - crop_w // 2)
+
+ best_cluster = min(clusters, key=lambda c: abs(c["center_x"] - crop_center))
+ snapped = max(0, min(best_cluster["center_x"] - crop_w // 2, width - crop_w))
+ return snapped
+
+
+def _upgrade_speaker_mappings(face_map: dict) -> dict:
+ """
+ Invalidate stale speaker-to-cluster mappings from old caches.
+
+ Old caches used "first-to-speak = left" which breaks when the host
+ speaks first but sits on the right. Rather than trying to re-compute
+ mappings from clip-scoped data, we clear the mappings entirely so the
+ clip-local tracking decides which face to follow based on visual
+ evidence (which is more reliable for any single clip).
+
+ Cluster positions are kept — they're still useful for dead-zone
+ clamping and safe defaults.
+ """
+ face_map = dict(face_map)
+ face_map["speaker_mappings"] = {}
+ face_map["_mappings_v2"] = True
+ return face_map
+
+
def _track_and_crop(
input_path: str,
output_path: str,
@@ -1170,8 +1365,14 @@ def _track_and_crop(
cap.release()
return None
- # ── Dense face sampling (~10 fps) ────────────────────────────
+ # ── Dense face sampling (~10 fps, 2× at clip start) ──────────
sample_step = max(1, int(fps / 10))
+ # Sample every frame for the first 0.5s, then every other frame
+ # until 1s, to maximise the chance of catching a face before gestures.
+ dense_end_frame = int(fps * 0.5)
+ semi_dense_end_frame = int(fps * 1.0)
+ dense_step = 1
+ semi_dense_step = max(1, sample_step // 2)
detections = [] # [(time, faces), ...]
frame_idx = 0
@@ -1184,7 +1385,12 @@ def _track_and_crop(
t = frame_idx / fps
faces = detect_faces(detector, frame, width, height)
detections.append((t, faces))
- frame_idx += sample_step
+ if frame_idx < dense_end_frame:
+ frame_idx += dense_step
+ elif frame_idx < semi_dense_end_frame:
+ frame_idx += semi_dense_step
+ else:
+ frame_idx += sample_step
cap.release()
@@ -1241,6 +1447,13 @@ def _track_and_crop(
tracked_detections = _assign_face_tracks(detections, width)
+ # Upgrade stale face_map speaker mappings: re-compute from cached
+ # observations using observation-based evidence instead of the old
+ # "first-to-speak = left" heuristic. This runs in-memory only and
+ # does not modify the cached file.
+ if face_map and face_map.get("is_split_screen") and not face_map.get("_mappings_v2"):
+ face_map = _upgrade_speaker_mappings(face_map)
+
speaker_side = _resolve_speaker_sides(segments, tracked_detections, width, face_map)
speaker_anchor_x = {}
if face_map:
@@ -1276,6 +1489,216 @@ def _track_and_crop(
# crop_h < height due to aspect-ratio clamping).
crop_y = max(0, (height - crop_h) // 2)
+ # ── Mixed-layout shortcut ───────────────────────────────────
+ # Riverside-style recordings switch between split-screen and
+ # single-person views unpredictably. The segment/keyframe
+ # pipeline assumes stable face positions within a turn, which
+ # breaks on every layout transition. For mixed layouts, use
+ # simple per-frame largest-face following instead.
+ is_mixed = face_map.get("is_mixed_layout", False) if face_map else False
+ if is_mixed and not is_mixed: # disabled — checking below
+ pass
+ if is_mixed:
+ # Build a time→speaker lookup from segments
+ def _speaker_at(t_sec: float) -> str | None:
+ for s_start, s_end, s_spk in segments:
+ if s_start <= t_sec <= s_end:
+ return s_spk
+ return None
+
+ # Speaker-side mapping from face_map (cleared stale ones are empty)
+ mixed_side = {}
+ if face_map:
+ fm_clusters = face_map.get("clusters", [])
+ fm_mappings = face_map.get("speaker_mappings", {})
+ for spk, ci in fm_mappings.items():
+ if ci is not None and ci < len(fm_clusters):
+ mixed_side[spk] = "left" if fm_clusters[ci]["center_x"] < mid_x else "right"
+
+ # ── Collect per-frame face positions (speaker-aware) ──────
+ from statistics import median
+ face_points = [] # [(t, center_x), ...]
+ for t, faces in detections:
+ if not faces:
+ continue
+ speaker = _speaker_at(t)
+ side = mixed_side.get(speaker)
+
+ if len(faces) >= 2 and side:
+ best = min(faces, key=lambda f: f["cx"]) if side == "left" else max(faces, key=lambda f: f["cx"])
+ elif len(faces) == 1:
+ best = faces[0]
+ else:
+ best = max(faces, key=lambda f: f["fw"])
+ face_points.append((t, float(best["cx"])))
+
+ # ── Split into stable position runs ─────────────────────
+ # Each run = one locked camera position. Runs split on
+ # layout changes (split↔fullscreen jumps ~450px).
+ max_crop_x = max(0, width - crop_w)
+ def _crop_for(cx: float) -> int:
+ return max(0, min(int(cx - crop_w / 2), max_crop_x))
+
+ if face_points:
+ jump_thresh = max(180.0, crop_w * 0.35)
+ runs = [] # [(start_t, end_t, median_cx), ...]
+ run_start = face_points[0][0]
+ run_cxs = [face_points[0][1]]
+
+ for i in range(1, len(face_points)):
+ t, cx = face_points[i]
+ run_median = float(median(run_cxs))
+ if abs(cx - run_median) > jump_thresh:
+ # Layout changed — close current run, start new one
+ runs.append((run_start, face_points[i-1][0], float(median(run_cxs))))
+ run_start = t
+ run_cxs = [cx]
+ else:
+ run_cxs.append(cx)
+ runs.append((run_start, face_points[-1][0], float(median(run_cxs))))
+
+ # Merge very short runs (<0.8s) into their neighbors
+ if len(runs) > 1:
+ merged_runs = [runs[0]]
+ for r in runs[1:]:
+ if (r[1] - r[0]) < 0.8:
+ # Absorb into previous run
+ merged_runs[-1] = (merged_runs[-1][0], r[1], merged_runs[-1][2])
+ else:
+ merged_runs.append(r)
+ runs = merged_runs
+
+ if len(runs) <= 1:
+ # Single layout — static crop, no transitions needed
+ crop_x = _crop_for(runs[0][2]) if runs else max(0, (width - crop_w) // 2)
+ vf = f"crop={crop_w}:{crop_h}:{crop_x}:{crop_y},scale={target_w}:{target_h}"
+ return _run_ffmpeg_with_fallback(
+ cmd_parts_before_enc=["ffmpeg", "-y", "-i", input_path, "-vf", vf],
+ cmd_parts_after_enc=[
+ "-c:a", "aac", "-b:a", "192k", "-ar", "44100",
+ "-movflags", "+faststart",
+ ],
+ output_path=output_path, label="crop_mixed_static",
+ )
+
+ # Multiple layouts — crop each segment separately, join
+ # with cross-dissolve so transitions feel editorial.
+ import tempfile
+ work_dir = os.path.dirname(output_path) or "."
+ xfade_dur = 0.18 # Short dissolve
+ part_paths = []
+
+ try:
+ for ri, (r_start, r_end, r_cx) in enumerate(runs):
+ r_crop_x = _crop_for(r_cx)
+ # Pad segment slightly for xfade overlap
+ pad = xfade_dur if ri < len(runs) - 1 else 0
+ seg_start = max(0, r_start)
+ seg_end = min(duration, r_end + pad)
+ if seg_end - seg_start < 0.1:
+ continue
+
+ part_path = os.path.join(work_dir, f"_mixed_part_{ri}.mp4")
+ seg_vf = f"crop={crop_w}:{crop_h}:{r_crop_x}:{crop_y},scale={target_w}:{target_h}"
+ cmd = [
+ "ffmpeg", "-y",
+ "-ss", str(seg_start), "-t", str(seg_end - seg_start),
+ "-i", input_path,
+ "-vf", seg_vf,
+ "-c:v", "libx264", "-crf", "18", "-preset", "fast",
+ "-c:a", "aac", "-b:a", "192k",
+ "-avoid_negative_ts", "make_zero",
+ part_path,
+ ]
+ r = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
+ if r.returncode != 0:
+ continue
+ part_paths.append(part_path)
+
+ if len(part_paths) < 2:
+ # Fallback to static crop of longest run
+ best_run = max(runs, key=lambda r: r[1] - r[0])
+ crop_x = _crop_for(best_run[2])
+ vf = f"crop={crop_w}:{crop_h}:{crop_x}:{crop_y},scale={target_w}:{target_h}"
+ return _run_ffmpeg_with_fallback(
+ cmd_parts_before_enc=["ffmpeg", "-y", "-i", input_path, "-vf", vf],
+ cmd_parts_after_enc=["-c:a", "aac", "-b:a", "192k", "-ar", "44100", "-movflags", "+faststart"],
+ output_path=output_path, label="crop_mixed_fallback",
+ )
+
+ # Chain xfade filters between all segments
+ if len(part_paths) == 2:
+ # Simple 2-segment xfade
+ offset = max(0.01, (runs[0][1] - runs[0][0]) - xfade_dur)
+ cmd = [
+ "ffmpeg", "-y",
+ "-i", part_paths[0], "-i", part_paths[1],
+ "-filter_complex",
+ f"[0:v][1:v]xfade=transition=fade:duration={xfade_dur}:offset={offset:.3f}[v];"
+ f"[0:a][1:a]acrossfade=d={xfade_dur}[a]",
+ "-map", "[v]", "-map", "[a]",
+ "-c:v", "libx264", "-crf", "18", "-preset", "fast",
+ "-c:a", "aac", "-b:a", "192k",
+ "-movflags", "+faststart",
+ output_path,
+ ]
+ r = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
+ if r.returncode == 0:
+ return output_path
+ else:
+ # 3+ segments: chain xfades
+ inputs = []
+ for p in part_paths:
+ inputs.extend(["-i", p])
+
+ # Build filter chain: xfade each pair
+ filter_parts = []
+ audio_parts = []
+ prev_label = "[0:v]"
+ prev_audio = "[0:a]"
+ cumulative_offset = 0.0
+
+ for i in range(1, len(part_paths)):
+ seg_dur = runs[i-1][1] - runs[i-1][0]
+ cumulative_offset += max(0.01, seg_dur - xfade_dur)
+ out_label = f"[v{i}]" if i < len(part_paths) - 1 else "[v]"
+ out_audio = f"[a{i}]" if i < len(part_paths) - 1 else "[a]"
+ filter_parts.append(
+ f"{prev_label}[{i}:v]xfade=transition=fade:duration={xfade_dur}:offset={cumulative_offset:.3f}{out_label}"
+ )
+ audio_parts.append(
+ f"{prev_audio}[{i}:a]acrossfade=d={xfade_dur}{out_audio}"
+ )
+ prev_label = out_label
+ prev_audio = out_audio
+
+ filter_complex = ";".join(filter_parts + audio_parts)
+ cmd = ["ffmpeg", "-y"] + inputs + [
+ "-filter_complex", filter_complex,
+ "-map", "[v]", "-map", "[a]",
+ "-c:v", "libx264", "-crf", "18", "-preset", "fast",
+ "-c:a", "aac", "-b:a", "192k",
+ "-movflags", "+faststart",
+ output_path,
+ ]
+ r = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
+ if r.returncode == 0:
+ return output_path
+
+ # If xfade failed, fall back to static
+ best_run = max(runs, key=lambda r: r[1] - r[0])
+ crop_x = _crop_for(best_run[2])
+ vf = f"crop={crop_w}:{crop_h}:{crop_x}:{crop_y},scale={target_w}:{target_h}"
+ return _run_ffmpeg_with_fallback(
+ cmd_parts_before_enc=["ffmpeg", "-y", "-i", input_path, "-vf", vf],
+ cmd_parts_after_enc=["-c:a", "aac", "-b:a", "192k", "-ar", "44100", "-movflags", "+faststart"],
+ output_path=output_path, label="crop_mixed_xfade_fallback",
+ )
+ finally:
+ for p in part_paths:
+ if os.path.exists(p):
+ os.remove(p)
+
keyframes_x = []
if segment_tracks:
segment_targets = _choose_track_segment_targets(
@@ -1286,18 +1709,49 @@ def _track_and_crop(
crop_w=crop_w,
)
if segment_targets:
- default_x = segment_targets[0][2] if segment_targets[0][0] <= 0.3 else max(0, (width - crop_w) // 2)
+ default_x = segment_targets[0][2]
+ # Ensure default_x doesn't land in the split-screen dead zone
+ default_x = _clamp_away_from_dead_zone(
+ default_x, crop_w, width, face_map, has_any_split,
+ )
+ # Verify the opening position actually contains a face. If not,
+ # find the first real face detection and start there instead.
+ # This catches cases where face_map anchors are stale or wrong.
+ if segment_targets[0][0] > 0.5:
+ # First segment starts late — face detection failed at clip start.
+ # Find the earliest actual face position to validate default_x.
+ first_face_cx = None
+ for _t, _faces in tracked_detections:
+ if _faces:
+ first_face_cx = float(_faces[0]["cx"])
+ break
+ if first_face_cx is not None:
+ # If default_x puts the crop far from any early face, override
+ default_crop_center = default_x + crop_w // 2
+ if abs(default_crop_center - first_face_cx) > crop_w * 0.6:
+ default_x = max(0, min(int(first_face_cx - crop_w // 2), width - crop_w))
+ default_x = _clamp_away_from_dead_zone(
+ default_x, crop_w, width, face_map, has_any_split,
+ )
+
keyframes_x = _build_track_turn_keyframes(
segment_targets=segment_targets,
default_x=default_x,
+ crop_w=crop_w,
+ width=width,
+ face_map=face_map,
+ has_any_split=has_any_split,
)
if not keyframes_x:
- cam_x = float(width) / 2
+ first_speaker = segment_tracks[0][2] if segment_tracks else None
+ cam_x = _safe_default_center(
+ width, crop_w, face_map, has_any_split,
+ first_speaker, speaker_anchor_x,
+ )
first_target_x = None
if segment_tracks:
first_track_id = segment_tracks[0][3]
- first_speaker = segment_tracks[0][2]
for _, faces in tracked_detections:
face = next((f for f in faces if f["track_id"] == first_track_id), None)
if face is not None:
@@ -1324,6 +1778,8 @@ def _track_and_crop(
prev_t = 0.0
segment_index = 0
+ confirmed_speakers = set()
+ last_confirmed_speaker_t = {}
for t, faces in tracked_detections:
while segment_index + 1 < len(segment_tracks) and t > segment_tracks[segment_index][1]:
@@ -1341,13 +1797,20 @@ def _track_and_crop(
if chosen_face is not None:
target_x = float(chosen_face["cx"])
if speaker is not None:
+ confirmed_speakers.add(speaker)
+ last_confirmed_speaker_t[speaker] = t
prev_anchor = speaker_anchor_x.get(speaker, target_x)
speaker_anchor_x[speaker] = prev_anchor * 0.75 + target_x * 0.25
- elif speaker is not None and speaker in speaker_anchor_x:
+ elif (
+ speaker is not None
+ and speaker in speaker_anchor_x
+ and speaker in confirmed_speakers
+ and (t - last_confirmed_speaker_t.get(speaker, -9999.0)) <= 2.5
+ ):
target_x = float(speaker_anchor_x[speaker])
- elif faces:
- target_x = float(max(faces, key=lambda f: f["fw"])["cx"])
else:
+ # Hold position — picking the largest random face often grabs the
+ # wrong speaker and causes a jump.
target_x = None
cam_x = _update_tripod_camera(
@@ -1366,6 +1829,38 @@ def _track_and_crop(
prev_t = t
+ # ── Validate keyframes: guarantee every position shows a face ─
+ if keyframes_x and len(keyframes_x) >= 2:
+ max_crop_x = max(0, width - crop_w)
+
+ # Build a time→nearest-face-cx lookup from detections
+ def _nearest_face_cx(t_target: float, window: float = 1.5) -> float | None:
+ best_cx, best_dt = None, window + 1
+ for t, faces in detections:
+ dt = abs(t - t_target)
+ if dt > window or not faces:
+ continue
+ if dt < best_dt:
+ best_cx = float(max(faces, key=lambda f: f["fw"])["cx"])
+ best_dt = dt
+ return best_cx
+
+ validated = []
+ for kf_t, kf_x in keyframes_x:
+ face_cx = _nearest_face_cx(kf_t)
+ if face_cx is not None:
+ # Check if face is inside the crop window
+ crop_left = kf_x
+ crop_right = kf_x + crop_w
+ margin = crop_w * 0.15 # face should be at least 15% inside
+ if face_cx < crop_left + margin or face_cx > crop_right - margin:
+ # Face not in window — recompute crop to center on face
+ new_x = max(0, min(int(face_cx - crop_w // 2), max_crop_x))
+ validated.append((kf_t, new_x))
+ continue
+ validated.append((kf_t, kf_x))
+ keyframes_x = validated
+
# ── Build FFmpeg filter ──────────────────────────────────────
if not keyframes_x:
crop_x = max(0, (width - crop_w) // 2)
@@ -1376,8 +1871,25 @@ def _track_and_crop(
# Simplify keyframes to reduce expression complexity
keyframes_x = _simplify_keyframes(keyframes_x)
x_expr = _build_cam_expr(keyframes_x, duration, has_any_split)
- blur_filter = _build_motion_blur_filter(keyframes_x)
- zoom_filter = _build_motion_zoom_filter(keyframes_x, target_w=target_w, target_h=target_h)
+ if has_any_split:
+ # Split/single layout switches are the harshest reframes. Use a
+ # stronger, longer blur mask to hide the handoff while keeping
+ # focus locked on the active speaker.
+ blur_filter = _build_motion_blur_filter(
+ keyframes_x,
+ min_jump=40,
+ outer_sigma=1.8,
+ core_sigma=3.6,
+ outer_pad_before=0.05,
+ outer_pad_after=0.08,
+ core_pad_before=0.02,
+ core_pad_after=0.05,
+ )
+ # Zoom bumps can read as extra motion during speaker handoff.
+ zoom_filter = ""
+ else:
+ blur_filter = _build_motion_blur_filter(keyframes_x)
+ zoom_filter = _build_motion_zoom_filter(keyframes_x, target_w=target_w, target_h=target_h)
if x_expr is None:
xs = [x for _, x in keyframes_x]
med_x = sorted(xs)[len(xs) // 2]
@@ -2498,31 +3010,44 @@ def concat_outro(
label="outro_scale",
)
- # Try crossfade first (requires FFmpeg 4.3+)
- try:
- xfade_cmd = [
- "ffmpeg", "-y",
- "-i", input_path,
- "-i", outro_scaled,
- "-filter_complex",
+ # Try crossfade (requires FFmpeg 4.3+).
+ # acrossfade often fails on mismatched streams, so use afade on
+ # each input and amerge instead — more robust.
+ for audio_filter in [
+ # Option 1: fade out main audio, fade in outro audio, then amerge
+ (
f"[0:v][1:v]xfade=transition=fade:duration={crossfade_duration}:offset={fade_offset}[v];"
- f"[0:a][1:a]acrossfade=d={crossfade_duration}[a]",
- "-map", "[v]", "-map", "[a]",
- ]
- xfade_cmd += get_video_encode_flags()
- xfade_cmd += [
- "-c:a", "aac", "-b:a", "192k",
- "-movflags", "+faststart",
- output_path,
- ]
- result = subprocess.run(xfade_cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
- if result.returncode == 0:
- # Clean up
- if os.path.exists(outro_scaled):
- os.remove(outro_scaled)
- return output_path
- except Exception:
- pass
+ f"[0:a]afade=t=out:st={fade_offset}:d={crossfade_duration}[a0];"
+ f"[1:a]afade=t=in:st=0:d={crossfade_duration}[a1];"
+ f"[a0][a1]concat=n=2:v=0:a=1[a]"
+ ),
+ # Option 2: just video xfade, concat audio without crossfade
+ (
+ f"[0:v][1:v]xfade=transition=fade:duration={crossfade_duration}:offset={fade_offset}[v];"
+ f"[0:a][1:a]concat=n=2:v=0:a=1[a]"
+ ),
+ ]:
+ try:
+ xfade_cmd = [
+ "ffmpeg", "-y",
+ "-i", input_path,
+ "-i", outro_scaled,
+ "-filter_complex", audio_filter,
+ "-map", "[v]", "-map", "[a]",
+ ]
+ xfade_cmd += get_video_encode_flags()
+ xfade_cmd += [
+ "-c:a", "aac", "-b:a", "192k",
+ "-movflags", "+faststart",
+ output_path,
+ ]
+ result = subprocess.run(xfade_cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
+ if result.returncode == 0:
+ if os.path.exists(outro_scaled):
+ os.remove(outro_scaled)
+ return output_path
+ except Exception:
+ continue
# Fallback: hard cut concat
main_reenc = output_path + ".main_reenc.mp4"
diff --git a/tests/test_ai_fallback.py b/tests/test_ai_fallback.py
index 0917160..7dadb95 100644
--- a/tests/test_ai_fallback.py
+++ b/tests/test_ai_fallback.py
@@ -167,6 +167,74 @@ class AIFallbackTests(unittest.TestCase):
self.assertGreaterEqual(len(calls), 2)
self.assertEqual(calls[-1]["start"], 0.0)
+ def test_suggest_initial_with_claude_uses_bucket_strategy_for_long_episode(self):
+ segments = []
+ for i in range(20):
+ start = float(i * 330)
+ segments.append({
+ "start": start,
+ "end": start + 45.0,
+ "speaker": "SPEAKER_00",
+ "text": f"Segment {i} with enough text to count as a normal transcript block.",
+ })
+
+ calls = []
+
+ def fake_suggest(*, segments, top_n, exclude_clips=None, progress_callback=None, timeout=300):
+ calls.append({
+ "start": segments[0]["start"],
+ "end": segments[-1]["end"],
+ "top_n": top_n,
+ "timeout": timeout,
+ "exclude_len": len(exclude_clips or []),
+ })
+ first_start = segments[0]["start"]
+ return [{
+ "title": f"Clip {first_start}",
+ "start_second": first_start,
+ "end_second": first_start + 28.0,
+ "segments": [{"start": first_start, "end": first_start + 28.0}],
+ "duration": 28,
+ }]
+
+ with mock.patch.object(cs, "suggest_with_claude", side_effect=fake_suggest):
+ clips = cs.suggest_initial_with_claude(
+ segments=segments,
+ top_n=5,
+ )
+
+ self.assertIsNotNone(clips)
+ self.assertGreaterEqual(len(calls), 4)
+ self.assertTrue(all(call["timeout"] == 90 for call in calls[:4]))
+ self.assertTrue(all(call["end"] - call["start"] < 2500 for call in calls[:4]))
+ self.assertGreaterEqual(calls[1]["exclude_len"], 1)
+
+ def test_suggest_with_claude_reports_actual_timeout_limit(self):
+ segments = [
+ {"start": 0.0, "end": 20.0, "speaker": "SPEAKER_00", "text": "Short segment one."},
+ {"start": 20.0, "end": 40.0, "speaker": "SPEAKER_00", "text": "Short segment two."},
+ ]
+ progress = []
+
+ with mock.patch.object(
+ cs,
+ "_find_ai_cli_candidates",
+ return_value=[("/tmp/claude", "claude")],
+ ), mock.patch.object(
+ cs,
+ "_run_ai_command",
+ side_effect=subprocess.TimeoutExpired(cmd=["claude"], timeout=90),
+ ):
+ clips = cs.suggest_with_claude(
+ segments=segments,
+ top_n=1,
+ timeout=90,
+ progress_callback=lambda _pct, msg: progress.append(msg),
+ )
+
+ self.assertIsNone(clips)
+ self.assertIn("Claude timed out (90s limit)", progress)
+
def test_generate_clip_content_retries_with_codex(self):
clip = {
"title": "Power demand is exploding",
diff --git a/tests/test_clip_generator.py b/tests/test_clip_generator.py
index 330894c..4ede01f 100644
--- a/tests/test_clip_generator.py
+++ b/tests/test_clip_generator.py
@@ -98,6 +98,58 @@ class ClipGeneratorTests(unittest.TestCase):
self.assertIsNone(cg._remotion_available)
self.assertGreaterEqual(mock_run.call_count, 4)
+ def test_trim_weak_opening_cuts_initial_dead_air(self):
+ words = [
+ {"word": "We", "start": 3.2, "end": 3.4},
+ {"word": "built", "start": 3.4, "end": 3.8},
+ ]
+ trimmed = cg._trim_weak_opening(words, start_second=0.0, end_second=12.0)
+ self.assertEqual(trimmed, 3.0)
+
+ def test_trim_weak_opening_skips_setup_words(self):
+ words = [
+ {"word": "So", "start": 10.0, "end": 10.2},
+ {"word": "yeah", "start": 10.2, "end": 10.35},
+ {"word": "we", "start": 10.5, "end": 10.62},
+ {"word": "launched", "start": 10.62, "end": 10.95},
+ ]
+ trimmed = cg._trim_weak_opening(words, start_second=10.0, end_second=24.0)
+ self.assertGreaterEqual(trimmed, 10.35)
+
+ def test_trim_weak_opening_preserves_question_hook(self):
+ words = [
+ {"word": "Why?", "start": 5.0, "end": 5.2},
+ {"word": "would", "start": 5.2, "end": 5.35},
+ {"word": "anyone", "start": 5.35, "end": 5.6},
+ ]
+ trimmed = cg._trim_weak_opening(words, start_second=5.0, end_second=20.0)
+ self.assertEqual(trimmed, 5.0)
+
+ def test_select_problematic_scene_cuts_flags_tail_and_clusters(self):
+ cuts = [5.0, 10.0, 10.7, 18.4]
+ flagged = cg._select_problematic_scene_cuts(cuts, duration=20.0)
+ self.assertIn(10.0, flagged)
+ self.assertIn(10.7, flagged)
+ self.assertIn(18.4, flagged)
+ self.assertNotIn(5.0, flagged)
+
+ def test_select_problematic_scene_cuts_returns_empty_when_clean(self):
+ cuts = [3.0, 9.0, 15.0]
+ flagged = cg._select_problematic_scene_cuts(cuts, duration=30.0)
+ self.assertEqual(flagged, [])
+
+ def test_auto_fix_transition_jumps_is_bounded_by_max_passes(self):
+ with mock.patch.object(cg, "_get_media_duration", return_value=40.0), \
+ mock.patch.object(cg, "_detect_scene_cuts", return_value=[10.0, 10.6, 38.8]) as detect_mock, \
+ mock.patch.object(cg, "_apply_local_transition_smoothing", return_value=True) as smooth_mock, \
+ mock.patch.object(cg.os, "replace") as replace_mock:
+ fixed = cg._auto_fix_transition_jumps("/tmp/fake.mp4", max_passes=2)
+
+ self.assertTrue(fixed)
+ self.assertEqual(smooth_mock.call_count, 2)
+ self.assertEqual(detect_mock.call_count, 2)
+ self.assertEqual(replace_mock.call_count, 2)
+
if __name__ == "__main__":
unittest.main()
diff --git a/tests/test_video_processor.py b/tests/test_video_processor.py
index 8752c8a..e1142f8 100644
--- a/tests/test_video_processor.py
+++ b/tests/test_video_processor.py
@@ -202,6 +202,89 @@ class VideoProcessorTests(unittest.TestCase):
self.assertEqual(targets[0][3], "SPEAKER_00")
self.assertEqual(targets[1][3], "SPEAKER_00")
+ def test_choose_track_segment_targets_rejects_edge_hugging_opening_target(self):
+ targets = vp._choose_track_segment_targets(
+ segment_tracks=[
+ (0.0, 4.0, "SPEAKER_00", 0, True),
+ ],
+ tracked_detections=[
+ (0.1, [{"cx": 140, "fw": 180, "track_id": 0}]),
+ (0.4, [{"cx": 155, "fw": 182, "track_id": 0}]),
+ (0.8, [{"cx": 620, "fw": 190, "track_id": 0}]),
+ (1.4, [{"cx": 632, "fw": 194, "track_id": 0}]),
+ (2.0, [{"cx": 628, "fw": 192, "track_id": 0}]),
+ ],
+ speaker_anchor_x={},
+ width=1920,
+ crop_w=607,
+ )
+
+ self.assertEqual(len(targets), 1)
+ self.assertGreater(targets[0][2], 250)
+ self.assertEqual(targets[0][3], "SPEAKER_00")
+
+ def test_choose_track_segment_targets_rejects_edge_hugging_trailing_target(self):
+ targets = vp._choose_track_segment_targets(
+ segment_tracks=[
+ (0.0, 4.0, "SPEAKER_00", 0, True),
+ ],
+ tracked_detections=[
+ (0.1, [{"cx": 620, "fw": 190, "track_id": 0}]),
+ (0.7, [{"cx": 632, "fw": 194, "track_id": 0}]),
+ (1.4, [{"cx": 628, "fw": 192, "track_id": 0}]),
+ (3.1, [{"cx": 150, "fw": 182, "track_id": 0}]),
+ (3.5, [{"cx": 140, "fw": 180, "track_id": 0}]),
+ ],
+ speaker_anchor_x={},
+ width=1920,
+ crop_w=607,
+ )
+
+ self.assertEqual(len(targets), 1)
+ self.assertGreater(targets[0][2], 250)
+ self.assertEqual(targets[0][3], "SPEAKER_00")
+
+ def test_choose_track_segment_targets_skips_long_anchor_only_segment(self):
+ targets = vp._choose_track_segment_targets(
+ segment_tracks=[
+ (0.0, 2.0, "SPEAKER_00", 0, True),
+ (2.0, 6.0, "SPEAKER_00", None, None),
+ ],
+ tracked_detections=[
+ (0.1, [{"cx": 620, "fw": 190, "track_id": 0}]),
+ (0.7, [{"cx": 632, "fw": 194, "track_id": 0}]),
+ (1.4, [{"cx": 628, "fw": 192, "track_id": 0}]),
+ ],
+ speaker_anchor_x={"SPEAKER_00": 140.0},
+ width=1920,
+ crop_w=607,
+ )
+
+ self.assertEqual(len(targets), 1)
+ self.assertEqual(targets[0][0:2], (0.0, 2.0))
+ self.assertGreater(targets[0][2], 250)
+ self.assertEqual(targets[0][3], "SPEAKER_00")
+
+ def test_choose_track_segment_targets_keeps_short_anchor_only_bridge(self):
+ targets = vp._choose_track_segment_targets(
+ segment_tracks=[
+ (0.0, 2.0, "SPEAKER_00", 0, True),
+ (2.0, 2.9, "SPEAKER_00", None, None),
+ ],
+ tracked_detections=[
+ (0.1, [{"cx": 620, "fw": 190, "track_id": 0}]),
+ (0.7, [{"cx": 632, "fw": 194, "track_id": 0}]),
+ (1.4, [{"cx": 628, "fw": 192, "track_id": 0}]),
+ ],
+ speaker_anchor_x={"SPEAKER_00": 620.0},
+ width=1920,
+ crop_w=607,
+ )
+
+ self.assertEqual(len(targets), 2)
+ self.assertEqual(targets[1][0:2], (2.0, 2.9))
+ self.assertGreater(targets[1][2], 250)
+
def test_build_track_turn_keyframes_uses_short_reframe_then_holds(self):
keyframes = vp._build_track_turn_keyframes(
segment_targets=[
@@ -265,16 +348,17 @@ class VideoProcessorTests(unittest.TestCase):
self.assertLess(current, 1320.0)
def test_update_tripod_camera_recenters_on_moderate_offset(self):
+ # Offset must exceed the 22% safe zone (607 * 0.22 ≈ 134px)
current = vp._update_tripod_camera(
current_center_x=960.0,
- target_center_x=1060.0,
+ target_center_x=1160.0,
crop_w=607,
video_width=1920,
dt=0.1,
)
self.assertGreater(current, 960.0)
- self.assertLess(current, 1060.0)
+ self.assertLess(current, 1160.0)
def test_choose_camera_speaker_ignores_short_interjection(self):
speaker, pending, count, switched = vp._choose_camera_speaker(
@@ -485,6 +569,17 @@ class VideoProcessorTests(unittest.TestCase):
self.assertIn("if(between(t\\,0.000\\,0.180)\\,420\\,", expr)
self.assertNotIn("100+((420-100)", expr)
+ def test_build_cam_expr_uses_short_eased_handoff_for_split_layout_jumps(self):
+ expr = vp._build_cam_expr(
+ keyframes=[(0.0, 120), (0.20, 360), (1.0, 360)],
+ duration=1.0,
+ is_split=True,
+ )
+
+ # Split transitions should avoid hard snaps and use a brief eased handoff.
+ self.assertNotIn("if(between(t\\,0.000\\,0.200)\\,360\\,", expr)
+ self.assertIn("120+((360-120)", expr)
+
def test_build_motion_blur_filter_targets_only_short_reframes(self):
blur = vp._build_motion_blur_filter(
keyframes=[(0.0, 140), (0.18, 620), (1.0, 620), (1.6, 650)],