Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

fix: split-screen crop, speaker detection, thumbnail extraction, and caption readability

Nika Siradze1f85457bd80e
diff --git a/backend/cli.py b/backend/cli.py
index 4b58fc7..5d7fa1c 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -13,6 +13,13 @@ One-command processing:
import argparse
import json
import os
+
+# Load .env file (for HF_TOKEN, etc.)
+try:
+ from dotenv import load_dotenv
+ load_dotenv(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".env"))
+except ImportError:
+ pass
import sys
import time
@@ -154,6 +161,8 @@ def cmd_process(args):
config["time_adjust"] = args.time_adjust
if args.no_energy:
config["energy_boost"] = False
+ if getattr(args, "no_speakers", False):
+ config["no_speakers"] = True
if args.quality:
config["quality"] = args.quality
@@ -232,6 +241,7 @@ def cmd_process(args):
result = transcribe_file(
file_path=video_path,
model_size=config.get("whisper_model", "base"),
+ enable_diarization=not config.get("no_speakers", False),
progress_callback=_transcribe_progress,
)
_spin_stop.set()
@@ -242,6 +252,9 @@ def cmd_process(args):
# Check speaker data availability (needed for smart cropping)
speakers_in_words = set(w.get("speaker") for w in words if w.get("speaker"))
+ diarization_warning = result.get("diarization_warning")
+ if diarization_warning:
+ print(f" ⚠ {diarization_warning}")
if len(speakers_in_words) > 1:
print(f" Speakers detected: {len(speakers_in_words)} (crop will follow active speaker)")
elif len(speakers_in_words) == 1:
@@ -550,6 +563,26 @@ Output as clean markdown. No code fences around the whole thing."""
print(f"\n {gray}Clips rendered. For titles, descriptions & thumbnails:{reset}")
print(f" {gray}Open Claude Code and run {accent}/produce-shorts{reset}")
+ # ── What next? ──
+ print(f" {'─' * 45}")
+ print(f" {bold}What next?{reset}")
+ print(f" {accent}1{reset} Re-run with different settings")
+ print(f" {accent}2{reset} Open output folder")
+ print(f" {accent}q{reset} Done")
+ print()
+ sys.stdout.write(f" {accent}▸{reset} ")
+ sys.stdout.flush()
+ try:
+ next_choice = sys.stdin.readline().strip()
+ except (EOFError, KeyboardInterrupt):
+ print()
+ return
+
+ if next_choice == "1":
+ _interactive_process()
+ elif next_choice == "2":
+ import subprocess as _sp
+ _sp.run(["open", output_dir] if sys.platform == "darwin" else ["xdg-open", output_dir])
print()
@@ -1110,6 +1143,7 @@ def print_help():
print(f" {green}--outro{reset} {gray}<asset|path>{reset} Append outro video")
print(f" {green}--quality{reset} {gray}<level>{reset} low | medium | high | max")
print(f" {green}--no-energy{reset} Skip audio energy analysis")
+ print(f" {green}--no-speakers{reset} Skip speaker detection (faster)")
print()
print(f" {bold}Examples:{reset}")
print(f" {dim}${reset} podcli process episode.mp4")
@@ -1156,6 +1190,7 @@ def main():
proc.add_argument("--outro", help="Outro video (asset name or path)")
proc.add_argument("--time-adjust", type=float, help="Timestamp offset in seconds")
proc.add_argument("--no-energy", action="store_true", help="Skip audio energy analysis")
+ proc.add_argument("--no-speakers", action="store_true", help="Skip speaker detection (faster, uses face detection only)")
proc.add_argument("--quality", choices=["low", "medium", "high", "max"], help="Output quality (default: high)")
# ── presets ──
@@ -1249,14 +1284,13 @@ def interactive_menu():
print(f" {accent}q{reset} Quit")
print()
- _flush_stdin()
try:
- choice = ""
- while not choice:
- choice = input(f" {gray}>{reset} ").strip()
+ choice = input(f" {accent}▸{reset} ").strip()
except (EOFError, KeyboardInterrupt):
print()
return
+ if not choice:
+ return
if choice == "1":
_interactive_process()
@@ -1450,11 +1484,17 @@ def _interactive_process():
if outro:
print(f" {bold}Outro:{reset} ✓ {dim}{os.path.basename(outro)}{reset}")
print(f" {bold}Transcript:{reset} {'auto (Whisper)' if not transcript else os.path.basename(transcript)}")
+ print()
+ sys.stdout.write(f" {green}Ready!{reset} {bold}Press Enter to start{reset} {dim}(q to cancel){reset} ")
+ sys.stdout.flush()
try:
- input(f"\n {green}Ready!{reset} {bold}Press Enter to start{reset} {dim}(q to cancel){reset} ")
+ confirm = sys.stdin.readline().strip()
except (EOFError, KeyboardInterrupt):
print(f"\n {gray}Cancelled.{reset}")
return
+ if confirm.lower() == "q":
+ print(f" {gray}Cancelled.{reset}")
+ return
# Run
cmd = [sys.executable, os.path.abspath(__file__), "process", video]
@@ -1468,53 +1508,72 @@ def _interactive_process():
if outro:
cmd += ["--outro", outro]
- print(f"\n {green}▶{reset} Starting...\n")
+ # Clear screen before starting processing to avoid prompt artifacts
+ sys.stdout.write("\033[2J\033[H") # clear screen + move cursor to top
sys.stdout.flush()
+ print(f" {green}▶{reset} Starting...\n")
sys.stderr.flush()
- os.execv(sys.executable, cmd)
+ # Use subprocess instead of execv to avoid terminal state issues
+ import subprocess as _sp
+ exit_code = _sp.call(cmd)
+ sys.exit(exit_code)
def _interactive_assets():
"""Interactive asset management."""
+ from services.asset_store import register, unregister, list_assets
accent = "\033[38;2;212;135;74m"
gray = "\033[38;5;245m"
green = "\033[38;2;74;222;128m"
+ red = "\033[38;2;248;113;113m"
bold = "\033[1m"
dim = "\033[2m"
reset = "\033[0m"
- from services.asset_store import register, list_assets
+ # Show current assets first
+ current = list_assets()
+ if current:
+ print(f"\n {bold}Registered assets:{reset}")
+ for a in current:
+ exists = os.path.exists(a["path"])
+ icon = f"{green}✓{reset}" if exists else f"{red}✗{reset}"
+ print(f" {icon} {accent}{a['name']}{reset} {gray}({a['type']}) {os.path.basename(a['path'])}{reset}")
+ else:
+ print(f"\n {gray}No assets registered yet.{reset}")
+
+ print(f"\n {bold}Add asset{reset} {dim}(or press Enter to go back){reset}")
- print(f"\n {bold}Assets{reset}")
- print(f" {accent}1{reset} List registered assets")
- print(f" {accent}2{reset} Add new asset")
+ # Get name
+ sys.stdout.write(f" Name: ")
+ sys.stdout.flush()
try:
- choice = input(f" {gray}>{reset} ").strip()
+ name = sys.stdin.readline().strip()
except (EOFError, KeyboardInterrupt):
+ print()
+ return
+ if not name:
return
- if choice == "1":
- assets = list_assets()
- if not assets:
- print(f"\n {gray}No assets. Choose 2 to add one.{reset}\n")
- return
- print()
- for a in assets:
- exists = os.path.exists(a["path"])
- icon = f"{green}✓{reset}" if exists else f"\033[38;2;248;113;113m✗{reset}"
- print(f" {icon} {accent}{a['name']}{reset} {gray}({a['type']}) {os.path.basename(a['path'])}{reset}")
+ # Get path
+ sys.stdout.write(f" Path: ")
+ sys.stdout.flush()
+ try:
+ path = sys.stdin.readline().strip().strip("'\"")
+ except (EOFError, KeyboardInterrupt):
print()
+ return
+ if not path:
+ print(f" {gray}Cancelled.{reset}\n")
+ return
- elif choice == "2":
- try:
- name = input(f"\n {bold}Asset name{reset} {dim}(short, e.g. mylogo){reset}: ").strip()
- path = input(f" {bold}File path:{reset} ").strip().strip("'\"")
- if name and path:
- asset = register(name, path)
- print(f"\n {green}✓{reset} Registered {accent}{name}{reset} ({asset['type']})\n")
- except (EOFError, KeyboardInterrupt, FileNotFoundError) as e:
- print(f"\n {gray}{e}{reset}\n")
+ path = _clean_path(path)
+ try:
+ asset = register(name, path)
+ print(f"\n {green}✓{reset} Registered {accent}{name}{reset} ({asset['type']})")
+ print(f" {gray}{asset['path']}{reset}\n")
+ except FileNotFoundError as e:
+ print(f"\n {red}✗{reset} {e}\n", file=sys.stderr)
if __name__ == "__main__":
diff --git a/backend/config/caption_styles.py b/backend/config/caption_styles.py
index bfad697..230ee02 100644
--- a/backend/config/caption_styles.py
+++ b/backend/config/caption_styles.py
@@ -114,7 +114,7 @@ STYLES = {
"branded": {
"description": "Large bold text, 5-7 words wrapping across 2 lines, dark rounded pill on active word. Clean, no gradient.",
"font_name": DETECTED_FONT,
- "font_size": 72, # Slightly smaller for clean 2-line fit
+ "font_size": 80, # Large bold text for shorts
"primary_color": "&H00FFFFFF", # White (all words)
"active_color": "&H00FFFFFF", # White text on dark box
"active_box_color": "&H00000000", # Black pill bg
@@ -129,7 +129,7 @@ STYLES = {
"shadow_depth": 1, # Subtle shadow for text edge smoothing
"alignment": 2, # Bottom-center
"margin_v": 500, # Position: ~72% from top (natural lower-third)
- "words_per_chunk": 6, # Show 5-7 words across 2 lines
+ "words_per_chunk": 3, # Show 3 words at a time — readable on small screens
"uppercase": False, # Mixed case, natural capitalization
"gradient_overlay": False, # No gradient — clean direct-on-video
"gradient_opacity": 0.0,
diff --git a/backend/main.py b/backend/main.py
index 34311a2..68f28fa 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -7,7 +7,15 @@ and writes a JSON result to stdout. Progress events go to stderr.
"""
import json
+import os
import sys
+
+# Load .env file (for HF_TOKEN, etc.)
+try:
+ from dotenv import load_dotenv
+ load_dotenv(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", ".env"))
+except ImportError:
+ pass
import traceback
def emit_progress(task_id: str, stage: str, percent: int, message: str):
@@ -72,6 +80,7 @@ def handle_create_clip(task_id: str, params: dict):
logo_path=params.get("logo_path"),
outro_path=params.get("outro_path"),
clean_fillers=params.get("clean_fillers", True),
+ face_map=params.get("face_map"),
progress_callback=lambda pct, msg: emit_progress(task_id, "processing", pct, msg),
)
emit_result(task_id, "success", data=result)
@@ -105,6 +114,7 @@ def handle_batch_clips(task_id: str, params: dict):
logo_path=clip.get("logo_path") or params.get("logo_path"),
outro_path=params.get("outro_path"),
clean_fillers=params.get("clean_fillers", True),
+ face_map=params.get("face_map"),
progress_callback=lambda pct, msg, _i=i: emit_progress(
task_id, "batch", int((_i / total) * 100 + pct / total), msg
),
diff --git a/backend/services/caption_renderer.py b/backend/services/caption_renderer.py
index e13b366..052f8eb 100644
--- a/backend/services/caption_renderer.py
+++ b/backend/services/caption_renderer.py
@@ -51,7 +51,7 @@ ScaledBorderAndShadow: yes
[V4+ Styles]
Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
-Style: BrandedNormal,{style["font_name"]},{style["font_size"]},&H00FFFFFF,&H00FFFFFF,&H50000000,&H00000000,-1,0,0,0,100,100,2,0,1,2,1,{style["alignment"]},60,60,{style["margin_v"]},1
+Style: BrandedNormal,{style["font_name"]},{style["font_size"]},&H00FFFFFF,&H00FFFFFF,&H90000000,&H00000000,-1,0,0,0,100,100,2,0,1,1,1,{style["alignment"]},60,60,{style["margin_v"]},1
[Events]
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
@@ -508,9 +508,11 @@ def _render_branded(words: list[dict], style: dict, offset: float) -> str:
text = text[0].upper() + text[1:] if len(text) > 1 else text.upper()
normalized.append(text)
- # Measure word widths for pill positioning
- word_widths = _measure_text_widths(normalized, font_name, font_size, is_bold, spacing)
- space_width = _measure_text_widths([" "], font_name, font_size, is_bold, spacing)[0]
+ # Measure word widths for pill positioning.
+ # Use spacing=0 here — ASS Spacing is handled by libass internally,
+ # but we position words ourselves with \pos so we need pure font metrics.
+ word_widths = _measure_text_widths(normalized, font_name, font_size, is_bold, 0)
+ space_width = _measure_text_widths([" "], font_name, font_size, is_bold, 0)[0]
# Split into visual lines that fit within play_res_x - margins
max_line_w = play_res_x - 120 # 60px margin each side
diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.py
index 26e2811..6315637 100644
--- a/backend/services/clip_generator.py
+++ b/backend/services/clip_generator.py
@@ -46,15 +46,16 @@ def _snap_to_sentence_end(
if not clip_words:
return end_second
- last_word_text = clip_words[-1].get("word", "").strip()
+ last_clip_word = clip_words[-1]
+ last_word_text = last_clip_word.get("word", "").strip()
- # Already ends on a sentence boundary
+ # Already ends on a sentence boundary — extend to include full word
if last_word_text and last_word_text[-1] in SENTENCE_ENDINGS:
- return end_second
+ return max(end_second, last_clip_word["end"])
# Search forward: find next sentence-ending word within max_extension
- for w in transcript_words:
- if w["start"] >= end_second and w["end"] <= end_second + max_extension:
+ for w in sorted(transcript_words, key=lambda x: x["start"]):
+ if w["start"] >= last_clip_word["end"] and w["end"] <= end_second + max_extension:
text = w.get("word", "").strip()
if text and text[-1] in SENTENCE_ENDINGS:
return w["end"]
@@ -98,6 +99,7 @@ def generate_clip(
transcript_words: list[dict] = None,
title: str = "clip",
output_dir: Optional[str] = None,
+ face_map: dict = None,
logo_path: Optional[str] = None,
outro_path: Optional[str] = None,
clean_fillers: bool = True,
@@ -167,6 +169,7 @@ def generate_clip(
strategy=crop_strategy,
transcript_words=transcript_words,
clip_start=start_second,
+ face_map=face_map,
)
# Step 3: Generate captions + burn (with optional gradient & logo)
diff --git a/backend/services/face_analysis.py b/backend/services/face_analysis.py
new file mode 100644
index 0000000..bf88821
--- /dev/null
+++ b/backend/services/face_analysis.py
@@ -0,0 +1,213 @@
+"""
+Face analysis service — runs once during transcription, caches results.
+
+Detects face positions throughout the video and maps them to speakers.
+Clip generation uses this pre-computed data instead of re-scanning every time.
+
+Output: face_map dict with observations, clusters, speaker mappings, and
+pre-computed crop positions for each speaker segment.
+"""
+
+import os
+import sys
+from typing import Optional, Callable
+
+
+def analyze_faces(
+ video_path: str,
+ speaker_segments: list[dict],
+ duration: float,
+ progress_callback: Optional[Callable[[int, str], None]] = None,
+) -> Optional[dict]:
+ """
+ Analyze face positions throughout the video and map to speakers.
+
+ Args:
+ video_path: Path to the video file
+ speaker_segments: List of {speaker, start, end} dicts from diarization
+ duration: Video duration in seconds
+ progress_callback: Optional (percent, message) callback
+
+ Returns:
+ face_map dict or None if analysis fails:
+ {
+ "observations": [{time, face_center_x, face_width, confidence}, ...],
+ "clusters": [{center_x, count, crop_x}, ...],
+ "speaker_mappings": {speaker: cluster_index, ...},
+ "is_split_screen": bool,
+ "dominant_speaker": str or None,
+ "video_width": int,
+ "video_height": int,
+ }
+ """
+ try:
+ import cv2
+ import numpy as np
+ except ImportError:
+ print("Warning: OpenCV not available, skipping face analysis", file=sys.stderr)
+ return None
+
+ if progress_callback:
+ progress_callback(0, "Starting face analysis...")
+
+ cap = cv2.VideoCapture(video_path)
+ if not cap.isOpened():
+ return None
+
+ width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+ height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ fps = cap.get(cv2.CAP_PROP_FPS) or 30
+
+ # Load DNN face detector
+ backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ proto = os.path.join(backend_dir, "models", "deploy.prototxt")
+ model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")
+
+ if not (os.path.exists(proto) and os.path.exists(model)):
+ cap.release()
+ print("Warning: Face detection model not found, skipping", file=sys.stderr)
+ return None
+
+ detector = cv2.dnn.readNetFromCaffe(proto, model)
+
+ # Sample ~2 frames per second across the full video (enough for analysis)
+ sample_count = min(300, max(20, int(duration * 2)))
+ observations = []
+ faces_per_frame = []
+
+ if progress_callback:
+ progress_callback(10, f"Scanning {sample_count} frames for faces...")
+
+ for i in range(sample_count):
+ t = i * duration / sample_count
+ cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000)
+ ret, frame = cap.read()
+ if not ret:
+ continue
+
+ h, w = frame.shape[:2]
+ blob = cv2.dnn.blobFromImage(
+ cv2.resize(frame, (300, 300)), 1.0, (300, 300),
+ (104.0, 177.0, 123.0)
+ )
+ detector.setInput(blob)
+ detections = detector.forward()
+
+ frame_faces = 0
+ for j in range(detections.shape[2]):
+ conf = detections[0, 0, j, 2]
+ if conf > 0.5:
+ x1 = int(detections[0, 0, j, 3] * w)
+ x2 = int(detections[0, 0, j, 5] * w)
+ fw = x2 - x1
+ if fw < w * 0.04:
+ continue
+ cx = (x1 + x2) // 2
+ observations.append({
+ "time": round(t, 3),
+ "face_center_x": cx,
+ "face_width": fw,
+ "confidence": round(float(conf), 3),
+ })
+ frame_faces += 1
+ faces_per_frame.append(frame_faces)
+
+ if progress_callback and i % 20 == 0:
+ pct = 10 + int(60 * i / sample_count)
+ progress_callback(pct, f"Analyzing faces... {i}/{sample_count}")
+
+ cap.release()
+
+ if len(observations) < 3:
+ return None
+
+ if progress_callback:
+ progress_callback(75, "Clustering face positions...")
+
+ # Cluster faces by position
+ target_ratio = 1080 / 1920 # 9:16
+ crop_w = int(height * target_ratio)
+ positions = np.array([o["face_center_x"] for o in observations])
+ cluster_radius = width * 0.20
+
+ clusters_list = []
+ used = np.zeros(len(positions), dtype=bool)
+ sorted_idx = np.argsort(positions)
+
+ for idx in sorted_idx:
+ if used[idx]:
+ continue
+ mask = (np.abs(positions - positions[idx]) < cluster_radius) & ~used
+ if np.any(mask):
+ cluster_center = int(np.median(positions[mask]))
+ crop_x = max(0, min(cluster_center - crop_w // 2, width - crop_w))
+ clusters_list.append({
+ "center_x": cluster_center,
+ "count": int(np.sum(mask)),
+ "crop_x": crop_x,
+ })
+ used |= mask
+
+ if not clusters_list:
+ return None
+
+ clusters_list.sort(key=lambda c: c["center_x"])
+
+ # Detect split-screen
+ avg_faces = float(np.mean(faces_per_frame)) if faces_per_frame else 0
+ is_split_screen = avg_faces >= 1.5 and len(clusters_list) >= 2
+
+ # Map speakers to clusters
+ speakers = sorted(set(s.get("speaker", "") for s in speaker_segments if s.get("speaker")))
+ speaker_mappings = {}
+
+ if progress_callback:
+ progress_callback(85, "Mapping speakers to face positions...")
+
+ if is_split_screen and len(clusters_list) >= 2 and len(speakers) == 2:
+ # Split-screen: map by who speaks first → left face
+ speaker_first_time = {}
+ for seg in speaker_segments:
+ sp = seg.get("speaker")
+ if sp and sp not in speaker_first_time:
+ speaker_first_time[sp] = seg["start"]
+ speakers_sorted = sorted(speakers, key=lambda s: speaker_first_time.get(s, float("inf")))
+ speaker_mappings[speakers_sorted[0]] = 0 # left cluster
+ speaker_mappings[speakers_sorted[1]] = 1 # right cluster
+ elif len(clusters_list) >= 2 and len(speakers) >= 2:
+ # Non-split-screen: vote by which face is visible when speaker talks
+ obs_arr = [(o["time"], o["face_center_x"]) for o in observations]
+ for speaker in speakers:
+ sp_times = [s["start"] for s in speaker_segments if s.get("speaker") == speaker]
+ votes = [0] * len(clusters_list)
+ for t in sp_times:
+ for obs_t, obs_cx in obs_arr:
+ if abs(obs_t - t) < 1.5:
+ for ci, cl in enumerate(clusters_list):
+ if abs(obs_cx - cl["center_x"]) < cluster_radius:
+ votes[ci] += 1
+ if any(votes):
+ speaker_mappings[speaker] = int(np.argmax(votes))
+ elif len(speakers) == 1 and clusters_list:
+ speaker_mappings[speakers[0]] = 0
+
+ # Determine dominant speaker
+ speaker_durations = {}
+ for seg in speaker_segments:
+ sp = seg.get("speaker", "")
+ if sp:
+ speaker_durations[sp] = speaker_durations.get(sp, 0) + (seg["end"] - seg["start"])
+ dominant_speaker = max(speaker_durations, key=speaker_durations.get) if speaker_durations else None
+
+ if progress_callback:
+ progress_callback(100, "Face analysis complete")
+
+ return {
+ "observations": observations,
+ "clusters": clusters_list,
+ "speaker_mappings": speaker_mappings,
+ "is_split_screen": is_split_screen,
+ "dominant_speaker": dominant_speaker,
+ "video_width": width,
+ "video_height": height,
+ }
diff --git a/backend/services/speaker_detection.py b/backend/services/speaker_detection.py
index e389fa0..4e91e34 100644
--- a/backend/services/speaker_detection.py
+++ b/backend/services/speaker_detection.py
@@ -13,6 +13,20 @@ import subprocess
import tempfile
from typing import Optional, Callable
+# Fix torchaudio compatibility — speechbrain calls torchaudio.list_audio_backends()
+# which was removed in torchaudio >= 2.10. Monkey-patch before any import.
+try:
+ import torchaudio
+ if not hasattr(torchaudio, "list_audio_backends"):
+ torchaudio.list_audio_backends = lambda: ["ffmpeg"]
+except ImportError:
+ pass
+
+# Suppress noisy PyTorch warnings (std() degrees of freedom, etc.)
+import warnings
+warnings.filterwarnings("ignore", message="std\\(\\): degrees of freedom")
+warnings.filterwarnings("ignore", category=UserWarning, module="pyannote")
+
def extract_audio_wav(video_path: str, output_path: str) -> str:
"""Extract audio as 16kHz mono WAV (required by pyannote)."""
@@ -64,9 +78,15 @@ def run_diarization(
if not token:
msg = (
- "HF_TOKEN not set — speaker detection requires a HuggingFace token. "
- "Get one at https://huggingface.co/settings/tokens and set HF_TOKEN in your .env file. "
- "You must also accept terms at https://huggingface.co/pyannote/speaker-diarization-3.1"
+ "HF_TOKEN not set — speaker detection requires a HuggingFace token.\n"
+ "\n"
+ " Setup (one-time):\n"
+ " 1. Create a token at https://huggingface.co/settings/tokens\n"
+ " 2. Accept model terms at ALL THREE:\n"
+ " → https://huggingface.co/pyannote/speaker-diarization-3.1\n"
+ " → https://huggingface.co/pyannote/segmentation-3.0\n"
+ " → https://huggingface.co/pyannote/speaker-diarization-community-1\n"
+ " 3. Add to your .env file: HF_TOKEN=hf_your_token_here"
)
if progress_callback:
progress_callback(0, msg)
@@ -76,27 +96,65 @@ def run_diarization(
progress_callback(10, "Loading speaker diarization model...")
try:
- pipeline = Pipeline.from_pretrained(
- "pyannote/speaker-diarization-3.1",
- use_auth_token=token,
- )
+ # pyannote >= 3.1 uses 'token', older versions use 'use_auth_token'
+ try:
+ pipeline = Pipeline.from_pretrained(
+ "pyannote/speaker-diarization-3.1",
+ token=token,
+ )
+ except TypeError:
+ pipeline = Pipeline.from_pretrained(
+ "pyannote/speaker-diarization-3.1",
+ use_auth_token=token,
+ )
except Exception as e:
err_str = str(e).lower()
if "token" in err_str or "auth" in err_str or "403" in err_str or "401" in err_str:
msg = (
- f"HuggingFace auth failed: {e}. "
- "Make sure your HF_TOKEN is valid and you accepted the model terms at "
- "https://huggingface.co/pyannote/speaker-diarization-3.1"
+ f"HuggingFace auth failed: {e}\n"
+ "\n"
+ " Fix: Accept model terms at ALL THREE (while logged into HuggingFace):\n"
+ " → https://huggingface.co/pyannote/speaker-diarization-3.1\n"
+ " → https://huggingface.co/pyannote/segmentation-3.0\n"
+ " → https://huggingface.co/pyannote/speaker-diarization-community-1\n"
+ " Then verify your HF_TOKEN is valid."
)
if progress_callback:
progress_callback(0, msg)
raise PermissionError(msg) from e
raise
+ # Move pipeline to GPU if available (Apple Silicon MPS or CUDA)
+ import torch
+ device = "cpu"
+ if torch.cuda.is_available():
+ device = "cuda"
+ pipeline.to(torch.device("cuda"))
+ elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+ try:
+ pipeline.to(torch.device("mps"))
+ device = "mps"
+ except Exception:
+ pass # Some pyannote components don't support MPS, fall back to CPU
+
+ # Estimate duration for progress message
+ import wave
+ try:
+ with wave.open(audio_path, "rb") as wf:
+ audio_dur = wf.getnframes() / wf.getframerate()
+ if device != "cpu":
+ est_min = max(1, int(audio_dur / 600)) # GPU: ~2x faster
+ time_msg = f"Running speaker diarization on {device.upper()} (~{est_min}-{est_min * 2} min)..."
+ else:
+ est_min = max(1, int(audio_dur / 300))
+ time_msg = f"Running speaker diarization on CPU (~{est_min}-{est_min * 2} min for {int(audio_dur / 60)} min audio)..."
+ except Exception:
+ time_msg = f"Running speaker diarization on {device.upper()}..."
+
if progress_callback:
- progress_callback(30, "Running speaker diarization...")
+ progress_callback(30, time_msg)
- # Run diarization
+ # Run diarization — this is a blocking call, no progress updates from pyannote
diarization_params = {}
if num_speakers:
diarization_params["num_speakers"] = num_speakers
@@ -109,6 +167,10 @@ def run_diarization(
if progress_callback:
progress_callback(90, "Processing speaker segments...")
+ # pyannote >= 4.0 returns DiarizeOutput; extract the Annotation from it
+ if hasattr(diarization, "speaker_diarization"):
+ diarization = diarization.speaker_diarization
+
# Convert to simple list format
speaker_segments = []
for turn, _, speaker in diarization.itertracks(yield_label=True):
diff --git a/backend/services/thumbnail_ai.py b/backend/services/thumbnail_ai.py
index 7f1780f..5c90ed0 100644
--- a/backend/services/thumbnail_ai.py
+++ b/backend/services/thumbnail_ai.py
@@ -56,60 +56,74 @@ def _frame_sharpness(frame, face_roi=None) -> float:
def _face_expression_quality(frame, x1: int, y1: int, x2: int, y2: int) -> float:
"""
Estimate facial expression quality (0 to 1).
- Penalizes: closed/squinting eyes, wide open mouth, motion blur.
- Higher = better thumbnail candidate.
+ Rejects: looking down/away, closed eyes, wide open mouth, blur, bad angles.
+ Higher = better thumbnail candidate. Threshold should be >= 0.5.
"""
import cv2
import numpy as np
region = frame[y1:y2, x1:x2]
h, w = region.shape[:2]
- if h < 20 or w < 20:
- return 0.5 # Too small to analyze
+ if h < 30 or w < 30:
+ return 0.3 # Too small to analyze properly
gray = cv2.cvtColor(region, cv2.COLOR_BGR2GRAY)
- # 1) Eye region check (top 25-45% of face)
- eye_y1 = int(h * 0.25)
- eye_y2 = int(h * 0.45)
- eye_region = gray[eye_y1:eye_y2, :]
+ # 1) Face angle — reject extreme tilt (looking down, sideways)
+ # A front-facing face has the brightest area (forehead) in the top third.
+ # Looking down shifts brightness to forehead being occluded.
+ top_third = gray[:int(h * 0.33), :]
+ mid_third = gray[int(h * 0.33):int(h * 0.66), :]
+ bot_third = gray[int(h * 0.66):, :]
+ top_brightness = float(np.mean(top_third))
+ mid_brightness = float(np.mean(mid_third))
+ bot_brightness = float(np.mean(bot_third))
+
+ # Looking down: top third is much darker than bottom (forehead hidden, chin/neck visible)
+ if top_brightness < mid_brightness * 0.7:
+ return 0.2 # Looking down — terrible for thumbnails
+
+ # 2) Face symmetry — detect extreme side angles
+ # A frontal face has roughly symmetric left/right brightness
+ left_half = gray[:, :w // 2]
+ right_half = gray[:, w // 2:]
+ symmetry = 1.0 - abs(float(np.mean(left_half)) - float(np.mean(right_half))) / 128.0
+ if symmetry < 0.7:
+ return 0.3 # Extreme side angle
+
+ # 3) Eye region check (top 25-45% of face)
+ eye_region = gray[int(h * 0.25):int(h * 0.45), :]
if eye_region.size > 0:
eye_sharpness = cv2.Laplacian(eye_region, cv2.CV_64F).var()
- # Sharp eyes = open eyes (edges around iris/pupil)
- # Blurry eyes = blinking or closed
- if eye_sharpness < 10:
- return 0.3 # Eyes likely closed or blinking
+ if eye_sharpness < 15:
+ return 0.25 # Eyes closed or blinking
else:
- eye_sharpness = 50 # Can't measure, assume OK
+ eye_sharpness = 50
- # 2) Mouth region check (bottom 30% of face)
+ # 4) Mouth region check (bottom 30% of face)
mouth_region = gray[int(h * 0.65):, :]
+ mouth_penalty = 0.0
if mouth_region.size > 0:
mouth_var = cv2.Laplacian(mouth_region, cv2.CV_64F).var()
- # Very high variance in mouth = wide open (teeth, tongue visible)
- # Moderate variance = natural expression
- mouth_penalty = 0.0
if mouth_var > 200:
- mouth_penalty = 0.3 # Wide open mouth
+ mouth_penalty = 0.35 # Wide open mouth
elif mouth_var > 120:
- mouth_penalty = 0.15 # Slightly open
- else:
- mouth_penalty = 0.0
+ mouth_penalty = 0.15
- # 3) Overall face sharpness — catch motion blur
+ # 5) Overall face sharpness — catch motion blur
face_sharpness = cv2.Laplacian(gray, cv2.CV_64F).var()
- if face_sharpness < 20:
- return 0.35 # Motion blur
+ if face_sharpness < 25:
+ return 0.3 # Motion blur
- # 4) Face aspect ratio — detect extreme angles or distortion
+ # 6) Face bounding box aspect ratio
aspect = h / max(1, w)
- if aspect < 0.6 or aspect > 1.5:
- return 0.5 # Odd angle
+ if aspect < 0.7 or aspect > 1.4:
+ return 0.35 # Odd angle or distortion
- # Combine scores
- eye_score = min(1.0, eye_sharpness / 80.0)
- quality = eye_score * (1.0 - mouth_penalty)
- return max(0.2, min(1.0, quality))
+ # Combine: eye sharpness × symmetry × (1 - mouth penalty)
+ eye_score = min(1.0, eye_sharpness / 60.0)
+ quality = eye_score * symmetry * (1.0 - mouth_penalty)
+ return max(0.15, min(1.0, quality))
def extract_candidate_frames(
@@ -175,7 +189,7 @@ def extract_candidate_frames(
for j in range(detections.shape[2]):
conf = detections[0, 0, j, 2]
- if conf > 0.7:
+ if conf > 0.6:
x1 = max(0, int(detections[0, 0, j, 3] * w))
y1 = max(0, int(detections[0, 0, j, 4] * h))
x2 = min(w, int(detections[0, 0, j, 5] * w))
@@ -183,25 +197,27 @@ def extract_candidate_frames(
fw = x2 - x1
fh = y2 - y1
- if fw < w * 0.10:
+ if fw < w * 0.08:
continue
- # Split-screen check
+ # Split-screen check — only for landscape sources where
+ # a small centered face likely means a multi-cam layout.
+ # Portrait sources naturally have centered faces.
face_cx = (x1 + x2) / 2
- if 0.4 * w < face_cx < 0.6 * w and fw < w * 0.25:
+ if not is_portrait and 0.4 * w < face_cx < 0.6 * w and fw < w * 0.15:
continue
# Sharpness of face region (proxy for eyes open, good expression)
sharpness = _frame_sharpness(frame, (x1, y1, x2, y2))
# Skip very blurry faces (motion blur, out of focus)
- if sharpness < 15:
+ if sharpness < 12:
continue
# Face expression quality — filter blinking, open mouths
expression_quality = _face_expression_quality(frame, x1, y1, x2, y2)
- if expression_quality < 0.4:
- continue # Bad expression (blinking, mouth wide open, etc.)
+ if expression_quality < 0.45:
+ continue # Bad expression (looking down, blinking, mouth open, etc.)
# Face size as fraction of frame
face_area_pct = (fw * fh) / (w * h) * 100
@@ -514,8 +530,21 @@ def generate_variations(
frames_dir = os.path.join(output_dir, "_frames")
frames = extract_candidate_frames(video_path, frames_dir, count=n)
+ if not frames and video_path:
+ # Fallback: simpler face extraction with less aggressive filters
+ try:
+ from services.thumbnail_generator import extract_face_frame
+ fallback_path = os.path.join(output_dir, "_face_frame.png")
+ result = extract_face_frame(video_path, fallback_path,
+ target_width=cfg.get("width", 1080),
+ target_height=cfg.get("height", 1920))
+ if result:
+ frames = [result]
+ except Exception:
+ pass
+
if not frames:
- frames = [{"path": None}] # No photo — dark bg only
+ frames = [{"path": None}]
paths = []
for i in range(n):
diff --git a/backend/services/thumbnail_generator.py b/backend/services/thumbnail_generator.py
index a29e78a..f8e61c4 100644
--- a/backend/services/thumbnail_generator.py
+++ b/backend/services/thumbnail_generator.py
@@ -722,7 +722,7 @@ def generate_variations(
def thumbnail_to_video_frame(
thumbnail_path: str,
output_path: str,
- duration: float = 2.0,
+ duration: float = 0.8,
fade_in: float = 0.3,
fade_out: float = 0.3,
width: int = 1080,
diff --git a/backend/services/thumbnail_html.py b/backend/services/thumbnail_html.py
index f9d8815..c5150fb 100644
--- a/backend/services/thumbnail_html.py
+++ b/backend/services/thumbnail_html.py
@@ -500,7 +500,7 @@ def generate_variations(
def thumbnail_to_video_frame(
thumbnail_path: str,
output_path: str,
- duration: float = 2.0,
+ duration: float = 0.8,
width: int = 1080,
height: int = 1920,
) -> str:
diff --git a/backend/services/transcription.py b/backend/services/transcription.py
index f81332b..3dde47d 100644
--- a/backend/services/transcription.py
+++ b/backend/services/transcription.py
@@ -192,8 +192,23 @@ def transcribe_file(
else:
diarization_warning = "Speaker detection disabled"
+ # Run face analysis on the video (maps speakers to face positions)
+ # Run silently — no progress callbacks to avoid interfering with CLI output
+ face_map = None
+ try:
+ if progress_callback:
+ progress_callback(95, "Analyzing face positions...")
+ from services.face_analysis import analyze_faces
+ face_map = analyze_faces(
+ video_path=file_path,
+ speaker_segments=speaker_segments,
+ duration=duration,
+ )
+ except Exception as e:
+ print(f"Warning: face analysis failed: {e}", file=sys.stderr)
+
if progress_callback:
- progress_callback(100, "Transcription complete")
+ progress_callback(100, "Complete")
result_data = {
"transcript": result.get("text", "").strip(),
@@ -205,6 +220,9 @@ def transcribe_file(
"speaker_segments": speaker_segments,
}
+ if face_map:
+ result_data["face_map"] = face_map
+
if diarization_warning:
result_data["diarization_warning"] = diarization_warning
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index ecae0f8..1e4b972 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -118,6 +118,7 @@ def crop_to_vertical(
strategy: str = "face",
transcript_words: list = None,
clip_start: float = 0,
+ face_map: dict = None,
) -> str:
"""
Crop/scale video to 1080x1920 (9:16 vertical).
@@ -139,33 +140,40 @@ def crop_to_vertical(
source_ratio = width / height
if strategy in ("face", "speaker"):
- # Check if we have multi-speaker data — if so, do speaker-aware crop
- has_speakers = (
- transcript_words
- and len(set(w.get("speaker") for w in transcript_words if w.get("speaker"))) > 1
- )
+ crop_x_expr = None
+
+ # Fast path: use pre-computed face_map from transcription
+ # Only use if we have speaker mappings — without them, face_map
+ # can't know which face to follow in a multi-face layout
+ if face_map and face_map.get("clusters") and face_map.get("speaker_mappings"):
+ crop_x_expr = _use_face_map(
+ face_map, transcript_words, clip_start,
+ width, height, target_ratio,
+ )
- if has_speakers:
- crop_x_expr = _build_speaker_aware_crop(
- input_path, width, height, target_ratio,
- transcript_words, clip_start,
+ # Slow path: analyze video on-the-fly (no face_map cached)
+ if crop_x_expr is None:
+ has_speakers = (
+ transcript_words
+ and len(set(w.get("speaker") for w in transcript_words if w.get("speaker"))) > 1
)
- if crop_x_expr is not None:
- crop_h = height
- crop_w = int(crop_h * target_ratio)
- # crop_x_expr is either a static int or an FFmpeg expression for dynamic panning
- vf = f"crop={crop_w}:{crop_h}:{crop_x_expr}:0,scale={target_w}:{target_h}"
+
+ if has_speakers:
+ crop_x_expr = _build_speaker_aware_crop(
+ input_path, width, height, target_ratio,
+ transcript_words, clip_start,
+ )
else:
- strategy = "center"
+ # Single speaker: find face and center on it (static crop).
+ # Use _detect_face_center for a stable, centered position.
+ crop_x_expr = _detect_face_center(input_path, width, height, target_ratio)
+
+ if crop_x_expr is not None:
+ crop_h = height
+ crop_w = int(crop_h * target_ratio)
+ vf = f"crop={crop_w}:{crop_h}:{crop_x_expr}:0,scale={target_w}:{target_h}"
else:
- # Single speaker or no speaker data — use static face detection
- crop_x = _detect_face_offset(input_path, width, height, target_ratio)
- if crop_x is not None:
- crop_h = height
- crop_w = int(crop_h * target_ratio)
- vf = f"crop={crop_w}:{crop_h}:{crop_x}:0,scale={target_w}:{target_h}"
- else:
- strategy = "center"
+ strategy = "center"
if strategy == "center":
if source_ratio > target_ratio:
@@ -199,6 +207,130 @@ def crop_to_vertical(
)
+def _use_face_map(
+ face_map: dict,
+ transcript_words: list,
+ clip_start: float,
+ width: int,
+ height: int,
+ target_ratio: float,
+) -> Optional[str]:
+ """
+ Use pre-computed face_map from transcription to determine crop position.
+ Much faster than re-scanning the video.
+
+ For multi-speaker clips: builds speaker-aware panning expression.
+ For single-speaker: returns the dominant speaker's crop position.
+ """
+ clusters = face_map.get("clusters", [])
+ speaker_mappings = face_map.get("speaker_mappings", {})
+ dominant = face_map.get("dominant_speaker")
+ crop_w = int(height * target_ratio)
+
+ if not clusters:
+ return None
+
+ # Verify face_map was computed at the same resolution
+ map_w = face_map.get("video_width", width)
+ if map_w != width:
+ # Resolution mismatch — face_map positions don't apply
+ return None
+
+ # Check if clip has multiple speakers
+ speakers_in_clip = set()
+ if transcript_words:
+ speakers_in_clip = set(w.get("speaker") for w in transcript_words if w.get("speaker"))
+
+ if len(speakers_in_clip) < 2 or len(clusters) < 2:
+ # Single speaker — use their cluster or the dominant one
+ for sp in speakers_in_clip:
+ ci = speaker_mappings.get(sp)
+ if ci is not None and ci < len(clusters):
+ return str(clusters[ci]["crop_x"])
+ # Fallback to dominant speaker's position
+ if dominant and dominant in speaker_mappings:
+ ci = speaker_mappings[dominant]
+ if ci < len(clusters):
+ return str(clusters[ci]["crop_x"])
+ return str(clusters[0]["crop_x"])
+
+ # Multi-speaker: build panning expression from speaker segments
+ # Group words into speaker segments
+ segments = []
+ current_speaker = None
+ seg_start = 0
+ clip_end = max(w["end"] for w in transcript_words) if transcript_words else 0
+
+ for w in sorted(transcript_words, key=lambda x: x["start"]):
+ sp = w.get("speaker")
+ t = w["start"] - clip_start
+ if sp != current_speaker and sp is not None:
+ if current_speaker is not None:
+ segments.append((seg_start, t, current_speaker))
+ current_speaker = sp
+ seg_start = t
+
+ if current_speaker is not None:
+ segments.append((seg_start, clip_end - clip_start, current_speaker))
+
+ if not segments:
+ return str(clusters[0]["crop_x"])
+
+ # Merge short segments (<1.0s)
+ merged = []
+ for seg in segments:
+ if merged and seg[2] == merged[-1][2]:
+ merged[-1] = (merged[-1][0], seg[1], seg[2])
+ elif merged and (seg[1] - seg[0]) < 1.0:
+ merged[-1] = (merged[-1][0], seg[1], merged[-1][2])
+ else:
+ merged.append(list(seg))
+ segments = merged
+
+ # Default position
+ default_ci = speaker_mappings.get(dominant, 0)
+ default_x = clusters[min(default_ci, len(clusters) - 1)]["crop_x"]
+
+ # Build keyframes
+ pan_duration = 0.4
+ duration = segments[-1][1] if segments else 1.0
+ keyframes = []
+ prev_x = default_x
+
+ for start_t, end_t, speaker in segments:
+ ci = speaker_mappings.get(speaker)
+ target_x = clusters[ci]["crop_x"] if ci is not None and ci < len(clusters) else default_x
+
+ if target_x != prev_x:
+ keyframes.append((start_t, prev_x))
+ keyframes.append((start_t + pan_duration, target_x))
+ prev_x = target_x
+
+ if not keyframes:
+ return str(default_x)
+
+ # Build FFmpeg expression
+ expr_parts = []
+ for i in range(0, len(keyframes) - 1, 2):
+ t0, x0 = keyframes[i]
+ t1, x1 = keyframes[i + 1]
+ dt = max(0.01, t1 - t0)
+ expr_parts.append(
+ f"if(between(t\\,{t0:.2f}\\,{t1:.2f})\\,"
+ f"{x0}+(({x1}-{x0})*(t-{t0:.2f})/{dt:.2f})\\,"
+ )
+ next_t = keyframes[i + 2][0] if i + 2 < len(keyframes) else duration
+ expr_parts.append(
+ f"if(between(t\\,{t1:.2f}\\,{next_t:.2f})\\,"
+ f"{x1}\\,"
+ )
+
+ if len(expr_parts) > 30:
+ return str(default_x)
+
+ return "".join(expr_parts) + str(default_x) + ")" * len(expr_parts)
+
+
def _build_speaker_aware_crop(
video_path: str,
width: int,
@@ -778,6 +910,142 @@ def normalize_audio(
return output_path
+def _detect_face_center(
+ video_path: str,
+ width: int,
+ height: int,
+ target_ratio: float,
+) -> Optional[str]:
+ """
+ Simple static face detection — finds the dominant face and returns a
+ STATIC crop position that centers the face horizontally. No tracking,
+ no panning. The face stays centered for the entire clip.
+ """
+ try:
+ import cv2
+ import numpy as np
+
+ cap = cv2.VideoCapture(video_path)
+ if not cap.isOpened():
+ return None
+
+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+ fps = cap.get(cv2.CAP_PROP_FPS) or 30
+ duration = total_frames / fps
+ crop_w = int(height * target_ratio)
+
+ # Load DNN face detector
+ backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ proto = os.path.join(backend_dir, "models", "deploy.prototxt")
+ model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")
+ if not (os.path.exists(proto) and os.path.exists(model)):
+ cap.release()
+ return None
+
+ detector = cv2.dnn.readNetFromCaffe(proto, model)
+
+ # Sample frames across the clip
+ sample_count = min(40, max(10, int(duration * 2)))
+ face_positions = []
+
+ for i in range(sample_count):
+ t = i * duration / sample_count
+ cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000)
+ ret, frame = cap.read()
+ if not ret:
+ continue
+
+ h, w = frame.shape[:2]
+ blob = cv2.dnn.blobFromImage(
+ cv2.resize(frame, (300, 300)), 1.0, (300, 300),
+ (104.0, 177.0, 123.0)
+ )
+ detector.setInput(blob)
+ detections = detector.forward()
+
+ best_score = 0
+ best_cx = None
+ for j in range(detections.shape[2]):
+ conf = detections[0, 0, j, 2]
+ if conf > 0.5:
+ x1 = int(detections[0, 0, j, 3] * w)
+ x2 = int(detections[0, 0, j, 5] * w)
+ face_w = x2 - x1
+ if face_w < w * 0.05:
+ continue
+ score = conf * (face_w ** 2)
+ if score > best_score:
+ best_score = score
+ best_cx = (x1 + x2) // 2
+
+ if best_cx is not None:
+ face_positions.append(best_cx)
+
+ cap.release()
+
+ if not face_positions:
+ return None
+
+ # Cluster face positions to detect split-screen layouts.
+ # Use a tighter radius (15%) so faces at 25% and 75% don't merge.
+ positions = np.array(face_positions)
+ cluster_radius = width * 0.15
+
+ clusters = []
+ used = np.zeros(len(positions), dtype=bool)
+ sorted_idx = np.argsort(positions)
+
+ for idx in sorted_idx:
+ if used[idx]:
+ continue
+ mask = np.abs(positions - positions[idx]) < cluster_radius
+ mask &= ~used
+ if np.any(mask):
+ cluster_center = int(np.median(positions[mask]))
+ clusters.append({
+ "center_x": cluster_center,
+ "count": int(np.sum(mask)),
+ })
+ used |= mask
+
+ if len(clusters) >= 2:
+ # Split-screen: faces in distinct horizontal groups.
+ # Pick the cluster with the most detections, then crop to
+ # that HALF of the frame (not just center on face — centering
+ # still shows the seam between cameras).
+ clusters.sort(key=lambda c: c["count"], reverse=True)
+ best_cx = clusters[0]["center_x"]
+
+ # Snap to the half containing this face
+ if best_cx < width // 2:
+ # Left half: crop starts at 0
+ half_w = width // 2
+ face_x = best_cx # within the left half
+ else:
+ # Right half: crop starts at midpoint
+ half_w = width - width // 2
+ face_x = best_cx
+
+ crop_x = face_x - crop_w // 2
+ # Clamp to stay within the chosen half
+ if best_cx < width // 2:
+ crop_x = max(0, min(crop_x, half_w - crop_w))
+ else:
+ crop_x = max(width // 2, min(crop_x, width - crop_w))
+ else:
+ face_x = int(np.median(face_positions))
+ crop_x = face_x - crop_w // 2
+ crop_x = max(0, min(crop_x, width - crop_w))
+
+ return str(crop_x)
+
+ except ImportError:
+ return None
+ except Exception as e:
+ print(f"Warning: face center detection failed: {e}", file=sys.stderr)
+ return None
+
+
def _detect_face_offset(
video_path: str,
width: int,
@@ -923,7 +1191,9 @@ def _detect_face_offset(
return str(crop_x)
# --- Smooth the positions ---
- # 1) Convert face_center_x to crop_x (clamped)
+ # 1) Convert face_center_x to crop_x (clamped with margin).
+ # Face should be in the center 70% of crop window, never at edge.
+ margin = int(crop_w * 0.15) # 15% margin on each side
tracked_times = np.array([t for t, _ in tracked])
tracked_crop_x = np.array([
max(0, min(cx - crop_w // 2, width - crop_w))
@@ -950,17 +1220,18 @@ def _detect_face_offset(
smoothed = np.clip(np.round(smoothed), 0, width - crop_w).astype(int)
# --- Check if tracking is even needed ---
- # If the face barely moves (range < 5% of frame width), just use static
+ # If the face barely moves (range < 8% of frame width), just use static.
+ # Most podcast clips have speakers sitting still — tracking should be rare.
movement_range = int(smoothed.max() - smoothed.min())
- if movement_range < width * 0.05:
+ if movement_range < width * 0.08:
return str(int(np.median(smoothed)))
- # --- Build keyframes: only emit when position changes significantly ---
- # Use conservative thresholds: only pan when the speaker truly shifts
- # position, not for natural head movement.
+ # --- Build keyframes: only emit on MAJOR position changes ---
+ # Very conservative: only pan when speaker physically moves to a new
+ # position (leans far, switches seats). Normal head movement = static.
keyframes = [(tracked_times[0], int(smoothed[0]))]
- min_time_gap = 2.0 # Don't place keyframes closer than 2s
- min_px_change = 50 # Ignore movement smaller than 50px (~5% of frame)
+ min_time_gap = 3.0 # Don't place keyframes closer than 3s
+ min_px_change = 80 # Ignore movement smaller than 80px (~7% of frame)
for i in range(1, len(smoothed)):
t = tracked_times[i]
diff --git a/src/ui/web-server.ts b/src/ui/web-server.ts
index 5efe116..260c32b 100644
--- a/src/ui/web-server.ts
+++ b/src/ui/web-server.ts
@@ -578,7 +578,7 @@ app.post("/api/batch-clips", async (req, res) => {
executor
.execute(
"batch_clips",
- { video_path, clips, transcript_words, output_dir: paths.output, logo_path, outro_path },
+ { video_path, clips, transcript_words, output_dir: paths.output, logo_path, outro_path, face_map: (uiState.transcript as any)?.face_map },
(event) => {
job.progress = event.percent;
job.message = event.message;
@@ -1489,7 +1489,7 @@ app.post("/api/mcp/export", async (req, res) => {
executor
.execute(
"batch_clips",
- { video_path: videoPath, clips: styledClips, transcript_words: transcriptWords, output_dir: paths.output, logo_path: logoPath, outro_path: outroPath },
+ { video_path: videoPath, clips: styledClips, transcript_words: transcriptWords, output_dir: paths.output, logo_path: logoPath, outro_path: outroPath, face_map: (uiState.transcript as any)?.face_map },
(event) => {
job.progress = event.percent;
job.message = event.message;