Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
feat: unified face tracking with YuNet, deep PodStack integration, fix camera flashing
Nika Siradzea4e10547522b
diff --git a/backend/main.py b/backend/main.pyindex 68f28fa..be43e42 100644--- a/backend/main.py+++ b/backend/main.py@@ -81,6 +81,7 @@ def handle_create_clip(task_id: str, params: dict):outro_path=params.get("outro_path"),clean_fillers=params.get("clean_fillers", True),face_map=params.get("face_map"),+ keep_segments=params.get("keep_segments"),progress_callback=lambda pct, msg: emit_progress(task_id, "processing", pct, msg),)emit_result(task_id, "success", data=result)@@ -115,6 +116,7 @@ def handle_batch_clips(task_id: str, params: dict):outro_path=params.get("outro_path"),clean_fillers=params.get("clean_fillers", True),face_map=params.get("face_map"),+ keep_segments=clip.get("keep_segments"),progress_callback=lambda pct, msg, _i=i: emit_progress(task_id, "batch", int((_i / total) * 100 + pct / total), msg),diff --git a/backend/models/face_detection_yunet_2023mar.onnx b/backend/models/face_detection_yunet_2023mar.onnxnew file mode 100644index 0000000..f9beb30Binary files /dev/null and b/backend/models/face_detection_yunet_2023mar.onnx differdiff --git a/backend/services/audio_analyzer.py b/backend/services/audio_analyzer.pyindex ae2afea..097376f 100644--- a/backend/services/audio_analyzer.py+++ b/backend/services/audio_analyzer.py@@ -21,16 +21,18 @@ def extract_audio_energy(Returns list of {time, rms, peak} dicts, one per window."""- # Use ffmpeg astats filter to get per-frame audio stats+ # Use ffmpeg astats filter on audio only (-vn skips video decoding,+ # which is the main bottleneck on long podcasts).cmd = ["ffmpeg","-i", video_path,+ "-vn","-af", f"astats=metadata=1:reset={int(1/window_sec)},"f"ametadata=print:key=lavfi.astats.Overall.RMS_level:file=-","-f", "null", "-",]- result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=600)# Parse the ametadata output from stdout# Format: "frame:N pts:N pts_time:N.N\nlavfi.astats.Overall.RMS_level=N.N"@@ -76,14 +78,15 @@ def _fallback_energy(video_path: str) -> list[dict]:except (json.JSONDecodeError, KeyError):return []- # Use ebur128 filter which reliably outputs loudness per second+ # Use ebur128 filter on audio only (-vn skips slow video decoding)cmd = ["ffmpeg","-i", video_path,+ "-vn","-af", "ebur128=peak=true","-f", "null", "-",]- result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=600)energy_data = []# Parse ebur128 output from stderr: "t: N.N M: -N.N S: -N.N"diff --git a/backend/services/claude_suggest.py b/backend/services/claude_suggest.pyindex 53f5419..3a13516 100644--- a/backend/services/claude_suggest.py+++ b/backend/services/claude_suggest.py@@ -37,6 +37,31 @@ def _find_claude() -> Optional[str]:return None+def _load_existing_shorts(episodes_path: str) -> list[str]:+ """Extract existing short titles from episode database to avoid duplicates."""+ if not os.path.exists(episodes_path):+ return []+ try:+ with open(episodes_path) as f:+ content = f.read()+ # Parse lines that look like shorts entries: "1. [title] — [category]"+ shorts = []+ for line in content.split("\n"):+ line = line.strip()+ if line and (line.startswith("1.") or line.startswith("2.") or+ line.startswith("3.") or line.startswith("4.") or+ line.startswith("5.") or line.startswith("6.") or+ line.startswith("7.") or line.startswith("8.") or+ line.startswith("9.")):+ # Extract title between brackets or after number+ title = line.split("—")[0].strip().lstrip("0123456789.").strip()+ if title and title != "[Short title]":+ shorts.append(title)+ return shorts+ except Exception:+ return []++def _build_prompt(transcript_text: str, segment_count: int, duration_min: float, top_n: int) -> str:"""Build the prompt for Claude to extract clips.@@ -44,22 +69,40 @@ def _build_prompt(transcript_text: str, segment_count: int, duration_min: float,can't read project files."""- # Try to load knowledge base files inline+ # Load knowledge base files inline — prioritized by relevance to clip selectionkb_context = ""kb_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", ".podcli", "knowledge")- for fname in ["04-shorts-creation-guide.md", "05-title-formulas.md", "02-voice-and-tone.md"]:+ # (filename, max_chars) — higher priority files get more budget+ _kb_files = [+ ("04-shorts-creation-guide.md", 4000), # moment selection criteria, content types+ ("05-title-formulas.md", 3000), # title rules, shapes, banned openers+ ("02-voice-and-tone.md", 3000), # banned words, voice fingerprint, coffee test+ ("01-brand-identity.md", 1500), # show context, positioning, audience+ ("11-inspiration-channels.md", 2000), # viral hook patterns, reference styles+ ("12-quick-reference.md", 2000), # hook bank, title formulas, hashtags+ ("08-topics-themes.md", 1000), # topic areas, audience interest map+ ("00-master-instructions.md", 1500), # quality gate, auto-detection rules+ ]+ for fname, max_chars in _kb_files:fpath = os.path.join(kb_dir, fname)if os.path.exists(fpath):try:with open(fpath) as f:- content = f.read()- kb_context += f"\n--- {fname} ---\n{content[:2000]}\n"+ content = f.read().strip()+ # Skip template-only files (uncustomized placeholders)+ if content.count("[Your Show Name]") > 2 and len(content) < 500:+ continue+ kb_context += f"\n--- {fname} ---\n{content[:max_chars]}\n"except Exception:pass- return f"""You are the clip extraction engine for a podcast. Analyze this transcript and return the {top_n} best moments for YouTube Shorts.+ # Load existing shorts from episode database for duplicate avoidance+ episodes_path = os.path.join(kb_dir, "03-episodes-database.md")+ existing_shorts = _load_existing_shorts(episodes_path)++ return f"""You are a viral clip editor for TikTok and YouTube Shorts. Find the {top_n} most scroll-stopping moments in this podcast transcript.IMPORTANT: Return ONLY valid JSON. No markdown, no explanation, no code fences.@@ -67,30 +110,34 @@ TIMESTAMP FORMAT: All timestamps in the transcript are in SECONDS (e.g., [123.4sAll timestamps you return MUST be in SECONDS as numbers (e.g., 123.4), NOT minutes:seconds.DURATION RULES (CRITICAL):-- Target: 30-45 seconds per clip (ideal for YouTube Shorts)-- Maximum: 60 seconds (hard limit — YouTube Shorts cuts off at 60s)-- Minimum: 20 seconds (too short = no value)-- If a moment runs longer than 50 seconds, find a tighter cut point+- Target: 25-40 seconds (this is the viral sweet spot)+- Maximum: 50 seconds (absolute hard limit — anything longer loses viewers)+- Minimum: 15 seconds (too short = no payoff)+- SHORTER IS BETTER. A punchy 25s clip outperforms a 50s clip every time.+- If a thought takes longer than 40s, use segments to cut the filler in the middleCUTTING RULES (CRITICAL):- Cut TIGHT. Every second must earn its place.- Start at the exact moment the hook hits — no preamble, no "so", no "well"-- End the INSTANT the point lands — don't let it trail off-- If the speaker rambles before making a point, start AFTER the ramble-- If there's filler/tangent in the middle, pick the tighter segment that preserves the core idea-- The transcript timestamps are your scalpel — use them precisely+- End the MOMENT the point lands with a complete thought — don't trail off+- NEVER cut mid-sentence or mid-thought. The viewer must feel closure.+- The last sentence must feel like a natural ending, a punchline, or a mic-drop+- If there's filler/tangent in the middle, use multiple segments to skip it- A 30s clip with zero dead weight beats a 50s clip with 20s of fluff-MOMENT SELECTION:-- Must start with a strong hook (first 3 seconds grab attention)+MOMENT SELECTION (think like a TikTok editor):+- Would YOU stop scrolling for this? If no, skip it.+- First 3 seconds must HOOK — a bold claim, shocking number, or provocative question- Must make complete sense standalone — no "as I mentioned" or "going back to"-- Must end cleanly on a sentence boundary (not mid-thought)-- Single focused idea per clip — one concept, fully delivered-- Prioritize: surprising facts, bold claims, founder stories, counterintuitive insights, "aha" moments-- Variety across content types+- Must end on a COMPLETE THOUGHT — sentence boundary, natural pause, or mic-drop moment+- Single focused idea — one concept, fully delivered, no loose threads+- Prioritize: controversial takes, surprising numbers, founder war stories, "wait what?" moments, emotional peaks+- Skip: generic advice, obvious statements, context-dependent references{f"KNOWLEDGE BASE:{kb_context}" if kb_context else ""}+{f"EXISTING SHORTS (avoid duplicating these moments):{chr(10).join('- ' + s for s in existing_shorts)}" if existing_shorts else ""}+Score each moment on 4 dimensions (1-5 each):- standalone: Makes sense without episode context?- hook: Grabs attention in first 3 seconds?@@ -131,8 +178,9 @@ SEGMENTS RULES:- Example: speaker makes great point (10s), rambles (8s), delivers punchline (12s) → 2 segments, 22s totalRules:-- Final clip duration (sum of segments) MUST be 20-60 seconds (target 30-45s)-- Each segment must start and end on sentence boundaries+- Final clip duration (sum of segments) MUST be 15-50 seconds (target 25-40s)+- Each segment must start and end on COMPLETE SENTENCES — never mid-thought+- The LAST segment must end on a sentence that feels like a natural conclusion- Must make sense standalone when stitched together- Sort clips by timestamp order@@ -273,8 +321,8 @@ def suggest_with_claude(# Compute actual kept durationkept_duration = sum(seg["end"] - seg["start"] for seg in keep_segments)- # Reject clips shorter than 15 seconds (Claude returned bad timestamps)- if kept_duration < 15:+ # Reject clips outside the target duration range+ if kept_duration < 15 or kept_duration > 55:continuenormalized.append({@@ -285,6 +333,11 @@ def suggest_with_claude("duration": round(kept_duration),"score": total,"content_type": c.get("content_type", "unknown"),+ # MCP-aligned fields+ "reasoning": c.get("why", ""),+ "preview_text": c.get("quote", "")[:120],+ "suggested_caption_style": "hormozi",+ # Backwards compat"quote": c.get("quote", ""),"why": c.get("why", ""),"reasons": [c.get("content_type", "")],diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.pyindex 55fef71..123eda8 100644--- a/backend/services/clip_generator.py+++ b/backend/services/clip_generator.py@@ -95,7 +95,7 @@ def _build_tight_segments(words: list[dict],start_second: float,end_second: float,- silence_threshold: float = 1.5,+ silence_threshold: float = 0.8,) -> list[dict]:"""Build tight keep-segments from transcript words, cutting out:@@ -152,7 +152,7 @@ def _build_tight_segments(kept_duration = sum(s["end"] - s["start"] for s in segments)saved = original_duration - kept_duration- if saved < 1.5 or len(segments) < 2:+ if saved < 0.8 or len(segments) < 2:# Not worth the cuts — too little savedreturn [{"start": start_second, "end": end_second}]@@ -212,6 +212,16 @@ def generate_clip(keep_segments.sort(key=lambda s: s["start"])start_second = keep_segments[0]["start"]end_second = keep_segments[-1]["end"]++ # If Claude returned a single segment, still auto-trim pauses within it.+ # Multiple segments means Claude made deliberate editorial cuts — trust those.+ if len(keep_segments) == 1 and transcript_words and clean_fillers:+ auto_segments = _build_tight_segments(+ transcript_words, start_second, end_second,+ )+ if len(auto_segments) > 1:+ keep_segments = auto_segments+duration = sum(s["end"] - s["start"] for s in keep_segments)else:# Snap clip boundaries to sentence endings@@ -233,8 +243,8 @@ def generate_clip(keep_segments = Noneduration = end_second - start_second- if duration > 180:- raise ValueError(f"Clip too long ({duration:.0f}s). Max 180 seconds for shorts.")+ if duration > 60:+ raise ValueError(f"Clip too long ({duration:.0f}s). Max 60 seconds for shorts.")# Load style config for branded-specific settingsstyle_config = get_style(caption_style)@@ -284,7 +294,11 @@ def generate_clip(crop_clip_start = 0caption_time_offset = 0else:- crop_words = transcript_words+ # Filter words to just this clip's time range+ crop_words = [+ w for w in transcript_words+ if w["end"] > start_second and w["start"] < end_second+ ] if transcript_words else transcript_wordscrop_clip_start = start_secondcaption_time_offset = start_seconddiff --git a/backend/services/face_analysis.py b/backend/services/face_analysis.pyindex b15c5f8..78a4237 100644--- a/backend/services/face_analysis.py+++ b/backend/services/face_analysis.py@@ -43,6 +43,7 @@ def analyze_faces(try:import cv2import numpy as np+ from services.face_detector import create_detector, detect_facesexcept ImportError:print("Warning: OpenCV not available, skipping face analysis", file=sys.stderr)return None@@ -58,18 +59,13 @@ def analyze_faces(height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))fps = cap.get(cv2.CAP_PROP_FPS) or 30- # Load DNN face detector- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))- proto = os.path.join(backend_dir, "models", "deploy.prototxt")- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")-- if not (os.path.exists(proto) and os.path.exists(model)):+ # Create YuNet detector+ detector = create_detector(width, height)+ if detector is None:cap.release()print("Warning: Face detection model not found, skipping", file=sys.stderr)return None- detector = cv2.dnn.readNetFromCaffe(proto, model)-# Sample ~2 frames per second across the full video (enough for analysis)sample_count = min(300, max(20, int(duration * 2)))observations = []@@ -85,35 +81,18 @@ def analyze_faces(if not ret:continue- h, w = frame.shape[:2]- blob = cv2.dnn.blobFromImage(- cv2.resize(frame, (300, 300)), 1.0, (300, 300),- (104.0, 177.0, 123.0)- )- detector.setInput(blob)- detections = detector.forward()+ faces = detect_faces(detector, frame, width, height)frame_faces = 0- for j in range(detections.shape[2]):- conf = detections[0, 0, j, 2]- if conf > 0.5:- x1 = int(detections[0, 0, j, 3] * w)- x2 = int(detections[0, 0, j, 5] * w)- y1 = int(detections[0, 0, j, 4] * h)- y2 = int(detections[0, 0, j, 6] * h)- fw = x2 - x1- if fw < w * 0.04:- continue- cx = (x1 + x2) // 2- cy = (y1 + y2) // 2- observations.append({- "time": round(t, 3),- "face_center_x": cx,- "face_center_y": cy,- "face_width": fw,- "confidence": round(float(conf), 3),- })- frame_faces += 1+ for f in faces:+ observations.append({+ "time": round(t, 3),+ "face_center_x": f["cx"],+ "face_center_y": f["cy"],+ "face_width": f["fw"],+ "confidence": f["confidence"],+ })+ frame_faces += 1faces_per_frame.append(frame_faces)if progress_callback and i % 20 == 0:diff --git a/backend/services/face_detector.py b/backend/services/face_detector.pynew file mode 100644index 0000000..3b0bc38--- /dev/null+++ b/backend/services/face_detector.py@@ -0,0 +1,109 @@+"""+Shared face detector using OpenCV YuNet.++YuNet is a lightweight CNN face detector (227KB) built into OpenCV.+Much faster and more accurate than the old ResNet10 SSD, especially+on side profiles and partially occluded faces.++All face detection in the codebase should go through create_detector()+and detect_faces() to keep the model path and parameters in one place.+"""++import os+from typing import Optional++# Minimum face width as fraction of frame width (filters noise)+MIN_FACE_RATIO = 0.04++# Detection confidence threshold (0.5 balances accuracy vs recall;+# source 16:9 video scores higher than processed 9:16 clips)+CONFIDENCE_THRESHOLD = 0.5++# Max dimension for detection input — keeps inference fast.+# Coordinates are scaled back to original frame size.+_MAX_DIM = 640+++def _model_path() -> str:+ backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+ return os.path.join(backend_dir, "models", "face_detection_yunet_2023mar.onnx")+++def create_detector(frame_width: int, frame_height: int):+ """+ Create a YuNet face detector for the given frame dimensions.++ Internally uses a scaled-down input size for speed, but detect_faces()+ returns coordinates in the original frame space.++ Returns (detector, scale_x, scale_y) tuple, or None if model not found.+ """+ import cv2++ model = _model_path()+ if not os.path.exists(model):+ return None++ # Scale down to _MAX_DIM while preserving aspect ratio+ scale = min(_MAX_DIM / frame_width, _MAX_DIM / frame_height, 1.0)+ det_w = int(frame_width * scale)+ det_h = int(frame_height * scale)++ detector = cv2.FaceDetectorYN.create(+ model=model,+ config="",+ input_size=(det_w, det_h),+ score_threshold=CONFIDENCE_THRESHOLD,+ nms_threshold=0.3,+ top_k=10,+ )+ return (detector, det_w, det_h, frame_width / det_w, frame_height / det_h)+++def detect_faces(detector_tuple, frame, frame_width: int, frame_height: int) -> list[dict]:+ """+ Detect faces in a single frame using YuNet.++ detector_tuple: the return value of create_detector()+ Returns list of dicts: [{cx, cy, fw, fh, confidence}, ...]+ Coordinates are in original frame space.+ """+ import cv2++ detector, det_w, det_h, scale_x, scale_y = detector_tuple++ resized = cv2.resize(frame, (det_w, det_h))+ _, faces = detector.detect(resized)++ results = []+ if faces is None:+ return results++ for face in faces:+ # Scale bbox back to original frame coordinates+ x = int(face[0] * scale_x)+ y = int(face[1] * scale_y)+ w = int(face[2] * scale_x)+ h = int(face[3] * scale_y)+ conf = float(face[-1])++ if w < frame_width * MIN_FACE_RATIO:+ continue++ cx = x + w // 2+ cy = y + h // 2++ results.append({+ "cx": cx,+ "cy": cy,+ "fw": w,+ "fh": h,+ "confidence": round(conf, 3),+ })++ return results+++def is_available() -> bool:+ """Check if YuNet model file exists."""+ return os.path.exists(_model_path())diff --git a/backend/services/thumbnail_ai.py b/backend/services/thumbnail_ai.pyindex 5c90ed0..4ec0139 100644--- a/backend/services/thumbnail_ai.py+++ b/backend/services/thumbnail_ai.py@@ -158,15 +158,13 @@ def extract_candidate_frames(is_portrait = src_h > src_w # Already 9:16 or similar# Load face detector- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))- proto = os.path.join(backend_dir, "models", "deploy.prototxt")- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")+ from services.face_detector import create_detector, detect_faces+ detector = create_detector(src_w, src_h)- if not (os.path.exists(proto) and os.path.exists(model)):+ if detector is None:cap.release()return []- detector = cv2.dnn.readNetFromCaffe(proto, model)os.makedirs(output_dir, exist_ok=True)# Sample more frames for better candidates@@ -183,22 +181,19 @@ def extract_candidate_frames(continueh, w = frame.shape[:2]- blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))- detector.setInput(blob)- detections = detector.forward()-- for j in range(detections.shape[2]):- conf = detections[0, 0, j, 2]- if conf > 0.6:- x1 = max(0, int(detections[0, 0, j, 3] * w))- y1 = max(0, int(detections[0, 0, j, 4] * h))- x2 = min(w, int(detections[0, 0, j, 5] * w))- y2 = min(h, int(detections[0, 0, j, 6] * h))- fw = x2 - x1- fh = y2 - y1-- if fw < w * 0.08:- continue+ faces_detected = detect_faces(detector, frame, src_w, src_h)++ for f in faces_detected:+ conf = f["confidence"]+ x1 = max(0, f["cx"] - f["fw"] // 2)+ y1 = max(0, f["cy"] - f["fh"] // 2)+ x2 = min(w, f["cx"] + f["fw"] // 2)+ y2 = min(h, f["cy"] + f["fh"] // 2)+ fw = f["fw"]+ fh = f["fh"]++ if fw < w * 0.08:+ continue# Split-screen check — only for landscape sources where# a small centered face likely means a multi-cam layout.diff --git a/backend/services/thumbnail_generator.py b/backend/services/thumbnail_generator.pyindex 891d18b..08523f9 100644--- a/backend/services/thumbnail_generator.py+++ b/backend/services/thumbnail_generator.py@@ -251,16 +251,15 @@ def extract_face_frame(duration = total_frames / fps# Load face detector- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))- proto = os.path.join(backend_dir, "models", "deploy.prototxt")- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")+ from services.face_detector import create_detector, detect_faces+ src_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))+ src_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))+ detector = create_detector(src_w, src_h)- if not (os.path.exists(proto) and os.path.exists(model)):+ if detector is None:cap.release()return None- detector = cv2.dnn.readNetFromCaffe(proto, model)-best_frame = Nonebest_score = 0best_face_cx = None@@ -280,32 +279,25 @@ def extract_face_frame(continueh, w = frame.shape[:2]- blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))- detector.setInput(blob)- detections = detector.forward()-- for j in range(detections.shape[2]):- conf = detections[0, 0, j, 2]- if conf > 0.6:- x1 = int(detections[0, 0, j, 3] * w)- y1 = int(detections[0, 0, j, 4] * h)- x2 = int(detections[0, 0, j, 5] * w)- y2 = int(detections[0, 0, j, 6] * h)- face_w = x2 - x1- face_h = y2 - y1-- if face_w < w * 0.08:- continue-- # Score: confidence × face size (prefer larger/closer faces)- score = conf * (face_w * face_h)- if score > best_score:- best_score = score- best_frame = frame.copy()- best_face_cx = (x1 + x2) // 2- best_face_cy = (y1 + y2) // 2- best_face_w = face_w- best_face_h = face_h+ faces_detected = detect_faces(detector, frame, src_w, src_h)++ for f in faces_detected:+ conf = f["confidence"]+ face_w = f["fw"]+ face_h = f["fh"]++ if face_w < w * 0.08:+ continue++ # Score: confidence × face size (prefer larger/closer faces)+ score = conf * (face_w * face_h)+ if score > best_score:+ best_score = score+ best_frame = frame.copy()+ best_face_cx = f["cx"]+ best_face_cy = f["cy"]+ best_face_w = face_w+ best_face_h = face_hcap.release()diff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex e4c586f..81bbdf6 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -13,6 +13,9 @@ from typing import Optionalfrom services.encoder import get_video_encode_flagsimport sys+# Max time for any single FFmpeg call (seconds). Prevents infinite hangs.+_FFMPEG_TIMEOUT = 300+# Quality presets: name → (crf, preset)# Lower CRF = higher quality = larger file. 18 is visually lossless.QUALITY_PRESETS = {@@ -35,7 +38,7 @@ def _run_ffmpeg_with_fallback(cmd_parts_before_enc: list, cmd_parts_after_enc: lenc_flags = get_video_encode_flags()cmd = cmd_parts_before_enc + enc_flags + cmd_parts_after_enc + [output_path]- result = subprocess.run(cmd, capture_output=True, text=True)+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if result.returncode == 0:return output_path@@ -43,7 +46,7 @@ def _run_ffmpeg_with_fallback(cmd_parts_before_enc: list, cmd_parts_after_enc: lif enc_flags != CPU_FLAGS:print(f"Warning: HW encoder failed for {label}, falling back to libx264", file=sys.stderr)cmd_fallback = cmd_parts_before_enc + CPU_FLAGS + cmd_parts_after_enc + [output_path]- result2 = subprocess.run(cmd_fallback, capture_output=True, text=True)+ result2 = subprocess.run(cmd_fallback, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if result2.returncode == 0:return output_pathraise RuntimeError(f"FFmpeg {label} failed (both HW and CPU): {result2.stderr[-500:]}")@@ -51,6 +54,15 @@ def _run_ffmpeg_with_fallback(cmd_parts_before_enc: list, cmd_parts_after_enc: lraise RuntimeError(f"FFmpeg {label} failed: {result.stderr[-500:]}")+def _has_audio_stream(video_path: str) -> bool:+ """Check if a video file contains an audio stream."""+ try:+ info = get_video_info(video_path)+ return any(s.get("codec_type") == "audio" for s in info.get("streams", []))+ except Exception:+ return True # Assume yes if we can't check++def get_video_info(video_path: str) -> dict:"""Get video metadata via ffprobe."""cmd = [@@ -61,7 +73,7 @@ def get_video_info(video_path: str) -> dict:"-show_streams",video_path,]- result = subprocess.run(cmd, capture_output=True, text=True)+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if result.returncode != 0:raise RuntimeError(f"ffprobe failed: {result.stderr}")return json.loads(result.stdout)@@ -98,7 +110,7 @@ def cut_segment("-avoid_negative_ts", "make_zero",output_path,]- result = subprocess.run(cmd, capture_output=True, text=True)+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if result.returncode != 0:raise RuntimeError(f"FFmpeg cut failed: {result.stderr[-500:]}")return output_path@@ -146,7 +158,7 @@ def cut_multi_segment("-movflags", "+faststart",output_path,]- result = subprocess.run(cmd, capture_output=True, text=True)+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if result.returncode != 0:raise RuntimeError(f"FFmpeg concat failed: {result.stderr[-500:]}")@@ -190,38 +202,42 @@ def crop_to_vertical(source_ratio = width / heightif strategy in ("face", "speaker"):- # Split-screen: cut each speaker segment separately with static crop per half.- # Check face_map first (computed from full video), fall back to segment detection.- is_split = False- if face_map and face_map.get("is_split_screen"):- is_split = True- else:- is_split = _detect_split_screen(input_path, width, height)-- if is_split and transcript_words:- result = _crop_split_screen(- input_path, output_path,- width, height, target_w, target_h,- transcript_words, clip_start,- )- if result:- return result-- # Non-split-screen: single static face crop- crop_x = _detect_face_center(input_path, width, height, target_ratio)-- if crop_x is not None:- crop_w = int(height * target_ratio)- vf = f"crop={crop_w}:{height}:{crop_x}:0,scale={target_w}:{target_h}"- else:- strategy = "center"+ result = _track_and_crop(+ input_path, output_path,+ width, height, target_w, target_h,+ transcript_words, clip_start,+ )+ if result:+ return result+ strategy = "center"if strategy == "center":if source_ratio > target_ratio:- crop_h = height- crop_w = int(crop_h * target_ratio)- crop_x = (width - crop_w) // 2- vf = f"crop={crop_w}:{crop_h}:{crop_x}:0,scale={target_w}:{target_h}"+ # Wide source with no face detected: blurred background + sharp center.+ # Scales source to fill 9:16 height → blur → overlay sharp fit-to-width.+ vf_complex = (+ f"split[bg][fg];"+ f"[bg]scale=-2:{target_h},crop={target_w}:{target_h}:(iw-{target_w})/2:0,"+ f"boxblur=25:3[bbg];"+ f"[fg]scale={target_w}:-2[sfg];"+ f"[bbg][sfg]overlay=0:(H-h)/2[v]"+ )+ return _run_ffmpeg_with_fallback(+ cmd_parts_before_enc=[+ "ffmpeg", "-y",+ "-i", input_path,+ "-filter_complex", vf_complex,+ "-map", "[v]", "-map", "0:a?",+ ],+ cmd_parts_after_enc=[+ "-c:a", "aac",+ "-b:a", "192k",+ "-ar", "44100",+ "-movflags", "+faststart",+ ],+ output_path=output_path,+ label="crop_blur_bg",+ )else:crop_w = widthcrop_h = int(crop_w / target_ratio)@@ -252,7 +268,7 @@ def _detect_split_screen(video_path: str, width: int, height: int) -> bool:"""Quick check: is this a split-screen layout (two side-by-side cameras)?"""try:import cv2- import numpy as np+ from services.face_detector import create_detector, detect_facescap = cv2.VideoCapture(video_path)if not cap.isOpened():@@ -262,14 +278,11 @@ def _detect_split_screen(video_path: str, width: int, height: int) -> bool:total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))duration = total / fps- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))- proto = os.path.join(backend_dir, "models", "deploy.prototxt")- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")- if not (os.path.exists(proto) and os.path.exists(model)):+ detector = create_detector(width, height)+ if detector is None:cap.release()return False- detector = cv2.dnn.readNetFromCaffe(proto, model)mid_x = width // 2frames_with_two_faces = 0@@ -282,20 +295,15 @@ def _detect_split_screen(video_path: str, width: int, height: int) -> bool:if not ret:continue- h, w = frame.shape[:2]- blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))- detector.setInput(blob)- dets = detector.forward()+ faces = detect_faces(detector, frame, width, height)left = Falseright = False- for j in range(dets.shape[2]):- if dets[0, 0, j, 2] > 0.5:- cx = int((dets[0, 0, j, 3] + dets[0, 0, j, 5]) / 2 * w)- if cx < mid_x:- left = True- else:- right = True+ for f in faces:+ if f["cx"] < mid_x:+ left = True+ else:+ right = Truetotal_sampled += 1if left and right:@@ -308,6 +316,336 @@ def _detect_split_screen(video_path: str, width: int, height: int) -> bool:return False+def _build_cam_expr(keyframes: list, duration: float, is_split: bool, max_parts: int = 80):+ """+ Build an FFmpeg crop_x expression from SmoothedCameraman keyframes.++ keyframes: [(time, crop_x), ...]+ Returns expression string, or None if too complex.+ """+ if not keyframes:+ return None+ # Ensure t=0 covered+ if keyframes[0][0] > 0.05:+ keyframes = [(0, keyframes[0][1])] + keyframes+ if len(keyframes) == 1:+ return str(keyframes[0][1])++ parts = []+ for i in range(len(keyframes) - 1):+ t0, x0 = keyframes[i]+ t1, x1 = keyframes[i + 1]+ dt = max(0.01, t1 - t0)+ jump = abs(x1 - x0)++ if jump < 2:+ # Negligible movement — hold+ parts.append(f"if(between(t\\,{t0:.3f}\\,{t1:.3f})\\,{x0}\\,")+ elif jump > 150 or (is_split and jump > 50):+ # Large jump / split-screen speaker switch: instant cut+ cut_t = round(t1 - 0.01, 3)+ parts.append(f"if(between(t\\,{t0:.3f}\\,{cut_t:.3f})\\,{x0}\\,")+ parts.append(f"if(between(t\\,{cut_t:.3f}\\,{t1:.3f})\\,{x1}\\,")+ else:+ # Smooth linear interpolation (camera pan)+ parts.append(+ f"if(between(t\\,{t0:.3f}\\,{t1:.3f})\\,"+ f"{x0}+(({x1}-{x0})*(t-{t0:.3f})/{dt:.3f})\\,"+ )++ if len(parts) > max_parts:+ return None++ return "".join(parts) + str(keyframes[-1][1]) + ")" * len(parts)+++def _simplify_keyframes(keyframes: list, tolerance: int = 5) -> list:+ """Remove intermediate keyframes that lie on a line between neighbours."""+ if len(keyframes) <= 2:+ return keyframes+ result = [keyframes[0]]+ for i in range(1, len(keyframes) - 1):+ t_prev, x_prev = result[-1]+ t_curr, x_curr = keyframes[i]+ t_next, x_next = keyframes[i + 1]+ dt_total = t_next - t_prev+ if dt_total < 0.01:+ continue+ expected = x_prev + (x_next - x_prev) * (t_curr - t_prev) / dt_total+ if abs(x_curr - expected) > tolerance:+ result.append(keyframes[i])+ result.append(keyframes[-1])+ return result+++def _track_and_crop(+ input_path: str,+ output_path: str,+ width: int,+ height: int,+ target_w: int,+ target_h: int,+ transcript_words: list = None,+ clip_start: float = 0,+) -> Optional[str]:+ """+ Adaptive face tracking with exponential-smoothing camera.++ Works for both split-screen and single-camera layouts:+ 1. Dense face sampling (~10 fps) with YuNet+ 2. Transcript-driven speaker selection (who's talking → which face)+ 3. Exponential-smoothing camera with dead zone (no jitter)+ 4. Hard cuts only for split-screen speaker switches+ 5. Vertical position from median face-Y (static per clip)+ """+ try:+ import cv2+ from services.face_detector import create_detector, detect_faces+ except ImportError:+ return None++ target_ratio = target_w / target_h # 0.5625++ cap = cv2.VideoCapture(input_path)+ if not cap.isOpened():+ return None++ fps = cap.get(cv2.CAP_PROP_FPS) or 30+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))+ duration = total_frames / fps if total_frames > 0 else 0++ if duration < 0.5:+ cap.release()+ return None++ detector = create_detector(width, height)+ if detector is None:+ cap.release()+ return None++ # ── Dense face sampling (~10 fps) ────────────────────────────+ sample_step = max(1, int(fps / 10))+ detections = [] # [(time, faces), ...]++ frame_idx = 0+ while frame_idx < total_frames:+ cap.set(cv2.CAP_PROP_POS_FRAMES, frame_idx)+ ret, frame = cap.read()+ if not ret:+ frame_idx += sample_step+ continue+ t = frame_idx / fps+ faces = detect_faces(detector, frame, width, height)+ detections.append((t, faces))+ frame_idx += sample_step++ cap.release()++ if not detections:+ return None++ face_frames = sum(1 for _, faces in detections if faces)+ if face_frames < 3:+ return None++ # ── Detect layout from face data ────────────────────────────+ mid_x = width // 2+ split_count = sum(1 for _, faces in detections if len(faces) >= 2)+ has_any_split = split_count >= 3 # enough multi-face frames to build side mapping++ # ── Crop dimensions ──────────────────────────────────────────+ # Full-height crop: the 16:9 → 9:16 aspect conversion already+ # gives ~3× horizontal zoom. Adding vertical zoom on top clips+ # faces on laptop-angle cameras and shows ceiling. Keep it+ # simple: crop the full height, only pan horizontally.+ crop_h = height+ crop_w = int(crop_h * target_ratio)+ if crop_w > width:+ crop_w = width+ crop_h = int(crop_w / target_ratio)++ # ── Speaker segments from transcript ─────────────────────────+ segments = [] # [(start, end, speaker), ...]+ if transcript_words:+ cur_sp = None+ seg_start = 0.0+ for w in sorted(transcript_words, key=lambda x: x["start"]):+ sp = w.get("speaker")+ t = max(0.0, w["start"] - clip_start)+ if sp != cur_sp and sp is not None:+ if cur_sp is not None:+ segments.append((seg_start, t, cur_sp))+ cur_sp = sp+ seg_start = t+ if cur_sp:+ segments.append((seg_start, duration, cur_sp))+ # Merge short segments (<2s) — brief interjections ("yeah",+ # "right") shouldn't trigger camera moves. Absorb them into+ # the previous speaker's turn so the camera holds steady.+ merged = []+ for seg in segments:+ if merged and seg[2] == merged[-1][2]:+ merged[-1] = (merged[-1][0], seg[1], seg[2])+ elif merged and (seg[1] - seg[0]) < 2.0:+ merged[-1] = (merged[-1][0], seg[1], merged[-1][2])+ else:+ merged.append(list(seg))+ segments = merged++ speakers = sorted(set(seg[2] for seg in segments)) if segments else []++ def _speaker_at(t):+ for s, e, sp in segments:+ if s <= t <= e:+ return sp+ return segments[0][2] if segments else None++ # ── Speaker-to-side mapping ─────────────────────────────────+ # Built from ANY frames with 2+ faces — works for both+ # pure split-screen and Riverside-style mixed layouts.+ speaker_side = {} # speaker → "left" | "right"+ if has_any_split and len(speakers) >= 2:+ talk = {}+ first = {}+ for seg in segments:+ sp = seg[2]+ talk[sp] = talk.get(sp, 0) + (seg[1] - seg[0])+ if sp not in first:+ first[sp] = seg[0]+ top2 = sorted(speakers, key=lambda s: talk.get(s, 0), reverse=True)[:2]+ by_first = sorted(top2, key=lambda s: first.get(s, float("inf")))+ speaker_side[by_first[0]] = "left"+ if len(by_first) > 1:+ speaker_side[by_first[1]] = "right"+ for sp in speakers:+ if sp not in speaker_side:+ speaker_side[sp] = "left"++ # ── Face selection per frame ─────────────────────────────────+ # Per-frame layout detection: if THIS frame has 2+ faces,+ # pick the one on the active speaker's side. Handles Riverside+ # videos that switch between single-camera and split-screen.+ def _pick_face(faces, speaker):+ if not faces:+ return None+ if len(faces) >= 2 and speaker in speaker_side:+ side = speaker_side[speaker]+ return (min if side == "left" else max)(faces, key=lambda f: f["cx"])+ # Single face or no side mapping: use largest (most prominent)+ return max(faces, key=lambda f: f["fw"])++ # ── Exponential-smoothing camera ────────────────────────────+ # The camera "chases" the target face using exponential decay,+ # which gives natural ease-out motion (fast start, gradual+ # settling). Unlike fixed-speed approaches this is frame-rate+ # independent and feels organic.+ #+ # dead_zone: ignore face drift within this radius of the+ # current camera center — prevents micro-jitter+ # smooth_rate: convergence speed; at rate 4 the camera+ # reaches ~95 % of the target in ≈0.75 s+ # snap_distance: teleport instantly above this (speaker change+ # or layout switch — no sweeping across the seam)+ import math++ dead_zone = crop_w * 0.20+ smooth_rate = 4.0+ snap_distance = crop_w * 0.55++ # Initialise camera on the first detected face+ cam_x = float(width) / 2+ for _, faces in detections:+ if faces:+ cam_x = float(max(faces, key=lambda f: f["fw"])["cx"])+ break++ keyframes_x = []+ prev_t = 0.0+ prev_speaker = None+ last_snap_t = -10.0 # time of last force_snap (start with no cooldown)+ snap_cooldown = 2.0 # minimum seconds between force_snaps++ for t, faces in detections:+ speaker = _speaker_at(t) if segments else None+ face = _pick_face(faces, speaker)++ if face is None:+ prev_t = t+ continue++ target_x = float(face["cx"])+ diff = target_x - cam_x+ dt = max(0.01, t - prev_t)++ # Hard cut when the active speaker switches to a different+ # side. Works for pure split-screen AND mixed layouts+ # (Riverside-style single↔split transitions).+ # Cooldown prevents rapid ping-pong from brief interjections+ # that survive segment merging.+ force_snap = (+ speaker is not None+ and speaker != prev_speaker+ and prev_speaker is not None+ and speaker_side.get(speaker) != speaker_side.get(prev_speaker)+ and (t - last_snap_t) >= snap_cooldown+ )++ if force_snap or abs(diff) > snap_distance:+ # Teleport — speaker switch or very large jump+ cam_x = target_x+ if force_snap:+ last_snap_t = t+ elif abs(diff) > dead_zone:+ # Exponential smoothing: move a fraction of remaining+ # distance each step. alpha → 1 means instant, → 0 means+ # no movement. exp(-rate * dt) makes it time-step safe.+ alpha = 1.0 - math.exp(-smooth_rate * dt)+ cam_x += diff * alpha+ # else: face inside dead zone — camera holds position++ crop_x = int(cam_x - crop_w / 2)+ crop_x = max(0, min(crop_x, width - crop_w))++ if not keyframes_x or crop_x != keyframes_x[-1][1]:+ keyframes_x.append((round(t, 3), crop_x))++ prev_t = t+ if speaker is not None:+ prev_speaker = speaker++ # ── Vertical position ────────────────────────────────────────+ # Full-height crop → crop_y is always 0 (or centered if+ # crop_h < height due to aspect-ratio clamping).+ crop_y = max(0, (height - crop_h) // 2)++ # ── Build FFmpeg filter ──────────────────────────────────────+ if not keyframes_x:+ crop_x = max(0, (width - crop_w) // 2)+ vf = f"crop={crop_w}:{crop_h}:{crop_x}:{crop_y},scale={target_w}:{target_h}"+ elif len(keyframes_x) == 1:+ vf = f"crop={crop_w}:{crop_h}:{keyframes_x[0][1]}:{crop_y},scale={target_w}:{target_h}"+ else:+ # Simplify keyframes to reduce expression complexity+ keyframes_x = _simplify_keyframes(keyframes_x)+ x_expr = _build_cam_expr(keyframes_x, duration, has_any_split)+ if x_expr is None:+ # Too complex — fall back to median position+ xs = [x for _, x in keyframes_x]+ med_x = sorted(xs)[len(xs) // 2]+ vf = f"crop={crop_w}:{crop_h}:{med_x}:{crop_y},scale={target_w}:{target_h}"+ else:+ vf = f"crop={crop_w}:{crop_h}:{x_expr}:{crop_y},scale={target_w}:{target_h}"++ return _run_ffmpeg_with_fallback(+ cmd_parts_before_enc=["ffmpeg", "-y", "-i", input_path, "-vf", vf],+ cmd_parts_after_enc=[+ "-c:a", "aac", "-b:a", "192k", "-ar", "44100",+ "-movflags", "+faststart",+ ],+ output_path=output_path, label="crop_track",+ )++def _crop_split_screen(input_path: str,output_path: str,@@ -328,6 +666,7 @@ def _crop_split_screen(try:import cv2import numpy as np+ from services.face_detector import create_detector, detect_facestarget_ratio = target_w / target_hmid_x = width // 2@@ -341,20 +680,23 @@ def _crop_split_screen(total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))duration = total_frames / fps- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))- proto = os.path.join(backend_dir, "models", "deploy.prototxt")- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")- if not (os.path.exists(proto) and os.path.exists(model)):+ detector = create_detector(width, height)+ if detector is None:cap.release()return None- detector = cv2.dnn.readNetFromCaffe(proto, model)-- # Collect face positions per half+ # Collect face positions per half AND per-frame face counts+ # to distinguish split-screen frames from single-person framesleft_faces = [] # (cx_in_half, cy)right_faces = [] # (cx_in_half, cy)+ single_faces = [] # (cx_in_full, cy) — faces from single-person framessample_count = min(40, max(10, int(duration * 2)))+ split_frame_count = 0+ single_frame_count = 0+ # Record per-sample face count for per-segment layout lookup+ sample_face_counts = [] # (time, num_faces)+for i in range(sample_count):t = (i + 1) * duration / (sample_count + 1)cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000)@@ -362,63 +704,92 @@ def _crop_split_screen(if not ret:continue- h, w = frame.shape[:2]- blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))- detector.setInput(blob)- dets = detector.forward()+ faces = detect_faces(detector, frame, width, height)+ sample_face_counts.append((t, len(faces)))- for j in range(dets.shape[2]):- if dets[0, 0, j, 2] > 0.5:- cx = int((dets[0, 0, j, 3] + dets[0, 0, j, 5]) / 2 * w)- cy = int((dets[0, 0, j, 4] + dets[0, 0, j, 6]) / 2 * h)+ if len(faces) >= 2:+ # Split-screen frame+ split_frame_count += 1+ for f in faces:+ cx, cy = f["cx"], f["cy"]if cx < mid_x:left_faces.append((cx, cy))else:- right_faces.append((cx - mid_x, cy)) # relative to right half+ right_faces.append((cx - mid_x, cy))+ elif len(faces) == 1:+ # Single-person frame+ single_frame_count += 1+ single_faces.append((faces[0]["cx"], faces[0]["cy"]))cap.release()- if not left_faces and not right_faces:+ if not left_faces and not right_faces and not single_faces:return None- # Compute static crop for each half- # Within a 960x1080 half, we need a 9:16 crop+ is_mixed = split_frame_count >= 3 and single_frame_count >= 3++ # Compute static crop for each half (split-screen segments)half_w = mid_x- crop_w = int(height * target_ratio) # ~607 for 1080 height+ # Vertical zoom: 75% of frame height for tighter face framing+ adj_h = int(height * 0.75)+ adj_w = int(adj_h * target_ratio)+ if adj_w > half_w:+ # Zoomed crop wider than half — fall back to full height+ adj_h = height+ adj_w = int(height * target_ratio)++ def _compute_split_crop(faces, half_offset):+ """Compute crop for a split-screen half (zoomed to face)."""+ if not faces:+ cx_in_half = half_w // 2+ crop_x = max(0, min(cx_in_half - adj_w // 2, half_w - adj_w))+ return half_offset + crop_x, max(0, (height - adj_h) // 2), adj_h++ cx = int(np.median([f[0] for f in faces]))+ cy = int(np.median([f[1] for f in faces]))++ crop_x = cx - adj_w // 2+ crop_x = max(0, min(crop_x, half_w - adj_w))+ crop_x += half_offset- def _compute_crop(faces, half_offset):- """Compute static crop_x, crop_y for one camera half."""+ # Vertical: place face at ~33% from top+ crop_y = cy - int(adj_h * 0.33)+ crop_y = max(0, min(crop_y, height - adj_h))++ return crop_x, crop_y, adj_h++ def _compute_single_crop(faces):+ """Compute crop for single-person fullscreen (with same zoom for consistency)."""if not faces:- # No face — center crop within this half- return half_offset + (half_w - crop_w) // 2, 0, height+ return (width - adj_w) // 2, max(0, (height - adj_h) // 2), adj_hcx = int(np.median([f[0] for f in faces]))cy = int(np.median([f[1] for f in faces]))- # Horizontal: center crop on face within this half- crop_x = cx - crop_w // 2- crop_x = max(0, min(crop_x, half_w - crop_w))- crop_x += half_offset # offset to position within full frame+ crop_x = cx - adj_w // 2+ crop_x = max(0, min(crop_x, width - adj_w))- # Vertical: place face at ~33% from top with 75% zoom- adj_h = int(height * 0.75)crop_y = cy - int(adj_h * 0.33)crop_y = max(0, min(crop_y, height - adj_h))return crop_x, crop_y, adj_h- left_crop_x, left_crop_y, left_h = _compute_crop(left_faces, 0)- right_crop_x, right_crop_y, right_h = _compute_crop(right_faces, mid_x)+ left_crop_x, left_crop_y, left_h = _compute_split_crop(left_faces, 0)+ right_crop_x, right_crop_y, right_h = _compute_split_crop(right_faces, mid_x)+ single_crop_x, single_crop_y, single_h = _compute_single_crop(single_faces)left_w = int(left_h * target_ratio)right_w = int(right_h * target_ratio)+ single_w = int(single_h * target_ratio)# Map speakers to sides (first speaker by time = left)speakers = sorted(set(w.get("speaker") for w in transcript_words if w.get("speaker")))if len(speakers) < 2:- # Single speaker — use the side with more face detections- if len(left_faces) >= len(right_faces):+ # Single speaker — if mixed layout, prefer fullscreen crop; else use best half+ if is_mixed and single_faces:+ vf = f"crop={single_w}:{single_h}:{single_crop_x}:{single_crop_y},scale={target_w}:{target_h}"+ elif len(left_faces) >= len(right_faces):vf = f"crop={left_w}:{left_h}:{left_crop_x}:{left_crop_y},scale={target_w}:{target_h}"else:vf = f"crop={right_w}:{right_h}:{right_crop_x}:{right_crop_y},scale={target_w}:{target_h}"@@ -449,14 +820,14 @@ def _crop_split_screen(if sp not in speaker_side:speaker_side[sp] = speaker_side.get(top_2_by_first[0], "left")- # Build speaker segments+ # Build speaker segments (times relative to the cut segment, clamped to [0, duration])segments = []cur_speaker = Noneseg_start = 0for w in sorted(transcript_words, key=lambda x: x["start"]):sp = w.get("speaker")- t = w["start"] - clip_start+ t = max(0, min(w["start"] - clip_start, duration))if sp != cur_speaker and sp is not None:if cur_speaker is not None:segments.append((seg_start, t, cur_speaker))@@ -480,24 +851,54 @@ def _crop_split_screen(if not segments:return None+ # For mixed layouts, detect per-segment layout by sampling a frame+ # from the original (uncut) video at the segment's absolute time+ def _is_segment_split(seg_start_t):+ """Check if a segment's time falls in a split-screen or single-person region.+ Uses the pre-computed sample_face_counts instead of re-opening the video."""+ if not is_mixed:+ return True # Pure split-screen, all segments use half crops+ # Find the nearest sampled frame to this segment's start time+ best_dist = float("inf")+ best_count = 2 # default to split+ for t, n in sample_face_counts:+ dist = abs(t - seg_start_t)+ if dist < best_dist:+ best_dist = dist+ best_count = n+ return best_count >= 2+# Cut each segment with its speaker's crop, then concatwork_dir = os.path.dirname(output_path) or "."part_paths = []try:for i, (start_t, end_t, speaker) in enumerate(segments):- side = speaker_side.get(speaker, "left")- if side == "left":- cw, ch, cx, cy = left_w, left_h, left_crop_x, left_crop_y+ # Skip segments shorter than 1 frame (~0.04s at 24fps)+ if end_t - start_t < 0.05:+ continue++ # For mixed layouts: detect if this segment is split or single+ seg_is_split = _is_segment_split(start_t)++ if seg_is_split:+ side = speaker_side.get(speaker, "left")+ if side == "left":+ cw, ch, cx, cy = left_w, left_h, left_crop_x, left_crop_y+ else:+ cw, ch, cx, cy = right_w, right_h, right_crop_x, right_crop_yelse:- cw, ch, cx, cy = right_w, right_h, right_crop_x, right_crop_y+ # Single-person fullscreen — use full-frame crop (no vertical zoom)+ cw, ch, cx, cy = single_w, single_h, single_crop_x, single_crop_ypart_path = os.path.join(work_dir, f"_speaker_part_{i}.mp4")vf = f"crop={cw}:{ch}:{cx}:{cy},scale={target_w}:{target_h}"+ seg_duration = end_t - start_t+cmd = ["ffmpeg", "-y",- "-ss", str(start_t), "-t", str(end_t - start_t),+ "-ss", str(start_t), "-t", str(seg_duration),"-i", input_path,"-vf", vf,"-c:v", "libx264", "-crf", "18", "-preset", "fast",@@ -505,7 +906,7 @@ def _crop_split_screen("-avoid_negative_ts", "make_zero",part_path,]- r = subprocess.run(cmd, capture_output=True, text=True)+ r = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if r.returncode != 0:print(f"Warning: split-screen segment {i} failed: {r.stderr[-200:]}", file=sys.stderr)continue@@ -528,7 +929,7 @@ def _crop_split_screen("-movflags", "+faststart",output_path,]- r = subprocess.run(cmd, capture_output=True, text=True)+ r = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if r.returncode != 0:return None@@ -555,6 +956,7 @@ def _use_face_map(width: int,height: int,target_ratio: float,+ crop_h: int = None,) -> Optional[str]:"""Use pre-computed face_map from transcription to determine crop position.@@ -562,12 +964,13 @@ def _use_face_map(For multi-speaker clips: builds speaker-aware panning expression.For single-speaker: returns the dominant speaker's crop position.+ crop_h: if set, compute crop width from this height (for vertical zoom)."""clusters = face_map.get("clusters", [])speaker_mappings = face_map.get("speaker_mappings", {})dominant = face_map.get("dominant_speaker")is_split_screen = face_map.get("is_split_screen", False)- crop_w = int(height * target_ratio)+ crop_w = int((crop_h or height) * target_ratio)mid_x = width // 2seam_margin = 20@@ -579,14 +982,18 @@ def _use_face_map(if map_w != width:return None- # Apply seam margin to crop_x values for split-screen (copy to avoid mutating face_map)- if is_split_screen and len(clusters) >= 2:- clusters = [dict(cl) for cl in clusters]- for cl in clusters:- if cl["center_x"] < mid_x:- cl["crop_x"] = max(0, min(cl["crop_x"], mid_x - crop_w - seam_margin))+ # Recompute crop_x from center_x with current crop_w (may differ from+ # stored values if vertical zoom changes the crop width).+ clusters = [dict(cl) for cl in clusters]+ for cl in clusters:+ cx = cl["center_x"]+ if is_split_screen and len(clusters) >= 2:+ if cx < mid_x:+ cl["crop_x"] = max(0, min(cx - crop_w // 2, mid_x - crop_w - seam_margin))else:- cl["crop_x"] = max(mid_x + seam_margin, min(cl["crop_x"], width - crop_w))+ cl["crop_x"] = max(mid_x + seam_margin, min(cx - crop_w // 2, width - crop_w))+ else:+ cl["crop_x"] = max(0, min(cx - crop_w // 2, width - crop_w))# Check if clip has multiple speakersspeakers_in_clip = set()@@ -614,7 +1021,7 @@ def _use_face_map(for w in sorted(transcript_words, key=lambda x: x["start"]):sp = w.get("speaker")- t = w["start"] - clip_start+ t = max(0, w["start"] - clip_start)if sp != current_speaker and sp is not None:if current_speaker is not None:segments.append((seg_start, t, current_speaker))@@ -708,6 +1115,7 @@ def _build_speaker_aware_crop(try:import cv2import numpy as np+ from services.face_detector import create_detector, detect_facescap = cv2.VideoCapture(video_path)if not cap.isOpened():@@ -717,19 +1125,17 @@ def _build_speaker_aware_crop(fps = cap.get(cv2.CAP_PROP_FPS) or 30duration = total_frames / fps- # Load DNN face detector- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))- proto = os.path.join(backend_dir, "models", "deploy.prototxt")- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")- if not (os.path.exists(proto) and os.path.exists(model)):+ detector = create_detector(width, height)+ if detector is None:cap.release()return None- detector = cv2.dnn.readNetFromCaffe(proto, model)-# Zoom to 75% of frame height for tight face framingadjusted_h = int(height * 0.75)adj_w = int(adjusted_h * target_ratio)+ if adj_w > width:+ adjusted_h = height+ adj_w = int(height * target_ratio)mid_x = width // 2# ── Phase 1: Dense frame sampling ──────────────────────────────@@ -747,25 +1153,11 @@ def _build_speaker_aware_crop(if not ret:continue- h, w = frame.shape[:2]- blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))- detector.setInput(blob)- detections = detector.forward()+ faces_raw = detect_faces(detector, frame, width, height)faces = []- for j in range(detections.shape[2]):- conf = detections[0, 0, j, 2]- if conf > 0.5:- x1 = int(detections[0, 0, j, 3] * w)- x2 = int(detections[0, 0, j, 5] * w)- y1 = int(detections[0, 0, j, 4] * h)- y2 = int(detections[0, 0, j, 6] * h)- fw = x2 - x1- if fw < w * 0.04:- continue- cx = (x1 + x2) // 2- cy = (y1 + y2) // 2- faces.append((cx, cy, fw, float(conf)))+ for f in faces_raw:+ faces.append((f["cx"], f["cy"], f["fw"], f["confidence"]))frame_data.append((t, faces))@@ -796,7 +1188,7 @@ def _build_speaker_aware_crop(for w in sorted(transcript_words, key=lambda x: x["start"]):sp = w.get("speaker")- t = w["start"] - clip_start+ t = max(0, w["start"] - clip_start)if sp != current_speaker and sp is not None:if current_speaker is not None:segments.append((seg_start, t, current_speaker))@@ -998,6 +1390,18 @@ def _build_speaker_aware_crop(parts.append(f"if(between(t\\,{jump_t:.2f}\\,{t1:.2f})\\,{v1}\\,")+ elif value_jump > jump_threshold * 0.6:+ # Medium jump (significant lean/shift): fast reframe (~0.15s)+ pan_t = min(0.15, dt)+ mid_t = t0 + pan_t+ parts.append(+ f"if(between(t\\,{t0:.2f}\\,{mid_t:.2f})\\,"+ f"{v0}+(({v1}-{v0})*(t-{t0:.2f})/{pan_t:.2f})\\,"+ )+ if mid_t < t1:+ parts.append(+ f"if(between(t\\,{mid_t:.2f}\\,{t1:.2f})\\,{v1}\\,"+ )else:# Close keyframes with small movement: smooth interpolationparts.append(@@ -1045,7 +1449,7 @@ def _create_gradient_png(output_path: str, width: int = 1080, height: int = 1920"-frames:v", "1",output_path,]- result = subprocess.run(cmd, capture_output=True, text=True)+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if result.returncode != 0:raise RuntimeError(f"Gradient creation failed: {result.stderr[-300:]}")return output_path@@ -1115,6 +1519,10 @@ def burn_captions(filter_complex = ";".join(filter_parts)+ # Check if input has audio (some split-screen concat outputs may not)+ has_audio = _has_audio_stream(input_path)+ audio_map = ["-map", "0:a?", "-c:a", "copy"] if has_audio else []+# Run with HW encoder, fallback to CPU if it failsenc_flags = get_video_encode_flags()cmd = [@@ -1122,14 +1530,13 @@ def burn_captions(*inputs,"-filter_complex", filter_complex,"-map", "[out]",- "-map", "0:a",+ *audio_map,*enc_flags,- "-c:a", "copy","-movflags", "+faststart",output_path,]- result = subprocess.run(cmd, capture_output=True, text=True)+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)# Fallback to CPU if HW encoder failedif result.returncode != 0 and enc_flags != CPU_FLAGS:@@ -1139,13 +1546,12 @@ def burn_captions(*inputs,"-filter_complex", filter_complex,"-map", "[out]",- "-map", "0:a",+ *audio_map,*CPU_FLAGS,- "-c:a", "copy","-movflags", "+faststart",output_path,]- result = subprocess.run(cmd_fallback, capture_output=True, text=True)+ result = subprocess.run(cmd_fallback, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)# Clean up gradient fileif gradient_overlay and os.path.exists(gradient_path):@@ -1175,7 +1581,7 @@ def concat_outro("ffprobe", "-v", "quiet", "-print_format", "json","-show_format", input_path,]- probe = subprocess.run(probe_cmd, capture_output=True, text=True)+ probe = subprocess.run(probe_cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)try:main_duration = float(json.loads(probe.stdout)["format"]["duration"])except Exception:@@ -1217,7 +1623,7 @@ def concat_outro("-movflags", "+faststart",output_path,]- result = subprocess.run(xfade_cmd, capture_output=True, text=True)+ result = subprocess.run(xfade_cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if result.returncode == 0:# Clean upif os.path.exists(outro_scaled):@@ -1257,7 +1663,7 @@ def concat_outro("-movflags", "+faststart",output_path,]- result = subprocess.run(cmd, capture_output=True, text=True)+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if result.returncode != 0:raise RuntimeError(f"FFmpeg concat failed: {result.stderr[-500:]}")return output_path@@ -1283,7 +1689,7 @@ def normalize_audio("-af", f"loudnorm=I={target_lufs}:TP=-1.5:LRA=11:print_format=json","-f", "null", "-",]- result = subprocess.run(measure_cmd, capture_output=True, text=True)+ result = subprocess.run(measure_cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)# Try to parse loudnorm output from stderrstderr = result.stderr@@ -1331,7 +1737,7 @@ def normalize_audio("-movflags", "+faststart",output_path,]- result = subprocess.run(cmd, capture_output=True, text=True)+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)if result.returncode != 0:raise RuntimeError(f"FFmpeg normalize failed: {result.stderr[-500:]}")return output_path@@ -1354,6 +1760,7 @@ def _detect_face_y(try:import cv2import numpy as np+ from services.face_detector import create_detector, detect_facescap = cv2.VideoCapture(video_path)if not cap.isOpened():@@ -1363,21 +1770,18 @@ def _detect_face_y(fps = cap.get(cv2.CAP_PROP_FPS) or 30duration = total_frames / fps- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))- proto = os.path.join(backend_dir, "models", "deploy.prototxt")- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")- if not (os.path.exists(proto) and os.path.exists(model)):- cap.release()- return None-- detector = cv2.dnn.readNetFromCaffe(proto, model)-- # Try to evaluate crop_x as a static value for sampling+ # Try to evaluate crop_x as a static value for ROItry:crop_x = int(float(crop_x_expr))except (ValueError, TypeError):crop_x = None+ # Detect on full frame, then filter by crop region+ detector = create_detector(width, height)+ if detector is None:+ cap.release()+ return None+face_ys = []sample_count = min(20, max(5, int(duration)))@@ -1388,28 +1792,15 @@ def _detect_face_y(if not ret:continue- # If we know crop_x, only look at that column for faces- h, w = frame.shape[:2]- if crop_x is not None:- x1 = max(0, crop_x)- x2 = min(w, crop_x + crop_w)- roi = frame[:, x1:x2]- else:- roi = frame-- rh, rw = roi.shape[:2]- blob = cv2.dnn.blobFromImage(cv2.resize(roi, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))- detector.setInput(blob)- dets = detector.forward()-- for j in range(dets.shape[2]):- conf = dets[0, 0, j, 2]- if conf > 0.5:- y1 = int(dets[0, 0, j, 4] * rh)- y2 = int(dets[0, 0, j, 6] * rh)- fh = y2 - y1- if fh > rh * 0.05:- face_ys.append((y1 + y2) // 2)+ faces = detect_faces(detector, frame, width, height)++ for f in faces:+ # If we know crop_x, only use faces within the crop column+ if crop_x is not None:+ if f["cx"] < crop_x or f["cx"] > crop_x + crop_w:+ continue+ if f["fh"] > height * 0.05:+ face_ys.append(f["cy"])cap.release()@@ -1427,15 +1818,20 @@ def _detect_face_center(width: int,height: int,target_ratio: float,-) -> Optional[str]:+ crop_h: int = None,+) -> Optional[tuple]:"""Simple static face detection — finds the dominant face and returns aSTATIC crop position that centers the face horizontally. No tracking,no panning. The face stays centered for the entire clip.++ Returns (crop_x_str, median_face_cy) tuple, or None if no face detected.+ crop_h: if set, compute crop width from this height (for vertical zoom)."""try:import cv2import numpy as np+ from services.face_detector import create_detector, detect_facescap = cv2.VideoCapture(video_path)if not cap.isOpened():@@ -1444,21 +1840,17 @@ def _detect_face_center(total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))fps = cap.get(cv2.CAP_PROP_FPS) or 30duration = total_frames / fps- crop_w = int(height * target_ratio)+ crop_w = int((crop_h or height) * target_ratio)- # Load DNN face detector- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))- proto = os.path.join(backend_dir, "models", "deploy.prototxt")- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")- if not (os.path.exists(proto) and os.path.exists(model)):+ detector = create_detector(width, height)+ if detector is None:cap.release()return None- detector = cv2.dnn.readNetFromCaffe(proto, model)-# Sample frames across the clipsample_count = min(40, max(10, int(duration * 2)))face_positions = []+ face_cy_values = []for i in range(sample_count):t = i * duration / sample_count@@ -1467,37 +1859,30 @@ def _detect_face_center(if not ret:continue- h, w = frame.shape[:2]- blob = cv2.dnn.blobFromImage(- cv2.resize(frame, (300, 300)), 1.0, (300, 300),- (104.0, 177.0, 123.0)- )- detector.setInput(blob)- detections = detector.forward()+ faces = detect_faces(detector, frame, width, height)+ # Pick the best face (largest × most confident)best_score = 0best_cx = None- for j in range(detections.shape[2]):- conf = detections[0, 0, j, 2]- if conf > 0.5:- x1 = int(detections[0, 0, j, 3] * w)- x2 = int(detections[0, 0, j, 5] * w)- face_w = x2 - x1- if face_w < w * 0.05:- continue- score = conf * (face_w ** 2)- if score > best_score:- best_score = score- best_cx = (x1 + x2) // 2+ best_cy = None+ for f in faces:+ score = f["confidence"] * (f["fw"] ** 2)+ if score > best_score:+ best_score = score+ best_cx = f["cx"]+ best_cy = f["cy"]if best_cx is not None:face_positions.append(best_cx)+ face_cy_values.append(best_cy)cap.release()if not face_positions:return None+ median_cy = int(np.median(face_cy_values)) if face_cy_values else None+# Detect split-screen: check if faces appear in both halvespositions = np.array(face_positions)mid_x = width // 2@@ -1522,7 +1907,7 @@ def _detect_face_center(crop_x = face_x - crop_w // 2crop_x = max(0, min(crop_x, width - crop_w))- return str(crop_x)+ return (str(crop_x), median_cy)except ImportError:return None@@ -1536,7 +1921,8 @@ def _detect_face_offset(width: int,height: int,target_ratio: float,-) -> Optional[str]:+ crop_h: int = None,+) -> Optional[tuple]:"""Continuous face tracking — detects face position over time and returnsan FFmpeg expression that smoothly follows the dominant face.@@ -1544,11 +1930,13 @@ def _detect_face_offset(Samples frames throughout the clip, picks the dominant face cluster,then builds a smoothed panning timeline so the crop follows head movement.- Returns an FFmpeg crop-x expression string, or None if no face detected.+ Returns (x_expr, median_face_cy) tuple, or None if no face detected.+ crop_h: if set, compute crop width from this height (for vertical zoom)."""try:import cv2import numpy as np+ from services.face_detector import create_detector, detect_facescap = cv2.VideoCapture(video_path)if not cap.isOpened():@@ -1557,7 +1945,7 @@ def _detect_face_offset(total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))fps = cap.get(cv2.CAP_PROP_FPS) or 30duration = total_frames / fps- crop_w = int(height * target_ratio)+ crop_w = int((crop_h or height) * target_ratio)# Sample ~4 frames/sec for smooth tracking (more than before)sample_count = min(240, max(30, int(duration * 4)))@@ -1565,33 +1953,12 @@ def _detect_face_offset(# (time, face_center_x) for each detectiontimed_positions = []+ face_cy_values = []- # Load DNN face detector- dnn_detector = None- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))- bundled_proto = os.path.join(backend_dir, "models", "deploy.prototxt")- bundled_model = os.path.join(backend_dir, "models",- "res10_300x300_ssd_iter_140000.caffemodel")- cv2_proto = os.path.join(os.path.dirname(cv2.__file__), "data",- "deploy.prototxt")- cv2_model = os.path.join(os.path.dirname(cv2.__file__), "data",- "res10_300x300_ssd_iter_140000.caffemodel")- proto_path = bundled_proto if os.path.exists(bundled_proto) else cv2_proto- model_path = bundled_model if os.path.exists(bundled_model) else cv2_model- if os.path.exists(proto_path) and os.path.exists(model_path):- dnn_detector = cv2.dnn.readNetFromCaffe(proto_path, model_path)-- # Fallback: Haar cascades- cascades = []- if dnn_detector is None:- for cascade_name in [- "haarcascade_frontalface_default.xml",- "haarcascade_frontalface_alt2.xml",- "haarcascade_profileface.xml",- ]:- cascade_path = cv2.data.haarcascades + cascade_name- if os.path.exists(cascade_path):- cascades.append(cv2.CascadeClassifier(cascade_path))+ detector = create_detector(width, height)+ if detector is None:+ cap.release()+ return Nonefor t in sample_times:cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000)@@ -1599,53 +1966,33 @@ def _detect_face_offset(if not ret:continue- h, w = frame.shape[:2]+ faces = detect_faces(detector, frame, width, height)- if dnn_detector is not None:- blob = cv2.dnn.blobFromImage(- cv2.resize(frame, (300, 300)), 1.0, (300, 300),- (104.0, 177.0, 123.0)- )- dnn_detector.setInput(blob)- detections = dnn_detector.forward()-- best_score = 0- best_cx = None- for i in range(detections.shape[2]):- confidence = detections[0, 0, i, 2]- if confidence > 0.5:- x1 = int(detections[0, 0, i, 3] * w)- y1 = int(detections[0, 0, i, 4] * h)- x2 = int(detections[0, 0, i, 5] * w)- y2 = int(detections[0, 0, i, 6] * h)- face_w = x2 - x1- face_h = y2 - y1- if face_w < w * 0.05 or face_h < h * 0.05:- continue- score = confidence * (face_w ** 2)- if score > best_score:- best_score = score- best_cx = (x1 + x2) // 2-- if best_cx is not None:- timed_positions.append((t, best_cx))- else:- gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)- for cascade in cascades:- faces = cascade.detectMultiScale(- gray, 1.1, 5, minSize=(60, 60)- )- if len(faces) > 0:- largest = max(faces, key=lambda f: f[2] * f[3])- face_center_x = largest[0] + largest[2] // 2- timed_positions.append((t, face_center_x))- break+ # Pick the best face (largest × most confident)+ best_score = 0+ best_cx = None+ best_cy = None+ for f in faces:+ if f["fh"] < height * 0.05:+ continue+ score = f["confidence"] * (f["fw"] ** 2)+ if score > best_score:+ best_score = score+ best_cx = f["cx"]+ best_cy = f["cy"]++ if best_cx is not None:+ timed_positions.append((t, best_cx))+ face_cy_values.append(best_cy)cap.release()if len(timed_positions) < 3:return None+ # Median face Y for vertical positioning (computed once, used in all returns)+ median_cy = int(np.median(face_cy_values)) if face_cy_values else None+# --- Cluster to find dominant face ---all_cx = np.array([p[1] for p in timed_positions])cluster_radius = crop_w * 0.35@@ -1673,7 +2020,7 @@ def _detect_face_offset(# Too few points — use static medianmedian_x = int(np.median([cx for _, cx in tracked]))crop_x = max(0, min(median_x - crop_w // 2, width - crop_w))- return str(crop_x)+ return (str(crop_x), median_cy)# --- Smooth the positions ---# 1) Convert face_center_x to crop_x (clamped with margin).@@ -1705,18 +2052,18 @@ def _detect_face_offset(smoothed = np.clip(np.round(smoothed), 0, width - crop_w).astype(int)# --- Check if tracking is even needed ---- # If the face barely moves (range < 8% of frame width), just use static.+ # If the face barely moves (range < 20% of crop width), just use static.# Most podcast clips have speakers sitting still — tracking should be rare.movement_range = int(smoothed.max() - smoothed.min())- if movement_range < width * 0.08:- return str(int(np.median(smoothed)))+ if movement_range < crop_w * 0.20:+ return (str(int(np.median(smoothed))), median_cy)# --- Build keyframes: only emit on MAJOR position changes ---# Very conservative: only pan when speaker physically moves to a new# position (leans far, switches seats). Normal head movement = static.keyframes = [(tracked_times[0], int(smoothed[0]))]min_time_gap = 3.0 # Don't place keyframes closer than 3s- min_px_change = 80 # Ignore movement smaller than 80px (~7% of frame)+ min_px_change = max(40, int(crop_w * 0.13)) # Proportional safe zone (~80px at 1080p)for i in range(1, len(smoothed)):t = tracked_times[i]@@ -1732,7 +2079,7 @@ def _detect_face_offset(keyframes.append((t, x))if len(keyframes) < 2:- return str(keyframes[0][1])+ return (str(keyframes[0][1]), median_cy)# --- Build FFmpeg expression ---# Piecewise linear interpolation between keyframes.@@ -1760,13 +2107,13 @@ def _detect_face_offset(keyframes = keyframes[::step] + [keyframes[-1]]# Rebuild (recursive call would be cleaner but let's just return static)print(f"Warning: face tracking produced {len(expr_parts)} keyframes, using static", file=sys.stderr)- return str(int(np.median(smoothed)))+ return (str(int(np.median(smoothed))), median_cy)# Final value: last keyframe positionlast_x = keyframes[-1][1]expr = "".join(expr_parts) + str(last_x) + ")" * len(expr_parts)- return expr+ return (expr, median_cy)except ImportError:return Nonediff --git a/src/handlers/batch-clips.handler.ts b/src/handlers/batch-clips.handler.tsindex c30e12e..cf1e69e 100644--- a/src/handlers/batch-clips.handler.ts+++ b/src/handlers/batch-clips.handler.ts@@ -124,6 +124,8 @@ export async function handleBatchClips((s.suggested_caption_style as string) || settings.captionStyle || "hormozi",crop_strategy: settings.cropStrategy || "face",logo_path: settings.logoPath || null,+ // Preserve multi-cut segments from suggestion+ ...(Array.isArray(s.segments) && s.segments.length > 0 && { keep_segments: s.segments }),});if (input.export_selected) {diff --git a/src/handlers/create-clip.handler.ts b/src/handlers/create-clip.handler.tsindex 0971c93..94afad2 100644--- a/src/handlers/create-clip.handler.ts+++ b/src/handlers/create-clip.handler.ts@@ -149,6 +149,10 @@ export async function handleCreateClip((transcript?.words as Array<unknown>) ||[];+ // Pull multi-cut segments from suggestion (if available)+ const keepSegments =+ (suggestion?.segments as Array<{ start: number; end: number }>) || null;+// Validate required fieldsif (!videoPath) {return JSON.stringify({ error: "video_path is required (no video in session state)" });@@ -171,6 +175,7 @@ export async function handleCreateClip(clean_fillers: input.clean_fillers !== false,logo_path: logoPath,outro_path: outroPath,+ ...(keepSegments && { keep_segments: keepSegments }),});const data = result.data as unknown as ClipResult;diff --git a/src/handlers/suggest-clips.handler.ts b/src/handlers/suggest-clips.handler.tsindex 16e119b..9642f13 100644--- a/src/handlers/suggest-clips.handler.ts+++ b/src/handlers/suggest-clips.handler.ts@@ -38,6 +38,20 @@ export const suggestClipsToolDef = {type: "number",description: "End timestamp in seconds",},+ segments: {+ type: "array",+ description:+ "Multi-cut keep-ranges within the clip. Use to cut out filler/tangents " ++ "in the middle. If omitted, the full start→end range is used.",+ items: {+ type: "object",+ properties: {+ start: { type: "number" },+ end: { type: "number" },+ },+ required: ["start", "end"],+ },+ },reasoning: {type: "string",description: "Why this moment is viral-worthy",@@ -46,6 +60,15 @@ export const suggestClipsToolDef = {type: "string",description: "Brief text preview of what's said",},+ content_type: {+ type: "string",+ description:+ "Content classification: guest_story, technical_insight, market_landscape, business_strategy, hot_take",+ },+ score: {+ type: "number",+ description: "Virality score (0-20). Sum of standalone + hook + relevance + quotability (each 1-5).",+ },suggested_caption_style: {type: "string",enum: ["hormozi", "karaoke", "subtle", "branded"],@@ -67,24 +90,38 @@ export async function handleSuggestClips(title: string;start_second: number;end_second: number;+ segments?: Array<{ start: number; end: number }>;reasoning: string;preview_text?: string;+ content_type?: string;+ score?: number;suggested_caption_style?: string;}>;// Validate and enrich suggestions- const enriched = suggestions.map((s, i) => ({- clip_number: i + 1,- clip_id: randomUUID(),- title: s.title,- start_second: s.start_second,- end_second: s.end_second,- duration: Math.round((s.end_second - s.start_second) * 10) / 10,- reasoning: s.reasoning,- preview_text: s.preview_text || "",- suggested_caption_style: s.suggested_caption_style || "hormozi",- timestamp_display: `${formatTime(s.start_second)} → ${formatTime(s.end_second)}`,- }));+ const enriched = suggestions.map((s, i) => {+ // Compute duration from segments if available, otherwise from start/end+ const segments = s.segments?.filter((seg) => seg.end > seg.start) || [];+ const keptDuration = segments.length > 0+ ? segments.reduce((sum, seg) => sum + (seg.end - seg.start), 0)+ : s.end_second - s.start_second;++ return {+ clip_number: i + 1,+ clip_id: randomUUID(),+ title: s.title,+ start_second: s.start_second,+ end_second: s.end_second,+ segments: segments.length > 0 ? segments : [{ start: s.start_second, end: s.end_second }],+ duration: Math.round(keptDuration * 10) / 10,+ reasoning: s.reasoning,+ preview_text: s.preview_text || "",+ content_type: s.content_type || "unknown",+ score: s.score || 0,+ suggested_caption_style: s.suggested_caption_style || "hormozi",+ timestamp_display: `${formatTime(s.start_second)} → ${formatTime(s.end_second)}`,+ };+ });return JSON.stringify({clip_count: enriched.length,diff --git a/src/server.ts b/src/server.tsindex bc949ba..8828cbe 100644--- a/src/server.ts+++ b/src/server.ts@@ -194,8 +194,14 @@ export function createServer(): McpServer {title: z.string(),start_second: z.number(),end_second: z.number(),+ segments: z+ .array(z.object({ start: z.number(), end: z.number() }))+ .optional()+ .describe("Multi-cut keep-ranges. Omit for a single continuous clip."),reasoning: z.string(),preview_text: z.string().optional(),+ content_type: z.string().optional(),+ score: z.number().optional(),suggested_caption_style: z.enum(["hormozi", "karaoke", "subtle", "branded"]).optional(),@@ -299,6 +305,7 @@ export function createServer(): McpServer {}// Resolve clip_number from UI state BEFORE routing+ let keepSegments: Array<{ start: number; end: number }> | null = null;if (params.clip_number != null && (params.start_second == null || params.end_second == null)) {const uiState = await readUIState();const suggestions = (uiState?.suggestions as Array<Record<string, unknown>>) || [];@@ -324,6 +331,11 @@ export function createServer(): McpServer {const transcript = uiState?.transcript as Record<string, unknown> | null;if (transcript?.words) params.transcript_words = transcript.words as any;}+ // Pull multi-cut segments from suggestion+ const segs = suggestion.segments as Array<{ start: number; end: number }> | undefined;+ if (segs && segs.length > 0) {+ keepSegments = segs;+ }}// Check for duplicates@@ -358,6 +370,7 @@ export function createServer(): McpServer {title: (params.title || "clip") as string,caption_style: params.caption_style || "hormozi",crop_strategy: params.crop_strategy || "face",+ ...(keepSegments && { segments: keepSegments }),}],transcript_words: params.transcript_words,logo_path: params.logo_path || null,@@ -510,6 +523,7 @@ export function createServer(): McpServer {title: s.title || `clip_${i + 1}`,caption_style: (s.suggested_caption_style as string) || settings.captionStyle || "hormozi",crop_strategy: settings.cropStrategy || "face",+ ...(Array.isArray(s.segments) && s.segments.length > 0 && { keep_segments: s.segments }),})) as any;} else if (params.clip_numbers) {resolvedClips = (params.clip_numbers as number[])@@ -522,6 +536,7 @@ export function createServer(): McpServer {title: s.title || `clip_${n}`,caption_style: (s.suggested_caption_style as string) || settings.captionStyle || "hormozi",crop_strategy: settings.cropStrategy || "face",+ ...(Array.isArray(s.segments) && s.segments.length > 0 && { keep_segments: s.segments }),};}) as any;}diff --git a/src/ui/web-server.ts b/src/ui/web-server.tsindex 260c32b..5b9219b 100644--- a/src/ui/web-server.ts+++ b/src/ui/web-server.ts@@ -1253,11 +1253,59 @@ app.post("/api/claude-suggest", async (req, res) => {const dur = (uiState.transcript as any)?.duration || 0;const durationMin = Math.round(dur / 60);- const prompt = `You are the clip extraction engine for a podcast. Analyze this transcript and return the ${top_n} best moments for YouTube Shorts.+ // Load knowledge base files for context+ const kbDir = join(process.cwd(), ".podcli", "knowledge");+ let kbContext = "";+ const kbFiles: [string, number][] = [+ ["04-shorts-creation-guide.md", 4000],+ ["05-title-formulas.md", 3000],+ ["02-voice-and-tone.md", 3000],+ ["01-brand-identity.md", 1500],+ ["11-inspiration-channels.md", 2000],+ ["12-quick-reference.md", 2000],+ ];+ for (const [fname, maxChars] of kbFiles) {+ const fpath = join(kbDir, fname);+ if (existsSync(fpath)) {+ try {+ const content = readFileSync(fpath, "utf-8").trim();+ if (content.split("[Your Show Name]").length > 3 && content.length < 500) continue;+ kbContext += `\n--- ${fname} ---\n${content.slice(0, maxChars)}\n`;+ } catch {}+ }+ }++ const prompt = `You are a viral clip editor for TikTok and YouTube Shorts. Find the ${top_n} most scroll-stopping moments in this podcast transcript.IMPORTANT: Return ONLY valid JSON. No markdown, no explanation, no code fences.-Use the knowledge base in .podcli/knowledge/ for voice rules, moment selection criteria, content types, and existing episodes.+TIMESTAMP FORMAT: All timestamps in the transcript are in SECONDS (e.g., [123.4s]).+All timestamps you return MUST be in SECONDS as numbers (e.g., 123.4), NOT minutes:seconds.++DURATION RULES (CRITICAL):+- Target: 25-40 seconds (this is the viral sweet spot)+- Maximum: 50 seconds (absolute hard limit — anything longer loses viewers)+- Minimum: 15 seconds (too short = no payoff)+- SHORTER IS BETTER. A punchy 25s clip outperforms a 50s clip every time.+- If a thought takes longer than 40s, use segments to cut the filler in the middle++CUTTING RULES (CRITICAL):+- Cut TIGHT. Every second must earn its place.+- Start at the exact moment the hook hits — no preamble, no "so", no "well"+- End the MOMENT the point lands with a complete thought — don't trail off+- NEVER cut mid-sentence or mid-thought. The viewer must feel closure.+- If there's filler/tangent in the middle, use multiple segments to skip it++MOMENT SELECTION (think like a TikTok editor):+- Would YOU stop scrolling for this? If no, skip it.+- First 3 seconds must HOOK — a bold claim, shocking number, or provocative question+- Must make complete sense standalone — no "as I mentioned" or "going back to"+- Must end on a COMPLETE THOUGHT — sentence boundary, natural pause, or mic-drop moment+- Single focused idea — one concept, fully delivered, no loose threads+- Prioritize: controversial takes, surprising numbers, founder war stories, "wait what?" moments, emotional peaks+- Skip: generic advice, obvious statements, context-dependent references++${kbContext ? `KNOWLEDGE BASE:${kbContext}` : ""}Score each moment on 4 dimensions (1-5 each):- standalone: Makes sense without episode context?@@ -1267,9 +1315,20 @@ Score each moment on 4 dimensions (1-5 each):Classify each as: guest_story | technical_insight | market_landscape | business_strategy | hot_take-Return JSON: { "clips": [{ "title": "...", "start_second": N, "end_second": N, "duration": N, "content_type": "...", "scores": {"standalone":N,"hook":N,"relevance":N,"quotability":N}, "total_score": N, "quote": "...", "why": "..." }] }+Return this exact JSON structure:+{ "clips": [{ "title": "First strong sentence from the moment", "start_second": 123.4, "end_second": 168.4, "segments": [{"start": 123.4, "end": 140.0}, {"start": 145.2, "end": 168.4}], "duration": 40, "content_type": "guest_story", "scores": {"standalone": 4, "hook": 5, "relevance": 4, "quotability": 3}, "total_score": 16, "quote": "The key quote from this moment", "why": "One sentence on why this works as a short" }] }++SEGMENTS RULES:+- "segments" is an array of keep-ranges within the clip. Use it to CUT OUT dead weight.+- If the moment is clean with no filler, use a single segment: [{"start": X, "end": Y}]+- If there's a ramble/tangent/filler in the middle, split into multiple segments that skip it+- "duration" = total kept time (sum of all segment lengths), NOT end - start-Rules: 20-90 seconds, sentence boundaries, standalone, variety.+Rules:+- Final clip duration (sum of segments) MUST be 15-50 seconds (target 25-40s)+- Each segment must start and end on COMPLETE SENTENCES+- Must make sense standalone when stitched together+- Sort clips by timestamp orderTranscript (${segCount} segments, ~${durationMin} min):@@ -1300,16 +1359,30 @@ ${transcriptText}`;}const data = JSON.parse(result.substring(jsonStart, jsonEnd));- const clips = (data.clips || []).map((c: any) => ({- title: c.title || "Untitled",- start_second: c.start_second || 0,- end_second: c.end_second || 0,- duration: c.duration || 0,- reasoning: c.why || "",- content_type: c.content_type || "unknown",- scores: c.scores || {},- total_score: c.total_score || 0,- }));+ const clips = (data.clips || []).map((c: any) => {+ // Parse and validate segments+ const rawSegs = c.segments || [];+ const segments = rawSegs+ .filter((s: any) => s.end > s.start)+ .map((s: any) => ({ start: s.start, end: s.end }));+ const keptDuration = segments.length > 0+ ? segments.reduce((sum: number, s: any) => sum + (s.end - s.start), 0)+ : (c.end_second || 0) - (c.start_second || 0);++ return {+ title: c.title || "Untitled",+ start_second: c.start_second || 0,+ end_second: c.end_second || 0,+ segments: segments.length > 0 ? segments : [{ start: c.start_second || 0, end: c.end_second || 0 }],+ duration: Math.round(keptDuration),+ reasoning: c.why || "",+ preview_text: (c.quote || "").slice(0, 120),+ content_type: c.content_type || "unknown",+ score: c.total_score || 0,+ scores: c.scores || {},+ suggested_caption_style: "hormozi",+ };+ }).filter((c: any) => c.duration >= 15 && c.duration <= 55);// Auto-push to UI state as suggestionsif (clips.length > 0) {@@ -1318,7 +1391,12 @@ ${transcriptText}`;title: c.title,start_second: c.start_second,end_second: c.end_second,+ segments: c.segments,reasoning: c.reasoning,+ preview_text: c.preview_text,+ content_type: c.content_type,+ score: c.score,+ suggested_caption_style: c.suggested_caption_style,}));(uiState as any).phase = "review";broadcastSSE("state-sync", uiState);@@ -1465,6 +1543,8 @@ app.post("/api/mcp/export", async (req, res) => {title: (c.title || "clip").slice(0, 40),caption_style: c.caption_style || captionStyle,crop_strategy: c.crop_strategy || cropStrategy,+ // Preserve multi-cut segments from suggestions+ ...(Array.isArray(c.segments) && c.segments.length > 0 && { keep_segments: c.segments }),}));const jobId = uuidv4();