Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

feat: unified face tracking with YuNet, deep PodStack integration, fix camera flashing

Nika Siradzea4e10547522b
diff --git a/backend/main.py b/backend/main.py
index 68f28fa..be43e42 100644
--- a/backend/main.py
+++ b/backend/main.py
@@ -81,6 +81,7 @@ def handle_create_clip(task_id: str, params: dict):
outro_path=params.get("outro_path"),
clean_fillers=params.get("clean_fillers", True),
face_map=params.get("face_map"),
+ keep_segments=params.get("keep_segments"),
progress_callback=lambda pct, msg: emit_progress(task_id, "processing", pct, msg),
)
emit_result(task_id, "success", data=result)
@@ -115,6 +116,7 @@ def handle_batch_clips(task_id: str, params: dict):
outro_path=params.get("outro_path"),
clean_fillers=params.get("clean_fillers", True),
face_map=params.get("face_map"),
+ keep_segments=clip.get("keep_segments"),
progress_callback=lambda pct, msg, _i=i: emit_progress(
task_id, "batch", int((_i / total) * 100 + pct / total), msg
),
diff --git a/backend/models/face_detection_yunet_2023mar.onnx b/backend/models/face_detection_yunet_2023mar.onnx
new file mode 100644
index 0000000..f9beb30
Binary files /dev/null and b/backend/models/face_detection_yunet_2023mar.onnx differ
diff --git a/backend/services/audio_analyzer.py b/backend/services/audio_analyzer.py
index ae2afea..097376f 100644
--- a/backend/services/audio_analyzer.py
+++ b/backend/services/audio_analyzer.py
@@ -21,16 +21,18 @@ def extract_audio_energy(
Returns list of {time, rms, peak} dicts, one per window.
"""
- # Use ffmpeg astats filter to get per-frame audio stats
+ # Use ffmpeg astats filter on audio only (-vn skips video decoding,
+ # which is the main bottleneck on long podcasts).
cmd = [
"ffmpeg",
"-i", video_path,
+ "-vn",
"-af", f"astats=metadata=1:reset={int(1/window_sec)},"
f"ametadata=print:key=lavfi.astats.Overall.RMS_level:file=-",
"-f", "null", "-",
]
- result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=600)
# Parse the ametadata output from stdout
# Format: "frame:N pts:N pts_time:N.N\nlavfi.astats.Overall.RMS_level=N.N"
@@ -76,14 +78,15 @@ def _fallback_energy(video_path: str) -> list[dict]:
except (json.JSONDecodeError, KeyError):
return []
- # Use ebur128 filter which reliably outputs loudness per second
+ # Use ebur128 filter on audio only (-vn skips slow video decoding)
cmd = [
"ffmpeg",
"-i", video_path,
+ "-vn",
"-af", "ebur128=peak=true",
"-f", "null", "-",
]
- result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=600)
energy_data = []
# Parse ebur128 output from stderr: "t: N.N M: -N.N S: -N.N"
diff --git a/backend/services/claude_suggest.py b/backend/services/claude_suggest.py
index 53f5419..3a13516 100644
--- a/backend/services/claude_suggest.py
+++ b/backend/services/claude_suggest.py
@@ -37,6 +37,31 @@ def _find_claude() -> Optional[str]:
return None
+def _load_existing_shorts(episodes_path: str) -> list[str]:
+ """Extract existing short titles from episode database to avoid duplicates."""
+ if not os.path.exists(episodes_path):
+ return []
+ try:
+ with open(episodes_path) as f:
+ content = f.read()
+ # Parse lines that look like shorts entries: "1. [title] — [category]"
+ shorts = []
+ for line in content.split("\n"):
+ line = line.strip()
+ if line and (line.startswith("1.") or line.startswith("2.") or
+ line.startswith("3.") or line.startswith("4.") or
+ line.startswith("5.") or line.startswith("6.") or
+ line.startswith("7.") or line.startswith("8.") or
+ line.startswith("9.")):
+ # Extract title between brackets or after number
+ title = line.split("—")[0].strip().lstrip("0123456789.").strip()
+ if title and title != "[Short title]":
+ shorts.append(title)
+ return shorts
+ except Exception:
+ return []
+
+
def _build_prompt(transcript_text: str, segment_count: int, duration_min: float, top_n: int) -> str:
"""Build the prompt for Claude to extract clips.
@@ -44,22 +69,40 @@ def _build_prompt(transcript_text: str, segment_count: int, duration_min: float,
can't read project files.
"""
- # Try to load knowledge base files inline
+ # Load knowledge base files inline — prioritized by relevance to clip selection
kb_context = ""
kb_dir = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "..", "..", ".podcli", "knowledge"
)
- for fname in ["04-shorts-creation-guide.md", "05-title-formulas.md", "02-voice-and-tone.md"]:
+ # (filename, max_chars) — higher priority files get more budget
+ _kb_files = [
+ ("04-shorts-creation-guide.md", 4000), # moment selection criteria, content types
+ ("05-title-formulas.md", 3000), # title rules, shapes, banned openers
+ ("02-voice-and-tone.md", 3000), # banned words, voice fingerprint, coffee test
+ ("01-brand-identity.md", 1500), # show context, positioning, audience
+ ("11-inspiration-channels.md", 2000), # viral hook patterns, reference styles
+ ("12-quick-reference.md", 2000), # hook bank, title formulas, hashtags
+ ("08-topics-themes.md", 1000), # topic areas, audience interest map
+ ("00-master-instructions.md", 1500), # quality gate, auto-detection rules
+ ]
+ for fname, max_chars in _kb_files:
fpath = os.path.join(kb_dir, fname)
if os.path.exists(fpath):
try:
with open(fpath) as f:
- content = f.read()
- kb_context += f"\n--- {fname} ---\n{content[:2000]}\n"
+ content = f.read().strip()
+ # Skip template-only files (uncustomized placeholders)
+ if content.count("[Your Show Name]") > 2 and len(content) < 500:
+ continue
+ kb_context += f"\n--- {fname} ---\n{content[:max_chars]}\n"
except Exception:
pass
- return f"""You are the clip extraction engine for a podcast. Analyze this transcript and return the {top_n} best moments for YouTube Shorts.
+ # Load existing shorts from episode database for duplicate avoidance
+ episodes_path = os.path.join(kb_dir, "03-episodes-database.md")
+ existing_shorts = _load_existing_shorts(episodes_path)
+
+ return f"""You are a viral clip editor for TikTok and YouTube Shorts. Find the {top_n} most scroll-stopping moments in this podcast transcript.
IMPORTANT: Return ONLY valid JSON. No markdown, no explanation, no code fences.
@@ -67,30 +110,34 @@ TIMESTAMP FORMAT: All timestamps in the transcript are in SECONDS (e.g., [123.4s
All timestamps you return MUST be in SECONDS as numbers (e.g., 123.4), NOT minutes:seconds.
DURATION RULES (CRITICAL):
-- Target: 30-45 seconds per clip (ideal for YouTube Shorts)
-- Maximum: 60 seconds (hard limit — YouTube Shorts cuts off at 60s)
-- Minimum: 20 seconds (too short = no value)
-- If a moment runs longer than 50 seconds, find a tighter cut point
+- Target: 25-40 seconds (this is the viral sweet spot)
+- Maximum: 50 seconds (absolute hard limit — anything longer loses viewers)
+- Minimum: 15 seconds (too short = no payoff)
+- SHORTER IS BETTER. A punchy 25s clip outperforms a 50s clip every time.
+- If a thought takes longer than 40s, use segments to cut the filler in the middle
CUTTING RULES (CRITICAL):
- Cut TIGHT. Every second must earn its place.
- Start at the exact moment the hook hits — no preamble, no "so", no "well"
-- End the INSTANT the point lands — don't let it trail off
-- If the speaker rambles before making a point, start AFTER the ramble
-- If there's filler/tangent in the middle, pick the tighter segment that preserves the core idea
-- The transcript timestamps are your scalpel — use them precisely
+- End the MOMENT the point lands with a complete thought — don't trail off
+- NEVER cut mid-sentence or mid-thought. The viewer must feel closure.
+- The last sentence must feel like a natural ending, a punchline, or a mic-drop
+- If there's filler/tangent in the middle, use multiple segments to skip it
- A 30s clip with zero dead weight beats a 50s clip with 20s of fluff
-MOMENT SELECTION:
-- Must start with a strong hook (first 3 seconds grab attention)
+MOMENT SELECTION (think like a TikTok editor):
+- Would YOU stop scrolling for this? If no, skip it.
+- First 3 seconds must HOOK — a bold claim, shocking number, or provocative question
- Must make complete sense standalone — no "as I mentioned" or "going back to"
-- Must end cleanly on a sentence boundary (not mid-thought)
-- Single focused idea per clip — one concept, fully delivered
-- Prioritize: surprising facts, bold claims, founder stories, counterintuitive insights, "aha" moments
-- Variety across content types
+- Must end on a COMPLETE THOUGHT — sentence boundary, natural pause, or mic-drop moment
+- Single focused idea — one concept, fully delivered, no loose threads
+- Prioritize: controversial takes, surprising numbers, founder war stories, "wait what?" moments, emotional peaks
+- Skip: generic advice, obvious statements, context-dependent references
{f"KNOWLEDGE BASE:{kb_context}" if kb_context else ""}
+{f"EXISTING SHORTS (avoid duplicating these moments):{chr(10).join('- ' + s for s in existing_shorts)}" if existing_shorts else ""}
+
Score each moment on 4 dimensions (1-5 each):
- standalone: Makes sense without episode context?
- hook: Grabs attention in first 3 seconds?
@@ -131,8 +178,9 @@ SEGMENTS RULES:
- Example: speaker makes great point (10s), rambles (8s), delivers punchline (12s) → 2 segments, 22s total
Rules:
-- Final clip duration (sum of segments) MUST be 20-60 seconds (target 30-45s)
-- Each segment must start and end on sentence boundaries
+- Final clip duration (sum of segments) MUST be 15-50 seconds (target 25-40s)
+- Each segment must start and end on COMPLETE SENTENCES — never mid-thought
+- The LAST segment must end on a sentence that feels like a natural conclusion
- Must make sense standalone when stitched together
- Sort clips by timestamp order
@@ -273,8 +321,8 @@ def suggest_with_claude(
# Compute actual kept duration
kept_duration = sum(seg["end"] - seg["start"] for seg in keep_segments)
- # Reject clips shorter than 15 seconds (Claude returned bad timestamps)
- if kept_duration < 15:
+ # Reject clips outside the target duration range
+ if kept_duration < 15 or kept_duration > 55:
continue
normalized.append({
@@ -285,6 +333,11 @@ def suggest_with_claude(
"duration": round(kept_duration),
"score": total,
"content_type": c.get("content_type", "unknown"),
+ # MCP-aligned fields
+ "reasoning": c.get("why", ""),
+ "preview_text": c.get("quote", "")[:120],
+ "suggested_caption_style": "hormozi",
+ # Backwards compat
"quote": c.get("quote", ""),
"why": c.get("why", ""),
"reasons": [c.get("content_type", "")],
diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.py
index 55fef71..123eda8 100644
--- a/backend/services/clip_generator.py
+++ b/backend/services/clip_generator.py
@@ -95,7 +95,7 @@ def _build_tight_segments(
words: list[dict],
start_second: float,
end_second: float,
- silence_threshold: float = 1.5,
+ silence_threshold: float = 0.8,
) -> list[dict]:
"""
Build tight keep-segments from transcript words, cutting out:
@@ -152,7 +152,7 @@ def _build_tight_segments(
kept_duration = sum(s["end"] - s["start"] for s in segments)
saved = original_duration - kept_duration
- if saved < 1.5 or len(segments) < 2:
+ if saved < 0.8 or len(segments) < 2:
# Not worth the cuts — too little saved
return [{"start": start_second, "end": end_second}]
@@ -212,6 +212,16 @@ def generate_clip(
keep_segments.sort(key=lambda s: s["start"])
start_second = keep_segments[0]["start"]
end_second = keep_segments[-1]["end"]
+
+ # If Claude returned a single segment, still auto-trim pauses within it.
+ # Multiple segments means Claude made deliberate editorial cuts — trust those.
+ if len(keep_segments) == 1 and transcript_words and clean_fillers:
+ auto_segments = _build_tight_segments(
+ transcript_words, start_second, end_second,
+ )
+ if len(auto_segments) > 1:
+ keep_segments = auto_segments
+
duration = sum(s["end"] - s["start"] for s in keep_segments)
else:
# Snap clip boundaries to sentence endings
@@ -233,8 +243,8 @@ def generate_clip(
keep_segments = None
duration = end_second - start_second
- if duration > 180:
- raise ValueError(f"Clip too long ({duration:.0f}s). Max 180 seconds for shorts.")
+ if duration > 60:
+ raise ValueError(f"Clip too long ({duration:.0f}s). Max 60 seconds for shorts.")
# Load style config for branded-specific settings
style_config = get_style(caption_style)
@@ -284,7 +294,11 @@ def generate_clip(
crop_clip_start = 0
caption_time_offset = 0
else:
- crop_words = transcript_words
+ # Filter words to just this clip's time range
+ crop_words = [
+ w for w in transcript_words
+ if w["end"] > start_second and w["start"] < end_second
+ ] if transcript_words else transcript_words
crop_clip_start = start_second
caption_time_offset = start_second
diff --git a/backend/services/face_analysis.py b/backend/services/face_analysis.py
index b15c5f8..78a4237 100644
--- a/backend/services/face_analysis.py
+++ b/backend/services/face_analysis.py
@@ -43,6 +43,7 @@ def analyze_faces(
try:
import cv2
import numpy as np
+ from services.face_detector import create_detector, detect_faces
except ImportError:
print("Warning: OpenCV not available, skipping face analysis", file=sys.stderr)
return None
@@ -58,18 +59,13 @@ def analyze_faces(
height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
fps = cap.get(cv2.CAP_PROP_FPS) or 30
- # Load DNN face detector
- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- proto = os.path.join(backend_dir, "models", "deploy.prototxt")
- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")
-
- if not (os.path.exists(proto) and os.path.exists(model)):
+ # Create YuNet detector
+ detector = create_detector(width, height)
+ if detector is None:
cap.release()
print("Warning: Face detection model not found, skipping", file=sys.stderr)
return None
- detector = cv2.dnn.readNetFromCaffe(proto, model)
-
# Sample ~2 frames per second across the full video (enough for analysis)
sample_count = min(300, max(20, int(duration * 2)))
observations = []
@@ -85,35 +81,18 @@ def analyze_faces(
if not ret:
continue
- h, w = frame.shape[:2]
- blob = cv2.dnn.blobFromImage(
- cv2.resize(frame, (300, 300)), 1.0, (300, 300),
- (104.0, 177.0, 123.0)
- )
- detector.setInput(blob)
- detections = detector.forward()
+ faces = detect_faces(detector, frame, width, height)
frame_faces = 0
- for j in range(detections.shape[2]):
- conf = detections[0, 0, j, 2]
- if conf > 0.5:
- x1 = int(detections[0, 0, j, 3] * w)
- x2 = int(detections[0, 0, j, 5] * w)
- y1 = int(detections[0, 0, j, 4] * h)
- y2 = int(detections[0, 0, j, 6] * h)
- fw = x2 - x1
- if fw < w * 0.04:
- continue
- cx = (x1 + x2) // 2
- cy = (y1 + y2) // 2
- observations.append({
- "time": round(t, 3),
- "face_center_x": cx,
- "face_center_y": cy,
- "face_width": fw,
- "confidence": round(float(conf), 3),
- })
- frame_faces += 1
+ for f in faces:
+ observations.append({
+ "time": round(t, 3),
+ "face_center_x": f["cx"],
+ "face_center_y": f["cy"],
+ "face_width": f["fw"],
+ "confidence": f["confidence"],
+ })
+ frame_faces += 1
faces_per_frame.append(frame_faces)
if progress_callback and i % 20 == 0:
diff --git a/backend/services/face_detector.py b/backend/services/face_detector.py
new file mode 100644
index 0000000..3b0bc38
--- /dev/null
+++ b/backend/services/face_detector.py
@@ -0,0 +1,109 @@
+"""
+Shared face detector using OpenCV YuNet.
+
+YuNet is a lightweight CNN face detector (227KB) built into OpenCV.
+Much faster and more accurate than the old ResNet10 SSD, especially
+on side profiles and partially occluded faces.
+
+All face detection in the codebase should go through create_detector()
+and detect_faces() to keep the model path and parameters in one place.
+"""
+
+import os
+from typing import Optional
+
+# Minimum face width as fraction of frame width (filters noise)
+MIN_FACE_RATIO = 0.04
+
+# Detection confidence threshold (0.5 balances accuracy vs recall;
+# source 16:9 video scores higher than processed 9:16 clips)
+CONFIDENCE_THRESHOLD = 0.5
+
+# Max dimension for detection input — keeps inference fast.
+# Coordinates are scaled back to original frame size.
+_MAX_DIM = 640
+
+
+def _model_path() -> str:
+ backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+ return os.path.join(backend_dir, "models", "face_detection_yunet_2023mar.onnx")
+
+
+def create_detector(frame_width: int, frame_height: int):
+ """
+ Create a YuNet face detector for the given frame dimensions.
+
+ Internally uses a scaled-down input size for speed, but detect_faces()
+ returns coordinates in the original frame space.
+
+ Returns (detector, scale_x, scale_y) tuple, or None if model not found.
+ """
+ import cv2
+
+ model = _model_path()
+ if not os.path.exists(model):
+ return None
+
+ # Scale down to _MAX_DIM while preserving aspect ratio
+ scale = min(_MAX_DIM / frame_width, _MAX_DIM / frame_height, 1.0)
+ det_w = int(frame_width * scale)
+ det_h = int(frame_height * scale)
+
+ detector = cv2.FaceDetectorYN.create(
+ model=model,
+ config="",
+ input_size=(det_w, det_h),
+ score_threshold=CONFIDENCE_THRESHOLD,
+ nms_threshold=0.3,
+ top_k=10,
+ )
+ return (detector, det_w, det_h, frame_width / det_w, frame_height / det_h)
+
+
+def detect_faces(detector_tuple, frame, frame_width: int, frame_height: int) -> list[dict]:
+ """
+ Detect faces in a single frame using YuNet.
+
+ detector_tuple: the return value of create_detector()
+ Returns list of dicts: [{cx, cy, fw, fh, confidence}, ...]
+ Coordinates are in original frame space.
+ """
+ import cv2
+
+ detector, det_w, det_h, scale_x, scale_y = detector_tuple
+
+ resized = cv2.resize(frame, (det_w, det_h))
+ _, faces = detector.detect(resized)
+
+ results = []
+ if faces is None:
+ return results
+
+ for face in faces:
+ # Scale bbox back to original frame coordinates
+ x = int(face[0] * scale_x)
+ y = int(face[1] * scale_y)
+ w = int(face[2] * scale_x)
+ h = int(face[3] * scale_y)
+ conf = float(face[-1])
+
+ if w < frame_width * MIN_FACE_RATIO:
+ continue
+
+ cx = x + w // 2
+ cy = y + h // 2
+
+ results.append({
+ "cx": cx,
+ "cy": cy,
+ "fw": w,
+ "fh": h,
+ "confidence": round(conf, 3),
+ })
+
+ return results
+
+
+def is_available() -> bool:
+ """Check if YuNet model file exists."""
+ return os.path.exists(_model_path())
diff --git a/backend/services/thumbnail_ai.py b/backend/services/thumbnail_ai.py
index 5c90ed0..4ec0139 100644
--- a/backend/services/thumbnail_ai.py
+++ b/backend/services/thumbnail_ai.py
@@ -158,15 +158,13 @@ def extract_candidate_frames(
is_portrait = src_h > src_w # Already 9:16 or similar
# Load face detector
- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- proto = os.path.join(backend_dir, "models", "deploy.prototxt")
- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")
+ from services.face_detector import create_detector, detect_faces
+ detector = create_detector(src_w, src_h)
- if not (os.path.exists(proto) and os.path.exists(model)):
+ if detector is None:
cap.release()
return []
- detector = cv2.dnn.readNetFromCaffe(proto, model)
os.makedirs(output_dir, exist_ok=True)
# Sample more frames for better candidates
@@ -183,22 +181,19 @@ def extract_candidate_frames(
continue
h, w = frame.shape[:2]
- blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))
- detector.setInput(blob)
- detections = detector.forward()
-
- for j in range(detections.shape[2]):
- conf = detections[0, 0, j, 2]
- if conf > 0.6:
- x1 = max(0, int(detections[0, 0, j, 3] * w))
- y1 = max(0, int(detections[0, 0, j, 4] * h))
- x2 = min(w, int(detections[0, 0, j, 5] * w))
- y2 = min(h, int(detections[0, 0, j, 6] * h))
- fw = x2 - x1
- fh = y2 - y1
-
- if fw < w * 0.08:
- continue
+ faces_detected = detect_faces(detector, frame, src_w, src_h)
+
+ for f in faces_detected:
+ conf = f["confidence"]
+ x1 = max(0, f["cx"] - f["fw"] // 2)
+ y1 = max(0, f["cy"] - f["fh"] // 2)
+ x2 = min(w, f["cx"] + f["fw"] // 2)
+ y2 = min(h, f["cy"] + f["fh"] // 2)
+ fw = f["fw"]
+ fh = f["fh"]
+
+ if fw < w * 0.08:
+ continue
# Split-screen check — only for landscape sources where
# a small centered face likely means a multi-cam layout.
diff --git a/backend/services/thumbnail_generator.py b/backend/services/thumbnail_generator.py
index 891d18b..08523f9 100644
--- a/backend/services/thumbnail_generator.py
+++ b/backend/services/thumbnail_generator.py
@@ -251,16 +251,15 @@ def extract_face_frame(
duration = total_frames / fps
# Load face detector
- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- proto = os.path.join(backend_dir, "models", "deploy.prototxt")
- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")
+ from services.face_detector import create_detector, detect_faces
+ src_w = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
+ src_h = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ detector = create_detector(src_w, src_h)
- if not (os.path.exists(proto) and os.path.exists(model)):
+ if detector is None:
cap.release()
return None
- detector = cv2.dnn.readNetFromCaffe(proto, model)
-
best_frame = None
best_score = 0
best_face_cx = None
@@ -280,32 +279,25 @@ def extract_face_frame(
continue
h, w = frame.shape[:2]
- blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))
- detector.setInput(blob)
- detections = detector.forward()
-
- for j in range(detections.shape[2]):
- conf = detections[0, 0, j, 2]
- if conf > 0.6:
- x1 = int(detections[0, 0, j, 3] * w)
- y1 = int(detections[0, 0, j, 4] * h)
- x2 = int(detections[0, 0, j, 5] * w)
- y2 = int(detections[0, 0, j, 6] * h)
- face_w = x2 - x1
- face_h = y2 - y1
-
- if face_w < w * 0.08:
- continue
-
- # Score: confidence × face size (prefer larger/closer faces)
- score = conf * (face_w * face_h)
- if score > best_score:
- best_score = score
- best_frame = frame.copy()
- best_face_cx = (x1 + x2) // 2
- best_face_cy = (y1 + y2) // 2
- best_face_w = face_w
- best_face_h = face_h
+ faces_detected = detect_faces(detector, frame, src_w, src_h)
+
+ for f in faces_detected:
+ conf = f["confidence"]
+ face_w = f["fw"]
+ face_h = f["fh"]
+
+ if face_w < w * 0.08:
+ continue
+
+ # Score: confidence × face size (prefer larger/closer faces)
+ score = conf * (face_w * face_h)
+ if score > best_score:
+ best_score = score
+ best_frame = frame.copy()
+ best_face_cx = f["cx"]
+ best_face_cy = f["cy"]
+ best_face_w = face_w
+ best_face_h = face_h
cap.release()
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index e4c586f..81bbdf6 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -13,6 +13,9 @@ from typing import Optional
from services.encoder import get_video_encode_flags
import sys
+# Max time for any single FFmpeg call (seconds). Prevents infinite hangs.
+_FFMPEG_TIMEOUT = 300
+
# Quality presets: name → (crf, preset)
# Lower CRF = higher quality = larger file. 18 is visually lossless.
QUALITY_PRESETS = {
@@ -35,7 +38,7 @@ def _run_ffmpeg_with_fallback(cmd_parts_before_enc: list, cmd_parts_after_enc: l
enc_flags = get_video_encode_flags()
cmd = cmd_parts_before_enc + enc_flags + cmd_parts_after_enc + [output_path]
- result = subprocess.run(cmd, capture_output=True, text=True)
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if result.returncode == 0:
return output_path
@@ -43,7 +46,7 @@ def _run_ffmpeg_with_fallback(cmd_parts_before_enc: list, cmd_parts_after_enc: l
if enc_flags != CPU_FLAGS:
print(f"Warning: HW encoder failed for {label}, falling back to libx264", file=sys.stderr)
cmd_fallback = cmd_parts_before_enc + CPU_FLAGS + cmd_parts_after_enc + [output_path]
- result2 = subprocess.run(cmd_fallback, capture_output=True, text=True)
+ result2 = subprocess.run(cmd_fallback, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if result2.returncode == 0:
return output_path
raise RuntimeError(f"FFmpeg {label} failed (both HW and CPU): {result2.stderr[-500:]}")
@@ -51,6 +54,15 @@ def _run_ffmpeg_with_fallback(cmd_parts_before_enc: list, cmd_parts_after_enc: l
raise RuntimeError(f"FFmpeg {label} failed: {result.stderr[-500:]}")
+def _has_audio_stream(video_path: str) -> bool:
+ """Check if a video file contains an audio stream."""
+ try:
+ info = get_video_info(video_path)
+ return any(s.get("codec_type") == "audio" for s in info.get("streams", []))
+ except Exception:
+ return True # Assume yes if we can't check
+
+
def get_video_info(video_path: str) -> dict:
"""Get video metadata via ffprobe."""
cmd = [
@@ -61,7 +73,7 @@ def get_video_info(video_path: str) -> dict:
"-show_streams",
video_path,
]
- result = subprocess.run(cmd, capture_output=True, text=True)
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if result.returncode != 0:
raise RuntimeError(f"ffprobe failed: {result.stderr}")
return json.loads(result.stdout)
@@ -98,7 +110,7 @@ def cut_segment(
"-avoid_negative_ts", "make_zero",
output_path,
]
- result = subprocess.run(cmd, capture_output=True, text=True)
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if result.returncode != 0:
raise RuntimeError(f"FFmpeg cut failed: {result.stderr[-500:]}")
return output_path
@@ -146,7 +158,7 @@ def cut_multi_segment(
"-movflags", "+faststart",
output_path,
]
- result = subprocess.run(cmd, capture_output=True, text=True)
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if result.returncode != 0:
raise RuntimeError(f"FFmpeg concat failed: {result.stderr[-500:]}")
@@ -190,38 +202,42 @@ def crop_to_vertical(
source_ratio = width / height
if strategy in ("face", "speaker"):
- # Split-screen: cut each speaker segment separately with static crop per half.
- # Check face_map first (computed from full video), fall back to segment detection.
- is_split = False
- if face_map and face_map.get("is_split_screen"):
- is_split = True
- else:
- is_split = _detect_split_screen(input_path, width, height)
-
- if is_split and transcript_words:
- result = _crop_split_screen(
- input_path, output_path,
- width, height, target_w, target_h,
- transcript_words, clip_start,
- )
- if result:
- return result
-
- # Non-split-screen: single static face crop
- crop_x = _detect_face_center(input_path, width, height, target_ratio)
-
- if crop_x is not None:
- crop_w = int(height * target_ratio)
- vf = f"crop={crop_w}:{height}:{crop_x}:0,scale={target_w}:{target_h}"
- else:
- strategy = "center"
+ result = _track_and_crop(
+ input_path, output_path,
+ width, height, target_w, target_h,
+ transcript_words, clip_start,
+ )
+ if result:
+ return result
+ strategy = "center"
if strategy == "center":
if source_ratio > target_ratio:
- crop_h = height
- crop_w = int(crop_h * target_ratio)
- crop_x = (width - crop_w) // 2
- vf = f"crop={crop_w}:{crop_h}:{crop_x}:0,scale={target_w}:{target_h}"
+ # Wide source with no face detected: blurred background + sharp center.
+ # Scales source to fill 9:16 height → blur → overlay sharp fit-to-width.
+ vf_complex = (
+ f"split[bg][fg];"
+ f"[bg]scale=-2:{target_h},crop={target_w}:{target_h}:(iw-{target_w})/2:0,"
+ f"boxblur=25:3[bbg];"
+ f"[fg]scale={target_w}:-2[sfg];"
+ f"[bbg][sfg]overlay=0:(H-h)/2[v]"
+ )
+ return _run_ffmpeg_with_fallback(
+ cmd_parts_before_enc=[
+ "ffmpeg", "-y",
+ "-i", input_path,
+ "-filter_complex", vf_complex,
+ "-map", "[v]", "-map", "0:a?",
+ ],
+ cmd_parts_after_enc=[
+ "-c:a", "aac",
+ "-b:a", "192k",
+ "-ar", "44100",
+ "-movflags", "+faststart",
+ ],
+ output_path=output_path,
+ label="crop_blur_bg",
+ )
else:
crop_w = width
crop_h = int(crop_w / target_ratio)
@@ -252,7 +268,7 @@ def _detect_split_screen(video_path: str, width: int, height: int) -> bool:
"""Quick check: is this a split-screen layout (two side-by-side cameras)?"""
try:
import cv2
- import numpy as np
+ from services.face_detector import create_detector, detect_faces
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
@@ -262,14 +278,11 @@ def _detect_split_screen(video_path: str, width: int, height: int) -> bool:
total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
duration = total / fps
- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- proto = os.path.join(backend_dir, "models", "deploy.prototxt")
- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")
- if not (os.path.exists(proto) and os.path.exists(model)):
+ detector = create_detector(width, height)
+ if detector is None:
cap.release()
return False
- detector = cv2.dnn.readNetFromCaffe(proto, model)
mid_x = width // 2
frames_with_two_faces = 0
@@ -282,20 +295,15 @@ def _detect_split_screen(video_path: str, width: int, height: int) -> bool:
if not ret:
continue
- h, w = frame.shape[:2]
- blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))
- detector.setInput(blob)
- dets = detector.forward()
+ faces = detect_faces(detector, frame, width, height)
left = False
right = False
- for j in range(dets.shape[2]):
- if dets[0, 0, j, 2] > 0.5:
- cx = int((dets[0, 0, j, 3] + dets[0, 0, j, 5]) / 2 * w)
- if cx < mid_x:
- left = True
- else:
- right = True
+ for f in faces:
+ if f["cx"] < mid_x:
+ left = True
+ else:
+ right = True
total_sampled += 1
if left and right:
@@ -308,6 +316,336 @@ def _detect_split_screen(video_path: str, width: int, height: int) -> bool:
return False
+def _build_cam_expr(keyframes: list, duration: float, is_split: bool, max_parts: int = 80):
+ """
+ Build an FFmpeg crop_x expression from SmoothedCameraman keyframes.
+
+ keyframes: [(time, crop_x), ...]
+ Returns expression string, or None if too complex.
+ """
+ if not keyframes:
+ return None
+ # Ensure t=0 covered
+ if keyframes[0][0] > 0.05:
+ keyframes = [(0, keyframes[0][1])] + keyframes
+ if len(keyframes) == 1:
+ return str(keyframes[0][1])
+
+ parts = []
+ for i in range(len(keyframes) - 1):
+ t0, x0 = keyframes[i]
+ t1, x1 = keyframes[i + 1]
+ dt = max(0.01, t1 - t0)
+ jump = abs(x1 - x0)
+
+ if jump < 2:
+ # Negligible movement — hold
+ parts.append(f"if(between(t\\,{t0:.3f}\\,{t1:.3f})\\,{x0}\\,")
+ elif jump > 150 or (is_split and jump > 50):
+ # Large jump / split-screen speaker switch: instant cut
+ cut_t = round(t1 - 0.01, 3)
+ parts.append(f"if(between(t\\,{t0:.3f}\\,{cut_t:.3f})\\,{x0}\\,")
+ parts.append(f"if(between(t\\,{cut_t:.3f}\\,{t1:.3f})\\,{x1}\\,")
+ else:
+ # Smooth linear interpolation (camera pan)
+ parts.append(
+ f"if(between(t\\,{t0:.3f}\\,{t1:.3f})\\,"
+ f"{x0}+(({x1}-{x0})*(t-{t0:.3f})/{dt:.3f})\\,"
+ )
+
+ if len(parts) > max_parts:
+ return None
+
+ return "".join(parts) + str(keyframes[-1][1]) + ")" * len(parts)
+
+
+def _simplify_keyframes(keyframes: list, tolerance: int = 5) -> list:
+ """Remove intermediate keyframes that lie on a line between neighbours."""
+ if len(keyframes) <= 2:
+ return keyframes
+ result = [keyframes[0]]
+ for i in range(1, len(keyframes) - 1):
+ t_prev, x_prev = result[-1]
+ t_curr, x_curr = keyframes[i]
+ t_next, x_next = keyframes[i + 1]
+ dt_total = t_next - t_prev
+ if dt_total < 0.01:
+ continue
+ expected = x_prev + (x_next - x_prev) * (t_curr - t_prev) / dt_total
+ if abs(x_curr - expected) > tolerance:
+ result.append(keyframes[i])
+ result.append(keyframes[-1])
+ return result
+
+
+def _track_and_crop(
+ input_path: str,
+ output_path: str,
+ width: int,
+ height: int,
+ target_w: int,
+ target_h: int,
+ transcript_words: list = None,
+ clip_start: float = 0,
+) -> Optional[str]:
+ """
+ Adaptive face tracking with exponential-smoothing camera.
+
+ Works for both split-screen and single-camera layouts:
+ 1. Dense face sampling (~10 fps) with YuNet
+ 2. Transcript-driven speaker selection (who's talking → which face)
+ 3. Exponential-smoothing camera with dead zone (no jitter)
+ 4. Hard cuts only for split-screen speaker switches
+ 5. Vertical position from median face-Y (static per clip)
+ """
+ try:
+ import cv2
+ from services.face_detector import create_detector, detect_faces
+ except ImportError:
+ return None
+
+ target_ratio = target_w / target_h # 0.5625
+
+ cap = cv2.VideoCapture(input_path)
+ if not cap.isOpened():
+ return None
+
+ fps = cap.get(cv2.CAP_PROP_FPS) or 30
+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
+ duration = total_frames / fps if total_frames > 0 else 0
+
+ if duration < 0.5:
+ cap.release()
+ return None
+
+ detector = create_detector(width, height)
+ if detector is None:
+ cap.release()
+ return None
+
+ # ── Dense face sampling (~10 fps) ────────────────────────────
+ sample_step = max(1, int(fps / 10))
+ detections = [] # [(time, faces), ...]
+
+ frame_idx = 0
+ while frame_idx < total_frames:
+ cap.set(cv2.CAP_PROP_POS_FRAMES, frame_idx)
+ ret, frame = cap.read()
+ if not ret:
+ frame_idx += sample_step
+ continue
+ t = frame_idx / fps
+ faces = detect_faces(detector, frame, width, height)
+ detections.append((t, faces))
+ frame_idx += sample_step
+
+ cap.release()
+
+ if not detections:
+ return None
+
+ face_frames = sum(1 for _, faces in detections if faces)
+ if face_frames < 3:
+ return None
+
+ # ── Detect layout from face data ────────────────────────────
+ mid_x = width // 2
+ split_count = sum(1 for _, faces in detections if len(faces) >= 2)
+ has_any_split = split_count >= 3 # enough multi-face frames to build side mapping
+
+ # ── Crop dimensions ──────────────────────────────────────────
+ # Full-height crop: the 16:9 → 9:16 aspect conversion already
+ # gives ~3× horizontal zoom. Adding vertical zoom on top clips
+ # faces on laptop-angle cameras and shows ceiling. Keep it
+ # simple: crop the full height, only pan horizontally.
+ crop_h = height
+ crop_w = int(crop_h * target_ratio)
+ if crop_w > width:
+ crop_w = width
+ crop_h = int(crop_w / target_ratio)
+
+ # ── Speaker segments from transcript ─────────────────────────
+ segments = [] # [(start, end, speaker), ...]
+ if transcript_words:
+ cur_sp = None
+ seg_start = 0.0
+ for w in sorted(transcript_words, key=lambda x: x["start"]):
+ sp = w.get("speaker")
+ t = max(0.0, w["start"] - clip_start)
+ if sp != cur_sp and sp is not None:
+ if cur_sp is not None:
+ segments.append((seg_start, t, cur_sp))
+ cur_sp = sp
+ seg_start = t
+ if cur_sp:
+ segments.append((seg_start, duration, cur_sp))
+ # Merge short segments (<2s) — brief interjections ("yeah",
+ # "right") shouldn't trigger camera moves. Absorb them into
+ # the previous speaker's turn so the camera holds steady.
+ merged = []
+ for seg in segments:
+ if merged and seg[2] == merged[-1][2]:
+ merged[-1] = (merged[-1][0], seg[1], seg[2])
+ elif merged and (seg[1] - seg[0]) < 2.0:
+ merged[-1] = (merged[-1][0], seg[1], merged[-1][2])
+ else:
+ merged.append(list(seg))
+ segments = merged
+
+ speakers = sorted(set(seg[2] for seg in segments)) if segments else []
+
+ def _speaker_at(t):
+ for s, e, sp in segments:
+ if s <= t <= e:
+ return sp
+ return segments[0][2] if segments else None
+
+ # ── Speaker-to-side mapping ─────────────────────────────────
+ # Built from ANY frames with 2+ faces — works for both
+ # pure split-screen and Riverside-style mixed layouts.
+ speaker_side = {} # speaker → "left" | "right"
+ if has_any_split and len(speakers) >= 2:
+ talk = {}
+ first = {}
+ for seg in segments:
+ sp = seg[2]
+ talk[sp] = talk.get(sp, 0) + (seg[1] - seg[0])
+ if sp not in first:
+ first[sp] = seg[0]
+ top2 = sorted(speakers, key=lambda s: talk.get(s, 0), reverse=True)[:2]
+ by_first = sorted(top2, key=lambda s: first.get(s, float("inf")))
+ speaker_side[by_first[0]] = "left"
+ if len(by_first) > 1:
+ speaker_side[by_first[1]] = "right"
+ for sp in speakers:
+ if sp not in speaker_side:
+ speaker_side[sp] = "left"
+
+ # ── Face selection per frame ─────────────────────────────────
+ # Per-frame layout detection: if THIS frame has 2+ faces,
+ # pick the one on the active speaker's side. Handles Riverside
+ # videos that switch between single-camera and split-screen.
+ def _pick_face(faces, speaker):
+ if not faces:
+ return None
+ if len(faces) >= 2 and speaker in speaker_side:
+ side = speaker_side[speaker]
+ return (min if side == "left" else max)(faces, key=lambda f: f["cx"])
+ # Single face or no side mapping: use largest (most prominent)
+ return max(faces, key=lambda f: f["fw"])
+
+ # ── Exponential-smoothing camera ────────────────────────────
+ # The camera "chases" the target face using exponential decay,
+ # which gives natural ease-out motion (fast start, gradual
+ # settling). Unlike fixed-speed approaches this is frame-rate
+ # independent and feels organic.
+ #
+ # dead_zone: ignore face drift within this radius of the
+ # current camera center — prevents micro-jitter
+ # smooth_rate: convergence speed; at rate 4 the camera
+ # reaches ~95 % of the target in ≈0.75 s
+ # snap_distance: teleport instantly above this (speaker change
+ # or layout switch — no sweeping across the seam)
+ import math
+
+ dead_zone = crop_w * 0.20
+ smooth_rate = 4.0
+ snap_distance = crop_w * 0.55
+
+ # Initialise camera on the first detected face
+ cam_x = float(width) / 2
+ for _, faces in detections:
+ if faces:
+ cam_x = float(max(faces, key=lambda f: f["fw"])["cx"])
+ break
+
+ keyframes_x = []
+ prev_t = 0.0
+ prev_speaker = None
+ last_snap_t = -10.0 # time of last force_snap (start with no cooldown)
+ snap_cooldown = 2.0 # minimum seconds between force_snaps
+
+ for t, faces in detections:
+ speaker = _speaker_at(t) if segments else None
+ face = _pick_face(faces, speaker)
+
+ if face is None:
+ prev_t = t
+ continue
+
+ target_x = float(face["cx"])
+ diff = target_x - cam_x
+ dt = max(0.01, t - prev_t)
+
+ # Hard cut when the active speaker switches to a different
+ # side. Works for pure split-screen AND mixed layouts
+ # (Riverside-style single↔split transitions).
+ # Cooldown prevents rapid ping-pong from brief interjections
+ # that survive segment merging.
+ force_snap = (
+ speaker is not None
+ and speaker != prev_speaker
+ and prev_speaker is not None
+ and speaker_side.get(speaker) != speaker_side.get(prev_speaker)
+ and (t - last_snap_t) >= snap_cooldown
+ )
+
+ if force_snap or abs(diff) > snap_distance:
+ # Teleport — speaker switch or very large jump
+ cam_x = target_x
+ if force_snap:
+ last_snap_t = t
+ elif abs(diff) > dead_zone:
+ # Exponential smoothing: move a fraction of remaining
+ # distance each step. alpha → 1 means instant, → 0 means
+ # no movement. exp(-rate * dt) makes it time-step safe.
+ alpha = 1.0 - math.exp(-smooth_rate * dt)
+ cam_x += diff * alpha
+ # else: face inside dead zone — camera holds position
+
+ crop_x = int(cam_x - crop_w / 2)
+ crop_x = max(0, min(crop_x, width - crop_w))
+
+ if not keyframes_x or crop_x != keyframes_x[-1][1]:
+ keyframes_x.append((round(t, 3), crop_x))
+
+ prev_t = t
+ if speaker is not None:
+ prev_speaker = speaker
+
+ # ── Vertical position ────────────────────────────────────────
+ # Full-height crop → crop_y is always 0 (or centered if
+ # crop_h < height due to aspect-ratio clamping).
+ crop_y = max(0, (height - crop_h) // 2)
+
+ # ── Build FFmpeg filter ──────────────────────────────────────
+ if not keyframes_x:
+ crop_x = max(0, (width - crop_w) // 2)
+ vf = f"crop={crop_w}:{crop_h}:{crop_x}:{crop_y},scale={target_w}:{target_h}"
+ elif len(keyframes_x) == 1:
+ vf = f"crop={crop_w}:{crop_h}:{keyframes_x[0][1]}:{crop_y},scale={target_w}:{target_h}"
+ else:
+ # Simplify keyframes to reduce expression complexity
+ keyframes_x = _simplify_keyframes(keyframes_x)
+ x_expr = _build_cam_expr(keyframes_x, duration, has_any_split)
+ if x_expr is None:
+ # Too complex — fall back to median position
+ xs = [x for _, x in keyframes_x]
+ med_x = sorted(xs)[len(xs) // 2]
+ vf = f"crop={crop_w}:{crop_h}:{med_x}:{crop_y},scale={target_w}:{target_h}"
+ else:
+ vf = f"crop={crop_w}:{crop_h}:{x_expr}:{crop_y},scale={target_w}:{target_h}"
+
+ return _run_ffmpeg_with_fallback(
+ cmd_parts_before_enc=["ffmpeg", "-y", "-i", input_path, "-vf", vf],
+ cmd_parts_after_enc=[
+ "-c:a", "aac", "-b:a", "192k", "-ar", "44100",
+ "-movflags", "+faststart",
+ ],
+ output_path=output_path, label="crop_track",
+ )
+
+
def _crop_split_screen(
input_path: str,
output_path: str,
@@ -328,6 +666,7 @@ def _crop_split_screen(
try:
import cv2
import numpy as np
+ from services.face_detector import create_detector, detect_faces
target_ratio = target_w / target_h
mid_x = width // 2
@@ -341,20 +680,23 @@ def _crop_split_screen(
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
duration = total_frames / fps
- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- proto = os.path.join(backend_dir, "models", "deploy.prototxt")
- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")
- if not (os.path.exists(proto) and os.path.exists(model)):
+ detector = create_detector(width, height)
+ if detector is None:
cap.release()
return None
- detector = cv2.dnn.readNetFromCaffe(proto, model)
-
- # Collect face positions per half
+ # Collect face positions per half AND per-frame face counts
+ # to distinguish split-screen frames from single-person frames
left_faces = [] # (cx_in_half, cy)
right_faces = [] # (cx_in_half, cy)
+ single_faces = [] # (cx_in_full, cy) — faces from single-person frames
sample_count = min(40, max(10, int(duration * 2)))
+ split_frame_count = 0
+ single_frame_count = 0
+ # Record per-sample face count for per-segment layout lookup
+ sample_face_counts = [] # (time, num_faces)
+
for i in range(sample_count):
t = (i + 1) * duration / (sample_count + 1)
cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000)
@@ -362,63 +704,92 @@ def _crop_split_screen(
if not ret:
continue
- h, w = frame.shape[:2]
- blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))
- detector.setInput(blob)
- dets = detector.forward()
+ faces = detect_faces(detector, frame, width, height)
+ sample_face_counts.append((t, len(faces)))
- for j in range(dets.shape[2]):
- if dets[0, 0, j, 2] > 0.5:
- cx = int((dets[0, 0, j, 3] + dets[0, 0, j, 5]) / 2 * w)
- cy = int((dets[0, 0, j, 4] + dets[0, 0, j, 6]) / 2 * h)
+ if len(faces) >= 2:
+ # Split-screen frame
+ split_frame_count += 1
+ for f in faces:
+ cx, cy = f["cx"], f["cy"]
if cx < mid_x:
left_faces.append((cx, cy))
else:
- right_faces.append((cx - mid_x, cy)) # relative to right half
+ right_faces.append((cx - mid_x, cy))
+ elif len(faces) == 1:
+ # Single-person frame
+ single_frame_count += 1
+ single_faces.append((faces[0]["cx"], faces[0]["cy"]))
cap.release()
- if not left_faces and not right_faces:
+ if not left_faces and not right_faces and not single_faces:
return None
- # Compute static crop for each half
- # Within a 960x1080 half, we need a 9:16 crop
+ is_mixed = split_frame_count >= 3 and single_frame_count >= 3
+
+ # Compute static crop for each half (split-screen segments)
half_w = mid_x
- crop_w = int(height * target_ratio) # ~607 for 1080 height
+ # Vertical zoom: 75% of frame height for tighter face framing
+ adj_h = int(height * 0.75)
+ adj_w = int(adj_h * target_ratio)
+ if adj_w > half_w:
+ # Zoomed crop wider than half — fall back to full height
+ adj_h = height
+ adj_w = int(height * target_ratio)
+
+ def _compute_split_crop(faces, half_offset):
+ """Compute crop for a split-screen half (zoomed to face)."""
+ if not faces:
+ cx_in_half = half_w // 2
+ crop_x = max(0, min(cx_in_half - adj_w // 2, half_w - adj_w))
+ return half_offset + crop_x, max(0, (height - adj_h) // 2), adj_h
+
+ cx = int(np.median([f[0] for f in faces]))
+ cy = int(np.median([f[1] for f in faces]))
+
+ crop_x = cx - adj_w // 2
+ crop_x = max(0, min(crop_x, half_w - adj_w))
+ crop_x += half_offset
- def _compute_crop(faces, half_offset):
- """Compute static crop_x, crop_y for one camera half."""
+ # Vertical: place face at ~33% from top
+ crop_y = cy - int(adj_h * 0.33)
+ crop_y = max(0, min(crop_y, height - adj_h))
+
+ return crop_x, crop_y, adj_h
+
+ def _compute_single_crop(faces):
+ """Compute crop for single-person fullscreen (with same zoom for consistency)."""
if not faces:
- # No face — center crop within this half
- return half_offset + (half_w - crop_w) // 2, 0, height
+ return (width - adj_w) // 2, max(0, (height - adj_h) // 2), adj_h
cx = int(np.median([f[0] for f in faces]))
cy = int(np.median([f[1] for f in faces]))
- # Horizontal: center crop on face within this half
- crop_x = cx - crop_w // 2
- crop_x = max(0, min(crop_x, half_w - crop_w))
- crop_x += half_offset # offset to position within full frame
+ crop_x = cx - adj_w // 2
+ crop_x = max(0, min(crop_x, width - adj_w))
- # Vertical: place face at ~33% from top with 75% zoom
- adj_h = int(height * 0.75)
crop_y = cy - int(adj_h * 0.33)
crop_y = max(0, min(crop_y, height - adj_h))
return crop_x, crop_y, adj_h
- left_crop_x, left_crop_y, left_h = _compute_crop(left_faces, 0)
- right_crop_x, right_crop_y, right_h = _compute_crop(right_faces, mid_x)
+ left_crop_x, left_crop_y, left_h = _compute_split_crop(left_faces, 0)
+ right_crop_x, right_crop_y, right_h = _compute_split_crop(right_faces, mid_x)
+ single_crop_x, single_crop_y, single_h = _compute_single_crop(single_faces)
left_w = int(left_h * target_ratio)
right_w = int(right_h * target_ratio)
+ single_w = int(single_h * target_ratio)
# Map speakers to sides (first speaker by time = left)
speakers = sorted(set(w.get("speaker") for w in transcript_words if w.get("speaker")))
if len(speakers) < 2:
- # Single speaker — use the side with more face detections
- if len(left_faces) >= len(right_faces):
+ # Single speaker — if mixed layout, prefer fullscreen crop; else use best half
+ if is_mixed and single_faces:
+ vf = f"crop={single_w}:{single_h}:{single_crop_x}:{single_crop_y},scale={target_w}:{target_h}"
+ elif len(left_faces) >= len(right_faces):
vf = f"crop={left_w}:{left_h}:{left_crop_x}:{left_crop_y},scale={target_w}:{target_h}"
else:
vf = f"crop={right_w}:{right_h}:{right_crop_x}:{right_crop_y},scale={target_w}:{target_h}"
@@ -449,14 +820,14 @@ def _crop_split_screen(
if sp not in speaker_side:
speaker_side[sp] = speaker_side.get(top_2_by_first[0], "left")
- # Build speaker segments
+ # Build speaker segments (times relative to the cut segment, clamped to [0, duration])
segments = []
cur_speaker = None
seg_start = 0
for w in sorted(transcript_words, key=lambda x: x["start"]):
sp = w.get("speaker")
- t = w["start"] - clip_start
+ t = max(0, min(w["start"] - clip_start, duration))
if sp != cur_speaker and sp is not None:
if cur_speaker is not None:
segments.append((seg_start, t, cur_speaker))
@@ -480,24 +851,54 @@ def _crop_split_screen(
if not segments:
return None
+ # For mixed layouts, detect per-segment layout by sampling a frame
+ # from the original (uncut) video at the segment's absolute time
+ def _is_segment_split(seg_start_t):
+ """Check if a segment's time falls in a split-screen or single-person region.
+ Uses the pre-computed sample_face_counts instead of re-opening the video."""
+ if not is_mixed:
+ return True # Pure split-screen, all segments use half crops
+ # Find the nearest sampled frame to this segment's start time
+ best_dist = float("inf")
+ best_count = 2 # default to split
+ for t, n in sample_face_counts:
+ dist = abs(t - seg_start_t)
+ if dist < best_dist:
+ best_dist = dist
+ best_count = n
+ return best_count >= 2
+
# Cut each segment with its speaker's crop, then concat
work_dir = os.path.dirname(output_path) or "."
part_paths = []
try:
for i, (start_t, end_t, speaker) in enumerate(segments):
- side = speaker_side.get(speaker, "left")
- if side == "left":
- cw, ch, cx, cy = left_w, left_h, left_crop_x, left_crop_y
+ # Skip segments shorter than 1 frame (~0.04s at 24fps)
+ if end_t - start_t < 0.05:
+ continue
+
+ # For mixed layouts: detect if this segment is split or single
+ seg_is_split = _is_segment_split(start_t)
+
+ if seg_is_split:
+ side = speaker_side.get(speaker, "left")
+ if side == "left":
+ cw, ch, cx, cy = left_w, left_h, left_crop_x, left_crop_y
+ else:
+ cw, ch, cx, cy = right_w, right_h, right_crop_x, right_crop_y
else:
- cw, ch, cx, cy = right_w, right_h, right_crop_x, right_crop_y
+ # Single-person fullscreen — use full-frame crop (no vertical zoom)
+ cw, ch, cx, cy = single_w, single_h, single_crop_x, single_crop_y
part_path = os.path.join(work_dir, f"_speaker_part_{i}.mp4")
vf = f"crop={cw}:{ch}:{cx}:{cy},scale={target_w}:{target_h}"
+ seg_duration = end_t - start_t
+
cmd = [
"ffmpeg", "-y",
- "-ss", str(start_t), "-t", str(end_t - start_t),
+ "-ss", str(start_t), "-t", str(seg_duration),
"-i", input_path,
"-vf", vf,
"-c:v", "libx264", "-crf", "18", "-preset", "fast",
@@ -505,7 +906,7 @@ def _crop_split_screen(
"-avoid_negative_ts", "make_zero",
part_path,
]
- r = subprocess.run(cmd, capture_output=True, text=True)
+ r = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if r.returncode != 0:
print(f"Warning: split-screen segment {i} failed: {r.stderr[-200:]}", file=sys.stderr)
continue
@@ -528,7 +929,7 @@ def _crop_split_screen(
"-movflags", "+faststart",
output_path,
]
- r = subprocess.run(cmd, capture_output=True, text=True)
+ r = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if r.returncode != 0:
return None
@@ -555,6 +956,7 @@ def _use_face_map(
width: int,
height: int,
target_ratio: float,
+ crop_h: int = None,
) -> Optional[str]:
"""
Use pre-computed face_map from transcription to determine crop position.
@@ -562,12 +964,13 @@ def _use_face_map(
For multi-speaker clips: builds speaker-aware panning expression.
For single-speaker: returns the dominant speaker's crop position.
+ crop_h: if set, compute crop width from this height (for vertical zoom).
"""
clusters = face_map.get("clusters", [])
speaker_mappings = face_map.get("speaker_mappings", {})
dominant = face_map.get("dominant_speaker")
is_split_screen = face_map.get("is_split_screen", False)
- crop_w = int(height * target_ratio)
+ crop_w = int((crop_h or height) * target_ratio)
mid_x = width // 2
seam_margin = 20
@@ -579,14 +982,18 @@ def _use_face_map(
if map_w != width:
return None
- # Apply seam margin to crop_x values for split-screen (copy to avoid mutating face_map)
- if is_split_screen and len(clusters) >= 2:
- clusters = [dict(cl) for cl in clusters]
- for cl in clusters:
- if cl["center_x"] < mid_x:
- cl["crop_x"] = max(0, min(cl["crop_x"], mid_x - crop_w - seam_margin))
+ # Recompute crop_x from center_x with current crop_w (may differ from
+ # stored values if vertical zoom changes the crop width).
+ clusters = [dict(cl) for cl in clusters]
+ for cl in clusters:
+ cx = cl["center_x"]
+ if is_split_screen and len(clusters) >= 2:
+ if cx < mid_x:
+ cl["crop_x"] = max(0, min(cx - crop_w // 2, mid_x - crop_w - seam_margin))
else:
- cl["crop_x"] = max(mid_x + seam_margin, min(cl["crop_x"], width - crop_w))
+ cl["crop_x"] = max(mid_x + seam_margin, min(cx - crop_w // 2, width - crop_w))
+ else:
+ cl["crop_x"] = max(0, min(cx - crop_w // 2, width - crop_w))
# Check if clip has multiple speakers
speakers_in_clip = set()
@@ -614,7 +1021,7 @@ def _use_face_map(
for w in sorted(transcript_words, key=lambda x: x["start"]):
sp = w.get("speaker")
- t = w["start"] - clip_start
+ t = max(0, w["start"] - clip_start)
if sp != current_speaker and sp is not None:
if current_speaker is not None:
segments.append((seg_start, t, current_speaker))
@@ -708,6 +1115,7 @@ def _build_speaker_aware_crop(
try:
import cv2
import numpy as np
+ from services.face_detector import create_detector, detect_faces
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
@@ -717,19 +1125,17 @@ def _build_speaker_aware_crop(
fps = cap.get(cv2.CAP_PROP_FPS) or 30
duration = total_frames / fps
- # Load DNN face detector
- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- proto = os.path.join(backend_dir, "models", "deploy.prototxt")
- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")
- if not (os.path.exists(proto) and os.path.exists(model)):
+ detector = create_detector(width, height)
+ if detector is None:
cap.release()
return None
- detector = cv2.dnn.readNetFromCaffe(proto, model)
-
# Zoom to 75% of frame height for tight face framing
adjusted_h = int(height * 0.75)
adj_w = int(adjusted_h * target_ratio)
+ if adj_w > width:
+ adjusted_h = height
+ adj_w = int(height * target_ratio)
mid_x = width // 2
# ── Phase 1: Dense frame sampling ──────────────────────────────
@@ -747,25 +1153,11 @@ def _build_speaker_aware_crop(
if not ret:
continue
- h, w = frame.shape[:2]
- blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))
- detector.setInput(blob)
- detections = detector.forward()
+ faces_raw = detect_faces(detector, frame, width, height)
faces = []
- for j in range(detections.shape[2]):
- conf = detections[0, 0, j, 2]
- if conf > 0.5:
- x1 = int(detections[0, 0, j, 3] * w)
- x2 = int(detections[0, 0, j, 5] * w)
- y1 = int(detections[0, 0, j, 4] * h)
- y2 = int(detections[0, 0, j, 6] * h)
- fw = x2 - x1
- if fw < w * 0.04:
- continue
- cx = (x1 + x2) // 2
- cy = (y1 + y2) // 2
- faces.append((cx, cy, fw, float(conf)))
+ for f in faces_raw:
+ faces.append((f["cx"], f["cy"], f["fw"], f["confidence"]))
frame_data.append((t, faces))
@@ -796,7 +1188,7 @@ def _build_speaker_aware_crop(
for w in sorted(transcript_words, key=lambda x: x["start"]):
sp = w.get("speaker")
- t = w["start"] - clip_start
+ t = max(0, w["start"] - clip_start)
if sp != current_speaker and sp is not None:
if current_speaker is not None:
segments.append((seg_start, t, current_speaker))
@@ -998,6 +1390,18 @@ def _build_speaker_aware_crop(
parts.append(
f"if(between(t\\,{jump_t:.2f}\\,{t1:.2f})\\,{v1}\\,"
)
+ elif value_jump > jump_threshold * 0.6:
+ # Medium jump (significant lean/shift): fast reframe (~0.15s)
+ pan_t = min(0.15, dt)
+ mid_t = t0 + pan_t
+ parts.append(
+ f"if(between(t\\,{t0:.2f}\\,{mid_t:.2f})\\,"
+ f"{v0}+(({v1}-{v0})*(t-{t0:.2f})/{pan_t:.2f})\\,"
+ )
+ if mid_t < t1:
+ parts.append(
+ f"if(between(t\\,{mid_t:.2f}\\,{t1:.2f})\\,{v1}\\,"
+ )
else:
# Close keyframes with small movement: smooth interpolation
parts.append(
@@ -1045,7 +1449,7 @@ def _create_gradient_png(output_path: str, width: int = 1080, height: int = 1920
"-frames:v", "1",
output_path,
]
- result = subprocess.run(cmd, capture_output=True, text=True)
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if result.returncode != 0:
raise RuntimeError(f"Gradient creation failed: {result.stderr[-300:]}")
return output_path
@@ -1115,6 +1519,10 @@ def burn_captions(
filter_complex = ";".join(filter_parts)
+ # Check if input has audio (some split-screen concat outputs may not)
+ has_audio = _has_audio_stream(input_path)
+ audio_map = ["-map", "0:a?", "-c:a", "copy"] if has_audio else []
+
# Run with HW encoder, fallback to CPU if it fails
enc_flags = get_video_encode_flags()
cmd = [
@@ -1122,14 +1530,13 @@ def burn_captions(
*inputs,
"-filter_complex", filter_complex,
"-map", "[out]",
- "-map", "0:a",
+ *audio_map,
*enc_flags,
- "-c:a", "copy",
"-movflags", "+faststart",
output_path,
]
- result = subprocess.run(cmd, capture_output=True, text=True)
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
# Fallback to CPU if HW encoder failed
if result.returncode != 0 and enc_flags != CPU_FLAGS:
@@ -1139,13 +1546,12 @@ def burn_captions(
*inputs,
"-filter_complex", filter_complex,
"-map", "[out]",
- "-map", "0:a",
+ *audio_map,
*CPU_FLAGS,
- "-c:a", "copy",
"-movflags", "+faststart",
output_path,
]
- result = subprocess.run(cmd_fallback, capture_output=True, text=True)
+ result = subprocess.run(cmd_fallback, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
# Clean up gradient file
if gradient_overlay and os.path.exists(gradient_path):
@@ -1175,7 +1581,7 @@ def concat_outro(
"ffprobe", "-v", "quiet", "-print_format", "json",
"-show_format", input_path,
]
- probe = subprocess.run(probe_cmd, capture_output=True, text=True)
+ probe = subprocess.run(probe_cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
try:
main_duration = float(json.loads(probe.stdout)["format"]["duration"])
except Exception:
@@ -1217,7 +1623,7 @@ def concat_outro(
"-movflags", "+faststart",
output_path,
]
- result = subprocess.run(xfade_cmd, capture_output=True, text=True)
+ result = subprocess.run(xfade_cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if result.returncode == 0:
# Clean up
if os.path.exists(outro_scaled):
@@ -1257,7 +1663,7 @@ def concat_outro(
"-movflags", "+faststart",
output_path,
]
- result = subprocess.run(cmd, capture_output=True, text=True)
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if result.returncode != 0:
raise RuntimeError(f"FFmpeg concat failed: {result.stderr[-500:]}")
return output_path
@@ -1283,7 +1689,7 @@ def normalize_audio(
"-af", f"loudnorm=I={target_lufs}:TP=-1.5:LRA=11:print_format=json",
"-f", "null", "-",
]
- result = subprocess.run(measure_cmd, capture_output=True, text=True)
+ result = subprocess.run(measure_cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
# Try to parse loudnorm output from stderr
stderr = result.stderr
@@ -1331,7 +1737,7 @@ def normalize_audio(
"-movflags", "+faststart",
output_path,
]
- result = subprocess.run(cmd, capture_output=True, text=True)
+ result = subprocess.run(cmd, capture_output=True, text=True, timeout=_FFMPEG_TIMEOUT)
if result.returncode != 0:
raise RuntimeError(f"FFmpeg normalize failed: {result.stderr[-500:]}")
return output_path
@@ -1354,6 +1760,7 @@ def _detect_face_y(
try:
import cv2
import numpy as np
+ from services.face_detector import create_detector, detect_faces
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
@@ -1363,21 +1770,18 @@ def _detect_face_y(
fps = cap.get(cv2.CAP_PROP_FPS) or 30
duration = total_frames / fps
- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- proto = os.path.join(backend_dir, "models", "deploy.prototxt")
- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")
- if not (os.path.exists(proto) and os.path.exists(model)):
- cap.release()
- return None
-
- detector = cv2.dnn.readNetFromCaffe(proto, model)
-
- # Try to evaluate crop_x as a static value for sampling
+ # Try to evaluate crop_x as a static value for ROI
try:
crop_x = int(float(crop_x_expr))
except (ValueError, TypeError):
crop_x = None
+ # Detect on full frame, then filter by crop region
+ detector = create_detector(width, height)
+ if detector is None:
+ cap.release()
+ return None
+
face_ys = []
sample_count = min(20, max(5, int(duration)))
@@ -1388,28 +1792,15 @@ def _detect_face_y(
if not ret:
continue
- # If we know crop_x, only look at that column for faces
- h, w = frame.shape[:2]
- if crop_x is not None:
- x1 = max(0, crop_x)
- x2 = min(w, crop_x + crop_w)
- roi = frame[:, x1:x2]
- else:
- roi = frame
-
- rh, rw = roi.shape[:2]
- blob = cv2.dnn.blobFromImage(cv2.resize(roi, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))
- detector.setInput(blob)
- dets = detector.forward()
-
- for j in range(dets.shape[2]):
- conf = dets[0, 0, j, 2]
- if conf > 0.5:
- y1 = int(dets[0, 0, j, 4] * rh)
- y2 = int(dets[0, 0, j, 6] * rh)
- fh = y2 - y1
- if fh > rh * 0.05:
- face_ys.append((y1 + y2) // 2)
+ faces = detect_faces(detector, frame, width, height)
+
+ for f in faces:
+ # If we know crop_x, only use faces within the crop column
+ if crop_x is not None:
+ if f["cx"] < crop_x or f["cx"] > crop_x + crop_w:
+ continue
+ if f["fh"] > height * 0.05:
+ face_ys.append(f["cy"])
cap.release()
@@ -1427,15 +1818,20 @@ def _detect_face_center(
width: int,
height: int,
target_ratio: float,
-) -> Optional[str]:
+ crop_h: int = None,
+) -> Optional[tuple]:
"""
Simple static face detection — finds the dominant face and returns a
STATIC crop position that centers the face horizontally. No tracking,
no panning. The face stays centered for the entire clip.
+
+ Returns (crop_x_str, median_face_cy) tuple, or None if no face detected.
+ crop_h: if set, compute crop width from this height (for vertical zoom).
"""
try:
import cv2
import numpy as np
+ from services.face_detector import create_detector, detect_faces
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
@@ -1444,21 +1840,17 @@ def _detect_face_center(
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
fps = cap.get(cv2.CAP_PROP_FPS) or 30
duration = total_frames / fps
- crop_w = int(height * target_ratio)
+ crop_w = int((crop_h or height) * target_ratio)
- # Load DNN face detector
- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- proto = os.path.join(backend_dir, "models", "deploy.prototxt")
- model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")
- if not (os.path.exists(proto) and os.path.exists(model)):
+ detector = create_detector(width, height)
+ if detector is None:
cap.release()
return None
- detector = cv2.dnn.readNetFromCaffe(proto, model)
-
# Sample frames across the clip
sample_count = min(40, max(10, int(duration * 2)))
face_positions = []
+ face_cy_values = []
for i in range(sample_count):
t = i * duration / sample_count
@@ -1467,37 +1859,30 @@ def _detect_face_center(
if not ret:
continue
- h, w = frame.shape[:2]
- blob = cv2.dnn.blobFromImage(
- cv2.resize(frame, (300, 300)), 1.0, (300, 300),
- (104.0, 177.0, 123.0)
- )
- detector.setInput(blob)
- detections = detector.forward()
+ faces = detect_faces(detector, frame, width, height)
+ # Pick the best face (largest × most confident)
best_score = 0
best_cx = None
- for j in range(detections.shape[2]):
- conf = detections[0, 0, j, 2]
- if conf > 0.5:
- x1 = int(detections[0, 0, j, 3] * w)
- x2 = int(detections[0, 0, j, 5] * w)
- face_w = x2 - x1
- if face_w < w * 0.05:
- continue
- score = conf * (face_w ** 2)
- if score > best_score:
- best_score = score
- best_cx = (x1 + x2) // 2
+ best_cy = None
+ for f in faces:
+ score = f["confidence"] * (f["fw"] ** 2)
+ if score > best_score:
+ best_score = score
+ best_cx = f["cx"]
+ best_cy = f["cy"]
if best_cx is not None:
face_positions.append(best_cx)
+ face_cy_values.append(best_cy)
cap.release()
if not face_positions:
return None
+ median_cy = int(np.median(face_cy_values)) if face_cy_values else None
+
# Detect split-screen: check if faces appear in both halves
positions = np.array(face_positions)
mid_x = width // 2
@@ -1522,7 +1907,7 @@ def _detect_face_center(
crop_x = face_x - crop_w // 2
crop_x = max(0, min(crop_x, width - crop_w))
- return str(crop_x)
+ return (str(crop_x), median_cy)
except ImportError:
return None
@@ -1536,7 +1921,8 @@ def _detect_face_offset(
width: int,
height: int,
target_ratio: float,
-) -> Optional[str]:
+ crop_h: int = None,
+) -> Optional[tuple]:
"""
Continuous face tracking — detects face position over time and returns
an FFmpeg expression that smoothly follows the dominant face.
@@ -1544,11 +1930,13 @@ def _detect_face_offset(
Samples frames throughout the clip, picks the dominant face cluster,
then builds a smoothed panning timeline so the crop follows head movement.
- Returns an FFmpeg crop-x expression string, or None if no face detected.
+ Returns (x_expr, median_face_cy) tuple, or None if no face detected.
+ crop_h: if set, compute crop width from this height (for vertical zoom).
"""
try:
import cv2
import numpy as np
+ from services.face_detector import create_detector, detect_faces
cap = cv2.VideoCapture(video_path)
if not cap.isOpened():
@@ -1557,7 +1945,7 @@ def _detect_face_offset(
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
fps = cap.get(cv2.CAP_PROP_FPS) or 30
duration = total_frames / fps
- crop_w = int(height * target_ratio)
+ crop_w = int((crop_h or height) * target_ratio)
# Sample ~4 frames/sec for smooth tracking (more than before)
sample_count = min(240, max(30, int(duration * 4)))
@@ -1565,33 +1953,12 @@ def _detect_face_offset(
# (time, face_center_x) for each detection
timed_positions = []
+ face_cy_values = []
- # Load DNN face detector
- dnn_detector = None
- backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
- bundled_proto = os.path.join(backend_dir, "models", "deploy.prototxt")
- bundled_model = os.path.join(backend_dir, "models",
- "res10_300x300_ssd_iter_140000.caffemodel")
- cv2_proto = os.path.join(os.path.dirname(cv2.__file__), "data",
- "deploy.prototxt")
- cv2_model = os.path.join(os.path.dirname(cv2.__file__), "data",
- "res10_300x300_ssd_iter_140000.caffemodel")
- proto_path = bundled_proto if os.path.exists(bundled_proto) else cv2_proto
- model_path = bundled_model if os.path.exists(bundled_model) else cv2_model
- if os.path.exists(proto_path) and os.path.exists(model_path):
- dnn_detector = cv2.dnn.readNetFromCaffe(proto_path, model_path)
-
- # Fallback: Haar cascades
- cascades = []
- if dnn_detector is None:
- for cascade_name in [
- "haarcascade_frontalface_default.xml",
- "haarcascade_frontalface_alt2.xml",
- "haarcascade_profileface.xml",
- ]:
- cascade_path = cv2.data.haarcascades + cascade_name
- if os.path.exists(cascade_path):
- cascades.append(cv2.CascadeClassifier(cascade_path))
+ detector = create_detector(width, height)
+ if detector is None:
+ cap.release()
+ return None
for t in sample_times:
cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000)
@@ -1599,53 +1966,33 @@ def _detect_face_offset(
if not ret:
continue
- h, w = frame.shape[:2]
+ faces = detect_faces(detector, frame, width, height)
- if dnn_detector is not None:
- blob = cv2.dnn.blobFromImage(
- cv2.resize(frame, (300, 300)), 1.0, (300, 300),
- (104.0, 177.0, 123.0)
- )
- dnn_detector.setInput(blob)
- detections = dnn_detector.forward()
-
- best_score = 0
- best_cx = None
- for i in range(detections.shape[2]):
- confidence = detections[0, 0, i, 2]
- if confidence > 0.5:
- x1 = int(detections[0, 0, i, 3] * w)
- y1 = int(detections[0, 0, i, 4] * h)
- x2 = int(detections[0, 0, i, 5] * w)
- y2 = int(detections[0, 0, i, 6] * h)
- face_w = x2 - x1
- face_h = y2 - y1
- if face_w < w * 0.05 or face_h < h * 0.05:
- continue
- score = confidence * (face_w ** 2)
- if score > best_score:
- best_score = score
- best_cx = (x1 + x2) // 2
-
- if best_cx is not None:
- timed_positions.append((t, best_cx))
- else:
- gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
- for cascade in cascades:
- faces = cascade.detectMultiScale(
- gray, 1.1, 5, minSize=(60, 60)
- )
- if len(faces) > 0:
- largest = max(faces, key=lambda f: f[2] * f[3])
- face_center_x = largest[0] + largest[2] // 2
- timed_positions.append((t, face_center_x))
- break
+ # Pick the best face (largest × most confident)
+ best_score = 0
+ best_cx = None
+ best_cy = None
+ for f in faces:
+ if f["fh"] < height * 0.05:
+ continue
+ score = f["confidence"] * (f["fw"] ** 2)
+ if score > best_score:
+ best_score = score
+ best_cx = f["cx"]
+ best_cy = f["cy"]
+
+ if best_cx is not None:
+ timed_positions.append((t, best_cx))
+ face_cy_values.append(best_cy)
cap.release()
if len(timed_positions) < 3:
return None
+ # Median face Y for vertical positioning (computed once, used in all returns)
+ median_cy = int(np.median(face_cy_values)) if face_cy_values else None
+
# --- Cluster to find dominant face ---
all_cx = np.array([p[1] for p in timed_positions])
cluster_radius = crop_w * 0.35
@@ -1673,7 +2020,7 @@ def _detect_face_offset(
# Too few points — use static median
median_x = int(np.median([cx for _, cx in tracked]))
crop_x = max(0, min(median_x - crop_w // 2, width - crop_w))
- return str(crop_x)
+ return (str(crop_x), median_cy)
# --- Smooth the positions ---
# 1) Convert face_center_x to crop_x (clamped with margin).
@@ -1705,18 +2052,18 @@ def _detect_face_offset(
smoothed = np.clip(np.round(smoothed), 0, width - crop_w).astype(int)
# --- Check if tracking is even needed ---
- # If the face barely moves (range < 8% of frame width), just use static.
+ # If the face barely moves (range < 20% of crop width), just use static.
# Most podcast clips have speakers sitting still — tracking should be rare.
movement_range = int(smoothed.max() - smoothed.min())
- if movement_range < width * 0.08:
- return str(int(np.median(smoothed)))
+ if movement_range < crop_w * 0.20:
+ return (str(int(np.median(smoothed))), median_cy)
# --- Build keyframes: only emit on MAJOR position changes ---
# Very conservative: only pan when speaker physically moves to a new
# position (leans far, switches seats). Normal head movement = static.
keyframes = [(tracked_times[0], int(smoothed[0]))]
min_time_gap = 3.0 # Don't place keyframes closer than 3s
- min_px_change = 80 # Ignore movement smaller than 80px (~7% of frame)
+ min_px_change = max(40, int(crop_w * 0.13)) # Proportional safe zone (~80px at 1080p)
for i in range(1, len(smoothed)):
t = tracked_times[i]
@@ -1732,7 +2079,7 @@ def _detect_face_offset(
keyframes.append((t, x))
if len(keyframes) < 2:
- return str(keyframes[0][1])
+ return (str(keyframes[0][1]), median_cy)
# --- Build FFmpeg expression ---
# Piecewise linear interpolation between keyframes.
@@ -1760,13 +2107,13 @@ def _detect_face_offset(
keyframes = keyframes[::step] + [keyframes[-1]]
# Rebuild (recursive call would be cleaner but let's just return static)
print(f"Warning: face tracking produced {len(expr_parts)} keyframes, using static", file=sys.stderr)
- return str(int(np.median(smoothed)))
+ return (str(int(np.median(smoothed))), median_cy)
# Final value: last keyframe position
last_x = keyframes[-1][1]
expr = "".join(expr_parts) + str(last_x) + ")" * len(expr_parts)
- return expr
+ return (expr, median_cy)
except ImportError:
return None
diff --git a/src/handlers/batch-clips.handler.ts b/src/handlers/batch-clips.handler.ts
index c30e12e..cf1e69e 100644
--- a/src/handlers/batch-clips.handler.ts
+++ b/src/handlers/batch-clips.handler.ts
@@ -124,6 +124,8 @@ export async function handleBatchClips(
(s.suggested_caption_style as string) || settings.captionStyle || "hormozi",
crop_strategy: settings.cropStrategy || "face",
logo_path: settings.logoPath || null,
+ // Preserve multi-cut segments from suggestion
+ ...(Array.isArray(s.segments) && s.segments.length > 0 && { keep_segments: s.segments }),
});
if (input.export_selected) {
diff --git a/src/handlers/create-clip.handler.ts b/src/handlers/create-clip.handler.ts
index 0971c93..94afad2 100644
--- a/src/handlers/create-clip.handler.ts
+++ b/src/handlers/create-clip.handler.ts
@@ -149,6 +149,10 @@ export async function handleCreateClip(
(transcript?.words as Array<unknown>) ||
[];
+ // Pull multi-cut segments from suggestion (if available)
+ const keepSegments =
+ (suggestion?.segments as Array<{ start: number; end: number }>) || null;
+
// Validate required fields
if (!videoPath) {
return JSON.stringify({ error: "video_path is required (no video in session state)" });
@@ -171,6 +175,7 @@ export async function handleCreateClip(
clean_fillers: input.clean_fillers !== false,
logo_path: logoPath,
outro_path: outroPath,
+ ...(keepSegments && { keep_segments: keepSegments }),
});
const data = result.data as unknown as ClipResult;
diff --git a/src/handlers/suggest-clips.handler.ts b/src/handlers/suggest-clips.handler.ts
index 16e119b..9642f13 100644
--- a/src/handlers/suggest-clips.handler.ts
+++ b/src/handlers/suggest-clips.handler.ts
@@ -38,6 +38,20 @@ export const suggestClipsToolDef = {
type: "number",
description: "End timestamp in seconds",
},
+ segments: {
+ type: "array",
+ description:
+ "Multi-cut keep-ranges within the clip. Use to cut out filler/tangents " +
+ "in the middle. If omitted, the full start→end range is used.",
+ items: {
+ type: "object",
+ properties: {
+ start: { type: "number" },
+ end: { type: "number" },
+ },
+ required: ["start", "end"],
+ },
+ },
reasoning: {
type: "string",
description: "Why this moment is viral-worthy",
@@ -46,6 +60,15 @@ export const suggestClipsToolDef = {
type: "string",
description: "Brief text preview of what's said",
},
+ content_type: {
+ type: "string",
+ description:
+ "Content classification: guest_story, technical_insight, market_landscape, business_strategy, hot_take",
+ },
+ score: {
+ type: "number",
+ description: "Virality score (0-20). Sum of standalone + hook + relevance + quotability (each 1-5).",
+ },
suggested_caption_style: {
type: "string",
enum: ["hormozi", "karaoke", "subtle", "branded"],
@@ -67,24 +90,38 @@ export async function handleSuggestClips(
title: string;
start_second: number;
end_second: number;
+ segments?: Array<{ start: number; end: number }>;
reasoning: string;
preview_text?: string;
+ content_type?: string;
+ score?: number;
suggested_caption_style?: string;
}>;
// Validate and enrich suggestions
- const enriched = suggestions.map((s, i) => ({
- clip_number: i + 1,
- clip_id: randomUUID(),
- title: s.title,
- start_second: s.start_second,
- end_second: s.end_second,
- duration: Math.round((s.end_second - s.start_second) * 10) / 10,
- reasoning: s.reasoning,
- preview_text: s.preview_text || "",
- suggested_caption_style: s.suggested_caption_style || "hormozi",
- timestamp_display: `${formatTime(s.start_second)} → ${formatTime(s.end_second)}`,
- }));
+ const enriched = suggestions.map((s, i) => {
+ // Compute duration from segments if available, otherwise from start/end
+ const segments = s.segments?.filter((seg) => seg.end > seg.start) || [];
+ const keptDuration = segments.length > 0
+ ? segments.reduce((sum, seg) => sum + (seg.end - seg.start), 0)
+ : s.end_second - s.start_second;
+
+ return {
+ clip_number: i + 1,
+ clip_id: randomUUID(),
+ title: s.title,
+ start_second: s.start_second,
+ end_second: s.end_second,
+ segments: segments.length > 0 ? segments : [{ start: s.start_second, end: s.end_second }],
+ duration: Math.round(keptDuration * 10) / 10,
+ reasoning: s.reasoning,
+ preview_text: s.preview_text || "",
+ content_type: s.content_type || "unknown",
+ score: s.score || 0,
+ suggested_caption_style: s.suggested_caption_style || "hormozi",
+ timestamp_display: `${formatTime(s.start_second)} → ${formatTime(s.end_second)}`,
+ };
+ });
return JSON.stringify({
clip_count: enriched.length,
diff --git a/src/server.ts b/src/server.ts
index bc949ba..8828cbe 100644
--- a/src/server.ts
+++ b/src/server.ts
@@ -194,8 +194,14 @@ export function createServer(): McpServer {
title: z.string(),
start_second: z.number(),
end_second: z.number(),
+ segments: z
+ .array(z.object({ start: z.number(), end: z.number() }))
+ .optional()
+ .describe("Multi-cut keep-ranges. Omit for a single continuous clip."),
reasoning: z.string(),
preview_text: z.string().optional(),
+ content_type: z.string().optional(),
+ score: z.number().optional(),
suggested_caption_style: z
.enum(["hormozi", "karaoke", "subtle", "branded"])
.optional(),
@@ -299,6 +305,7 @@ export function createServer(): McpServer {
}
// Resolve clip_number from UI state BEFORE routing
+ let keepSegments: Array<{ start: number; end: number }> | null = null;
if (params.clip_number != null && (params.start_second == null || params.end_second == null)) {
const uiState = await readUIState();
const suggestions = (uiState?.suggestions as Array<Record<string, unknown>>) || [];
@@ -324,6 +331,11 @@ export function createServer(): McpServer {
const transcript = uiState?.transcript as Record<string, unknown> | null;
if (transcript?.words) params.transcript_words = transcript.words as any;
}
+ // Pull multi-cut segments from suggestion
+ const segs = suggestion.segments as Array<{ start: number; end: number }> | undefined;
+ if (segs && segs.length > 0) {
+ keepSegments = segs;
+ }
}
// Check for duplicates
@@ -358,6 +370,7 @@ export function createServer(): McpServer {
title: (params.title || "clip") as string,
caption_style: params.caption_style || "hormozi",
crop_strategy: params.crop_strategy || "face",
+ ...(keepSegments && { segments: keepSegments }),
}],
transcript_words: params.transcript_words,
logo_path: params.logo_path || null,
@@ -510,6 +523,7 @@ export function createServer(): McpServer {
title: s.title || `clip_${i + 1}`,
caption_style: (s.suggested_caption_style as string) || settings.captionStyle || "hormozi",
crop_strategy: settings.cropStrategy || "face",
+ ...(Array.isArray(s.segments) && s.segments.length > 0 && { keep_segments: s.segments }),
})) as any;
} else if (params.clip_numbers) {
resolvedClips = (params.clip_numbers as number[])
@@ -522,6 +536,7 @@ export function createServer(): McpServer {
title: s.title || `clip_${n}`,
caption_style: (s.suggested_caption_style as string) || settings.captionStyle || "hormozi",
crop_strategy: settings.cropStrategy || "face",
+ ...(Array.isArray(s.segments) && s.segments.length > 0 && { keep_segments: s.segments }),
};
}) as any;
}
diff --git a/src/ui/web-server.ts b/src/ui/web-server.ts
index 260c32b..5b9219b 100644
--- a/src/ui/web-server.ts
+++ b/src/ui/web-server.ts
@@ -1253,11 +1253,59 @@ app.post("/api/claude-suggest", async (req, res) => {
const dur = (uiState.transcript as any)?.duration || 0;
const durationMin = Math.round(dur / 60);
- const prompt = `You are the clip extraction engine for a podcast. Analyze this transcript and return the ${top_n} best moments for YouTube Shorts.
+ // Load knowledge base files for context
+ const kbDir = join(process.cwd(), ".podcli", "knowledge");
+ let kbContext = "";
+ const kbFiles: [string, number][] = [
+ ["04-shorts-creation-guide.md", 4000],
+ ["05-title-formulas.md", 3000],
+ ["02-voice-and-tone.md", 3000],
+ ["01-brand-identity.md", 1500],
+ ["11-inspiration-channels.md", 2000],
+ ["12-quick-reference.md", 2000],
+ ];
+ for (const [fname, maxChars] of kbFiles) {
+ const fpath = join(kbDir, fname);
+ if (existsSync(fpath)) {
+ try {
+ const content = readFileSync(fpath, "utf-8").trim();
+ if (content.split("[Your Show Name]").length > 3 && content.length < 500) continue;
+ kbContext += `\n--- ${fname} ---\n${content.slice(0, maxChars)}\n`;
+ } catch {}
+ }
+ }
+
+ const prompt = `You are a viral clip editor for TikTok and YouTube Shorts. Find the ${top_n} most scroll-stopping moments in this podcast transcript.
IMPORTANT: Return ONLY valid JSON. No markdown, no explanation, no code fences.
-Use the knowledge base in .podcli/knowledge/ for voice rules, moment selection criteria, content types, and existing episodes.
+TIMESTAMP FORMAT: All timestamps in the transcript are in SECONDS (e.g., [123.4s]).
+All timestamps you return MUST be in SECONDS as numbers (e.g., 123.4), NOT minutes:seconds.
+
+DURATION RULES (CRITICAL):
+- Target: 25-40 seconds (this is the viral sweet spot)
+- Maximum: 50 seconds (absolute hard limit — anything longer loses viewers)
+- Minimum: 15 seconds (too short = no payoff)
+- SHORTER IS BETTER. A punchy 25s clip outperforms a 50s clip every time.
+- If a thought takes longer than 40s, use segments to cut the filler in the middle
+
+CUTTING RULES (CRITICAL):
+- Cut TIGHT. Every second must earn its place.
+- Start at the exact moment the hook hits — no preamble, no "so", no "well"
+- End the MOMENT the point lands with a complete thought — don't trail off
+- NEVER cut mid-sentence or mid-thought. The viewer must feel closure.
+- If there's filler/tangent in the middle, use multiple segments to skip it
+
+MOMENT SELECTION (think like a TikTok editor):
+- Would YOU stop scrolling for this? If no, skip it.
+- First 3 seconds must HOOK — a bold claim, shocking number, or provocative question
+- Must make complete sense standalone — no "as I mentioned" or "going back to"
+- Must end on a COMPLETE THOUGHT — sentence boundary, natural pause, or mic-drop moment
+- Single focused idea — one concept, fully delivered, no loose threads
+- Prioritize: controversial takes, surprising numbers, founder war stories, "wait what?" moments, emotional peaks
+- Skip: generic advice, obvious statements, context-dependent references
+
+${kbContext ? `KNOWLEDGE BASE:${kbContext}` : ""}
Score each moment on 4 dimensions (1-5 each):
- standalone: Makes sense without episode context?
@@ -1267,9 +1315,20 @@ Score each moment on 4 dimensions (1-5 each):
Classify each as: guest_story | technical_insight | market_landscape | business_strategy | hot_take
-Return JSON: { "clips": [{ "title": "...", "start_second": N, "end_second": N, "duration": N, "content_type": "...", "scores": {"standalone":N,"hook":N,"relevance":N,"quotability":N}, "total_score": N, "quote": "...", "why": "..." }] }
+Return this exact JSON structure:
+{ "clips": [{ "title": "First strong sentence from the moment", "start_second": 123.4, "end_second": 168.4, "segments": [{"start": 123.4, "end": 140.0}, {"start": 145.2, "end": 168.4}], "duration": 40, "content_type": "guest_story", "scores": {"standalone": 4, "hook": 5, "relevance": 4, "quotability": 3}, "total_score": 16, "quote": "The key quote from this moment", "why": "One sentence on why this works as a short" }] }
+
+SEGMENTS RULES:
+- "segments" is an array of keep-ranges within the clip. Use it to CUT OUT dead weight.
+- If the moment is clean with no filler, use a single segment: [{"start": X, "end": Y}]
+- If there's a ramble/tangent/filler in the middle, split into multiple segments that skip it
+- "duration" = total kept time (sum of all segment lengths), NOT end - start
-Rules: 20-90 seconds, sentence boundaries, standalone, variety.
+Rules:
+- Final clip duration (sum of segments) MUST be 15-50 seconds (target 25-40s)
+- Each segment must start and end on COMPLETE SENTENCES
+- Must make sense standalone when stitched together
+- Sort clips by timestamp order
Transcript (${segCount} segments, ~${durationMin} min):
@@ -1300,16 +1359,30 @@ ${transcriptText}`;
}
const data = JSON.parse(result.substring(jsonStart, jsonEnd));
- const clips = (data.clips || []).map((c: any) => ({
- title: c.title || "Untitled",
- start_second: c.start_second || 0,
- end_second: c.end_second || 0,
- duration: c.duration || 0,
- reasoning: c.why || "",
- content_type: c.content_type || "unknown",
- scores: c.scores || {},
- total_score: c.total_score || 0,
- }));
+ const clips = (data.clips || []).map((c: any) => {
+ // Parse and validate segments
+ const rawSegs = c.segments || [];
+ const segments = rawSegs
+ .filter((s: any) => s.end > s.start)
+ .map((s: any) => ({ start: s.start, end: s.end }));
+ const keptDuration = segments.length > 0
+ ? segments.reduce((sum: number, s: any) => sum + (s.end - s.start), 0)
+ : (c.end_second || 0) - (c.start_second || 0);
+
+ return {
+ title: c.title || "Untitled",
+ start_second: c.start_second || 0,
+ end_second: c.end_second || 0,
+ segments: segments.length > 0 ? segments : [{ start: c.start_second || 0, end: c.end_second || 0 }],
+ duration: Math.round(keptDuration),
+ reasoning: c.why || "",
+ preview_text: (c.quote || "").slice(0, 120),
+ content_type: c.content_type || "unknown",
+ score: c.total_score || 0,
+ scores: c.scores || {},
+ suggested_caption_style: "hormozi",
+ };
+ }).filter((c: any) => c.duration >= 15 && c.duration <= 55);
// Auto-push to UI state as suggestions
if (clips.length > 0) {
@@ -1318,7 +1391,12 @@ ${transcriptText}`;
title: c.title,
start_second: c.start_second,
end_second: c.end_second,
+ segments: c.segments,
reasoning: c.reasoning,
+ preview_text: c.preview_text,
+ content_type: c.content_type,
+ score: c.score,
+ suggested_caption_style: c.suggested_caption_style,
}));
(uiState as any).phase = "review";
broadcastSSE("state-sync", uiState);
@@ -1465,6 +1543,8 @@ app.post("/api/mcp/export", async (req, res) => {
title: (c.title || "clip").slice(0, 40),
caption_style: c.caption_style || captionStyle,
crop_strategy: c.crop_strategy || cropStrategy,
+ // Preserve multi-cut segments from suggestions
+ ...(Array.isArray(c.segments) && c.segments.length > 0 && { keep_segments: c.segments }),
}));
const jobId = uuidv4();