Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
feat: rewrite split-screen crop — static per-speaker, no FFmpeg expressions
Nika Siradzec876a70f26a9
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex cff4747..46a68ed 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -190,66 +190,25 @@ def crop_to_vertical(source_ratio = width / heightif strategy in ("face", "speaker"):- crop_x_expr = None- crop_y_expr = None- adjusted_h = None-- # Fast path: use pre-computed face_map for simple layouts only.- # Split-screen and mixed layouts (Riverside-style recordings that switch- # between split and single-person views) need the slow path for- # per-frame adaptive tracking.- is_split = face_map.get("is_split_screen", False) if face_map else False- is_mixed = face_map.get("is_mixed_layout", False) if face_map else False- if not is_split and not is_mixed and face_map and face_map.get("clusters") and face_map.get("speaker_mappings"):- crop_x_expr = _use_face_map(- face_map, transcript_words, clip_start,- width, height, target_ratio,+ # Split-screen: cut each speaker segment separately with static crop per half.+ # Simple, reliable, no complex FFmpeg expressions.+ is_split = _detect_split_screen(input_path, width, height)++ if is_split and transcript_words:+ result = _crop_split_screen(+ input_path, output_path,+ width, height, target_w, target_h,+ transcript_words, clip_start,)+ if result:+ return result- # Slow path: analyze video for face positions (x AND y per speaker)- if crop_x_expr is None:- has_speakers = (- transcript_words- and len(set(w.get("speaker") for w in transcript_words if w.get("speaker"))) > 1- )-- if has_speakers:- result = _build_speaker_aware_crop(- input_path, width, height, target_ratio,- transcript_words, clip_start,- )- # Returns (x_expr, y_expr, crop_h) tuple or single string- if isinstance(result, tuple):- crop_x_expr, crop_y_expr, adjusted_h = result- else:- crop_x_expr = result- else:- crop_x_expr = _detect_face_center(input_path, width, height, target_ratio)+ # Non-split-screen: single static face crop+ crop_x = _detect_face_center(input_path, width, height, target_ratio)- if crop_x_expr is None:- print(f"Warning: all crop paths returned None for {input_path} ({width}x{height})", file=sys.stderr)-- if crop_x_expr is not None:- if crop_y_expr is not None:- # Per-speaker vertical framing: crop with dynamic x AND y- adj_w = int(adjusted_h * target_ratio)- vf = f"crop={adj_w}:{adjusted_h}:{crop_x_expr}:{crop_y_expr},scale={target_w}:{target_h}"- else:- # Static vertical crop with face Y detection- crop_w = int(height * target_ratio)- face_y = _detect_face_y(input_path, width, height, crop_w, crop_x_expr)- if face_y is not None:- adjusted_h = int(height * 0.75)- adj_y = face_y - int(adjusted_h * 0.33)- adj_y = max(0, min(adj_y, height - adjusted_h))- adj_w = int(adjusted_h * target_ratio)- vf = (- f"crop={crop_w}:{height}:{crop_x_expr}:0,"- f"crop={adj_w}:{adjusted_h}:(iw-{adj_w})/2:{adj_y},"- f"scale={target_w}:{target_h}"- )- else:- vf = f"crop={crop_w}:{height}:{crop_x_expr}:0,scale={target_w}:{target_h}"+ if crop_x is not None:+ crop_w = int(height * target_ratio)+ vf = f"crop={crop_w}:{height}:{crop_x}:0,scale={target_w}:{target_h}"else:strategy = "center"@@ -285,6 +244,306 @@ def crop_to_vertical()+def _detect_split_screen(video_path: str, width: int, height: int) -> bool:+ """Quick check: is this a split-screen layout (two side-by-side cameras)?"""+ try:+ import cv2+ import numpy as np++ cap = cv2.VideoCapture(video_path)+ if not cap.isOpened():+ return False++ fps = cap.get(cv2.CAP_PROP_FPS) or 30+ total = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))+ duration = total / fps++ backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+ proto = os.path.join(backend_dir, "models", "deploy.prototxt")+ model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")+ if not (os.path.exists(proto) and os.path.exists(model)):+ cap.release()+ return False++ detector = cv2.dnn.readNetFromCaffe(proto, model)+ mid_x = width // 2++ frames_with_two_faces = 0+ total_sampled = 0++ for i in range(min(20, max(5, int(duration)))):+ t = (i + 1) * duration / (min(20, max(5, int(duration))) + 1)+ cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000)+ ret, frame = cap.read()+ if not ret:+ continue++ h, w = frame.shape[:2]+ blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))+ detector.setInput(blob)+ dets = detector.forward()++ left = False+ right = False+ for j in range(dets.shape[2]):+ if dets[0, 0, j, 2] > 0.5:+ cx = int((dets[0, 0, j, 3] + dets[0, 0, j, 5]) / 2 * w)+ if cx < mid_x:+ left = True+ else:+ right = True++ total_sampled += 1+ if left and right:+ frames_with_two_faces += 1++ cap.release()+ return total_sampled > 0 and frames_with_two_faces / total_sampled >= 0.5++ except Exception:+ return False+++def _crop_split_screen(+ input_path: str,+ output_path: str,+ width: int,+ height: int,+ target_w: int,+ target_h: int,+ transcript_words: list,+ clip_start: float,+) -> Optional[str]:+ """+ Split-screen crop: cut each speaker segment with a static crop on their+ camera half. No complex expressions — just separate FFmpeg calls + concat.++ This is how production tools (Opus Clip, Descript, etc.) handle it:+ static crop per speaker, hard cut between segments.+ """+ try:+ import cv2+ import numpy as np++ target_ratio = target_w / target_h+ mid_x = width // 2++ # Detect face position in each half+ cap = cv2.VideoCapture(input_path)+ if not cap.isOpened():+ return None++ fps = cap.get(cv2.CAP_PROP_FPS) or 30+ total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))+ duration = total_frames / fps++ backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))+ proto = os.path.join(backend_dir, "models", "deploy.prototxt")+ model = os.path.join(backend_dir, "models", "res10_300x300_ssd_iter_140000.caffemodel")+ if not (os.path.exists(proto) and os.path.exists(model)):+ cap.release()+ return None++ detector = cv2.dnn.readNetFromCaffe(proto, model)++ # Collect face positions per half+ left_faces = [] # (cx_in_half, cy)+ right_faces = [] # (cx_in_half, cy)++ sample_count = min(40, max(10, int(duration * 2)))+ for i in range(sample_count):+ t = (i + 1) * duration / (sample_count + 1)+ cap.set(cv2.CAP_PROP_POS_MSEC, t * 1000)+ ret, frame = cap.read()+ if not ret:+ continue++ h, w = frame.shape[:2]+ blob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300), (104.0, 177.0, 123.0))+ detector.setInput(blob)+ dets = detector.forward()++ for j in range(dets.shape[2]):+ if dets[0, 0, j, 2] > 0.5:+ cx = int((dets[0, 0, j, 3] + dets[0, 0, j, 5]) / 2 * w)+ cy = int((dets[0, 0, j, 4] + dets[0, 0, j, 6]) / 2 * h)+ if cx < mid_x:+ left_faces.append((cx, cy))+ else:+ right_faces.append((cx - mid_x, cy)) # relative to right half++ cap.release()++ if not left_faces and not right_faces:+ return None++ # Compute static crop for each half+ # Within a 960x1080 half, we need a 9:16 crop+ half_w = mid_x+ crop_w = int(height * target_ratio) # ~607 for 1080 height++ def _compute_crop(faces, half_offset):+ """Compute static crop_x, crop_y for one camera half."""+ if not faces:+ # No face — center crop within this half+ return half_offset + (half_w - crop_w) // 2, 0, height++ cx = int(np.median([f[0] for f in faces]))+ cy = int(np.median([f[1] for f in faces]))++ # Horizontal: center crop on face within this half+ crop_x = cx - crop_w // 2+ crop_x = max(0, min(crop_x, half_w - crop_w))+ crop_x += half_offset # offset to position within full frame++ # Vertical: place face at ~33% from top with 75% zoom+ adj_h = int(height * 0.75)+ crop_y = cy - int(adj_h * 0.33)+ crop_y = max(0, min(crop_y, height - adj_h))++ return crop_x, crop_y, adj_h++ left_crop_x, left_crop_y, left_h = _compute_crop(left_faces, 0)+ right_crop_x, right_crop_y, right_h = _compute_crop(right_faces, mid_x)++ left_w = int(left_h * target_ratio)+ right_w = int(right_h * target_ratio)++ # Map speakers to sides (first speaker by time = left)+ speakers = sorted(set(w.get("speaker") for w in transcript_words if w.get("speaker")))++ if len(speakers) < 2:+ # Single speaker — use the side with more face detections+ if len(left_faces) >= len(right_faces):+ vf = f"crop={left_w}:{left_h}:{left_crop_x}:{left_crop_y},scale={target_w}:{target_h}"+ else:+ vf = f"crop={right_w}:{right_h}:{right_crop_x}:{right_crop_y},scale={target_w}:{target_h}"+ return _run_ffmpeg_with_fallback(+ cmd_parts_before_enc=["ffmpeg", "-y", "-i", input_path, "-vf", vf],+ cmd_parts_after_enc=["-c:a", "aac", "-b:a", "192k", "-ar", "44100", "-movflags", "+faststart"],+ output_path=output_path, label="crop_split",+ )++ # Map speakers: sort by talk time, first by appearance = left+ speaker_talk = {}+ speaker_first = {}+ for w in transcript_words:+ sp = w.get("speaker")+ if sp:+ speaker_talk[sp] = speaker_talk.get(sp, 0) + (w["end"] - w["start"])+ if sp not in speaker_first:+ speaker_first[sp] = w["start"]++ top_2 = sorted(speakers, key=lambda s: speaker_talk.get(s, 0), reverse=True)[:2]+ top_2_by_first = sorted(top_2, key=lambda s: speaker_first.get(s, float("inf")))++ speaker_side = {}+ speaker_side[top_2_by_first[0]] = "left"+ if len(top_2_by_first) > 1:+ speaker_side[top_2_by_first[1]] = "right"+ for sp in speakers:+ if sp not in speaker_side:+ speaker_side[sp] = speaker_side.get(top_2_by_first[0], "left")++ # Build speaker segments+ segments = []+ cur_speaker = None+ seg_start = 0++ for w in sorted(transcript_words, key=lambda x: x["start"]):+ sp = w.get("speaker")+ t = w["start"] - clip_start+ if sp != cur_speaker and sp is not None:+ if cur_speaker is not None:+ segments.append((seg_start, t, cur_speaker))+ cur_speaker = sp+ seg_start = t++ if cur_speaker:+ segments.append((seg_start, duration, cur_speaker))++ # Merge short segments (<1s)+ merged = []+ for seg in segments:+ if merged and seg[2] == merged[-1][2]:+ merged[-1] = (merged[-1][0], seg[1], seg[2])+ elif merged and (seg[1] - seg[0]) < 1.0:+ merged[-1] = (merged[-1][0], seg[1], merged[-1][2])+ else:+ merged.append(list(seg))+ segments = merged++ if not segments:+ return None++ # Cut each segment with its speaker's crop, then concat+ work_dir = os.path.dirname(output_path) or "."+ part_paths = []++ try:+ for i, (start_t, end_t, speaker) in enumerate(segments):+ side = speaker_side.get(speaker, "left")+ if side == "left":+ cw, ch, cx, cy = left_w, left_h, left_crop_x, left_crop_y+ else:+ cw, ch, cx, cy = right_w, right_h, right_crop_x, right_crop_y++ part_path = os.path.join(work_dir, f"_speaker_part_{i}.mp4")+ vf = f"crop={cw}:{ch}:{cx}:{cy},scale={target_w}:{target_h}"++ cmd = [+ "ffmpeg", "-y",+ "-ss", str(start_t), "-t", str(end_t - start_t),+ "-i", input_path,+ "-vf", vf,+ "-c:v", "libx264", "-crf", "18", "-preset", "fast",+ "-c:a", "aac", "-b:a", "192k",+ "-avoid_negative_ts", "make_zero",+ part_path,+ ]+ r = subprocess.run(cmd, capture_output=True, text=True)+ if r.returncode != 0:+ print(f"Warning: split-screen segment {i} failed: {r.stderr[-200:]}", file=sys.stderr)+ continue+ part_paths.append(part_path)++ if not part_paths:+ return None++ # Concat all parts+ concat_file = os.path.join(work_dir, "_speaker_concat.txt")+ with open(concat_file, "w") as f:+ for p in part_paths:+ f.write(f"file '{os.path.abspath(p)}'\n")++ cmd = [+ "ffmpeg", "-y",+ "-f", "concat", "-safe", "0",+ "-i", concat_file,+ "-c", "copy",+ "-movflags", "+faststart",+ output_path,+ ]+ r = subprocess.run(cmd, capture_output=True, text=True)+ if r.returncode != 0:+ return None++ return output_path++ finally:+ for p in part_paths:+ if os.path.exists(p):+ os.remove(p)+ concat_path = os.path.join(work_dir, "_speaker_concat.txt")+ if os.path.exists(concat_path):+ os.remove(concat_path)++ except Exception as e:+ import traceback+ print(f"Warning: split-screen crop failed: {e}\n{traceback.format_exc()}", file=sys.stderr)+ return None++def _use_face_map(face_map: dict,transcript_words: list,