Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
feat: per-speaker vertical framing with adaptive zoom
Nika Siradze7fddc14f20dc
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex 1b6e8ec..e30d4f2 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -191,6 +191,8 @@ def crop_to_vertical(if strategy in ("face", "speaker"):crop_x_expr = None+ crop_y_expr = None+ adjusted_h = None# Fast path: use pre-computed face_map from transcription# Only use if we have speaker mappings — without them, face_map@@ -209,40 +211,39 @@ def crop_to_vertical()if has_speakers:- crop_x_expr = _build_speaker_aware_crop(+ result = _build_speaker_aware_crop(input_path, width, height, target_ratio,transcript_words, clip_start,)+ # Returns (x_expr, y_expr, crop_h) tuple or single string+ if isinstance(result, tuple):+ crop_x_expr, crop_y_expr, adjusted_h = result+ else:+ crop_x_expr = resultelse:- # Single speaker: find face and center on it (static crop).- # Use _detect_face_center for a stable, centered position.crop_x_expr = _detect_face_center(input_path, width, height, target_ratio)if crop_x_expr is not None:- crop_w = int(height * target_ratio)- crop_h = height- crop_y = 0-- # Detect face Y — if face is significantly off-center vertically,- # apply a two-step crop: first horizontal (with speaker tracking),- # then vertical zoom to center face. This handles webcams where- # the person sits low/high in frame.- face_y = _detect_face_y(input_path, width, height, crop_w, crop_x_expr)- if face_y is not None and face_y != height // 2:- # Place face at ~33% from top (portrait framing standard).- # Crop vertically to remove excess background above/below.- # Use 78% of frame height for enough zoom without distortion.- adjusted_h = int(height * 0.78)- adj_y = face_y - int(adjusted_h * 0.33) # Face at 33% from top- adj_y = max(0, min(adj_y, height - adjusted_h))+ if crop_y_expr is not None:+ # Per-speaker vertical framing: crop with dynamic x AND yadj_w = int(adjusted_h * target_ratio)- vf = (- f"crop={crop_w}:{crop_h}:{crop_x_expr}:0,"- f"crop={adj_w}:{adjusted_h}:(iw-{adj_w})/2:{adj_y},"- f"scale={target_w}:{target_h}"- )+ vf = f"crop={adj_w}:{adjusted_h}:{crop_x_expr}:{crop_y_expr},scale={target_w}:{target_h}"else:- vf = f"crop={crop_w}:{crop_h}:{crop_x_expr}:0,scale={target_w}:{target_h}"+ # Static vertical crop with face Y detection+ crop_w = int(height * target_ratio)+ face_y = _detect_face_y(input_path, width, height, crop_w, crop_x_expr)+ if face_y is not None:+ adjusted_h = int(height * 0.78)+ adj_y = face_y - int(adjusted_h * 0.33)+ adj_y = max(0, min(adj_y, height - adjusted_h))+ adj_w = int(adjusted_h * target_ratio)+ vf = (+ f"crop={crop_w}:{height}:{crop_x_expr}:0,"+ f"crop={adj_w}:{adjusted_h}:(iw-{adj_w})/2:{adj_y},"+ f"scale={target_w}:{target_h}"+ )+ else:+ vf = f"crop={crop_w}:{height}:{crop_x_expr}:0,scale={target_w}:{target_h}"else:strategy = "center"@@ -462,7 +463,7 @@ def _build_speaker_aware_crop(# Sample frames and detect ALL facessample_count = min(80, max(30, int(duration * 4)))- face_observations = [] # (time, face_center_x, face_width)+ face_observations = [] # (time, face_center_x, face_width, face_center_y)faces_per_frame = [] # count of faces per sampled framefor i in range(sample_count):@@ -483,11 +484,14 @@ def _build_speaker_aware_crop(if conf > 0.5:x1 = int(detections[0, 0, j, 3] * w)x2 = int(detections[0, 0, j, 5] * w)+ y1 = int(detections[0, 0, j, 4] * h)+ y2 = int(detections[0, 0, j, 6] * h)fw = x2 - x1if fw < w * 0.04:continuecx = (x1 + x2) // 2- face_observations.append((t, cx, fw))+ cy = (y1 + y2) // 2+ face_observations.append((t, cx, fw, cy))frame_faces += 1faces_per_frame.append(frame_faces)@@ -501,26 +505,32 @@ def _build_speaker_aware_crop(# This is more robust than radius-based clustering which can merge# faces that are close to the center line.positions = np.array([obs[1] for obs in face_observations])+ y_positions = np.array([obs[3] for obs in face_observations])mid_x = width // 2- left_pos = positions[positions < mid_x]- right_pos = positions[positions >= mid_x]+ left_mask = positions < mid_x+ right_mask = positions >= mid_x+ left_pos = positions[left_mask]+ right_pos = positions[right_mask]# Seam margin: keep crop at least 20px away from the center line- # to avoid showing the split-screen divider at the frame edgeseam_margin = 20clusters = []if len(left_pos) >= 3:cx = int(np.median(left_pos))+ cy = int(np.median(y_positions[left_mask]))clusters.append({"center_x": cx,+ "center_y": cy,"count": len(left_pos),"crop_x": max(0, min(cx - crop_w // 2, mid_x - crop_w - seam_margin)),})if len(right_pos) >= 3:cx = int(np.median(right_pos))+ cy = int(np.median(y_positions[right_mask]))clusters.append({"center_x": cx,+ "center_y": cy,"count": len(right_pos),"crop_x": max(mid_x + seam_margin, min(cx - crop_w // 2, width - crop_w)),})@@ -638,63 +648,81 @@ def _build_speaker_aware_crop(speaker_durations[sp] = speaker_durations.get(sp, 0) + (end_t - start_t)dominant_speaker = max(speaker_durations, key=speaker_durations.get)default_cluster = speaker_to_cluster.get(dominant_speaker) or clusters[0]++ # Compute per-cluster vertical framing: crop_y (face at 33% from top)+ # and recompute crop_x for the narrower adjusted width.+ # Use adaptive zoom: if face is very low/high, zoom more aggressively.+ max_face_y = max(cl.get("center_y", height // 2) for cl in clusters)+ min_face_y = min(cl.get("center_y", height // 2) for cl in clusters)+ worst_offset = max(abs(max_face_y - height * 0.33), abs(min_face_y - height * 0.33))+ # Scale zoom: 78% height for centered faces, down to 60% for extreme cases+ zoom_factor = max(0.60, min(0.78, 1.0 - worst_offset / height))+ adjusted_h = int(height * zoom_factor)+ adj_w = int(adjusted_h * target_ratio)+ for cl in clusters:+ cy = cl.get("center_y", height // 2)+ crop_y = cy - int(adjusted_h * 0.33)+ cl["crop_y"] = max(0, min(crop_y, height - adjusted_h))+ # Recompute crop_x for the narrower adj_w+ cx = cl["center_x"]+ if cx < mid_x:+ cl["crop_x"] = max(0, min(cx - adj_w // 2, mid_x - adj_w - seam_margin))+ else:+ cl["crop_x"] = max(mid_x + seam_margin, min(cx - adj_w // 2, width - adj_w))default_x = default_cluster["crop_x"]+ default_y = default_cluster.get("crop_y", 0)- # Build a timeline: list of (time, target_x) keyframes.+ # Build a timeline: list of (time, target_x, target_y) keyframes.# Split-screen: instant cut (panning through the center seam looks bad).# Non-split-screen: smooth 400ms pan.pan_duration = 0.0 if is_split_screen else 0.4- keyframes = []+ keyframes = [] # (time, x, y)prev_x = default_x+ prev_y = default_yfor start_t, end_t, speaker in segments:cl = speaker_to_cluster.get(speaker)target_x = cl["crop_x"] if cl else default_x+ target_y = cl.get("crop_y", default_y) if cl else default_y- if target_x != prev_x:+ if target_x != prev_x or target_y != prev_y:if pan_duration > 0:- keyframes.append((start_t, prev_x))- keyframes.append((start_t + pan_duration, target_x))+ keyframes.append((start_t, prev_x, prev_y))+ keyframes.append((start_t + pan_duration, target_x, target_y))else:- # Instant cut — jump at speaker change- keyframes.append((start_t, prev_x))- keyframes.append((start_t + 0.01, target_x))+ keyframes.append((start_t, prev_x, prev_y))+ keyframes.append((start_t + 0.01, target_x, target_y))prev_x = target_x+ prev_y = target_yif not keyframes:- return str(default_x)-- # Build FFmpeg expression using chained lerp:- # For each keyframe pair, if t is in [t0, t1], lerp from x0 to x1- # Otherwise fall through to next pair or default- expr_parts = []- for i in range(0, len(keyframes) - 1, 2):- t0, x0 = keyframes[i]- t1, x1 = keyframes[i + 1]- # During transition: linear interpolation- expr_parts.append(- f"if(between(t\\,{t0:.2f}\\,{t1:.2f})\\,"- f"{x0}+(({x1}-{x0})*(t-{t0:.2f})/{max(0.01, t1 - t0):.2f})\\,"- )- # After transition until next one: hold at target- if i + 2 < len(keyframes):- next_t = keyframes[i + 2][0]- else:- next_t = duration- expr_parts.append(- f"if(between(t\\,{t1:.2f}\\,{next_t:.2f})\\,"- f"{x1}\\,"- )-- # Cap nesting depth to avoid FFmpeg expression parser limits- if len(expr_parts) > 30:- print(f"Warning: {len(expr_parts)} pan segments, simplifying to avoid FFmpeg limits", file=sys.stderr)- # Fallback: just use the dominant speaker position- return str(default_x)+ return str(default_x), str(default_y), adjusted_h++ # Build FFmpeg expressions for both X and Y+ def _build_expr(keyframes_xy, val_idx, default_val):+ parts = []+ for i in range(0, len(keyframes_xy) - 1, 2):+ t0, *vals0 = keyframes_xy[i]+ t1, *vals1 = keyframes_xy[i + 1]+ v0, v1 = vals0[val_idx], vals1[val_idx]+ dt = max(0.01, t1 - t0)+ parts.append(+ f"if(between(t\\,{t0:.2f}\\,{t1:.2f})\\,"+ f"{v0}+(({v1}-{v0})*(t-{t0:.2f})/{dt:.2f})\\,"+ )+ next_t = keyframes_xy[i + 2][0] if i + 2 < len(keyframes_xy) else duration+ parts.append(+ f"if(between(t\\,{t1:.2f}\\,{next_t:.2f})\\,"+ f"{v1}\\,"+ )+ if len(parts) > 30:+ return str(default_val)+ return "".join(parts) + str(default_val) + ")" * len(parts)- expr = "".join(expr_parts) + str(default_x) + ")" * len(expr_parts)+ x_expr = _build_expr(keyframes, 0, default_x)+ y_expr = _build_expr(keyframes, 1, default_y)- return expr+ return x_expr, y_expr, adjusted_hexcept ImportError:return None