Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
feat: branded captions with rounded pill backgrounds and fade animation
Nika Siradze594873a005ba
diff --git a/backend/config/caption_styles.py b/backend/config/caption_styles.pyindex 6b7ae8a..bfad697 100644--- a/backend/config/caption_styles.py+++ b/backend/config/caption_styles.py@@ -117,10 +117,11 @@ STYLES = {"font_size": 72, # Slightly smaller for clean 2-line fit"primary_color": "&H00FFFFFF", # White (all words)"active_color": "&H00FFFFFF", # White text on dark box- "active_box_color": "&H00101010", # Near-black pill bg (slightly lighter than pure black)- "active_box_padding_x": 20, # Horizontal padding inside pill+ "active_box_color": "&H00000000", # Black pill bg+ "active_box_alpha": "&H30", # Pill opacity (00=solid, FF=invisible) — 30 ≈ 81% opaque+ "active_box_padding_x": 10, # Horizontal padding inside pill"active_box_padding_y": 10, # Vertical padding inside pill- "active_box_rounding": 8, # Border rounding for pill shape+ "active_box_rounding": 15, # Border rounding for pill shape"outline_color": "&H00000000", # Black"back_color": "&H00000000", # Transparent"bold": True,diff --git a/backend/services/caption_renderer.py b/backend/services/caption_renderer.pyindex 3d194bb..e13b366 100644--- a/backend/services/caption_renderer.py+++ b/backend/services/caption_renderer.py@@ -51,7 +51,7 @@ ScaledBorderAndShadow: yes[V4+ Styles]Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding-Style: BrandedNormal,{style["font_name"]},{style["font_size"]},&H00FFFFFF,&H00FFFFFF,&H00000000,&H80000000,-1,0,0,0,100,100,2,0,3,0,{style["shadow_depth"]},{style["alignment"]},60,60,{style["margin_v"]},1+Style: BrandedNormal,{style["font_name"]},{style["font_size"]},&H00FFFFFF,&H00FFFFFF,&H50000000,&H00000000,-1,0,0,0,100,100,2,0,1,2,1,{style["alignment"]},60,60,{style["margin_v"]},1[Events]Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text@@ -254,13 +254,214 @@ def _normalize_case(text: str) -> str:return text.lower()+_libass_calibration_cache: dict[str, list[int]] = {}+++def _calibrate_libass_y(+ font_name: str, font_size: int, bold: bool, margin_v: int,+ play_res_x: int, play_res_y: int,+ lines: list, space_width: int, line_height: int,+) -> list[int]:+ """+ Render a probe frame with FFmpeg+libass to measure the exact Y position+ of each text line. Returns a list of Y-center values (one per line).+ """+ import subprocess+ import tempfile++ cache_key = f"{font_name}:{font_size}:{bold}:{margin_v}:{len(lines)}"+ if cache_key in _libass_calibration_cache:+ return _libass_calibration_cache[cache_key]++ # Build a minimal ASS with the same style and text layout+ bold_val = -1 if bold else 0+ display_lines = []+ for line in lines:+ display_lines.append(" ".join(t for _, t, _ in line))+ display_text = "\\N".join(display_lines)++ ass_content = (+ "[Script Info]\nScriptType: v4.00+\n"+ f"PlayResX: {play_res_x}\nPlayResY: {play_res_y}\n"+ "ScaledBorderAndShadow: yes\n\n"+ "[V4+ Styles]\n"+ "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, "+ "OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, "+ "ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, "+ "Alignment, MarginL, MarginR, MarginV, Encoding\n"+ f"Style: Cal,{font_name},{font_size},"+ f"&H00FFFFFF,&H00FFFFFF,&H00000000,&H00000000,"+ f"{bold_val},0,0,0,100,100,2,0,1,0,0,2,60,60,{margin_v},1\n\n"+ "[Events]\n"+ "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n"+ f"Dialogue: 0,0:00:00.00,0:00:01.00,Cal,,0,0,0,,{display_text}\n"+ )++ try:+ ass_file = tempfile.NamedTemporaryFile(suffix=".ass", delete=False, mode="w")+ ass_file.write(ass_content)+ ass_file.close()++ png_file = tempfile.NamedTemporaryFile(suffix=".png", delete=False)+ png_file.close()++ # Render one frame with grey background+ cmd = [+ "ffmpeg", "-y", "-f", "lavfi",+ "-i", f"color=0x808080:s={play_res_x}x{play_res_y}:d=1",+ "-vf", f"ass='{ass_file.name}'",+ "-frames:v", "1", png_file.name,+ ]+ subprocess.run(cmd, capture_output=True, text=True, timeout=10)++ # Analyze the rendered frame to find text line Y centers+ from PIL import Image+ import numpy as np++ img = Image.open(png_file.name).convert("L") # greyscale+ arr = np.array(img)++ # Background is 128 grey. White text > 200, black outline < 50.+ # Find rows with bright pixels (white text)+ bright = arr > 200+ rows_with_text = np.where(bright.any(axis=1))[0]++ if len(rows_with_text) < 2:+ # Fallback+ _libass_calibration_cache[cache_key] = []+ return []++ # Split into contiguous blocks (one per line)+ # Return list of (y_center, x_left, x_right) per line+ line_info = []+ block_start = rows_with_text[0]+ prev = rows_with_text[0]+ for r in rows_with_text[1:]:+ if r - prev > 8:+ y_center = (block_start + prev) // 2+ # Find X bounds for this line+ line_rows = arr[block_start:prev + 1]+ cols_with_text = np.where((line_rows > 200).any(axis=0))[0]+ x_left = int(cols_with_text[0]) if len(cols_with_text) else 0+ x_right = int(cols_with_text[-1]) if len(cols_with_text) else play_res_x+ line_info.append((y_center, x_left, x_right))+ block_start = r+ prev = r+ y_center = (block_start + prev) // 2+ line_rows = arr[block_start:prev + 1]+ cols_with_text = np.where((line_rows > 200).any(axis=0))[0]+ x_left = int(cols_with_text[0]) if len(cols_with_text) else 0+ x_right = int(cols_with_text[-1]) if len(cols_with_text) else play_res_x+ line_info.append((y_center, x_left, x_right))++ _libass_calibration_cache[cache_key] = line_info+ return line_info++ except Exception as e:+ print(f"Warning: libass calibration failed: {e}", file=sys.stderr)+ _libass_calibration_cache[cache_key] = []+ return []+ finally:+ try:+ os.unlink(ass_file.name)+ except Exception:+ pass+ try:+ os.unlink(png_file.name)+ except Exception:+ pass+++def _measure_text_widths(texts: list[str], font_name: str, font_size: int, bold: bool, spacing: int = 2) -> list[int]:+ """Measure pixel width of each text string using Pillow for accurate positioning.++ Uses fc-match to find the exact same font that libass will use, ensuring+ pixel-accurate pill positioning.+ """+ try:+ from PIL import ImageFont, ImageDraw, Image+ import subprocess++ font = None++ # 1) Use fc-match to find the exact font libass resolves (most accurate)+ try:+ style = "Bold" if bold else "Regular"+ result = subprocess.run(+ ["fc-match", f"{font_name}:{style}", "--format=%{file}"],+ capture_output=True, text=True, timeout=3,+ )+ if result.returncode == 0 and result.stdout.strip():+ fc_path = result.stdout.strip()+ if os.path.exists(fc_path):+ font = ImageFont.truetype(fc_path, font_size)+ except Exception:+ pass++ # 2) Fallback: try common paths (macOS, Linux, Windows)+ if font is None:+ candidates = [+ "/System/Library/Fonts/Supplemental/Arial Bold.ttf",+ "/System/Library/Fonts/Helvetica.ttc",+ "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",+ "/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf",+ "C:/Windows/Fonts/arialbd.ttf",+ ] if bold else [+ "/System/Library/Fonts/Supplemental/Arial.ttf",+ "/System/Library/Fonts/Helvetica.ttc",+ "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",+ "/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf",+ "C:/Windows/Fonts/arial.ttf",+ ]+ for path in candidates:+ if os.path.exists(path):+ try:+ idx = 1 if (bold and path.endswith(".ttc")) else 0+ font = ImageFont.truetype(path, font_size, index=idx)+ break+ except Exception:+ continue++ if font is None:+ return [int(len(t) * font_size * 0.6 + spacing * len(t)) for t in texts]++ draw = ImageDraw.Draw(Image.new("RGBA", (1, 1)))+ widths = []+ for text in texts:+ bbox = draw.textbbox((0, 0), text, font=font)+ # ASS Spacing adds extra pixels between each character+ w = bbox[2] - bbox[0] + spacing * max(0, len(text) - 1)+ widths.append(w)+ return widths+ except ImportError:+ return [int(len(t) * font_size * 0.6) for t in texts]+++def _rounded_rect_drawing(w: int, h: int, r: int) -> str:+ """Generate ASS \\p1 drawing commands for a rounded rectangle at origin."""+ r = min(r, w // 2, h // 2)+ # Cubic bezier control point offset for ~circular arcs: r * 0.55+ c = int(r * 0.55)+ return (+ f"m {r} 0 "+ f"l {w - r} 0 "+ f"b {w - r + c} 0 {w} {r - c} {w} {r} "+ f"l {w} {h - r} "+ f"b {w} {h - r + c} {w - r + c} {h} {w - r} {h} "+ f"l {r} {h} "+ f"b {r - c} {h} 0 {h - r + c} 0 {h - r} "+ f"l 0 {r} "+ f"b 0 {r - c} {r - c} 0 {r} 0"+ )++def _render_branded(words: list[dict], style: dict, offset: float) -> str:"""Branded style — large bold white text, dark rounded pill on active word only.- Matches the reference: clean white text directly on video, currently-spoken- word gets a near-black rounded rectangle background (pill shape). 5-7 words- shown at a time across 2 natural lines. No gradient overlay.+ Uses a two-layer approach:+ - Layer 0: \\p1 drawing of a rounded rectangle (pill) behind the active word+ - Layer 1: Text with all words visibleEach word's display is contiguous to the next word's start to preventflashing/gaps between words.@@ -269,13 +470,22 @@ def _render_branded(words: list[dict], style: dict, offset: float) -> str:events = []chunk_size = style.get("words_per_chunk", 6)- raw_box = style.get("active_box_color", "&H00101010")+ raw_box = style.get("active_box_color", "&H00000000")box_color = raw_box if raw_box.endswith("&") else raw_box + "&"+ box_alpha = style.get("active_box_alpha", "&H00")# Pill dimensions from style configpad_x = style.get("active_box_padding_x", 20)pad_y = style.get("active_box_padding_y", 10)- rounding = style.get("active_box_rounding", 8)+ rounding = style.get("active_box_rounding", 15)++ font_size = style.get("font_size", 72)+ font_name = style.get("font_name", "Arial")+ is_bold = style.get("bold", True)+ margin_v = style.get("margin_v", 500)+ play_res_x = 1080+ play_res_y = 1920+ spacing = 2 # ASS Spacing field from header# Group words into chunkschunks = []@@ -290,7 +500,7 @@ def _render_branded(words: list[dict], style: dict, offset: float) -> str:chunk_start = max(0, chunk[0]["start"] - offset)chunk_end = max(0, chunk[-1]["end"] - offset)- # Normalize casing: sentence-case (lowercase except acronyms/I, capitalize first word)+ # Normalize casingnormalized = []for j, w in enumerate(chunk):text = _normalize_case(w["word"])@@ -298,11 +508,68 @@ def _render_branded(words: list[dict], style: dict, offset: float) -> str:text = text[0].upper() + text[1:] if len(text) > 1 else text.upper()normalized.append(text)- # For each word: emit a dialogue line where THAT word has a dark pill bg+ # Measure word widths for pill positioning+ word_widths = _measure_text_widths(normalized, font_name, font_size, is_bold, spacing)+ space_width = _measure_text_widths([" "], font_name, font_size, is_bold, spacing)[0]++ # Split into visual lines that fit within play_res_x - margins+ max_line_w = play_res_x - 120 # 60px margin each side+ lines = [] # each line: list of (word_index, text, width)+ current_line = []+ current_w = 0+ for wi, text in enumerate(normalized):+ test_w = current_w + (space_width if current_line else 0) + word_widths[wi]+ if test_w > max_line_w and current_line:+ lines.append(current_line)+ current_line = [(wi, text, word_widths[wi])]+ current_w = word_widths[wi]+ else:+ current_line.append((wi, text, word_widths[wi]))+ current_w = test_w+ if current_line:+ lines.append(current_line)++ # Build display text with explicit \N line breaks+ display_lines = []+ for line in lines:+ display_lines.append(" ".join(t for _, t, _ in line))+ display_text = "\\N".join(display_lines)++ # Line height and vertical layout+ line_height = int(font_size * 1.2)+ total_text_h = line_height * len(lines)+ # Use explicit \pos for both text and pill — no reliance on libass layout.+ # This gives us full control over line spacing and pill sizing.+ # Position text block from bottom: last line baseline at (PlayResY - MarginV)+ # Use generous line spacing so pills have room for proper padding.+ pill_total_h = font_size + pad_y * 2 # pill height per line+ line_spacing = pill_total_h + 8 # pill height + 8px clearance between pills+ total_block_h = line_spacing * len(lines)+ # Block bottom aligns with MarginV+ block_bottom_y = play_res_y - margin_v+ # Each line's center Y:+ line_center_ys = []+ for li in range(len(lines)):+ # Position from bottom up: last line first+ y = block_bottom_y - (len(lines) - 1 - li) * line_spacing - font_size // 2+ line_center_ys.append(y)++ # Pre-compute line geometry — measure space width directly with the font,+ # use word widths without ASS Spacing inflation for tighter layout.+ raw_space_w = _measure_text_widths(["A A"], font_name, font_size, is_bold, 0)[0] - \+ _measure_text_widths(["AA"], font_name, font_size, is_bold, 0)[0]+ line_geometry = [] # (line_center_y, line_left_x, full_line_w, space_w)+ for li, line in enumerate(lines):+ sum_word_w = sum(ww for _, _, ww in line)+ full_line_w = sum_word_w + raw_space_w * max(0, len(line) - 1)+ line_left_x = (play_res_x - full_line_w) // 2+ line_geometry.append((line_center_ys[li], line_left_x, full_line_w, raw_space_w))++ # For each word: emit pill + text (both explicitly positioned)for wi, w in enumerate(chunk):w_start = max(0, w["start"] - offset)- # Contiguous: each word's display extends to next word's start+ # Contiguous timingif wi < len(chunk) - 1:w_end = max(0, chunk[wi + 1]["start"] - offset)else:@@ -311,31 +578,63 @@ def _render_branded(words: list[dict], style: dict, offset: float) -> str:if w_end <= w_start:w_end = w_start + 0.1- # Build line: all words visible, active one gets dark pill override- parts = []- for wj, text in enumerate(normalized):- if wj == wi:- # Active word: dark rounded pill background- # \4c = box fill color (BorderStyle=3 uses BackColour as box)- # \3c = box outline color (same as fill for solid pill)- # \xbord = horizontal padding, \ybord = vertical padding- # \bord = border thickness (creates rounding effect)- parts.append(- f"{{\\4c{box_color}\\3c{box_color}"- f"\\xbord{pad_x}\\ybord{pad_y}\\bord{rounding}"- f"\\shad0}}{text}"- f"{{\\4c&H00000000&\\3c&H00000000&"- f"\\xbord0\\ybord0\\bord0}}"- )- else:- parts.append(text)-- line_text = " ".join(parts)start_ts = seconds_to_ass(w_start)end_ts = seconds_to_ass(w_end)+ # Find which visual line this word is on+ word_line_idx = 0+ word_pos_in_line = 0+ for li, line in enumerate(lines):+ for pi, (idx, _, _) in enumerate(line):+ if idx == wi:+ word_line_idx = li+ word_pos_in_line = pi+ break++ line_cy, line_lx, _, actual_space_w = line_geometry[word_line_idx]++ # X position using derived natural spacing+ current_line = lines[word_line_idx]+ word_x = line_lx+ for k in range(word_pos_in_line):+ word_x += current_line[k][2] + actual_space_w+ word_x = int(word_x)++ word_w = word_widths[wi]+ word_cx = word_x + word_w // 2+ actual_pad_x = max(pad_x, int(font_size * 0.25)) # at least 25% of font size++ # Pill dimensions+ pill_w = word_w + actual_pad_x * 2+ pill_h = font_size + pad_y * 2+ pill_r = min(rounding, pill_h // 2)++ # Pill drawing centered at origin (for \an5 positioning)+ drawing = _rounded_rect_drawing(pill_w, pill_h, pill_r)+ # Shift drawing so its center is at (0,0) for \an5+ draw_offset_x = pill_w // 2+ draw_offset_y = pill_h // 2++ # Layer 0: Pill — positioned at word center with \an5events.append(- f"Dialogue: 0,{start_ts},{end_ts},BrandedNormal,,0,0,0,,{line_text}"+ f"Dialogue: 0,{start_ts},{end_ts},BrandedNormal,,0,0,0,,"+ f"{{\\an7\\pos({word_cx - draw_offset_x},{line_cy - draw_offset_y})"+ f"\\p1\\c{box_color}\\bord0\\shad0"+ f"\\1a&HFF&\\t(0,80,\\1a{box_alpha}&)"+ f"}}"+ f"{drawing}{{\\p0}}")+ # Layer 1: Each word positioned with \an5\pos for perfect pill alignment+ for li, line in enumerate(lines):+ lcy_l, llx_l, _, asp_w = line_geometry[li]+ wx = llx_l+ for pi, (_, text, ww) in enumerate(line):+ wcx = int(wx + ww // 2)+ events.append(+ f"Dialogue: 1,{start_ts},{end_ts},BrandedNormal,,0,0,0,,"+ f"{{\\an5\\pos({wcx},{lcy_l})}}{text}"+ )+ wx += ww + asp_w+return header + "\n".join(events) + "\n"diff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex 1eed0ea..913d5e2 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -771,13 +771,15 @@ def _detect_face_offset(width: int,height: int,target_ratio: float,-) -> Optional[int]:+) -> Optional[str]:"""- Detect face position across the video and return crop x-offset.- Uses DNN-based face detector (more robust than Haar cascades).- Strongly favors the largest/closest face and clusters positions- to lock onto the dominant speaker when multiple faces are visible.- Returns None if no face detected (caller should fall back to center).+ Continuous face tracking — detects face position over time and returns+ an FFmpeg expression that smoothly follows the dominant face.++ Samples frames throughout the clip, picks the dominant face cluster,+ then builds a smoothed panning timeline so the crop follows head movement.++ Returns an FFmpeg crop-x expression string, or None if no face detected."""try:import cv2@@ -790,17 +792,17 @@ def _detect_face_offset(total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))fps = cap.get(cv2.CAP_PROP_FPS) or 30duration = total_frames / fps+ crop_w = int(height * target_ratio)- # Sample more frames across the full clip for better coverage- sample_count = 60+ # Sample ~4 frames/sec for smooth tracking (more than before)+ sample_count = min(240, max(30, int(duration * 4)))sample_times = [i * duration / sample_count for i in range(sample_count)]- face_positions = []+ # (time, face_center_x) for each detection+ timed_positions = []- # Try DNN detector first (much more robust than Haar cascades)+ # Load DNN face detectordnn_detector = None-- # Look for bundled DNN model first, then fall back to OpenCV's data dirbackend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))bundled_proto = os.path.join(backend_dir, "models", "deploy.prototxt")bundled_model = os.path.join(backend_dir, "models",@@ -809,14 +811,12 @@ def _detect_face_offset("deploy.prototxt")cv2_model = os.path.join(os.path.dirname(cv2.__file__), "data","res10_300x300_ssd_iter_140000.caffemodel")-proto_path = bundled_proto if os.path.exists(bundled_proto) else cv2_protomodel_path = bundled_model if os.path.exists(bundled_model) else cv2_model-if os.path.exists(proto_path) and os.path.exists(model_path):dnn_detector = cv2.dnn.readNetFromCaffe(proto_path, model_path)- # Fallback: load multiple Haar cascades for better coverage+ # Fallback: Haar cascadescascades = []if dnn_detector is None:for cascade_name in [@@ -837,7 +837,6 @@ def _detect_face_offset(h, w = frame.shape[:2]if dnn_detector is not None:- # DNN-based detection: handles frontal, profile, angled facesblob = cv2.dnn.blobFromImage(cv2.resize(frame, (300, 300)), 1.0, (300, 300),(104.0, 177.0, 123.0)@@ -849,26 +848,23 @@ def _detect_face_offset(best_cx = Nonefor i in range(detections.shape[2]):confidence = detections[0, 0, i, 2]- if confidence > 0.5: # Higher threshold to reduce false positives+ if confidence > 0.5:x1 = int(detections[0, 0, i, 3] * w)y1 = int(detections[0, 0, i, 4] * h)x2 = int(detections[0, 0, i, 5] * w)y2 = int(detections[0, 0, i, 6] * h)face_w = x2 - x1face_h = y2 - y1- # Skip tiny detections (likely false positives)if face_w < w * 0.05 or face_h < h * 0.05:continue- # Quadratic face-size weighting: strongly favor larger/closer facesscore = confidence * (face_w ** 2)if score > best_score:best_score = scorebest_cx = (x1 + x2) // 2if best_cx is not None:- face_positions.append(best_cx)+ timed_positions.append((t, best_cx))else:- # Haar cascade fallback: try all loaded cascadesgray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)for cascade in cascades:faces = cascade.detectMultiScale(@@ -877,61 +873,135 @@ def _detect_face_offset(if len(faces) > 0:largest = max(faces, key=lambda f: f[2] * f[3])face_center_x = largest[0] + largest[2] // 2- face_positions.append(face_center_x)- break # found face with this cascade, move to next frame+ timed_positions.append((t, face_center_x))+ breakcap.release()- if not face_positions:+ if len(timed_positions) < 3:return None- # Cluster face positions to find the dominant speaker.- # When two speakers are visible, positions will form two groups.- # Pick the largest cluster (most frequently detected face).- crop_w = int(height * target_ratio)- cluster_radius = crop_w * 0.35 # positions within 35% of crop width = same face (allows for head movement)-- positions = np.array(face_positions)+ # --- Cluster to find dominant face ---+ all_cx = np.array([p[1] for p in timed_positions])+ cluster_radius = crop_w * 0.35- # Find all distinct clustersclusters = []- used = np.zeros(len(positions), dtype=bool)- sorted_idx = np.argsort(positions)-- for idx in sorted_idx:+ used = np.zeros(len(all_cx), dtype=bool)+ for idx in np.argsort(all_cx):if used[idx]:continue- mask = np.abs(positions - positions[idx]) < cluster_radius- mask &= ~used- cluster = positions[mask]- if len(cluster) > 0:- clusters.append(cluster)+ mask = (np.abs(all_cx - all_cx[idx]) < cluster_radius) & ~used+ if np.any(mask):+ clusters.append(np.where(mask)[0])used |= maskif not clusters:- clusters = [positions]+ return None- # Sort clusters by size (largest first)+ # Pick largest cluster (dominant face)clusters.sort(key=lambda c: len(c), reverse=True)+ dominant_indices = set(clusters[0])++ # Filter to only dominant face detections+ tracked = [(t, cx) for i, (t, cx) in enumerate(timed_positions) if i in dominant_indices]+ if len(tracked) < 3:+ # Too few points — use static median+ median_x = int(np.median([cx for _, cx in tracked]))+ crop_x = max(0, min(median_x - crop_w // 2, width - crop_w))+ return str(crop_x)++ # --- Smooth the positions ---+ # 1) Convert face_center_x to crop_x (clamped)+ tracked_times = np.array([t for t, _ in tracked])+ tracked_crop_x = np.array([+ max(0, min(cx - crop_w // 2, width - crop_w))+ for _, cx in tracked+ ], dtype=float)++ # 2) Fill gaps: if face wasn't detected for some frames, the tracked+ # array has gaps. That's fine — we interpolate between known points.++ # 3) Heavy smoothing to prevent jitter. Use a rolling average with a+ # window of ~1.5 seconds. This means small head bobs don't move the+ # crop, but sustained movement (leaning, shifting) does.+ if len(tracked_crop_x) > 5:+ sample_interval = tracked_times[-1] / len(tracked_times) if len(tracked_times) > 1 else 0.25+ window = max(3, int(1.5 / max(0.05, sample_interval)))+ # Pad edges so smoothing doesn't pull toward zero+ padded = np.pad(tracked_crop_x, (window // 2, window // 2), mode="edge")+ kernel = np.ones(window) / window+ smoothed = np.convolve(padded, kernel, mode="valid")[:len(tracked_crop_x)]+ else:+ smoothed = tracked_crop_x++ # 4) Round to ints and clamp+ smoothed = np.clip(np.round(smoothed), 0, width - crop_w).astype(int)++ # --- Check if tracking is even needed ---+ # If the face barely moves (range < 3% of frame width), just use static+ movement_range = int(smoothed.max() - smoothed.min())+ if movement_range < width * 0.03:+ return str(int(np.median(smoothed)))++ # --- Build keyframes: only emit when position changes significantly ---+ # Downsample smoothed track to keyframes at ~1s intervals,+ # or whenever the crop shifts by more than 20px.+ keyframes = [(tracked_times[0], int(smoothed[0]))]+ min_time_gap = 0.8 # Don't place keyframes closer than 0.8s+ min_px_change = 20 # Ignore movement smaller than 20px++ for i in range(1, len(smoothed)):+ t = tracked_times[i]+ x = int(smoothed[i])+ prev_t, prev_x = keyframes[-1]+ dt = t - prev_t+ dx = abs(x - prev_x)++ if dx >= min_px_change and dt >= min_time_gap:+ keyframes.append((t, x))+ elif i == len(smoothed) - 1:+ # Always include last point+ keyframes.append((t, x))++ if len(keyframes) < 2:+ return str(keyframes[0][1])++ # --- Build FFmpeg expression ---+ # Piecewise linear interpolation between keyframes.+ # if(lt(t, t1), lerp(t0→t1), if(lt(t, t2), lerp(t1→t2), ...))+ # This is simpler than between() chains and nests one level per segment.+ expr_parts = []+ for i in range(len(keyframes) - 1):+ t0, x0 = keyframes[i]+ t1, x1 = keyframes[i + 1]+ dt = max(0.01, t1 - t0)+ if x0 == x1:+ # No movement in this segment — constant+ expr_parts.append(f"if(lt(t\\,{t1:.2f})\\,{x0}\\,")+ else:+ # Linear interpolation+ expr_parts.append(+ f"if(lt(t\\,{t1:.2f})\\,"+ f"{x0}+(({x1}-{x0})*(t-{t0:.2f})/{dt:.2f})\\,"+ )- # Always pick the largest cluster (most frequently detected face).- # This ensures we lock onto the dominant speaker rather than- # splitting the crop between two equally-visible speakers.- best_cluster = clusters[0]-- avg_face_x = int(np.median(best_cluster))-- # Calculate crop window centered on face- crop_x = avg_face_x - crop_w // 2+ # Cap nesting depth+ if len(expr_parts) > 40:+ # Too many keyframes — subsample+ step = len(keyframes) // 20+ keyframes = keyframes[::step] + [keyframes[-1]]+ # Rebuild (recursive call would be cleaner but let's just return static)+ print(f"Warning: face tracking produced {len(expr_parts)} keyframes, using static", file=sys.stderr)+ return str(int(np.median(smoothed)))- # Clamp to valid range- crop_x = max(0, min(crop_x, width - crop_w))+ # Final value: last keyframe position+ last_x = keyframes[-1][1]+ expr = "".join(expr_parts) + str(last_x) + ")" * len(expr_parts)- return crop_x+ return exprexcept ImportError:- # OpenCV not installed, fall back to centerreturn Noneexcept Exception as e:- print(f"Warning: face detection failed: {e}", file=sys.stderr)+ print(f"Warning: face tracking failed: {e}", file=sys.stderr)return None