Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

feat: branded captions with rounded pill backgrounds and fade animation

Nika Siradze594873a005ba
diff --git a/backend/config/caption_styles.py b/backend/config/caption_styles.py
index 6b7ae8a..bfad697 100644
--- a/backend/config/caption_styles.py
+++ b/backend/config/caption_styles.py
@@ -117,10 +117,11 @@ STYLES = {
"font_size": 72, # Slightly smaller for clean 2-line fit
"primary_color": "&H00FFFFFF", # White (all words)
"active_color": "&H00FFFFFF", # White text on dark box
- "active_box_color": "&H00101010", # Near-black pill bg (slightly lighter than pure black)
- "active_box_padding_x": 20, # Horizontal padding inside pill
+ "active_box_color": "&H00000000", # Black pill bg
+ "active_box_alpha": "&H30", # Pill opacity (00=solid, FF=invisible) — 30 ≈ 81% opaque
+ "active_box_padding_x": 10, # Horizontal padding inside pill
"active_box_padding_y": 10, # Vertical padding inside pill
- "active_box_rounding": 8, # Border rounding for pill shape
+ "active_box_rounding": 15, # Border rounding for pill shape
"outline_color": "&H00000000", # Black
"back_color": "&H00000000", # Transparent
"bold": True,
diff --git a/backend/services/caption_renderer.py b/backend/services/caption_renderer.py
index 3d194bb..e13b366 100644
--- a/backend/services/caption_renderer.py
+++ b/backend/services/caption_renderer.py
@@ -51,7 +51,7 @@ ScaledBorderAndShadow: yes
[V4+ Styles]
Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV, Encoding
-Style: BrandedNormal,{style["font_name"]},{style["font_size"]},&H00FFFFFF,&H00FFFFFF,&H00000000,&H80000000,-1,0,0,0,100,100,2,0,3,0,{style["shadow_depth"]},{style["alignment"]},60,60,{style["margin_v"]},1
+Style: BrandedNormal,{style["font_name"]},{style["font_size"]},&H00FFFFFF,&H00FFFFFF,&H50000000,&H00000000,-1,0,0,0,100,100,2,0,1,2,1,{style["alignment"]},60,60,{style["margin_v"]},1
[Events]
Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text
@@ -254,13 +254,214 @@ def _normalize_case(text: str) -> str:
return text.lower()
+_libass_calibration_cache: dict[str, list[int]] = {}
+
+
+def _calibrate_libass_y(
+ font_name: str, font_size: int, bold: bool, margin_v: int,
+ play_res_x: int, play_res_y: int,
+ lines: list, space_width: int, line_height: int,
+) -> list[int]:
+ """
+ Render a probe frame with FFmpeg+libass to measure the exact Y position
+ of each text line. Returns a list of Y-center values (one per line).
+ """
+ import subprocess
+ import tempfile
+
+ cache_key = f"{font_name}:{font_size}:{bold}:{margin_v}:{len(lines)}"
+ if cache_key in _libass_calibration_cache:
+ return _libass_calibration_cache[cache_key]
+
+ # Build a minimal ASS with the same style and text layout
+ bold_val = -1 if bold else 0
+ display_lines = []
+ for line in lines:
+ display_lines.append(" ".join(t for _, t, _ in line))
+ display_text = "\\N".join(display_lines)
+
+ ass_content = (
+ "[Script Info]\nScriptType: v4.00+\n"
+ f"PlayResX: {play_res_x}\nPlayResY: {play_res_y}\n"
+ "ScaledBorderAndShadow: yes\n\n"
+ "[V4+ Styles]\n"
+ "Format: Name, Fontname, Fontsize, PrimaryColour, SecondaryColour, "
+ "OutlineColour, BackColour, Bold, Italic, Underline, StrikeOut, "
+ "ScaleX, ScaleY, Spacing, Angle, BorderStyle, Outline, Shadow, "
+ "Alignment, MarginL, MarginR, MarginV, Encoding\n"
+ f"Style: Cal,{font_name},{font_size},"
+ f"&H00FFFFFF,&H00FFFFFF,&H00000000,&H00000000,"
+ f"{bold_val},0,0,0,100,100,2,0,1,0,0,2,60,60,{margin_v},1\n\n"
+ "[Events]\n"
+ "Format: Layer, Start, End, Style, Name, MarginL, MarginR, MarginV, Effect, Text\n"
+ f"Dialogue: 0,0:00:00.00,0:00:01.00,Cal,,0,0,0,,{display_text}\n"
+ )
+
+ try:
+ ass_file = tempfile.NamedTemporaryFile(suffix=".ass", delete=False, mode="w")
+ ass_file.write(ass_content)
+ ass_file.close()
+
+ png_file = tempfile.NamedTemporaryFile(suffix=".png", delete=False)
+ png_file.close()
+
+ # Render one frame with grey background
+ cmd = [
+ "ffmpeg", "-y", "-f", "lavfi",
+ "-i", f"color=0x808080:s={play_res_x}x{play_res_y}:d=1",
+ "-vf", f"ass='{ass_file.name}'",
+ "-frames:v", "1", png_file.name,
+ ]
+ subprocess.run(cmd, capture_output=True, text=True, timeout=10)
+
+ # Analyze the rendered frame to find text line Y centers
+ from PIL import Image
+ import numpy as np
+
+ img = Image.open(png_file.name).convert("L") # greyscale
+ arr = np.array(img)
+
+ # Background is 128 grey. White text > 200, black outline < 50.
+ # Find rows with bright pixels (white text)
+ bright = arr > 200
+ rows_with_text = np.where(bright.any(axis=1))[0]
+
+ if len(rows_with_text) < 2:
+ # Fallback
+ _libass_calibration_cache[cache_key] = []
+ return []
+
+ # Split into contiguous blocks (one per line)
+ # Return list of (y_center, x_left, x_right) per line
+ line_info = []
+ block_start = rows_with_text[0]
+ prev = rows_with_text[0]
+ for r in rows_with_text[1:]:
+ if r - prev > 8:
+ y_center = (block_start + prev) // 2
+ # Find X bounds for this line
+ line_rows = arr[block_start:prev + 1]
+ cols_with_text = np.where((line_rows > 200).any(axis=0))[0]
+ x_left = int(cols_with_text[0]) if len(cols_with_text) else 0
+ x_right = int(cols_with_text[-1]) if len(cols_with_text) else play_res_x
+ line_info.append((y_center, x_left, x_right))
+ block_start = r
+ prev = r
+ y_center = (block_start + prev) // 2
+ line_rows = arr[block_start:prev + 1]
+ cols_with_text = np.where((line_rows > 200).any(axis=0))[0]
+ x_left = int(cols_with_text[0]) if len(cols_with_text) else 0
+ x_right = int(cols_with_text[-1]) if len(cols_with_text) else play_res_x
+ line_info.append((y_center, x_left, x_right))
+
+ _libass_calibration_cache[cache_key] = line_info
+ return line_info
+
+ except Exception as e:
+ print(f"Warning: libass calibration failed: {e}", file=sys.stderr)
+ _libass_calibration_cache[cache_key] = []
+ return []
+ finally:
+ try:
+ os.unlink(ass_file.name)
+ except Exception:
+ pass
+ try:
+ os.unlink(png_file.name)
+ except Exception:
+ pass
+
+
+def _measure_text_widths(texts: list[str], font_name: str, font_size: int, bold: bool, spacing: int = 2) -> list[int]:
+ """Measure pixel width of each text string using Pillow for accurate positioning.
+
+ Uses fc-match to find the exact same font that libass will use, ensuring
+ pixel-accurate pill positioning.
+ """
+ try:
+ from PIL import ImageFont, ImageDraw, Image
+ import subprocess
+
+ font = None
+
+ # 1) Use fc-match to find the exact font libass resolves (most accurate)
+ try:
+ style = "Bold" if bold else "Regular"
+ result = subprocess.run(
+ ["fc-match", f"{font_name}:{style}", "--format=%{file}"],
+ capture_output=True, text=True, timeout=3,
+ )
+ if result.returncode == 0 and result.stdout.strip():
+ fc_path = result.stdout.strip()
+ if os.path.exists(fc_path):
+ font = ImageFont.truetype(fc_path, font_size)
+ except Exception:
+ pass
+
+ # 2) Fallback: try common paths (macOS, Linux, Windows)
+ if font is None:
+ candidates = [
+ "/System/Library/Fonts/Supplemental/Arial Bold.ttf",
+ "/System/Library/Fonts/Helvetica.ttc",
+ "/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf",
+ "/usr/share/fonts/truetype/liberation/LiberationSans-Bold.ttf",
+ "C:/Windows/Fonts/arialbd.ttf",
+ ] if bold else [
+ "/System/Library/Fonts/Supplemental/Arial.ttf",
+ "/System/Library/Fonts/Helvetica.ttc",
+ "/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf",
+ "/usr/share/fonts/truetype/liberation/LiberationSans-Regular.ttf",
+ "C:/Windows/Fonts/arial.ttf",
+ ]
+ for path in candidates:
+ if os.path.exists(path):
+ try:
+ idx = 1 if (bold and path.endswith(".ttc")) else 0
+ font = ImageFont.truetype(path, font_size, index=idx)
+ break
+ except Exception:
+ continue
+
+ if font is None:
+ return [int(len(t) * font_size * 0.6 + spacing * len(t)) for t in texts]
+
+ draw = ImageDraw.Draw(Image.new("RGBA", (1, 1)))
+ widths = []
+ for text in texts:
+ bbox = draw.textbbox((0, 0), text, font=font)
+ # ASS Spacing adds extra pixels between each character
+ w = bbox[2] - bbox[0] + spacing * max(0, len(text) - 1)
+ widths.append(w)
+ return widths
+ except ImportError:
+ return [int(len(t) * font_size * 0.6) for t in texts]
+
+
+def _rounded_rect_drawing(w: int, h: int, r: int) -> str:
+ """Generate ASS \\p1 drawing commands for a rounded rectangle at origin."""
+ r = min(r, w // 2, h // 2)
+ # Cubic bezier control point offset for ~circular arcs: r * 0.55
+ c = int(r * 0.55)
+ return (
+ f"m {r} 0 "
+ f"l {w - r} 0 "
+ f"b {w - r + c} 0 {w} {r - c} {w} {r} "
+ f"l {w} {h - r} "
+ f"b {w} {h - r + c} {w - r + c} {h} {w - r} {h} "
+ f"l {r} {h} "
+ f"b {r - c} {h} 0 {h - r + c} 0 {h - r} "
+ f"l 0 {r} "
+ f"b 0 {r - c} {r - c} 0 {r} 0"
+ )
+
+
def _render_branded(words: list[dict], style: dict, offset: float) -> str:
"""
Branded style — large bold white text, dark rounded pill on active word only.
- Matches the reference: clean white text directly on video, currently-spoken
- word gets a near-black rounded rectangle background (pill shape). 5-7 words
- shown at a time across 2 natural lines. No gradient overlay.
+ Uses a two-layer approach:
+ - Layer 0: \\p1 drawing of a rounded rectangle (pill) behind the active word
+ - Layer 1: Text with all words visible
Each word's display is contiguous to the next word's start to prevent
flashing/gaps between words.
@@ -269,13 +470,22 @@ def _render_branded(words: list[dict], style: dict, offset: float) -> str:
events = []
chunk_size = style.get("words_per_chunk", 6)
- raw_box = style.get("active_box_color", "&H00101010")
+ raw_box = style.get("active_box_color", "&H00000000")
box_color = raw_box if raw_box.endswith("&") else raw_box + "&"
+ box_alpha = style.get("active_box_alpha", "&H00")
# Pill dimensions from style config
pad_x = style.get("active_box_padding_x", 20)
pad_y = style.get("active_box_padding_y", 10)
- rounding = style.get("active_box_rounding", 8)
+ rounding = style.get("active_box_rounding", 15)
+
+ font_size = style.get("font_size", 72)
+ font_name = style.get("font_name", "Arial")
+ is_bold = style.get("bold", True)
+ margin_v = style.get("margin_v", 500)
+ play_res_x = 1080
+ play_res_y = 1920
+ spacing = 2 # ASS Spacing field from header
# Group words into chunks
chunks = []
@@ -290,7 +500,7 @@ def _render_branded(words: list[dict], style: dict, offset: float) -> str:
chunk_start = max(0, chunk[0]["start"] - offset)
chunk_end = max(0, chunk[-1]["end"] - offset)
- # Normalize casing: sentence-case (lowercase except acronyms/I, capitalize first word)
+ # Normalize casing
normalized = []
for j, w in enumerate(chunk):
text = _normalize_case(w["word"])
@@ -298,11 +508,68 @@ def _render_branded(words: list[dict], style: dict, offset: float) -> str:
text = text[0].upper() + text[1:] if len(text) > 1 else text.upper()
normalized.append(text)
- # For each word: emit a dialogue line where THAT word has a dark pill bg
+ # Measure word widths for pill positioning
+ word_widths = _measure_text_widths(normalized, font_name, font_size, is_bold, spacing)
+ space_width = _measure_text_widths([" "], font_name, font_size, is_bold, spacing)[0]
+
+ # Split into visual lines that fit within play_res_x - margins
+ max_line_w = play_res_x - 120 # 60px margin each side
+ lines = [] # each line: list of (word_index, text, width)
+ current_line = []
+ current_w = 0
+ for wi, text in enumerate(normalized):
+ test_w = current_w + (space_width if current_line else 0) + word_widths[wi]
+ if test_w > max_line_w and current_line:
+ lines.append(current_line)
+ current_line = [(wi, text, word_widths[wi])]
+ current_w = word_widths[wi]
+ else:
+ current_line.append((wi, text, word_widths[wi]))
+ current_w = test_w
+ if current_line:
+ lines.append(current_line)
+
+ # Build display text with explicit \N line breaks
+ display_lines = []
+ for line in lines:
+ display_lines.append(" ".join(t for _, t, _ in line))
+ display_text = "\\N".join(display_lines)
+
+ # Line height and vertical layout
+ line_height = int(font_size * 1.2)
+ total_text_h = line_height * len(lines)
+ # Use explicit \pos for both text and pill — no reliance on libass layout.
+ # This gives us full control over line spacing and pill sizing.
+ # Position text block from bottom: last line baseline at (PlayResY - MarginV)
+ # Use generous line spacing so pills have room for proper padding.
+ pill_total_h = font_size + pad_y * 2 # pill height per line
+ line_spacing = pill_total_h + 8 # pill height + 8px clearance between pills
+ total_block_h = line_spacing * len(lines)
+ # Block bottom aligns with MarginV
+ block_bottom_y = play_res_y - margin_v
+ # Each line's center Y:
+ line_center_ys = []
+ for li in range(len(lines)):
+ # Position from bottom up: last line first
+ y = block_bottom_y - (len(lines) - 1 - li) * line_spacing - font_size // 2
+ line_center_ys.append(y)
+
+ # Pre-compute line geometry — measure space width directly with the font,
+ # use word widths without ASS Spacing inflation for tighter layout.
+ raw_space_w = _measure_text_widths(["A A"], font_name, font_size, is_bold, 0)[0] - \
+ _measure_text_widths(["AA"], font_name, font_size, is_bold, 0)[0]
+ line_geometry = [] # (line_center_y, line_left_x, full_line_w, space_w)
+ for li, line in enumerate(lines):
+ sum_word_w = sum(ww for _, _, ww in line)
+ full_line_w = sum_word_w + raw_space_w * max(0, len(line) - 1)
+ line_left_x = (play_res_x - full_line_w) // 2
+ line_geometry.append((line_center_ys[li], line_left_x, full_line_w, raw_space_w))
+
+ # For each word: emit pill + text (both explicitly positioned)
for wi, w in enumerate(chunk):
w_start = max(0, w["start"] - offset)
- # Contiguous: each word's display extends to next word's start
+ # Contiguous timing
if wi < len(chunk) - 1:
w_end = max(0, chunk[wi + 1]["start"] - offset)
else:
@@ -311,31 +578,63 @@ def _render_branded(words: list[dict], style: dict, offset: float) -> str:
if w_end <= w_start:
w_end = w_start + 0.1
- # Build line: all words visible, active one gets dark pill override
- parts = []
- for wj, text in enumerate(normalized):
- if wj == wi:
- # Active word: dark rounded pill background
- # \4c = box fill color (BorderStyle=3 uses BackColour as box)
- # \3c = box outline color (same as fill for solid pill)
- # \xbord = horizontal padding, \ybord = vertical padding
- # \bord = border thickness (creates rounding effect)
- parts.append(
- f"{{\\4c{box_color}\\3c{box_color}"
- f"\\xbord{pad_x}\\ybord{pad_y}\\bord{rounding}"
- f"\\shad0}}{text}"
- f"{{\\4c&H00000000&\\3c&H00000000&"
- f"\\xbord0\\ybord0\\bord0}}"
- )
- else:
- parts.append(text)
-
- line_text = " ".join(parts)
start_ts = seconds_to_ass(w_start)
end_ts = seconds_to_ass(w_end)
+ # Find which visual line this word is on
+ word_line_idx = 0
+ word_pos_in_line = 0
+ for li, line in enumerate(lines):
+ for pi, (idx, _, _) in enumerate(line):
+ if idx == wi:
+ word_line_idx = li
+ word_pos_in_line = pi
+ break
+
+ line_cy, line_lx, _, actual_space_w = line_geometry[word_line_idx]
+
+ # X position using derived natural spacing
+ current_line = lines[word_line_idx]
+ word_x = line_lx
+ for k in range(word_pos_in_line):
+ word_x += current_line[k][2] + actual_space_w
+ word_x = int(word_x)
+
+ word_w = word_widths[wi]
+ word_cx = word_x + word_w // 2
+ actual_pad_x = max(pad_x, int(font_size * 0.25)) # at least 25% of font size
+
+ # Pill dimensions
+ pill_w = word_w + actual_pad_x * 2
+ pill_h = font_size + pad_y * 2
+ pill_r = min(rounding, pill_h // 2)
+
+ # Pill drawing centered at origin (for \an5 positioning)
+ drawing = _rounded_rect_drawing(pill_w, pill_h, pill_r)
+ # Shift drawing so its center is at (0,0) for \an5
+ draw_offset_x = pill_w // 2
+ draw_offset_y = pill_h // 2
+
+ # Layer 0: Pill — positioned at word center with \an5
events.append(
- f"Dialogue: 0,{start_ts},{end_ts},BrandedNormal,,0,0,0,,{line_text}"
+ f"Dialogue: 0,{start_ts},{end_ts},BrandedNormal,,0,0,0,,"
+ f"{{\\an7\\pos({word_cx - draw_offset_x},{line_cy - draw_offset_y})"
+ f"\\p1\\c{box_color}\\bord0\\shad0"
+ f"\\1a&HFF&\\t(0,80,\\1a{box_alpha}&)"
+ f"}}"
+ f"{drawing}{{\\p0}}"
)
+ # Layer 1: Each word positioned with \an5\pos for perfect pill alignment
+ for li, line in enumerate(lines):
+ lcy_l, llx_l, _, asp_w = line_geometry[li]
+ wx = llx_l
+ for pi, (_, text, ww) in enumerate(line):
+ wcx = int(wx + ww // 2)
+ events.append(
+ f"Dialogue: 1,{start_ts},{end_ts},BrandedNormal,,0,0,0,,"
+ f"{{\\an5\\pos({wcx},{lcy_l})}}{text}"
+ )
+ wx += ww + asp_w
+
return header + "\n".join(events) + "\n"
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index 1eed0ea..913d5e2 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -771,13 +771,15 @@ def _detect_face_offset(
width: int,
height: int,
target_ratio: float,
-) -> Optional[int]:
+) -> Optional[str]:
"""
- Detect face position across the video and return crop x-offset.
- Uses DNN-based face detector (more robust than Haar cascades).
- Strongly favors the largest/closest face and clusters positions
- to lock onto the dominant speaker when multiple faces are visible.
- Returns None if no face detected (caller should fall back to center).
+ Continuous face tracking — detects face position over time and returns
+ an FFmpeg expression that smoothly follows the dominant face.
+
+ Samples frames throughout the clip, picks the dominant face cluster,
+ then builds a smoothed panning timeline so the crop follows head movement.
+
+ Returns an FFmpeg crop-x expression string, or None if no face detected.
"""
try:
import cv2
@@ -790,17 +792,17 @@ def _detect_face_offset(
total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
fps = cap.get(cv2.CAP_PROP_FPS) or 30
duration = total_frames / fps
+ crop_w = int(height * target_ratio)
- # Sample more frames across the full clip for better coverage
- sample_count = 60
+ # Sample ~4 frames/sec for smooth tracking (more than before)
+ sample_count = min(240, max(30, int(duration * 4)))
sample_times = [i * duration / sample_count for i in range(sample_count)]
- face_positions = []
+ # (time, face_center_x) for each detection
+ timed_positions = []
- # Try DNN detector first (much more robust than Haar cascades)
+ # Load DNN face detector
dnn_detector = None
-
- # Look for bundled DNN model first, then fall back to OpenCV's data dir
backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
bundled_proto = os.path.join(backend_dir, "models", "deploy.prototxt")
bundled_model = os.path.join(backend_dir, "models",
@@ -809,14 +811,12 @@ def _detect_face_offset(
"deploy.prototxt")
cv2_model = os.path.join(os.path.dirname(cv2.__file__), "data",
"res10_300x300_ssd_iter_140000.caffemodel")
-
proto_path = bundled_proto if os.path.exists(bundled_proto) else cv2_proto
model_path = bundled_model if os.path.exists(bundled_model) else cv2_model
-
if os.path.exists(proto_path) and os.path.exists(model_path):
dnn_detector = cv2.dnn.readNetFromCaffe(proto_path, model_path)
- # Fallback: load multiple Haar cascades for better coverage
+ # Fallback: Haar cascades
cascades = []
if dnn_detector is None:
for cascade_name in [
@@ -837,7 +837,6 @@ def _detect_face_offset(
h, w = frame.shape[:2]
if dnn_detector is not None:
- # DNN-based detection: handles frontal, profile, angled faces
blob = cv2.dnn.blobFromImage(
cv2.resize(frame, (300, 300)), 1.0, (300, 300),
(104.0, 177.0, 123.0)
@@ -849,26 +848,23 @@ def _detect_face_offset(
best_cx = None
for i in range(detections.shape[2]):
confidence = detections[0, 0, i, 2]
- if confidence > 0.5: # Higher threshold to reduce false positives
+ if confidence > 0.5:
x1 = int(detections[0, 0, i, 3] * w)
y1 = int(detections[0, 0, i, 4] * h)
x2 = int(detections[0, 0, i, 5] * w)
y2 = int(detections[0, 0, i, 6] * h)
face_w = x2 - x1
face_h = y2 - y1
- # Skip tiny detections (likely false positives)
if face_w < w * 0.05 or face_h < h * 0.05:
continue
- # Quadratic face-size weighting: strongly favor larger/closer faces
score = confidence * (face_w ** 2)
if score > best_score:
best_score = score
best_cx = (x1 + x2) // 2
if best_cx is not None:
- face_positions.append(best_cx)
+ timed_positions.append((t, best_cx))
else:
- # Haar cascade fallback: try all loaded cascades
gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY)
for cascade in cascades:
faces = cascade.detectMultiScale(
@@ -877,61 +873,135 @@ def _detect_face_offset(
if len(faces) > 0:
largest = max(faces, key=lambda f: f[2] * f[3])
face_center_x = largest[0] + largest[2] // 2
- face_positions.append(face_center_x)
- break # found face with this cascade, move to next frame
+ timed_positions.append((t, face_center_x))
+ break
cap.release()
- if not face_positions:
+ if len(timed_positions) < 3:
return None
- # Cluster face positions to find the dominant speaker.
- # When two speakers are visible, positions will form two groups.
- # Pick the largest cluster (most frequently detected face).
- crop_w = int(height * target_ratio)
- cluster_radius = crop_w * 0.35 # positions within 35% of crop width = same face (allows for head movement)
-
- positions = np.array(face_positions)
+ # --- Cluster to find dominant face ---
+ all_cx = np.array([p[1] for p in timed_positions])
+ cluster_radius = crop_w * 0.35
- # Find all distinct clusters
clusters = []
- used = np.zeros(len(positions), dtype=bool)
- sorted_idx = np.argsort(positions)
-
- for idx in sorted_idx:
+ used = np.zeros(len(all_cx), dtype=bool)
+ for idx in np.argsort(all_cx):
if used[idx]:
continue
- mask = np.abs(positions - positions[idx]) < cluster_radius
- mask &= ~used
- cluster = positions[mask]
- if len(cluster) > 0:
- clusters.append(cluster)
+ mask = (np.abs(all_cx - all_cx[idx]) < cluster_radius) & ~used
+ if np.any(mask):
+ clusters.append(np.where(mask)[0])
used |= mask
if not clusters:
- clusters = [positions]
+ return None
- # Sort clusters by size (largest first)
+ # Pick largest cluster (dominant face)
clusters.sort(key=lambda c: len(c), reverse=True)
+ dominant_indices = set(clusters[0])
+
+ # Filter to only dominant face detections
+ tracked = [(t, cx) for i, (t, cx) in enumerate(timed_positions) if i in dominant_indices]
+ if len(tracked) < 3:
+ # Too few points — use static median
+ median_x = int(np.median([cx for _, cx in tracked]))
+ crop_x = max(0, min(median_x - crop_w // 2, width - crop_w))
+ return str(crop_x)
+
+ # --- Smooth the positions ---
+ # 1) Convert face_center_x to crop_x (clamped)
+ tracked_times = np.array([t for t, _ in tracked])
+ tracked_crop_x = np.array([
+ max(0, min(cx - crop_w // 2, width - crop_w))
+ for _, cx in tracked
+ ], dtype=float)
+
+ # 2) Fill gaps: if face wasn't detected for some frames, the tracked
+ # array has gaps. That's fine — we interpolate between known points.
+
+ # 3) Heavy smoothing to prevent jitter. Use a rolling average with a
+ # window of ~1.5 seconds. This means small head bobs don't move the
+ # crop, but sustained movement (leaning, shifting) does.
+ if len(tracked_crop_x) > 5:
+ sample_interval = tracked_times[-1] / len(tracked_times) if len(tracked_times) > 1 else 0.25
+ window = max(3, int(1.5 / max(0.05, sample_interval)))
+ # Pad edges so smoothing doesn't pull toward zero
+ padded = np.pad(tracked_crop_x, (window // 2, window // 2), mode="edge")
+ kernel = np.ones(window) / window
+ smoothed = np.convolve(padded, kernel, mode="valid")[:len(tracked_crop_x)]
+ else:
+ smoothed = tracked_crop_x
+
+ # 4) Round to ints and clamp
+ smoothed = np.clip(np.round(smoothed), 0, width - crop_w).astype(int)
+
+ # --- Check if tracking is even needed ---
+ # If the face barely moves (range < 3% of frame width), just use static
+ movement_range = int(smoothed.max() - smoothed.min())
+ if movement_range < width * 0.03:
+ return str(int(np.median(smoothed)))
+
+ # --- Build keyframes: only emit when position changes significantly ---
+ # Downsample smoothed track to keyframes at ~1s intervals,
+ # or whenever the crop shifts by more than 20px.
+ keyframes = [(tracked_times[0], int(smoothed[0]))]
+ min_time_gap = 0.8 # Don't place keyframes closer than 0.8s
+ min_px_change = 20 # Ignore movement smaller than 20px
+
+ for i in range(1, len(smoothed)):
+ t = tracked_times[i]
+ x = int(smoothed[i])
+ prev_t, prev_x = keyframes[-1]
+ dt = t - prev_t
+ dx = abs(x - prev_x)
+
+ if dx >= min_px_change and dt >= min_time_gap:
+ keyframes.append((t, x))
+ elif i == len(smoothed) - 1:
+ # Always include last point
+ keyframes.append((t, x))
+
+ if len(keyframes) < 2:
+ return str(keyframes[0][1])
+
+ # --- Build FFmpeg expression ---
+ # Piecewise linear interpolation between keyframes.
+ # if(lt(t, t1), lerp(t0→t1), if(lt(t, t2), lerp(t1→t2), ...))
+ # This is simpler than between() chains and nests one level per segment.
+ expr_parts = []
+ for i in range(len(keyframes) - 1):
+ t0, x0 = keyframes[i]
+ t1, x1 = keyframes[i + 1]
+ dt = max(0.01, t1 - t0)
+ if x0 == x1:
+ # No movement in this segment — constant
+ expr_parts.append(f"if(lt(t\\,{t1:.2f})\\,{x0}\\,")
+ else:
+ # Linear interpolation
+ expr_parts.append(
+ f"if(lt(t\\,{t1:.2f})\\,"
+ f"{x0}+(({x1}-{x0})*(t-{t0:.2f})/{dt:.2f})\\,"
+ )
- # Always pick the largest cluster (most frequently detected face).
- # This ensures we lock onto the dominant speaker rather than
- # splitting the crop between two equally-visible speakers.
- best_cluster = clusters[0]
-
- avg_face_x = int(np.median(best_cluster))
-
- # Calculate crop window centered on face
- crop_x = avg_face_x - crop_w // 2
+ # Cap nesting depth
+ if len(expr_parts) > 40:
+ # Too many keyframes — subsample
+ step = len(keyframes) // 20
+ keyframes = keyframes[::step] + [keyframes[-1]]
+ # Rebuild (recursive call would be cleaner but let's just return static)
+ print(f"Warning: face tracking produced {len(expr_parts)} keyframes, using static", file=sys.stderr)
+ return str(int(np.median(smoothed)))
- # Clamp to valid range
- crop_x = max(0, min(crop_x, width - crop_w))
+ # Final value: last keyframe position
+ last_x = keyframes[-1][1]
+ expr = "".join(expr_parts) + str(last_x) + ")" * len(expr_parts)
- return crop_x
+ return expr
except ImportError:
- # OpenCV not installed, fall back to center
return None
except Exception as e:
- print(f"Warning: face detection failed: {e}", file=sys.stderr)
+ print(f"Warning: face tracking failed: {e}", file=sys.stderr)
return None