Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

feat: AI CLI fallback chain, smarter thumbnails, improved camera tracking

Nika Siradze85f2a618810b
diff --git a/backend/cli.py b/backend/cli.py
index 5f71993..1f32fdd 100644
--- a/backend/cli.py
+++ b/backend/cli.py
@@ -366,12 +366,12 @@ def cmd_process(args):
top_n = config.get("top_clips", 5)
clips = None
- # Try Claude first (uses PodStack knowledge base for intelligent selection)
- from services.claude_suggest import suggest_with_claude, _find_ai_cli
+ # Try an AI CLI first (uses PodStack knowledge base for intelligent selection)
+ from services.claude_suggest import suggest_with_claude, _engine_label, _find_ai_cli
ai_path, ai_engine = _find_ai_cli()
if ai_path:
- ai_label = "Claude" if ai_engine == "claude" else "Codex"
+ ai_label = _engine_label(ai_engine)
print(f" [3/4] Selecting moments with {ai_label} (PodStack)...")
clips = suggest_with_claude(
segments=segments,
@@ -379,9 +379,10 @@ def cmd_process(args):
progress_callback=lambda pct, msg: print(f" {msg}") if msg else None,
)
if clips:
- print(f" ✓ {ai_label} selected {len(clips)} clips")
+ actual_engine = next((c.get("_ai_engine") for c in clips if c.get("_ai_engine")), ai_engine)
+ print(f" ✓ {_engine_label(actual_engine)} selected {len(clips)} clips")
else:
- print(f" ⚠ {ai_label} unavailable, falling back to heuristics")
+ print(" ⚠ AI CLI unavailable, falling back to heuristics")
else:
print(f" [3/4] Scoring clips (heuristic mode)...")
print(f" ℹ Install Claude Code or Codex for smarter selection")
@@ -421,7 +422,7 @@ def cmd_process(args):
# Check if AI CLI is available for per-clip content generation
from services.claude_suggest import _find_ai_cli
- _ai_cli_path, _ai_engine = _find_ai_cli()
+ _ai_cli_path, _ = _find_ai_cli()
# Pre-load thumbnail tools if enabled
_thumb_gen = None
@@ -444,7 +445,7 @@ def cmd_process(args):
except Exception:
_thumb_enabled = False
- _ai_label = f" (+ titles via {'Claude' if _ai_engine == 'claude' else 'Codex'})" if _ai_cli_path else ""
+ _ai_label = " (+ AI titles)" if _ai_cli_path else ""
print(f"\n [4/4] Exporting {len(clips)} clips{_ai_label} to {output_dir}/")
results = []
t0 = time.time()
@@ -764,10 +765,10 @@ def _print_clips(clips: list):
def _find_moment_with_claude(description: str, segments: list, existing_clips: list) -> list:
"""Use Claude to find a specific moment described by the user."""
- from services.claude_suggest import _find_ai_cli
+ from services.claude_suggest import _find_ai_cli_candidates, _run_ai_command
- cli_path, engine = _find_ai_cli()
- if not cli_path:
+ candidates = _find_ai_cli_candidates()
+ if not candidates:
print(" ⚠ No AI CLI available for moment search")
return []
@@ -830,83 +831,83 @@ Transcript:
prompt_file = f.name
try:
- if engine == "codex":
- output_file = prompt_file + ".out"
- result = subprocess.run(
- [cli_path, "exec", "--full-auto", "-o", output_file, prompt],
- capture_output=True, text=True, cwd=project_dir, timeout=300,
- )
- if os.path.exists(output_file):
- with open(output_file) as fh:
- result = subprocess.CompletedProcess(
- args=result.args, returncode=result.returncode,
- stdout=fh.read(), stderr=result.stderr,
- )
- try:
- os.unlink(output_file)
- except Exception:
- pass
- else:
- result = subprocess.run(
- f'cat "{prompt_file}" | "{cli_path}" --print -p -',
- capture_output=True, text=True, cwd=project_dir, timeout=300, shell=True,
- )
+ for idx, (cli_path, engine) in enumerate(candidates):
+ if idx > 0:
+ print(f" ⚠ Retrying moment search with {'Claude' if engine == 'claude' else 'Codex'}")
+ try:
+ result = _run_ai_command(
+ cli_path=cli_path,
+ engine=engine,
+ prompt=prompt,
+ prompt_file=prompt_file,
+ project_dir=project_dir,
+ timeout=300,
+ )
+ except Exception:
+ continue
- if result.returncode != 0:
- return []
+ if result.returncode != 0 or not result.stdout.strip():
+ continue
- response = result.stdout.strip()
- if "```" in response:
- import re
- fence_match = re.search(r"```(?:json)?\s*\n?(.*?)\n?\s*```", response, re.DOTALL)
- if fence_match:
- response = fence_match.group(1).strip()
-
- json_start = response.find("{")
- if json_start >= 0:
- decoder = _json.JSONDecoder()
- data, _ = decoder.raw_decode(response, json_start)
- else:
- data = _json.loads(response)
-
- found = []
- for c in data.get("clips", []):
- scores = c.get("scores", {})
- total = sum(scores.values()) if scores else c.get("total_score", 0)
- raw_segments = c.get("segments", [])
- keep_segments = []
- for seg in raw_segments:
- s = round(float(seg.get("start", 0)), 1)
- e = round(float(seg.get("end", 0)), 1)
- if e > s:
- keep_segments.append({"start": s, "end": e})
-
- start_sec = round(float(c.get("start_second", 0)), 1)
- end_sec = round(float(c.get("end_second", 0)), 1)
- if not keep_segments and end_sec > start_sec:
- keep_segments = [{"start": start_sec, "end": end_sec}]
-
- kept_duration = sum(seg["end"] - seg["start"] for seg in keep_segments)
- if kept_duration < 10 or kept_duration > 90:
+ response = result.stdout.strip()
+ if "```" in response:
+ import re
+ fence_match = re.search(r"```(?:json)?\s*\n?(.*?)\n?\s*```", response, re.DOTALL)
+ if fence_match:
+ response = fence_match.group(1).strip()
+
+ try:
+ json_start = response.find("{")
+ if json_start >= 0:
+ decoder = _json.JSONDecoder()
+ data, _ = decoder.raw_decode(response, json_start)
+ else:
+ data = _json.loads(response)
+ except Exception:
continue
- found.append({
- "title": c.get("title", "Untitled")[:55],
- "start_second": keep_segments[0]["start"] if keep_segments else start_sec,
- "end_second": keep_segments[-1]["end"] if keep_segments else end_sec,
- "segments": keep_segments,
- "duration": round(kept_duration),
- "score": total,
- "content_type": c.get("content_type", "unknown"),
- "reasoning": c.get("why", ""),
- "preview_text": c.get("quote", "")[:120],
- "suggested_caption_style": "hormozi",
- "quote": c.get("quote", ""),
- "why": c.get("why", ""),
- "reasons": [c.get("content_type", "")],
- "preview": c.get("quote", "")[:120],
- })
- return found
+ found = []
+ for c in data.get("clips", []):
+ scores = c.get("scores", {})
+ total = sum(scores.values()) if scores else c.get("total_score", 0)
+ raw_segments = c.get("segments", [])
+ keep_segments = []
+ for seg in raw_segments:
+ s = round(float(seg.get("start", 0)), 1)
+ e = round(float(seg.get("end", 0)), 1)
+ if e > s:
+ keep_segments.append({"start": s, "end": e})
+
+ start_sec = round(float(c.get("start_second", 0)), 1)
+ end_sec = round(float(c.get("end_second", 0)), 1)
+ if not keep_segments and end_sec > start_sec:
+ keep_segments = [{"start": start_sec, "end": end_sec}]
+
+ kept_duration = sum(seg["end"] - seg["start"] for seg in keep_segments)
+ if kept_duration < 10 or kept_duration > 90:
+ continue
+
+ found.append({
+ "title": c.get("title", "Untitled")[:55],
+ "start_second": keep_segments[0]["start"] if keep_segments else start_sec,
+ "end_second": keep_segments[-1]["end"] if keep_segments else end_sec,
+ "segments": keep_segments,
+ "duration": round(kept_duration),
+ "score": total,
+ "content_type": c.get("content_type", "unknown"),
+ "reasoning": c.get("why", ""),
+ "preview_text": c.get("quote", "")[:120],
+ "suggested_caption_style": "hormozi",
+ "quote": c.get("quote", ""),
+ "why": c.get("why", ""),
+ "reasons": [c.get("content_type", "")],
+ "preview": c.get("quote", "")[:120],
+ })
+
+ if found:
+ return found
+
+ return []
except Exception as e:
print(f" ⚠ Search error: {e}")
@@ -1020,7 +1021,7 @@ def _review_clips(clips: list, segments: list, energy_scores: list | None, confi
elif action == "more":
top_n = config.get("top_clips", 5)
from services.claude_suggest import suggest_with_claude
- with _Spinner(f"Asking Claude for {top_n} more suggestions...") as sp:
+ with _Spinner(f"Asking AI for {top_n} more suggestions...") as sp:
more_clips = suggest_with_claude(
segments=segments,
top_n=top_n,
@@ -1299,7 +1300,7 @@ def _post_render_loop(
elif action == "more":
top_n = config.get("top_clips", 5)
- print(f"\n Asking Claude for more suggestions...")
+ print(f"\n Asking AI for more suggestions...")
from services.claude_suggest import suggest_with_claude
more_clips = suggest_with_claude(
segments=segments,
diff --git a/backend/scripts/remotion_screenshot.cjs b/backend/scripts/remotion_screenshot.cjs
new file mode 100644
index 0000000..db219f3
--- /dev/null
+++ b/backend/scripts/remotion_screenshot.cjs
@@ -0,0 +1,86 @@
+#!/usr/bin/env node
+
+const path = require("node:path");
+const rendererRoot = path.resolve(__dirname, "..", "..", "node_modules", "@remotion", "renderer", "dist");
+const {openBrowser} = require("@remotion/renderer");
+const {screenshot} = require(path.join(rendererRoot, "puppeteer-screenshot.js"));
+
+const [htmlPath, outputPath, widthArg, heightArg, waitMsArg] = process.argv.slice(2);
+
+if (!htmlPath || !outputPath || !widthArg || !heightArg) {
+ console.error(
+ "Usage: remotion_screenshot.cjs <html_path> <output_path> <width> <height> [wait_ms]",
+ );
+ process.exit(1);
+}
+
+const width = Number(widthArg);
+const height = Number(heightArg);
+const waitMs = Number(waitMsArg || "1500");
+
+const waitForAssets = async (page) => {
+ await page
+ .evaluate(async () => {
+ const images = Array.from(document.images || []);
+ await Promise.all(
+ images.map((img) => {
+ if (img.complete) {
+ return Promise.resolve();
+ }
+
+ return new Promise((resolve) => {
+ img.addEventListener("load", resolve, {once: true});
+ img.addEventListener("error", resolve, {once: true});
+ });
+ }),
+ );
+
+ if (document.fonts?.ready) {
+ try {
+ await document.fonts.ready;
+ } catch {
+ }
+ }
+ })
+ .catch(() => undefined);
+};
+
+(async () => {
+ const browser = await openBrowser("chrome", {logLevel: "error"});
+ try {
+ const page = await browser.newPage({
+ context: () => null,
+ logLevel: "error",
+ indent: false,
+ pageIndex: 0,
+ onBrowserLog: () => undefined,
+ onLog: () => undefined,
+ });
+ await page.setViewport({width, height, deviceScaleFactor: 1});
+ await page.goto({
+ url: `file://${path.resolve(htmlPath)}`,
+ timeout: Math.max(30000, waitMs + 10000),
+ options: {
+ waitUntil: "load",
+ },
+ });
+ await waitForAssets(page);
+ if (waitMs > 0) {
+ await new Promise((resolve) => setTimeout(resolve, waitMs));
+ }
+ await screenshot({
+ page,
+ path: outputPath,
+ type: "png",
+ width,
+ height,
+ scale: 1,
+ });
+ await page.close();
+ } finally {
+ await browser.close({silent: true});
+ }
+})().catch((err) => {
+ console.error(err && (err.stack || err.message) ? err.stack || err.message : String(err));
+ process.exit(1);
+});
diff --git a/backend/services/claude_suggest.py b/backend/services/claude_suggest.py
index a66dcac..f070e22 100644
--- a/backend/services/claude_suggest.py
+++ b/backend/services/claude_suggest.py
@@ -29,31 +29,93 @@ def _find_cli(name: str, extra_paths: list[str] = None) -> Optional[str]:
return None
-def _find_ai_cli() -> tuple[Optional[str], str]:
- """
- Find the best available AI CLI.
+def _find_ai_cli_candidates() -> list[tuple[str, str]]:
+ """Find all available AI CLIs in preference order."""
+ candidates = []
- Returns (path, engine) where engine is "claude" or "codex".
- Returns (None, "") if neither is available.
- """
- # Prefer Claude Code
claude = _find_cli("claude", [
os.path.expanduser("~/.local/bin/claude"),
"/usr/local/bin/claude",
"/opt/homebrew/bin/claude",
])
if claude:
- return claude, "claude"
+ candidates.append((claude, "claude"))
- # Fall back to Codex
codex = _find_cli("codex", [
"/usr/local/bin/codex",
"/opt/homebrew/bin/codex",
])
if codex:
- return codex, "codex"
+ candidates.append((codex, "codex"))
+
+ return candidates
+
+
+def _find_ai_cli() -> tuple[Optional[str], str]:
+ """
+ Find the best available AI CLI.
- return None, ""
+ Returns (path, engine) where engine is "claude" or "codex".
+ Returns (None, "") if neither is available.
+ """
+ candidates = _find_ai_cli_candidates()
+ return candidates[0] if candidates else (None, "")
+
+
+def _engine_label(engine: str) -> str:
+ """Human-readable name for an AI engine id."""
+ if engine == "claude":
+ return "Claude"
+ if engine == "codex":
+ return "Codex"
+ return "AI"
+
+
+def _run_ai_command(
+ cli_path: str,
+ engine: str,
+ prompt: str,
+ prompt_file: str,
+ project_dir: str,
+ timeout: int,
+) -> subprocess.CompletedProcess:
+ """Execute one AI CLI prompt and return the completed process."""
+ if engine == "codex":
+ output_file = prompt_file + ".out"
+ result = subprocess.run(
+ [
+ cli_path, "exec",
+ "--full-auto",
+ "-o", output_file,
+ prompt,
+ ],
+ capture_output=True,
+ text=True,
+ cwd=project_dir,
+ timeout=timeout,
+ )
+ if os.path.exists(output_file):
+ with open(output_file) as f:
+ result = subprocess.CompletedProcess(
+ args=result.args,
+ returncode=result.returncode,
+ stdout=f.read(),
+ stderr=result.stderr,
+ )
+ try:
+ os.unlink(output_file)
+ except Exception:
+ pass
+ return result
+
+ return subprocess.run(
+ f'cat "{prompt_file}" | "{cli_path}" --print -p -',
+ capture_output=True,
+ text=True,
+ cwd=project_dir,
+ timeout=timeout,
+ shell=True,
+ )
def _load_existing_shorts(episodes_path: str) -> list[str]:
@@ -216,15 +278,15 @@ def suggest_with_claude(
"""
Use an AI CLI (Claude Code or Codex) to extract the best clip moments.
- Tries Claude Code first, falls back to Codex. Returns None if neither
- is available.
+ Tries available AI CLIs in preference order and retries on runtime failure.
+ Returns None if neither succeeds.
"""
- cli_path, engine = _find_ai_cli()
- if not cli_path:
+ candidates = _find_ai_cli_candidates()
+ if not candidates:
return None
if progress_callback:
- label = "Claude" if engine == "claude" else "Codex"
+ label = _engine_label(candidates[0][1])
progress_callback(0, f"Preparing transcript for {label}...")
# Build transcript text from segments
@@ -255,84 +317,10 @@ def suggest_with_claude(
prompt_file = f.name
if progress_callback:
- label = "Claude" if engine == "claude" else "Codex"
- progress_callback(20, f"Asking {label} to analyze transcript...")
+ first_label = _engine_label(candidates[0][1])
+ progress_callback(20, f"Asking {first_label} to analyze transcript...")
try:
- # Run the AI CLI to analyze the transcript
- if engine == "codex":
- # Codex: use `codex exec` with --output-last-message to capture response
- output_file = prompt_file + ".out"
- result = subprocess.run(
- [
- cli_path, "exec",
- "--full-auto",
- "-o", output_file,
- prompt,
- ],
- capture_output=True,
- text=True,
- cwd=project_dir,
- timeout=300,
- )
- # Read response from output file
- if os.path.exists(output_file):
- with open(output_file) as f:
- result = subprocess.CompletedProcess(
- args=result.args,
- returncode=result.returncode,
- stdout=f.read(),
- stderr=result.stderr,
- )
- try:
- os.unlink(output_file)
- except Exception:
- pass
- else:
- # Claude: pipe prompt via stdin with --print mode
- result = subprocess.run(
- f'cat "{prompt_file}" | "{cli_path}" --print -p -',
- capture_output=True,
- text=True,
- cwd=project_dir,
- timeout=300,
- shell=True,
- )
-
- if result.returncode != 0:
- if progress_callback:
- progress_callback(0, f"{label} returned error: {result.stderr[:200]}")
- return None
-
- if progress_callback:
- progress_callback(80, f"Parsing {label}'s suggestions...")
-
- # Parse JSON from Claude's response
- response = result.stdout.strip()
-
- # Strip markdown code fences if present
- if "```" in response:
- import re
- fence_match = re.search(r"```(?:json)?\s*\n?(.*?)\n?\s*```", response, re.DOTALL)
- if fence_match:
- response = fence_match.group(1).strip()
-
- # Use raw_decode to extract the first complete JSON object
- # (safer than rfind which can grab a stray "}" after the JSON)
- json_start = response.find("{")
- if json_start >= 0:
- decoder = json.JSONDecoder()
- data, _ = decoder.raw_decode(response, json_start)
- else:
- data = json.loads(response)
-
- clips = data.get("clips", [])
-
- if not clips:
- return None
-
- # Normalize to the format the CLI expects
- normalized = []
def _parse_seconds(val) -> float:
"""Parse a timestamp value — handles both 123.4 and '2:03' formats."""
if isinstance(val, (int, float)):
@@ -349,71 +337,115 @@ def suggest_with_claude(
except ValueError:
return 0.0
- for c in clips:
- scores = c.get("scores", {})
- total = sum(scores.values()) if scores else c.get("total_score", 0)
+ for idx, (cli_path, engine) in enumerate(candidates):
+ label = _engine_label(engine)
+ if idx > 0 and progress_callback:
+ progress_callback(0, f"Retrying with {label}...")
+ progress_callback(20, f"Asking {label} to analyze transcript...")
- # Parse segments (multi-cut ranges) or fall back to single range
- raw_segments = c.get("segments", [])
- keep_segments = []
- for seg in raw_segments:
- s = round(_parse_seconds(seg.get("start", 0)), 1)
- e = round(_parse_seconds(seg.get("end", 0)), 1)
- if e > s:
- keep_segments.append({"start": s, "end": e})
+ try:
+ result = _run_ai_command(
+ cli_path=cli_path,
+ engine=engine,
+ prompt=prompt,
+ prompt_file=prompt_file,
+ project_dir=project_dir,
+ timeout=300,
+ )
+ except subprocess.TimeoutExpired:
+ if progress_callback:
+ progress_callback(0, f"{label} timed out (5 min limit)")
+ continue
+ except Exception as e:
+ if progress_callback:
+ progress_callback(0, f"{label} error: {e}")
+ continue
- start_sec = round(_parse_seconds(c.get("start_second", 0)), 1)
- end_sec = round(_parse_seconds(c.get("end_second", 0)), 1)
+ if result.returncode != 0 or not result.stdout.strip():
+ if progress_callback:
+ detail = (result.stderr or "no response").strip()[:200]
+ progress_callback(0, f"{label} returned error: {detail}")
+ continue
- # Fall back to single segment if none provided
- if not keep_segments and end_sec > start_sec:
- keep_segments = [{"start": start_sec, "end": end_sec}]
+ if progress_callback:
+ progress_callback(80, f"Parsing {label}'s suggestions...")
- # Compute actual kept duration
- kept_duration = sum(seg["end"] - seg["start"] for seg in keep_segments)
+ try:
+ response = result.stdout.strip()
+ if "```" in response:
+ import re
+ fence_match = re.search(r"```(?:json)?\s*\n?(.*?)\n?\s*```", response, re.DOTALL)
+ if fence_match:
+ response = fence_match.group(1).strip()
+
+ json_start = response.find("{")
+ if json_start >= 0:
+ decoder = json.JSONDecoder()
+ data, _ = decoder.raw_decode(response, json_start)
+ else:
+ data = json.loads(response)
+ except json.JSONDecodeError as e:
+ if progress_callback:
+ progress_callback(0, f"Could not parse {label}'s response as JSON: {e}")
+ continue
- # Reject clips outside the target duration range
- if kept_duration < 15 or kept_duration > 55:
+ clips = data.get("clips", [])
+ if not clips:
+ if progress_callback:
+ progress_callback(0, f"{label} returned no clips")
continue
- normalized.append({
- "title": c.get("title", "Untitled")[:55],
- "start_second": keep_segments[0]["start"] if keep_segments else start_sec,
- "end_second": keep_segments[-1]["end"] if keep_segments else end_sec,
- "segments": keep_segments,
- "duration": round(kept_duration),
- "score": total,
- "content_type": c.get("content_type", "unknown"),
- # MCP-aligned fields
- "reasoning": c.get("why", ""),
- "preview_text": c.get("quote", "")[:120],
- "suggested_caption_style": "hormozi",
- # Backwards compat
- "quote": c.get("quote", ""),
- "why": c.get("why", ""),
- "reasons": [c.get("content_type", "")],
- "preview": c.get("quote", "")[:120],
- })
-
- # Sort by timestamp
- normalized.sort(key=lambda x: x["start_second"])
-
- if progress_callback:
- progress_callback(100, f"{label} suggested {len(normalized)} clips")
-
- return normalized
-
- except json.JSONDecodeError as e:
- if progress_callback:
- progress_callback(0, f"Could not parse {label}'s response as JSON: {e}")
- return None
- except subprocess.TimeoutExpired:
- if progress_callback:
- progress_callback(0, f"{label} timed out (5 min limit)")
- return None
- except Exception as e:
- if progress_callback:
- progress_callback(0, f"{label} error: {e}")
+ normalized = []
+ for c in clips:
+ scores = c.get("scores", {})
+ total = sum(scores.values()) if scores else c.get("total_score", 0)
+
+ raw_segments = c.get("segments", [])
+ keep_segments = []
+ for seg in raw_segments:
+ s = round(_parse_seconds(seg.get("start", 0)), 1)
+ e = round(_parse_seconds(seg.get("end", 0)), 1)
+ if e > s:
+ keep_segments.append({"start": s, "end": e})
+
+ start_sec = round(_parse_seconds(c.get("start_second", 0)), 1)
+ end_sec = round(_parse_seconds(c.get("end_second", 0)), 1)
+
+ if not keep_segments and end_sec > start_sec:
+ keep_segments = [{"start": start_sec, "end": end_sec}]
+
+ kept_duration = sum(seg["end"] - seg["start"] for seg in keep_segments)
+ if kept_duration < 15 or kept_duration > 55:
+ continue
+
+ normalized.append({
+ "title": c.get("title", "Untitled")[:55],
+ "start_second": keep_segments[0]["start"] if keep_segments else start_sec,
+ "end_second": keep_segments[-1]["end"] if keep_segments else end_sec,
+ "segments": keep_segments,
+ "duration": round(kept_duration),
+ "score": total,
+ "content_type": c.get("content_type", "unknown"),
+ "reasoning": c.get("why", ""),
+ "preview_text": c.get("quote", "")[:120],
+ "suggested_caption_style": "hormozi",
+ "quote": c.get("quote", ""),
+ "why": c.get("why", ""),
+ "reasons": [c.get("content_type", "")],
+ "preview": c.get("quote", "")[:120],
+ "_ai_engine": engine,
+ })
+
+ normalized.sort(key=lambda x: x["start_second"])
+
+ if normalized:
+ if progress_callback:
+ progress_callback(100, f"{label} suggested {len(normalized)} clips")
+ return normalized
+
+ if progress_callback:
+ progress_callback(0, f"{label} returned no usable clips")
+
return None
finally:
# Clean up temp file
diff --git a/backend/services/content_generator.py b/backend/services/content_generator.py
index cccf120..a0635b9 100644
--- a/backend/services/content_generator.py
+++ b/backend/services/content_generator.py
@@ -10,7 +10,7 @@ import subprocess
import tempfile
from typing import Optional, Callable
-from services.claude_suggest import _find_ai_cli
+from services.claude_suggest import _engine_label, _find_ai_cli_candidates, _run_ai_command
def _load_kb_context() -> str:
@@ -56,11 +56,11 @@ def generate_clip_content(
Returns:
dict with raw_text, titles, description, tags, hashtags, or None if AI unavailable
"""
- cli_path, engine = _find_ai_cli()
- if not cli_path:
+ candidates = _find_ai_cli_candidates()
+ if not candidates:
return None
- label = "Claude" if engine == "claude" else "Codex"
+ label = _engine_label(candidates[0][1])
if progress_callback:
progress_callback(0, f"Generating content via {label}...")
@@ -119,94 +119,87 @@ HASHTAGS:
prompt_file = f.name
try:
- if progress_callback:
- progress_callback(30, f"Asking {label} for titles & descriptions...")
-
- if engine == "codex":
- out_file = prompt_file + ".out"
- cr = subprocess.run(
- [cli_path, "exec", "--full-auto", "-o", out_file, prompt[:4000]],
- capture_output=True, text=True, cwd=project_dir, timeout=120,
- )
- if os.path.exists(out_file):
- with open(out_file) as of:
- cr = subprocess.CompletedProcess(
- args=cr.args, returncode=cr.returncode,
- stdout=of.read(), stderr=cr.stderr,
- )
- try:
- os.unlink(out_file)
- except Exception:
- pass
- else:
- cr = subprocess.run(
- f'cat "{prompt_file}" | "{cli_path}" --print -p -',
- capture_output=True, text=True, cwd=project_dir,
- timeout=120, shell=True,
- )
-
- if cr.returncode != 0 or not cr.stdout.strip():
- return None
-
- raw_text = cr.stdout.strip()
-
- if progress_callback:
- progress_callback(90, "Parsing content...")
-
- # Parse structured sections from the response
- result = {
- "raw_text": raw_text,
- "titles": [],
- "top_pick": "",
- "description": "",
- "tags": "",
- "hashtags": "",
- "engine": engine,
- }
-
- lines = raw_text.split("\n")
- section = ""
- for line in lines:
- stripped = line.strip()
- if not stripped:
- continue
- upper = stripped.upper()
- if upper.startswith("TITLES"):
- section = "titles"
- continue
- elif upper.startswith("TOP PICK"):
- result["top_pick"] = stripped
- section = ""
- continue
- elif upper.startswith("DESCRIPTION"):
- section = "description"
+ for idx, (cli_path, engine) in enumerate(candidates):
+ label = _engine_label(engine)
+ if progress_callback:
+ if idx > 0:
+ progress_callback(0, f"Retrying content generation with {label}...")
+ progress_callback(30, f"Asking {label} for titles & descriptions...")
+
+ try:
+ cr = _run_ai_command(
+ cli_path=cli_path,
+ engine=engine,
+ prompt=prompt[:4000] if engine == "codex" else prompt,
+ prompt_file=prompt_file,
+ project_dir=project_dir,
+ timeout=120,
+ )
+ except subprocess.TimeoutExpired:
continue
- elif upper.startswith("TAGS"):
- section = "tags"
+ except Exception:
continue
- elif upper.startswith("HASHTAGS"):
- section = "hashtags"
+
+ if cr.returncode != 0 or not cr.stdout.strip():
continue
- if section == "titles" and stripped[0:1].isdigit() and ". " in stripped[:4]:
- result["titles"].append(stripped)
- elif section == "description":
- result["description"] += stripped + "\n"
- elif section == "tags":
- result["tags"] = stripped
- elif section == "hashtags":
- result["hashtags"] = stripped
+ raw_text = cr.stdout.strip()
+
+ if progress_callback:
+ progress_callback(90, "Parsing content...")
+
+ result = {
+ "raw_text": raw_text,
+ "titles": [],
+ "top_pick": "",
+ "description": "",
+ "tags": "",
+ "hashtags": "",
+ "engine": engine,
+ }
+
+ lines = raw_text.split("\n")
+ section = ""
+ for line in lines:
+ stripped = line.strip()
+ if not stripped:
+ continue
+ upper = stripped.upper()
+ if upper.startswith("TITLES"):
+ section = "titles"
+ continue
+ elif upper.startswith("TOP PICK"):
+ result["top_pick"] = stripped
+ section = ""
+ continue
+ elif upper.startswith("DESCRIPTION"):
+ section = "description"
+ continue
+ elif upper.startswith("TAGS"):
+ section = "tags"
+ continue
+ elif upper.startswith("HASHTAGS"):
+ section = "hashtags"
+ continue
- result["description"] = result["description"].strip()
+ if section == "titles" and stripped[0:1].isdigit() and ". " in stripped[:4]:
+ result["titles"].append(stripped)
+ elif section == "description":
+ result["description"] += stripped + "\n"
+ elif section == "tags":
+ result["tags"] = stripped
+ elif section == "hashtags":
+ result["hashtags"] = stripped
+
+ result["description"] = result["description"].strip()
+ if not result["titles"] and not result["description"]:
+ continue
- if progress_callback:
- progress_callback(100, f"Content ready ({len(result['titles'])} titles)")
+ if progress_callback:
+ progress_callback(100, f"Content ready ({len(result['titles'])} titles)")
- return result
+ return result
- except subprocess.TimeoutExpired:
- return None
- except Exception:
return None
finally:
try:
diff --git a/backend/services/thumbnail_ai.py b/backend/services/thumbnail_ai.py
index 8aa8a91..aa4a24a 100644
--- a/backend/services/thumbnail_ai.py
+++ b/backend/services/thumbnail_ai.py
@@ -366,14 +366,15 @@ def ask_claude_for_layout(
config: Optional[dict] = None,
) -> Optional[dict]:
"""
- Ask Claude to generate ALL layout values for the thumbnail.
- Claude sees the frame info and decides everything dynamically.
+ Ask an available AI CLI to generate layout values for the thumbnail.
+ The model sees the frame info and decides everything dynamically.
Returns dict with line1, line2, box_y, photo_object_position, etc.
"""
- from services.claude_suggest import _find_ai_cli
- claude_path, _engine = _find_ai_cli()
- if not claude_path:
+ from services.claude_suggest import _find_ai_cli_candidates, _run_ai_command
+
+ candidates = _find_ai_cli_candidates()
+ if not candidates:
return None
cfg = config or _load_brand_config()
@@ -404,7 +405,11 @@ Return ONLY valid JSON with these fields:
}}
RULES:
-- Split the title into 2 impactful lines. Line 1 = setup, Line 2 = payoff.
+- Rewrite the title into thumbnail copy, not a literal transcript sentence.
+- Split it into 2 impactful lines. Line 1 = setup, Line 2 = payoff.
+- Keep the combined copy to 4-8 words total whenever possible.
+- Keep each line short enough to fit comfortably on a Shorts thumbnail. Hard max: 24 characters per line.
+- Drop filler words and trailing clauses. Keep numbers, nouns, and the strongest claim.
- box_y: position the text box so it does NOT overlap the face. If face is high (y<40%), use 80-85%. If face is centered (y~50%), use 75-78%. If face is low, use 68-72%.
- photo_object_position: CSS value to show the face well. The photo is already 1080x1920 with face centered, so "center top" usually works. Adjust if face_y is unusual.
- Font sizes: for short text (1-2 words per line) use 96-110px. For medium (3-4 words) use 80-96px. For long (5+ words) use 64-80px. Line 2 should be 85-95% of Line 1.
@@ -412,28 +417,44 @@ RULES:
project_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")
+ prompt_file = None
try:
with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False, dir=project_dir) as f:
f.write(prompt)
prompt_file = f.name
- result = subprocess.run(
- f'cat "{prompt_file}" | "{claude_path}" --print -p -',
- capture_output=True, text=True, shell=True, timeout=30,
- cwd=project_dir,
- )
- os.unlink(prompt_file)
+ for cli_path, engine in candidates:
+ try:
+ result = _run_ai_command(
+ cli_path=cli_path,
+ engine=engine,
+ prompt=prompt,
+ prompt_file=prompt_file,
+ project_dir=project_dir,
+ timeout=30,
+ )
+ except Exception:
+ continue
- if result.returncode != 0:
- return None
+ if result.returncode != 0 or not result.stdout.strip():
+ continue
- text = result.stdout.strip()
- start = text.find("{")
- end = text.rfind("}") + 1
- if start >= 0 and end > start:
- return json.loads(text[start:end])
+ text = result.stdout.strip()
+ start = text.find("{")
+ end = text.rfind("}") + 1
+ if start >= 0 and end > start:
+ try:
+ return json.loads(text[start:end])
+ except json.JSONDecodeError:
+ continue
except Exception:
pass
+ finally:
+ if prompt_file:
+ try:
+ os.unlink(prompt_file)
+ except Exception:
+ pass
return None
@@ -450,7 +471,7 @@ def generate_thumbnail_with_template(
Template + AI layout. Claude decides all dynamic values per frame.
Template provides consistent visual structure.
"""
- from services.thumbnail_html import generate_thumbnail, _load_config
+ from services.thumbnail_html import generate_thumbnail, _load_config, _prepare_thumbnail_lines
cfg = _load_config()
if config:
@@ -460,8 +481,11 @@ def generate_thumbnail_with_template(
layout = ask_claude_for_layout(title, frame_path, frame_info, logo_path, cfg)
if layout:
- line1 = layout.get("line1", title).strip().strip("/").strip()
- line2 = layout.get("line2", "").strip().strip("/").strip()
+ line1, line2 = _prepare_thumbnail_lines(
+ title=title,
+ line1=layout.get("line1", title),
+ line2=layout.get("line2", ""),
+ )
# Apply Claude's CSS decisions to config
if layout.get("box_y"):
cfg["box_y"] = layout["box_y"]
@@ -473,15 +497,7 @@ def generate_thumbnail_with_template(
if layout.get("line2_font_size"):
cfg["line2_font_size"] = layout["line2_font_size"]
else:
- # Fallback
- if " / " in title:
- parts = title.split(" / ", 1)
- line1, line2 = parts[0].strip(), parts[1].strip()
- else:
- words = title.split()
- mid = len(words) // 2 or 1
- line1 = " ".join(words[:mid])
- line2 = " ".join(words[mid:])
+ line1, line2 = _prepare_thumbnail_lines(title=title)
return generate_thumbnail(
line1, line2, output_path,
diff --git a/backend/services/thumbnail_html.py b/backend/services/thumbnail_html.py
index dff1c78..4ddf09f 100644
--- a/backend/services/thumbnail_html.py
+++ b/backend/services/thumbnail_html.py
@@ -1,5 +1,5 @@
"""
-Thumbnail generator using HTML/CSS + Playwright screenshot.
+Thumbnail generator using HTML/CSS + headless browser screenshot.
Renders thumbnails via a headless browser for pixel-perfect CSS control
over fonts, letter-spacing, line-height, and highlight styling.
@@ -10,12 +10,25 @@ brand-specific values live in .podcli/thumbnail-config.json.
import json
import os
+import re
+import shutil
import subprocess
import sys
import tempfile
from typing import Optional
+_THUMBNAIL_LEADING_FILLERS = {
+ "a", "an", "the", "we", "i", "they", "he", "she", "it", "there", "this", "that",
+ "these", "those", "our", "your", "their", "my", "so", "well", "and", "but",
+}
+_THUMBNAIL_LINE_STOPWORDS = {
+ "a", "an", "and", "as", "at", "for", "from", "in", "into", "of", "on",
+ "or", "that", "the", "to", "with",
+}
+_THUMBNAIL_SPLIT_SEPARATORS = (" / ", " — ", " – ", " - ", ": ", "; ", ", ")
+
+
def _load_config() -> dict:
"""Load thumbnail config, merged with defaults.
@@ -92,6 +105,8 @@ def _load_config() -> dict:
"variations": 3,
"variation_offset_up": "3%",
"variation_offset_down": "2%",
+ "playwright_wait_ms": 1500,
+ "playwright_timeout_ms": 150000,
}
config_path = os.path.join(
@@ -107,6 +122,194 @@ def _load_config() -> dict:
return defaults
+def _playwright_cli_candidates() -> list[list[str]]:
+ """Return Playwright CLI commands in preference order."""
+ repo_root = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")
+ local_bin = os.path.join(repo_root, "node_modules", ".bin", "playwright")
+
+ candidates = []
+ if os.path.exists(local_bin):
+ candidates.append([local_bin])
+
+ global_bin = shutil.which("playwright")
+ if global_bin:
+ candidates.append([global_bin])
+
+ npx_bin = shutil.which("npx")
+ if npx_bin:
+ candidates.append([npx_bin, "--no-install", "playwright"])
+ candidates.append([npx_bin, "playwright"])
+
+ deduped = []
+ seen = set()
+ for cmd in candidates:
+ key = tuple(cmd)
+ if key not in seen:
+ seen.add(key)
+ deduped.append(cmd)
+ return deduped
+
+
+def _remotion_screenshot_script_path() -> str:
+ """Return the local Remotion screenshot helper script path."""
+ return os.path.join(
+ os.path.dirname(os.path.abspath(__file__)),
+ "..",
+ "scripts",
+ "remotion_screenshot.cjs",
+ )
+
+
+def _build_remotion_screenshot_command(
+ script_path: str,
+ html_path: str,
+ output_path: str,
+ width: int,
+ height: int,
+ wait_ms: int,
+) -> Optional[list[str]]:
+ """Build a Remotion-backed screenshot command if the repo can run it."""
+ node_bin = shutil.which("node")
+ repo_root = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")
+ renderer_pkg = os.path.join(repo_root, "node_modules", "@remotion", "renderer", "package.json")
+ if not node_bin or not os.path.exists(script_path) or not os.path.exists(renderer_pkg):
+ return None
+
+ return [
+ node_bin,
+ script_path,
+ html_path,
+ output_path,
+ str(width),
+ str(height),
+ str(wait_ms),
+ ]
+
+
+def _build_playwright_screenshot_command(
+ cli_cmd: list[str],
+ html_path: str,
+ output_path: str,
+ width: int,
+ height: int,
+ wait_ms: int,
+) -> list[str]:
+ """Build one Playwright screenshot command."""
+ return [
+ *cli_cmd,
+ "screenshot",
+ "--viewport-size", f"{width}, {height}",
+ "--wait-for-timeout", str(wait_ms),
+ f"file://{html_path}",
+ output_path,
+ ]
+
+
+def _compact_thumbnail_title(title: str, max_words: int = 7, max_chars: int = 42) -> str:
+ """Turn transcript-like title text into compact thumbnail copy."""
+ text = re.sub(r"\s+", " ", (title or "")).strip().strip("/").strip()
+ if not text:
+ return ""
+
+ replacements = [
+ (r"(?i)\b(\d+)\s+megawatts?\b", r"\1MW"),
+ (r"(?i)\b(\d+)\s+gigawatts?\b", r"\1GW"),
+ (r"(?i)\b(\d+)\s+kilowatts?\b", r"\1kW"),
+ (r"(?i)\b(\d+)\s+million\b", r"\1M"),
+ (r"(?i)\b(\d+)\s+billion\b", r"\1B"),
+ (r"(?i)\b(\d+)\s+trillion\b", r"\1T"),
+ ]
+ for pattern, repl in replacements:
+ text = re.sub(pattern, repl, text)
+
+ if len(text) > max_chars:
+ for sep in _THUMBNAIL_SPLIT_SEPARATORS[1:]:
+ if sep in text:
+ head = text.split(sep, 1)[0].strip()
+ if len(head.split()) >= 3:
+ text = head
+ break
+
+ words = text.split()
+ while len(words) > 4 and words and words[0].lower().strip(".,!?") in _THUMBNAIL_LEADING_FILLERS:
+ words = words[1:]
+
+ if len(words) > max_words:
+ words = words[:max_words]
+
+ compact = " ".join(words).strip(" -—–,:;.!?")
+ if len(compact) > max_chars:
+ compact = compact[:max_chars].rsplit(" ", 1)[0].strip(" -—–,:;.!?")
+ return compact
+
+
+def _split_thumbnail_title(title: str, max_line_chars: int = 24) -> tuple[str, str]:
+ """Split compact thumbnail copy into two punchy lines."""
+ text = _compact_thumbnail_title(title)
+ if not text:
+ return "", ""
+
+ for sep in _THUMBNAIL_SPLIT_SEPARATORS:
+ if sep in text:
+ left, right = [part.strip() for part in text.split(sep, 1)]
+ if left and right:
+ return left, right
+
+ words = text.split()
+ if len(words) <= 2:
+ return words[0] if words else "", " ".join(words[1:])
+
+ best_split = 1
+ best_score = float("inf")
+ for i in range(1, len(words)):
+ left_words = words[:i]
+ right_words = words[i:]
+ left = " ".join(left_words)
+ right = " ".join(right_words)
+ score = 0.0
+
+ score += max(0, len(left) - max_line_chars) * 5
+ score += max(0, len(right) - max_line_chars) * 5
+ score += abs(len(left_words) - 3) * 2.5
+ score += abs(len(right_words) - 4) * 2.0
+ score += abs(len(left) - len(right)) * 0.35
+ if left_words[-1].lower().strip(".,!?") in _THUMBNAIL_LINE_STOPWORDS:
+ score += 8
+ if right_words[0].lower().strip(".,!?") in _THUMBNAIL_LINE_STOPWORDS:
+ score += 8
+
+ if score < best_score:
+ best_score = score
+ best_split = i
+
+ return " ".join(words[:best_split]), " ".join(words[best_split:])
+
+
+def _prepare_thumbnail_lines(
+ title: str,
+ line1: Optional[str] = None,
+ line2: Optional[str] = None,
+ max_line_chars: int = 24,
+) -> tuple[str, str]:
+ """Normalize AI or fallback title text into compact thumbnail lines."""
+ raw_line1 = re.sub(r"\s+", " ", (line1 or "")).strip().strip("/").strip()
+ raw_line2 = re.sub(r"\s+", " ", (line2 or "")).strip().strip("/").strip()
+
+ if raw_line1 and raw_line2:
+ total_words = len((raw_line1 + " " + raw_line2).split())
+ if (
+ len(raw_line1) <= max_line_chars
+ and len(raw_line2) <= max_line_chars
+ and total_words <= 8
+ ):
+ return raw_line1, raw_line2
+ title = f"{raw_line1} / {raw_line2}"
+ elif raw_line1:
+ title = raw_line1
+
+ return _split_thumbnail_title(title, max_line_chars=max_line_chars)
+
+
def _build_html(
line1: str,
line2: str,
@@ -402,7 +605,7 @@ def generate_thumbnail(
variation: int = 0,
face_info: Optional[dict] = None,
) -> str:
- """Generate a single thumbnail via HTML + Playwright screenshot."""
+ """Generate a single thumbnail via HTML + headless browser screenshot."""
cfg = _load_config()
if config:
cfg.update(config)
@@ -419,22 +622,57 @@ def generate_thumbnail(
try:
w = cfg["width"]
h = cfg["height"]
-
- result = subprocess.run(
- [
- "npx", "playwright", "screenshot",
- "--viewport-size", f"{w}, {h}",
- "--wait-for-timeout", "1500",
- f"file://{tmp_html.name}",
- output_path,
- ],
- capture_output=True, text=True, timeout=60,
+ wait_ms = int(cfg.get("playwright_wait_ms", 1500))
+ timeout_s = max(30, int(cfg.get("playwright_timeout_ms", 150000)) // 1000)
+ errors = []
+ commands: list[list[str]] = []
+
+ remotion_cmd = _build_remotion_screenshot_command(
+ script_path=_remotion_screenshot_script_path(),
+ html_path=tmp_html.name,
+ output_path=output_path,
+ width=w,
+ height=h,
+ wait_ms=wait_ms,
)
-
- if result.returncode != 0:
- raise RuntimeError(f"Screenshot failed: {result.stderr[:300]}")
-
- return output_path
+ if remotion_cmd:
+ commands.append(remotion_cmd)
+
+ for cli_cmd in _playwright_cli_candidates():
+ commands.append(
+ _build_playwright_screenshot_command(
+ cli_cmd=cli_cmd,
+ html_path=tmp_html.name,
+ output_path=output_path,
+ width=w,
+ height=h,
+ wait_ms=wait_ms,
+ )
+ )
+
+ if not commands:
+ raise RuntimeError("No browser screenshot command available")
+
+ for cmd in commands:
+ cmd_label = " ".join(cmd[:3]) if len(cmd) > 3 else " ".join(cmd)
+ try:
+ result = subprocess.run(
+ cmd,
+ capture_output=True,
+ text=True,
+ timeout=timeout_s,
+ )
+ except subprocess.TimeoutExpired:
+ errors.append(f"{cmd_label} timed out after {timeout_s} seconds")
+ continue
+
+ if result.returncode == 0:
+ return output_path
+
+ detail = (result.stderr or result.stdout or "unknown error").strip()[:300]
+ errors.append(f"{cmd_label} failed: {detail}")
+
+ raise RuntimeError("Screenshot failed: " + " | ".join(errors))
finally:
os.unlink(tmp_html.name)
@@ -468,28 +706,7 @@ def generate_variations(
except Exception:
pass
- # Title split — if title contains " / " use that as the split point
- # (PodStack /plan-thumbnails outputs "LINE 1 / LINE 2" format)
- if " / " in title:
- parts = title.split(" / ", 1)
- line1 = parts[0].strip()
- line2 = parts[1].strip()
- else:
- # Auto-split: balance character count across two lines
- words = title.split()
- if len(words) <= 2:
- line1 = words[0] if words else ""
- line2 = " ".join(words[1:])
- else:
- best_split = len(words) // 2
- best_diff = float("inf")
- for i in range(1, len(words)):
- diff = abs(len(" ".join(words[:i])) - len(" ".join(words[i:])))
- if diff < best_diff:
- best_diff = diff
- best_split = i
- line1 = " ".join(words[:best_split])
- line2 = " ".join(words[best_split:])
+ line1, line2 = _prepare_thumbnail_lines(title)
paths = []
n = cfg.get("variations", 3)
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index 45813cd..ae3451b 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -202,10 +202,60 @@ def crop_to_vertical(
source_ratio = width / height
if strategy in ("face", "speaker"):
+ speakers_in_clip = {
+ w.get("speaker") for w in (transcript_words or []) if w.get("speaker")
+ }
+ # Any clip with speaker labels should prefer clip-local tracking over the
+ # episode-wide face_map. Global speaker→side mappings are too coarse for
+ # monologues and mixed-layout edits; they can pin a single-speaker clip
+ # to the wrong person for the whole render.
+ if speakers_in_clip:
+ result = _track_and_crop(
+ input_path, output_path,
+ width, height, target_w, target_h,
+ transcript_words, clip_start,
+ face_map=face_map,
+ )
+ if result:
+ return result
+
+ if face_map:
+ crop_h = height
+ crop_y = max(0, (height - crop_h) // 2)
+ x_expr = _use_face_map(
+ face_map=face_map,
+ transcript_words=transcript_words,
+ clip_start=clip_start,
+ width=width,
+ height=height,
+ target_ratio=target_ratio,
+ crop_h=crop_h,
+ )
+ if x_expr:
+ crop_w = int(crop_h * target_ratio)
+ crop_w = min(crop_w, width)
+ vf = f"crop={crop_w}:{crop_h}:{x_expr}:{crop_y},scale={target_w}:{target_h}"
+ return _run_ffmpeg_with_fallback(
+ cmd_parts_before_enc=[
+ "ffmpeg", "-y",
+ "-i", input_path,
+ "-vf", vf,
+ ],
+ cmd_parts_after_enc=[
+ "-c:a", "aac",
+ "-b:a", "192k",
+ "-ar", "44100",
+ "-movflags", "+faststart",
+ ],
+ output_path=output_path,
+ label="crop_face_map",
+ )
+
result = _track_and_crop(
input_path, output_path,
width, height, target_w, target_h,
transcript_words, clip_start,
+ face_map=face_map,
)
if result:
return result
@@ -331,26 +381,50 @@ def _build_cam_expr(keyframes: list, duration: float, is_split: bool, max_parts:
if len(keyframes) == 1:
return str(keyframes[0][1])
+ def _eased_move_expr(start_t: float, move_t: float, start_x: int, end_x: int) -> str:
+ # Smootherstep gives a softer start/landing than smoothstep while still
+ # finishing inside the bounded reframe window.
+ progress = f"((t-{start_t:.3f})/{move_t:.3f})"
+ eased = (
+ f"((6*({progress})*({progress})*({progress})*({progress})*({progress}))"
+ f"-(15*({progress})*({progress})*({progress})*({progress}))"
+ f"+(10*({progress})*({progress})*({progress})))"
+ )
+ return f"{start_x}+(({end_x}-{start_x})*{eased})"
+
parts = []
for i in range(len(keyframes) - 1):
t0, x0 = keyframes[i]
t1, x1 = keyframes[i + 1]
dt = max(0.01, t1 - t0)
jump = abs(x1 - x0)
+ blurred_cut_jump = 90 if is_split else 180
+ quick_reframe_jump = 40 if is_split else 90
if jump < 2:
# Negligible movement — hold
parts.append(f"if(between(t\\,{t0:.3f}\\,{t1:.3f})\\,{x0}\\,")
- elif jump > 150 or (is_split and jump > 50):
- # Large jump / split-screen speaker switch: instant cut
- cut_t = round(t1 - 0.01, 3)
- parts.append(f"if(between(t\\,{t0:.3f}\\,{cut_t:.3f})\\,{x0}\\,")
- parts.append(f"if(between(t\\,{cut_t:.3f}\\,{t1:.3f})\\,{x1}\\,")
+ elif jump >= blurred_cut_jump:
+ # Very large jumps look worse as animated crop slides than as a
+ # short blurred cut. The blur/zoom filters carry the transition.
+ parts.append(f"if(between(t\\,{t0:.3f}\\,{t1:.3f})\\,{x1}\\,")
+ elif jump >= quick_reframe_jump:
+ # Moderate jump: use a short bounded reframe, but keep it much
+ # tighter than a literal camera pan.
+ pan_t = min(0.14 if is_split else 0.18, dt)
+ pan_end_t = round(t0 + pan_t, 3)
+ parts.append(
+ f"if(between(t\\,{t0:.3f}\\,{pan_end_t:.3f})\\,"
+ f"{_eased_move_expr(t0, pan_t, x0, x1)}\\,"
+ )
+ if pan_end_t < t1:
+ parts.append(f"if(between(t\\,{pan_end_t:.3f}\\,{t1:.3f})\\,{x1}\\,")
else:
- # Smooth linear interpolation (camera pan)
+ # Small pans still benefit from eased starts/stops so they read like
+ # a camera operator, not a value sliding on rails.
parts.append(
f"if(between(t\\,{t0:.3f}\\,{t1:.3f})\\,"
- f"{x0}+(({x1}-{x0})*(t-{t0:.3f})/{dt:.3f})\\,"
+ f"{_eased_move_expr(t0, dt, x0, x1)}\\,"
)
if len(parts) > max_parts:
@@ -359,6 +433,120 @@ def _build_cam_expr(keyframes: list, duration: float, is_split: bool, max_parts:
return "".join(parts) + str(keyframes[-1][1]) + ")" * len(parts)
+def _motion_windows_from_keyframes(
+ keyframes: list,
+ min_jump: int = 60,
+ max_window_duration: float = 0.5,
+ max_windows: int = 16,
+) -> list[tuple[float, float]]:
+ """Find short reframe windows worth accenting."""
+ windows = []
+ for i in range(len(keyframes) - 1):
+ t0, x0 = keyframes[i]
+ t1, x1 = keyframes[i + 1]
+ if abs(x1 - x0) < min_jump:
+ continue
+ dt = t1 - t0
+ if dt <= 0.01 or dt > max_window_duration:
+ continue
+ windows.append((t0, t1))
+
+ if not windows or len(windows) > max_windows:
+ return []
+ return windows
+
+
+def _expand_motion_windows(
+ windows: list[tuple[float, float]],
+ pad_before: float = 0.03,
+ pad_after: float = 0.05,
+) -> list[tuple[float, float]]:
+ """Pad and merge motion windows so the blur eases in and out."""
+ if not windows:
+ return []
+
+ expanded: list[tuple[float, float]] = []
+ for t0, t1 in windows:
+ start = max(0.0, t0 - pad_before)
+ end = t1 + pad_after
+ if expanded and start <= expanded[-1][1] + 0.01:
+ expanded[-1] = (expanded[-1][0], max(expanded[-1][1], end))
+ else:
+ expanded.append((start, end))
+ return expanded
+
+
+def _build_motion_blur_filter(
+ keyframes: list,
+ min_jump: int = 60,
+ max_window_duration: float = 0.5,
+ max_windows: int = 16,
+ sigma: float = 5.4,
+ steps: int = 2,
+ pad_before: float = 0.03,
+ pad_after: float = 0.05,
+) -> str:
+ """
+ Add a stronger full-frame blur only while the camera is actively reframing.
+
+ The goal is to hide the crop move, not just soften it slightly.
+ """
+ windows = _expand_motion_windows(_motion_windows_from_keyframes(
+ keyframes=keyframes,
+ min_jump=min_jump,
+ max_window_duration=max_window_duration,
+ max_windows=max_windows,
+ ), pad_before=pad_before, pad_after=pad_after)
+ if not windows:
+ return ""
+
+ enable_expr = "+".join(
+ f"between(t\\,{t0:.3f}\\,{t1:.3f})"
+ for t0, t1 in windows
+ )
+ return f",gblur=sigma={sigma:.1f}:steps={steps}:enable='{enable_expr}'"
+
+
+def _build_motion_zoom_filter(
+ keyframes: list,
+ target_w: int,
+ target_h: int,
+ min_jump: int = 60,
+ max_window_duration: float = 0.5,
+ max_windows: int = 16,
+ max_zoom: float = 0.018,
+) -> str:
+ """
+ Add a tiny center zoom bump during short reframes.
+
+ The goal is not visible punch-in. It just helps the move read as an
+ intentional editorial transition instead of a hard crop shove.
+ """
+ windows = _motion_windows_from_keyframes(
+ keyframes=keyframes,
+ min_jump=min_jump,
+ max_window_duration=max_window_duration,
+ max_windows=max_windows,
+ )
+ if not windows:
+ return ""
+
+ bumps = []
+ for t0, t1 in windows:
+ dt = max(0.01, t1 - t0)
+ progress = f"((t-{t0:.3f})/{dt:.3f})"
+ bumps.append(
+ f"(16*pow({progress}\\,2)*pow((1-{progress})\\,2)*between(t\\,{t0:.3f}\\,{t1:.3f}))"
+ )
+
+ zoom_expr = "+".join(bumps)
+ return (
+ f",scale=w='iw*(1+{max_zoom:.4f}*({zoom_expr}))'"
+ f":h='ih*(1+{max_zoom:.4f}*({zoom_expr}))'"
+ f":eval=frame,crop={target_w}:{target_h}:(iw-{target_w})/2:(ih-{target_h})/2"
+ )
+
+
def _simplify_keyframes(keyframes: list, tolerance: int = 5) -> list:
"""Remove intermediate keyframes that lie on a line between neighbours."""
if len(keyframes) <= 2:
@@ -378,6 +566,519 @@ def _simplify_keyframes(keyframes: list, tolerance: int = 5) -> list:
return result
+def _resolve_speaker_sides(
+ segments: list,
+ detections: list,
+ width: int,
+ face_map: dict = None,
+) -> dict:
+ """
+ Build left/right speaker hints for mixed split-screen clips.
+ Prefer precomputed face_map mappings, then fall back to first-speaker order.
+ """
+ speaker_side = {}
+ mid_x = width // 2
+
+ if face_map:
+ clusters = face_map.get("clusters", [])
+ mappings = face_map.get("speaker_mappings", {})
+ for speaker, cluster_index in mappings.items():
+ if cluster_index is None or cluster_index >= len(clusters):
+ continue
+ speaker_side[speaker] = "left" if clusters[cluster_index]["center_x"] < mid_x else "right"
+
+ split_count = sum(1 for _, faces in detections if len(faces) >= 2)
+ has_any_split = split_count >= 3
+ speakers = sorted(set(seg[2] for seg in segments)) if segments else []
+
+ if has_any_split and len(speakers) >= 2 and len(speaker_side) < 2:
+ talk = {}
+ first = {}
+ for start_t, end_t, speaker in segments:
+ talk[speaker] = talk.get(speaker, 0) + (end_t - start_t)
+ if speaker not in first:
+ first[speaker] = start_t
+ top2 = sorted(speakers, key=lambda s: talk.get(s, 0), reverse=True)[:2]
+ by_first = sorted(top2, key=lambda s: first.get(s, float("inf")))
+ speaker_side[by_first[0]] = "left"
+ if len(by_first) > 1:
+ speaker_side[by_first[1]] = "right"
+
+ return speaker_side
+
+
+def _assign_face_tracks(
+ detections: list,
+ width: int,
+ max_gap: float = 1.2,
+) -> list:
+ """
+ Give detected faces stable local track ids.
+
+ This mirrors the useful part of OpenShorts: a lightweight identity pass
+ based on horizontal continuity. We only need clip-local stickiness, not
+ perfect biometric identity.
+ """
+ known_tracks = []
+ next_track_id = 0
+ tracked_detections = []
+
+ for t, faces in detections:
+ known_tracks = [track for track in known_tracks if (t - track["last_t"]) <= max_gap]
+ used_track_ids = set()
+ tracked_faces = []
+
+ for face in sorted(faces, key=lambda f: f["fw"], reverse=True):
+ best_track = None
+ best_dist = None
+ match_radius = max(width * 0.10, face["fw"] * 1.6)
+
+ for track in known_tracks:
+ if track["id"] in used_track_ids:
+ continue
+ dx = abs(face["cx"] - track["cx"])
+ dy = abs(face.get("cy", 0) - track["cy"])
+ size_ratio = abs(face["fw"] - track["fw"]) / max(face["fw"], track["fw"], 1.0)
+ allowed_dist = max(match_radius, track["fw"] * 1.6)
+ if dx > allowed_dist:
+ continue
+ score = dx + dy * 0.35 + size_ratio * width * 0.4
+ if best_dist is None or score < best_dist:
+ best_track = track
+ best_dist = score
+
+ if best_track is None:
+ track_id = next_track_id
+ next_track_id += 1
+ known_tracks.append({
+ "id": track_id,
+ "cx": float(face["cx"]),
+ "cy": float(face.get("cy", 0)),
+ "fw": float(face["fw"]),
+ "last_t": t,
+ })
+ else:
+ track_id = best_track["id"]
+ best_track["cx"] = float(face["cx"])
+ best_track["cy"] = float(face.get("cy", 0))
+ best_track["fw"] = float(face["fw"])
+ best_track["last_t"] = t
+
+ used_track_ids.add(track_id)
+ tracked_face = dict(face)
+ tracked_face["track_id"] = track_id
+ tracked_faces.append(tracked_face)
+
+ tracked_faces.sort(key=lambda f: f["cx"])
+ tracked_detections.append((t, tracked_faces))
+
+ return tracked_detections
+
+
+def _choose_segment_tracks(
+ segments: list,
+ tracked_detections: list,
+ speaker_side: dict,
+ speaker_anchor_x: dict,
+ width: int,
+) -> tuple[list, dict, dict]:
+ """
+ Choose one persistent visual track for each merged speaker segment.
+
+ The selected track is later followed for the whole turn. Other visible
+ faces in the same turn are ignored, which prevents reaction shots from
+ stealing the camera.
+ """
+ from statistics import median
+
+ segment_tracks = []
+ learned_anchor_x = dict(speaker_anchor_x or {})
+ learned_side = dict(speaker_side or {})
+ last_track_for_speaker = {}
+
+ for start_t, end_t, speaker in segments:
+ candidates = {}
+ for t, faces in tracked_detections:
+ if t < start_t or t > end_t:
+ continue
+ is_split_frame = len(faces) >= 2
+ for face in faces:
+ data = candidates.setdefault(face["track_id"], {
+ "frames": 0,
+ "split_frames": 0,
+ "solo_frames": 0,
+ "cxs": [],
+ "fws": [],
+ "first_t": t,
+ })
+ data["frames"] += 1
+ data["split_frames"] += 1 if is_split_frame else 0
+ data["solo_frames"] += 0 if is_split_frame else 1
+ data["cxs"].append(face["cx"])
+ data["fws"].append(face["fw"])
+ data["first_t"] = min(data["first_t"], t)
+
+ if not candidates:
+ segment_tracks.append((start_t, end_t, speaker, None, None))
+ continue
+
+ prev_track_id = last_track_for_speaker.get(speaker)
+ hint_anchor = learned_anchor_x.get(speaker)
+ hint_side = learned_side.get(speaker)
+
+ def _local_score(item):
+ track_id, data = item
+ median_x = float(median(data["cxs"]))
+ median_fw = float(median(data["fws"]))
+ score = (
+ data["frames"] * 1.0
+ + data["split_frames"] * 2.4
+ + data["solo_frames"] * 0.35
+ + median_fw / 140.0
+ )
+
+ if prev_track_id == track_id:
+ score += 2.5
+
+ score += max(0.0, 1.2 - max(0.0, data["first_t"] - start_t))
+ return score
+
+ def _score(item):
+ track_id, data = item
+ score = _local_score(item)
+ median_x = float(median(data["cxs"]))
+
+ if hint_side:
+ on_hint_side = median_x < (width / 2) if hint_side == "left" else median_x >= (width / 2)
+ score += 1.1 if on_hint_side else -0.8
+
+ if hint_anchor is not None:
+ score -= min(1.8, abs(median_x - hint_anchor) / max(width * 0.30, 1))
+ return score
+
+ local_best_track_id, local_best_data = max(candidates.items(), key=_local_score)
+ chosen_track_id, chosen_data = max(candidates.items(), key=_score)
+
+ local_best_score = _local_score((local_best_track_id, local_best_data))
+ hinted_score = _score((chosen_track_id, chosen_data))
+ local_scored_with_hints = _score((local_best_track_id, local_best_data))
+ if (
+ local_best_track_id != chosen_track_id
+ and local_best_score >= _local_score((chosen_track_id, chosen_data)) + 1.5
+ and local_scored_with_hints >= hinted_score - 0.8
+ ):
+ chosen_track_id, chosen_data = local_best_track_id, local_best_data
+
+ chosen_x = float(median(chosen_data["cxs"]))
+
+ segment_tracks.append((start_t, end_t, speaker, chosen_track_id, chosen_x))
+ last_track_for_speaker[speaker] = chosen_track_id
+
+ prev_anchor = learned_anchor_x.get(speaker)
+ if prev_anchor is None or abs(chosen_x - prev_anchor) > width * 0.18:
+ learned_anchor_x[speaker] = chosen_x
+ else:
+ learned_anchor_x[speaker] = prev_anchor * 0.65 + chosen_x * 0.35
+ learned_side[speaker] = "left" if learned_anchor_x[speaker] < (width / 2) else "right"
+
+ return segment_tracks, learned_anchor_x, learned_side
+
+
+def _update_tripod_camera(
+ current_center_x: float,
+ target_center_x: float | None,
+ crop_w: int,
+ video_width: int,
+ dt: float,
+ force_snap: bool = False,
+) -> float:
+ """
+ Heavy-tripod camera movement inspired by OpenShorts.
+
+ The camera stays still while the subject remains inside a safe zone, then
+ moves at a bounded speed instead of constantly chasing every face twitch.
+ """
+ half_crop = crop_w / 2.0
+ min_center = half_crop
+ max_center = max(half_crop, video_width - half_crop)
+
+ if target_center_x is None:
+ return min(max(current_center_x, min_center), max_center)
+
+ if force_snap:
+ current_center_x = target_center_x
+ else:
+ diff = target_center_x - current_center_x
+ # Keep the hold zone fairly tight so the speaker stays composed near
+ # center, but still ignore ordinary head movement and detector noise.
+ safe_zone_radius = crop_w * 0.14
+ if abs(diff) > safe_zone_radius:
+ slow_speed = 72.0
+ fast_speed = 360.0
+ speed = fast_speed if abs(diff) > crop_w * 0.5 else slow_speed
+ step = min(abs(diff), speed * max(dt, 0.01))
+ current_center_x += step if diff > 0 else -step
+
+ return min(max(current_center_x, min_center), max_center)
+
+
+def _choose_track_segment_targets(
+ segment_tracks: list,
+ tracked_detections: list,
+ speaker_anchor_x: dict,
+ width: int,
+ crop_w: int,
+) -> list[tuple[float, float, int, str | None]]:
+ """
+ Choose one horizontal crop target per speaker turn.
+
+ This keeps the speaker-tracked path from "rolling" across the frame.
+ We pick a stable representative position for the chosen visual track, then
+ let FFmpeg do one short reframe at the segment boundary.
+ """
+ from statistics import median
+
+ max_crop_x = max(0, width - crop_w)
+
+ def _crop_x_for_center(center_x: float) -> int:
+ crop_x = int(round(center_x - crop_w / 2.0))
+ return max(0, min(crop_x, max_crop_x))
+
+ def _pick_representative_center(points: list[tuple[float, float]]) -> float:
+ first_x = points[0][1]
+ median_x = float(median([cx for _, cx in points]))
+ seed_x = first_x * 0.72 + median_x * 0.28
+ return min(points, key=lambda p: abs(p[1] - seed_x))[1]
+
+ def _split_points_into_stable_runs(
+ points: list[tuple[float, float]],
+ end_t: float,
+ ) -> list[tuple[float, float, list[tuple[float, float]]]]:
+ if not points:
+ return []
+
+ jump_threshold = max(120.0, crop_w * 0.22)
+ gap_threshold = 0.55
+ runs = []
+
+ run_start_t = points[0][0]
+ run_points = [points[0]]
+ candidate_points: list[tuple[float, float]] = []
+
+ def _run_center(ps: list[tuple[float, float]]) -> float:
+ return float(median([cx for _, cx in ps]))
+
+ for point in points[1:]:
+ gap = point[0] - run_points[-1][0]
+ stable_center = _run_center(run_points)
+ far_from_run = abs(point[1] - stable_center) > jump_threshold
+ gap_reposition = gap > gap_threshold and abs(point[1] - stable_center) > jump_threshold * 0.45
+ should_probe_new_run = far_from_run or gap_reposition
+
+ if not should_probe_new_run:
+ if candidate_points:
+ run_points.extend(candidate_points)
+ candidate_points = []
+ run_points.append(point)
+ continue
+
+ candidate_points.append(point)
+
+ candidate_center = _run_center(candidate_points)
+ candidate_far = abs(candidate_center - stable_center) > jump_threshold * 0.85
+ enough_evidence = len(candidate_points) >= 2 or gap > gap_threshold
+ if candidate_far and enough_evidence:
+ split_t = candidate_points[0][0]
+ runs.append((run_start_t, split_t, run_points))
+ run_start_t = split_t
+ run_points = candidate_points[:]
+ candidate_points = []
+
+ if candidate_points:
+ candidate_center = _run_center(candidate_points)
+ stable_center = _run_center(run_points)
+ if abs(candidate_center - stable_center) > jump_threshold and len(candidate_points) >= 2:
+ split_t = candidate_points[0][0]
+ runs.append((run_start_t, split_t, run_points))
+ run_start_t = split_t
+ run_points = candidate_points[:]
+ else:
+ run_points.extend(candidate_points)
+
+ runs.append((run_start_t, end_t, run_points))
+ return runs
+
+ segment_targets = []
+ for start_t, end_t, speaker, track_id, _ in segment_tracks:
+ points = []
+ for t, faces in tracked_detections:
+ if t < start_t or t > end_t:
+ continue
+ face = next((f for f in faces if f["track_id"] == track_id), None)
+ if face is not None:
+ points.append((t, float(face["cx"])))
+
+ if points:
+ for idx, (sub_start_t, sub_end_t, run_points) in enumerate(_split_points_into_stable_runs(points, end_t)):
+ if idx == 0:
+ sub_start_t = start_t
+ center_x = _pick_representative_center(run_points)
+ segment_targets.append((sub_start_t, sub_end_t, _crop_x_for_center(center_x), speaker))
+ elif speaker is not None and speaker in speaker_anchor_x:
+ center_x = float(speaker_anchor_x[speaker])
+ segment_targets.append((start_t, end_t, _crop_x_for_center(center_x), speaker))
+ else:
+ continue
+
+ return segment_targets
+
+
+def _build_track_turn_keyframes(
+ segment_targets: list,
+ default_x: int,
+) -> list[tuple[float, int]]:
+ """
+ Build short bounded reframes between stable speaker-turn targets.
+
+ The camera should hold through the turn, then move quickly onto the new
+ speaker instead of slowly chasing the face for seconds.
+ """
+ keyframes = [(0.0, default_x)]
+ prev_x = default_x
+ prev_speaker = None
+ left_edge_margin = 90
+
+ for target in segment_targets:
+ if len(target) >= 4:
+ start_t, end_t, target_x, speaker = target[:4]
+ else:
+ start_t, end_t, target_x = target[:3]
+ speaker = None
+
+ if abs(target_x - prev_x) <= 6:
+ prev_speaker = speaker
+ continue
+
+ seg_duration = max(0.01, end_t - start_t)
+ jump = abs(target_x - prev_x)
+ same_speaker_continuation = prev_speaker is not None and speaker == prev_speaker
+ if same_speaker_continuation and prev_x <= left_edge_margin and jump > 180:
+ hold_t = max(0.0, round(start_t - 0.01, 3))
+ if hold_t > keyframes[-1][0]:
+ keyframes.append((hold_t, prev_x))
+ keyframes.append((round(start_t, 3), target_x))
+ prev_x = target_x
+ prev_speaker = speaker
+ continue
+
+ if same_speaker_continuation:
+ if jump > 180:
+ reframe_t = min(0.18, max(0.10, seg_duration * 0.15))
+ else:
+ reframe_t = min(0.14, max(0.08, seg_duration * 0.12))
+ elif jump > 180:
+ reframe_t = min(0.30, max(0.16, seg_duration * 0.24))
+ else:
+ reframe_t = min(0.22, max(0.12, seg_duration * 0.18))
+
+ settle_t = min(end_t, round(start_t + reframe_t, 3))
+ keyframes.append((start_t, prev_x))
+ keyframes.append((settle_t, target_x))
+ prev_x = target_x
+ prev_speaker = speaker
+
+ return keyframes
+
+
+def _pick_tracking_face(
+ faces: list,
+ speaker: str,
+ speaker_side: dict,
+ width: int,
+ last_target_x: float | None,
+ last_speaker: str | None,
+ speaker_anchor_x: dict,
+ has_any_split: bool,
+) -> tuple[dict | None, bool]:
+ """
+ Pick a tracking target for the active speaker.
+
+ Returns (face_dict_or_none, reliable_selection).
+ reliable_selection means the face is trustworthy enough to update a
+ speaker anchor, not merely good enough to keep the camera steady.
+ """
+ if not faces:
+ return None, False
+
+ mid_x = width // 2
+ side_hint = speaker_side.get(speaker)
+ anchor_x = speaker_anchor_x.get(speaker)
+
+ if len(faces) >= 2:
+ if side_hint == "left":
+ return min(faces, key=lambda f: f["cx"]), True
+ if side_hint == "right":
+ return max(faces, key=lambda f: f["cx"]), True
+ if last_target_x is not None:
+ return min(faces, key=lambda f: abs(f["cx"] - last_target_x)), False
+ return max(faces, key=lambda f: f["fw"]), False
+
+ face = faces[0]
+ if not has_any_split or side_hint is None:
+ return face, True
+
+ side_ok = face["cx"] < mid_x if side_hint == "left" else face["cx"] >= mid_x
+ near_anchor = anchor_x is not None and abs(face["cx"] - anchor_x) < width * 0.18
+
+ # On a fresh speaker turn, accept the solo face so we can move with the cut.
+ if speaker != last_speaker:
+ return face, side_ok or near_anchor
+
+ # Same speaker continuing: reject large contradictory jumps that usually
+ # come from reaction shots or layout inserts of the other person.
+ if last_target_x is not None and abs(face["cx"] - last_target_x) > width * 0.22 and not (side_ok or near_anchor):
+ return None, False
+
+ return face, side_ok or near_anchor
+
+
+def _choose_camera_speaker(
+ transcript_speaker: str | None,
+ transcript_duration: float,
+ active_speaker: str | None,
+ pending_speaker: str | None,
+ pending_count: int,
+ min_turn_duration: float = 2.6,
+ confirmation_frames: int = 3,
+) -> tuple[str | None, str | None, int, bool]:
+ """
+ Stabilize diarization before the camera switches.
+
+ Returns (camera_speaker, pending_speaker, pending_count, switched_now).
+ """
+ if transcript_speaker is None:
+ return active_speaker, pending_speaker, pending_count, False
+
+ if active_speaker is None:
+ return transcript_speaker, None, 0, True
+
+ if transcript_speaker == active_speaker:
+ return active_speaker, None, 0, False
+
+ # Brief interjections should not move the camera.
+ if transcript_duration < min_turn_duration:
+ return active_speaker, None, 0, False
+
+ if pending_speaker != transcript_speaker:
+ return active_speaker, transcript_speaker, 1, False
+
+ pending_count += 1
+ if pending_count < confirmation_frames:
+ return active_speaker, pending_speaker, pending_count, False
+
+ return transcript_speaker, None, 0, True
+
+
def _track_and_crop(
input_path: str,
output_path: str,
@@ -387,16 +1088,17 @@ def _track_and_crop(
target_h: int,
transcript_words: list = None,
clip_start: float = 0,
+ face_map: dict = None,
) -> Optional[str]:
"""
- Adaptive face tracking with exponential-smoothing camera.
+ Adaptive face tracking with sticky visual tracks and heavy-tripod movement.
Works for both split-screen and single-camera layouts:
1. Dense face sampling (~10 fps) with YuNet
- 2. Transcript-driven speaker selection (who's talking → which face)
- 3. Exponential-smoothing camera with dead zone (no jitter)
- 4. Hard cuts only for split-screen speaker switches
- 5. Vertical position from median face-Y (static per clip)
+ 2. Clip-local face tracks (OpenShorts-style identity stickiness)
+ 3. One chosen visual track per merged speaker turn
+ 4. Heavy-tripod camera movement with a safe zone
+ 5. Vertical position fixed for full-height crop
"""
try:
import cv2
@@ -492,134 +1194,133 @@ def _track_and_crop(
merged.append(list(seg))
segments = merged
- speakers = sorted(set(seg[2] for seg in segments)) if segments else []
+ tracked_detections = _assign_face_tracks(detections, width)
- def _speaker_at(t):
- for s, e, sp in segments:
- if s <= t <= e:
- return sp
- return segments[0][2] if segments else None
-
- # ── Speaker-to-side mapping ─────────────────────────────────
- # Built from ANY frames with 2+ faces — works for both
- # pure split-screen and Riverside-style mixed layouts.
- speaker_side = {} # speaker → "left" | "right"
- if has_any_split and len(speakers) >= 2:
- talk = {}
- first = {}
- for seg in segments:
- sp = seg[2]
- talk[sp] = talk.get(sp, 0) + (seg[1] - seg[0])
- if sp not in first:
- first[sp] = seg[0]
- top2 = sorted(speakers, key=lambda s: talk.get(s, 0), reverse=True)[:2]
- by_first = sorted(top2, key=lambda s: first.get(s, float("inf")))
- speaker_side[by_first[0]] = "left"
- if len(by_first) > 1:
- speaker_side[by_first[1]] = "right"
- for sp in speakers:
- if sp not in speaker_side:
- speaker_side[sp] = "left"
-
- # ── Face selection per frame ─────────────────────────────────
- # Per-frame layout detection: if THIS frame has 2+ faces,
- # pick the one on the active speaker's side. Handles Riverside
- # videos that switch between single-camera and split-screen.
- def _pick_face(faces, speaker):
- if not faces:
- return None
- if len(faces) >= 2 and speaker in speaker_side:
- side = speaker_side[speaker]
- return (min if side == "left" else max)(faces, key=lambda f: f["cx"])
- # Single face or no side mapping: use largest (most prominent)
- return max(faces, key=lambda f: f["fw"])
-
- # ── Exponential-smoothing camera ────────────────────────────
- # The camera "chases" the target face using exponential decay,
- # which gives natural ease-out motion (fast start, gradual
- # settling). Unlike fixed-speed approaches this is frame-rate
- # independent and feels organic.
- #
- # dead_zone: ignore face drift within this radius of the
- # current camera center — prevents micro-jitter
- # smooth_rate: convergence speed; at rate 4 the camera
- # reaches ~95 % of the target in ≈0.75 s
- # snap_distance: teleport instantly above this (speaker change
- # or layout switch — no sweeping across the seam)
- import math
-
- dead_zone = crop_w * 0.20
- smooth_rate = 4.0
- snap_distance = crop_w * 0.55
-
- # Initialise camera on the first detected face
- cam_x = float(width) / 2
- for _, faces in detections:
- if faces:
- cam_x = float(max(faces, key=lambda f: f["fw"])["cx"])
- break
-
- keyframes_x = []
- prev_t = 0.0
- prev_speaker = None
- last_snap_t = -10.0 # time of last force_snap (start with no cooldown)
- snap_cooldown = 2.0 # minimum seconds between force_snaps
-
- for t, faces in detections:
- speaker = _speaker_at(t) if segments else None
- face = _pick_face(faces, speaker)
-
- if face is None:
- prev_t = t
- continue
-
- target_x = float(face["cx"])
- diff = target_x - cam_x
- dt = max(0.01, t - prev_t)
-
- # Hard cut when the active speaker switches to a different
- # side. Works for pure split-screen AND mixed layouts
- # (Riverside-style single↔split transitions).
- # Cooldown prevents rapid ping-pong from brief interjections
- # that survive segment merging.
- force_snap = (
- speaker is not None
- and speaker != prev_speaker
- and prev_speaker is not None
- and speaker_side.get(speaker) != speaker_side.get(prev_speaker)
- and (t - last_snap_t) >= snap_cooldown
+ speaker_side = _resolve_speaker_sides(segments, tracked_detections, width, face_map)
+ speaker_anchor_x = {}
+ if face_map:
+ clusters = face_map.get("clusters", [])
+ mappings = face_map.get("speaker_mappings", {})
+ for speaker, cluster_index in mappings.items():
+ if cluster_index is None or cluster_index >= len(clusters):
+ continue
+ speaker_anchor_x[speaker] = float(clusters[cluster_index]["center_x"])
+
+ segment_tracks = []
+ if segments:
+ segment_tracks, speaker_anchor_x, speaker_side = _choose_segment_tracks(
+ segments=segments,
+ tracked_detections=tracked_detections,
+ speaker_side=speaker_side,
+ speaker_anchor_x=speaker_anchor_x,
+ width=width,
)
- if force_snap or abs(diff) > snap_distance:
- # Smooth 2-second pan to the other speaker instead of instant snap
- # Rate 1.5 with dt~0.1 gives alpha~0.14 per step → reaches 95% in ~2s
- pan_alpha = 1.0 - math.exp(-1.5 * dt)
- cam_x += diff * pan_alpha
- if force_snap:
- last_snap_t = t
- elif abs(diff) > dead_zone:
- # Exponential smoothing: move a fraction of remaining
- # distance each step. alpha → 1 means instant, → 0 means
- # no movement. exp(-rate * dt) makes it time-step safe.
- alpha = 1.0 - math.exp(-smooth_rate * dt)
- cam_x += diff * alpha
- # else: face inside dead zone — camera holds position
-
- crop_x = int(cam_x - crop_w / 2)
- crop_x = max(0, min(crop_x, width - crop_w))
-
- if not keyframes_x or crop_x != keyframes_x[-1][1]:
- keyframes_x.append((round(t, 3), crop_x))
-
- prev_t = t
- if speaker is not None:
- prev_speaker = speaker
+ fallback_track_id = None
+ if not segment_tracks:
+ track_counts = {}
+ for _, faces in tracked_detections:
+ for face in faces:
+ track_id = face["track_id"]
+ track_counts[track_id] = track_counts.get(track_id, 0) + 1
+ if track_counts:
+ fallback_track_id = max(track_counts, key=lambda tid: track_counts[tid])
# ── Vertical position ────────────────────────────────────────
# Full-height crop → crop_y is always 0 (or centered if
# crop_h < height due to aspect-ratio clamping).
crop_y = max(0, (height - crop_h) // 2)
+ keyframes_x = []
+ if segment_tracks:
+ segment_targets = _choose_track_segment_targets(
+ segment_tracks=segment_tracks,
+ tracked_detections=tracked_detections,
+ speaker_anchor_x=speaker_anchor_x,
+ width=width,
+ crop_w=crop_w,
+ )
+ if segment_targets:
+ default_x = segment_targets[0][2] if segment_targets[0][0] <= 0.3 else max(0, (width - crop_w) // 2)
+ keyframes_x = _build_track_turn_keyframes(
+ segment_targets=segment_targets,
+ default_x=default_x,
+ )
+
+ if not keyframes_x:
+ cam_x = float(width) / 2
+ first_target_x = None
+ if segment_tracks:
+ first_track_id = segment_tracks[0][3]
+ first_speaker = segment_tracks[0][2]
+ for _, faces in tracked_detections:
+ face = next((f for f in faces if f["track_id"] == first_track_id), None)
+ if face is not None:
+ first_target_x = float(face["cx"])
+ break
+ if first_target_x is None and first_speaker is not None:
+ first_target_x = speaker_anchor_x.get(first_speaker)
+ elif fallback_track_id is not None:
+ for _, faces in tracked_detections:
+ face = next((f for f in faces if f["track_id"] == fallback_track_id), None)
+ if face is not None:
+ first_target_x = float(face["cx"])
+ break
+
+ if first_target_x is not None:
+ cam_x = _update_tripod_camera(
+ current_center_x=cam_x,
+ target_center_x=first_target_x,
+ crop_w=crop_w,
+ video_width=width,
+ dt=0.0,
+ force_snap=True,
+ )
+
+ prev_t = 0.0
+ segment_index = 0
+
+ for t, faces in tracked_detections:
+ while segment_index + 1 < len(segment_tracks) and t > segment_tracks[segment_index][1]:
+ segment_index += 1
+
+ speaker = None
+ target_track_id = fallback_track_id
+ if segment_tracks:
+ _, _, speaker, target_track_id, _ = segment_tracks[segment_index]
+
+ chosen_face = None
+ if target_track_id is not None:
+ chosen_face = next((f for f in faces if f["track_id"] == target_track_id), None)
+
+ if chosen_face is not None:
+ target_x = float(chosen_face["cx"])
+ if speaker is not None:
+ prev_anchor = speaker_anchor_x.get(speaker, target_x)
+ speaker_anchor_x[speaker] = prev_anchor * 0.75 + target_x * 0.25
+ elif speaker is not None and speaker in speaker_anchor_x:
+ target_x = float(speaker_anchor_x[speaker])
+ elif faces:
+ target_x = float(max(faces, key=lambda f: f["fw"])["cx"])
+ else:
+ target_x = None
+
+ cam_x = _update_tripod_camera(
+ current_center_x=cam_x,
+ target_center_x=target_x,
+ crop_w=crop_w,
+ video_width=width,
+ dt=max(0.01, t - prev_t),
+ )
+
+ crop_x = int(cam_x - crop_w / 2)
+ crop_x = max(0, min(crop_x, width - crop_w))
+
+ if not keyframes_x or crop_x != keyframes_x[-1][1]:
+ keyframes_x.append((round(t, 3), crop_x))
+
+ prev_t = t
+
# ── Build FFmpeg filter ──────────────────────────────────────
if not keyframes_x:
crop_x = max(0, (width - crop_w) // 2)
@@ -630,13 +1331,14 @@ def _track_and_crop(
# Simplify keyframes to reduce expression complexity
keyframes_x = _simplify_keyframes(keyframes_x)
x_expr = _build_cam_expr(keyframes_x, duration, has_any_split)
+ blur_filter = _build_motion_blur_filter(keyframes_x)
+ zoom_filter = _build_motion_zoom_filter(keyframes_x, target_w=target_w, target_h=target_h)
if x_expr is None:
- # Too complex — fall back to median position
xs = [x for _, x in keyframes_x]
med_x = sorted(xs)[len(xs) // 2]
- vf = f"crop={crop_w}:{crop_h}:{med_x}:{crop_y},scale={target_w}:{target_h}"
+ vf = f"crop={crop_w}:{crop_h}:{med_x}:{crop_y}{blur_filter},scale={target_w}:{target_h}{zoom_filter}"
else:
- vf = f"crop={crop_w}:{crop_h}:{x_expr}:{crop_y},scale={target_w}:{target_h}"
+ vf = f"crop={crop_w}:{crop_h}:{x_expr}:{crop_y}{blur_filter},scale={target_w}:{target_h}{zoom_filter}"
return _run_ffmpeg_with_fallback(
cmd_parts_before_enc=["ffmpeg", "-y", "-i", input_path, "-vf", vf],
@@ -1102,6 +1804,7 @@ def _build_speaker_aware_crop(
target_ratio: float,
transcript_words: list,
clip_start: float,
+ face_map: dict = None,
) -> Optional[str]:
"""
Build a speaker-aware crop that pans to whoever is speaking.
@@ -1132,8 +1835,10 @@ def _build_speaker_aware_crop(
cap.release()
return None
- # Zoom to 75% of frame height for tight face framing
- adjusted_h = int(height * 0.75)
+ # Full-height crop is the safest baseline for podcasts. The horizontal
+ # crop already creates a strong zoom on 16:9 sources; extra vertical
+ # zoom made solo and mixed-layout shots feel too aggressive.
+ adjusted_h = height
adj_w = int(adjusted_h * target_ratio)
if adj_w > width:
adjusted_h = height
@@ -1227,7 +1932,15 @@ def _build_speaker_aware_crop(
# In split-screen: left face belongs to one speaker, right to another
speaker_side = {} # speaker → "left" or "right"
- if has_split and len(speakers) >= 2:
+ if face_map:
+ clusters = face_map.get("clusters", [])
+ mappings = face_map.get("speaker_mappings", {})
+ for sp, ci in mappings.items():
+ if ci is None or ci >= len(clusters):
+ continue
+ speaker_side[sp] = "left" if clusters[ci]["center_x"] < mid_x else "right"
+
+ if has_split and len(speakers) >= 2 and len(speaker_side) < 2:
# Use first-word heuristic: first speaker to talk → left face
speaker_talk_time = {}
speaker_first_word = {}
@@ -1285,12 +1998,11 @@ def _build_speaker_aware_crop(
"""Compute crop_x and crop_y for a face at (cx, cy)."""
crop_x = cx - adj_w // 2
crop_x = max(0, min(crop_x, width - adj_w))
- crop_y = cy - int(adjusted_h * 0.33)
- crop_y = max(0, min(crop_y, height - adjusted_h))
+ crop_y = max(0, (height - adjusted_h) // 2)
return crop_x, crop_y
# Build per-timestamp crop positions
- timed_crops = [] # (time, crop_x, crop_y, speaker)
+ timed_crops = [] # (time, crop_x, crop_y, speaker, reliable)
for t, faces in frame_data:
if not faces:
continue
@@ -1300,7 +2012,8 @@ def _build_speaker_aware_crop(
continue
cx, cy, fw, conf = face
crop_x, crop_y = _crop_xy_for_face(cx, cy)
- timed_crops.append((t, crop_x, crop_y, speaker))
+ reliable = len(faces) >= 2 or speaker not in speaker_side
+ timed_crops.append((t, crop_x, crop_y, speaker, reliable))
if not timed_crops:
return None
@@ -1312,45 +2025,34 @@ def _build_speaker_aware_crop(
speaker_durations[sp] = speaker_durations.get(sp, 0) + (end_t - start_t)
dominant_speaker = max(speaker_durations, key=speaker_durations.get)
- dom_crops = [(cx, cy) for _, cx, cy, sp in timed_crops if sp == dominant_speaker]
+ dom_crops = [(cx, cy) for _, cx, cy, sp, _ in timed_crops if sp == dominant_speaker]
default_x = int(np.median([cx for cx, cy in dom_crops])) if dom_crops else width // 2 - adj_w // 2
default_y = int(np.median([cy for cx, cy in dom_crops])) if dom_crops else 0
- x_keyframes = []
- y_keyframes = []
-
- for start_t, end_t, speaker in segments:
- # Get crop positions for frames within this segment
- seg_crops = [(t, cx, cy) for t, cx, cy, sp in timed_crops if start_t <= t <= end_t]
-
- if seg_crops:
- for t, cx, cy in seg_crops:
- x_keyframes.append((t, cx))
- y_keyframes.append((t, cy))
- else:
- # No frames in segment — use speaker's median from all frames
- sp_crops = [(cx, cy) for _, cx, cy, sp in timed_crops if sp == speaker]
- if sp_crops:
- med_x = int(np.median([cx for cx, cy in sp_crops]))
- med_y = int(np.median([cy for cx, cy in sp_crops]))
- x_keyframes.append((start_t, med_x))
- y_keyframes.append((start_t, med_y))
- else:
- x_keyframes.append((start_t, default_x))
- y_keyframes.append((start_t, default_y))
-
- # Deduplicate and sort both keyframe lists
- def _smooth_keyframes(kf, min_interval=0.5):
- kf.sort(key=lambda k: k[0])
- smoothed = []
- for t, v in kf:
- if smoothed and abs(t - smoothed[-1][0]) < min_interval:
- continue
- smoothed.append((t, v))
- return smoothed
+ segment_targets = _choose_segment_targets(
+ segments=segments,
+ timed_crops=timed_crops,
+ speakers=speakers,
+ default_x=default_x,
+ default_y=default_y,
+ max_crop_x=max(0, width - adj_w),
+ preferred_margin=max(24, int(adj_w * 0.10)),
+ )
- x_keyframes = _smooth_keyframes(x_keyframes)
- y_keyframes = _smooth_keyframes(y_keyframes)
+ # Start locked on the first actual speaker target when available.
+ if segment_targets and segment_targets[0][0] <= 0.3:
+ default_x = segment_targets[0][2]
+ default_y = segment_targets[0][3]
+
+ # Build transition-aware keyframes from segment targets. This keeps the
+ # crop stable within a speaker turn and only moves around turn changes.
+ x_keyframes, y_keyframes = _build_transition_keyframes(
+ segment_targets=segment_targets,
+ default_x=default_x,
+ default_y=default_y,
+ adj_w=adj_w,
+ adjusted_h=adjusted_h,
+ )
if not x_keyframes and not y_keyframes:
return str(default_x), str(default_y), adjusted_h
@@ -1381,10 +2083,9 @@ def _build_speaker_aware_crop(
dt = max(0.01, t1 - t0)
value_jump = abs(v1 - v0)
- if dt > 1.5 or value_jump > jump_threshold:
- # Large time gap OR large value jump (layout change):
- # hold previous value, then instant-cut to next.
- # Avoids interpolating through the split-screen seam.
+ if dt > 1.5:
+ # Large unsupported gap: hold until the next reliable
+ # keyframe instead of inventing motion through missing data.
jump_t = t1 - 0.01
parts.append(
f"if(between(t\\,{t0:.2f}\\,{jump_t:.2f})\\,{v0}\\,"
@@ -1392,9 +2093,21 @@ def _build_speaker_aware_crop(
parts.append(
f"if(between(t\\,{jump_t:.2f}\\,{t1:.2f})\\,{v1}\\,"
)
+ elif value_jump > jump_threshold:
+ # Host/guest switches should move with intent, not snap.
+ pan_t = min(0.30, dt)
+ mid_t = t0 + pan_t
+ parts.append(
+ f"if(between(t\\,{t0:.2f}\\,{mid_t:.2f})\\,"
+ f"{v0}+(({v1}-{v0})*(t-{t0:.2f})/{pan_t:.2f})\\,"
+ )
+ if mid_t < t1:
+ parts.append(
+ f"if(between(t\\,{mid_t:.2f}\\,{t1:.2f})\\,{v1}\\,"
+ )
elif value_jump > jump_threshold * 0.6:
- # Medium jump (significant lean/shift): fast reframe (~0.15s)
- pan_t = min(0.15, dt)
+ # Medium jump: still move briskly, but slower than a cut.
+ pan_t = min(0.24, dt)
mid_t = t0 + pan_t
parts.append(
f"if(between(t\\,{t0:.2f}\\,{mid_t:.2f})\\,"
@@ -1429,6 +2142,138 @@ def _build_speaker_aware_crop(
return None
+def _choose_segment_targets(
+ segments: list,
+ timed_crops: list,
+ speakers: list,
+ default_x: int,
+ default_y: int,
+ max_crop_x: int,
+ preferred_margin: int,
+) -> list:
+ """
+ Choose one crop target per speaker segment.
+
+ timed_crops entries are (time, crop_x, crop_y, speaker, reliable).
+ Reliable crops come from frames where speaker identity is trustworthy,
+ e.g. split-screen frames with known side mapping.
+ """
+ from statistics import median
+
+ def _pick_representative(points: list[tuple]) -> tuple[int, int]:
+ first_x = points[0][1]
+ first_y = points[0][2]
+ median_x = int(median([cx for _, cx, _ in points]))
+ median_y = int(median([cy for _, _, cy in points]))
+
+ # Bias toward how the speaker first appears in the turn so the camera
+ # starts in the right place, then settle around the segment median.
+ seed_x = int(round(first_x * 0.7 + median_x * 0.3))
+ seed_y = int(round(first_y * 0.7 + median_y * 0.3))
+
+ def _score(point):
+ _, cx, cy = point
+ edge_margin = min(cx, max_crop_x - cx) if max_crop_x > 0 else preferred_margin
+ edge_penalty = max(0, preferred_margin - edge_margin) * 3
+ return abs(cx - seed_x) + abs(cy - seed_y) * 0.35 + edge_penalty
+
+ _, best_x, best_y = min(points, key=_score)
+ return int(best_x), int(best_y)
+
+ speaker_anchors = {}
+ for speaker in speakers:
+ reliable = [
+ (t, cx, cy)
+ for t, cx, cy, sp, is_reliable in timed_crops
+ if sp == speaker and is_reliable
+ ]
+ if reliable:
+ speaker_anchors[speaker] = _pick_representative(reliable)
+
+ segment_targets = []
+ for start_t, end_t, speaker in segments:
+ reliable_seg_crops = [
+ (t, cx, cy)
+ for t, cx, cy, sp, is_reliable in timed_crops
+ if start_t <= t <= end_t and sp == speaker and is_reliable
+ ]
+ seg_crops = [
+ (t, cx, cy)
+ for t, cx, cy, sp, _ in timed_crops
+ if start_t <= t <= end_t and sp == speaker
+ ]
+
+ if reliable_seg_crops:
+ med_x, med_y = _pick_representative(reliable_seg_crops)
+ elif seg_crops and speaker in speaker_anchors:
+ local_x, local_y = _pick_representative(seg_crops)
+ anchor_x, anchor_y = speaker_anchors[speaker]
+ local_is_centerish = preferred_margin <= local_x <= max_crop_x - preferred_margin
+ local_overrides_anchor = (
+ start_t <= 0.35
+ or (local_is_centerish and abs(local_x - anchor_x) > max(80, int(max_crop_x * 0.18)))
+ )
+ if local_overrides_anchor:
+ med_x, med_y = local_x, local_y
+ else:
+ med_x, med_y = anchor_x, anchor_y
+ elif speaker in speaker_anchors:
+ med_x, med_y = speaker_anchors[speaker]
+ elif seg_crops:
+ med_x, med_y = _pick_representative(seg_crops)
+ else:
+ sp_crops = [(t, cx, cy) for t, cx, cy, sp, _ in timed_crops if sp == speaker]
+ if sp_crops:
+ med_x, med_y = _pick_representative(sp_crops)
+ else:
+ med_x = default_x
+ med_y = default_y
+
+ segment_targets.append((start_t, end_t, med_x, med_y))
+
+ return segment_targets
+
+
+def _build_transition_keyframes(
+ segment_targets: list,
+ default_x: int,
+ default_y: int,
+ adj_w: int,
+ adjusted_h: int,
+) -> tuple[list, list]:
+ """
+ Convert per-segment crop targets into eased transition keyframes.
+ """
+ x_keyframes = [(0.0, default_x)]
+ y_keyframes = [(0.0, default_y)]
+ prev_x = default_x
+ prev_y = default_y
+
+ for start_t, end_t, target_x, target_y in segment_targets:
+ seg_duration = max(0.01, end_t - start_t)
+ delta_x = abs(target_x - prev_x)
+ delta_y = abs(target_y - prev_y)
+
+ if delta_x <= 6 and delta_y <= 6:
+ continue
+
+ x_keyframes.append((start_t, prev_x))
+ y_keyframes.append((start_t, prev_y))
+
+ if delta_x > adj_w * 0.33 or delta_y > adjusted_h * 0.12:
+ pan_t = min(0.38, seg_duration * 0.45)
+ else:
+ pan_t = min(0.26, seg_duration * 0.35)
+
+ settle_t = min(end_t, start_t + max(0.08, pan_t))
+ x_keyframes.append((settle_t, target_x))
+ y_keyframes.append((settle_t, target_y))
+ prev_x = target_x
+ prev_y = target_y
+
+ return x_keyframes, y_keyframes
+
+
def _create_gradient_png(output_path: str, width: int = 1080, height: int = 1920, opacity: float = 0.7) -> str:
"""
Create a transparent-to-black gradient PNG for the bottom 50% of the frame.
diff --git a/remotion/src/components/BrandedCaptions.tsx b/remotion/src/components/BrandedCaptions.tsx
index f217814..beeb939 100644
--- a/remotion/src/components/BrandedCaptions.tsx
+++ b/remotion/src/components/BrandedCaptions.tsx
@@ -21,17 +21,35 @@ interface Chunk {
end: number;
}
+const MAX_CHARS_PER_CHUNK = 18;
+
function buildChunks(words: Word[], perChunk: number): Chunk[] {
const chunks: Chunk[] = [];
let i = 0;
while (i < words.length) {
- let end = Math.min(i + perChunk, words.length);
+ let end = i;
+ let charCount = 0;
+
+ while (end < words.length && end - i < perChunk) {
+ const nextWord = words[end].word.trim();
+ const nextChars = charCount === 0 ? nextWord.length : charCount + 1 + nextWord.length;
+
+ if (end > i && nextChars > MAX_CHARS_PER_CHUNK) {
+ break;
+ }
+
+ charCount = nextChars;
+ end += 1;
+ }
+
+ if (end === i) {
+ end = i + 1;
+ }
- // Only absorb 1 leftover word, not 2
const remaining = words.length - end;
- if (remaining === 1) {
- end = words.length;
+ if (remaining === 1 && end - i > 2) {
+ end -= 1;
}
const slice = words.slice(i, end);
@@ -47,11 +65,13 @@ function buildChunks(words: Word[], perChunk: number): Chunk[] {
}
function splitIntoLines(words: Word[]): [Word[], Word[]] {
- if (words.length <= 3) {
+ if (words.length <= 2) {
return [words, []];
}
- const mid = Math.ceil(words.length / 2);
- return [words.slice(0, mid), words.slice(mid)];
+ if (words.length === 3) {
+ return [words.slice(0, 2), words.slice(2)];
+ }
+ return [words.slice(0, 2), words.slice(2)];
}
/**
@@ -114,6 +134,7 @@ const CaptionLine: React.FC<{
color: style.color,
textShadow: "0 0 40px rgba(0, 0, 0, 0.15), 0 0 80px rgba(0, 0, 0, 0.1), 0 0 150px rgba(0, 0, 0, 0.08)",
lineHeight: 1.25,
+ whiteSpace: "nowrap",
}}
>
{words.map((word, i) => {
diff --git a/remotion/src/types.ts b/remotion/src/types.ts
index 4f092c9..07fae7c 100644
--- a/remotion/src/types.ts
+++ b/remotion/src/types.ts
@@ -71,7 +71,7 @@ export const STYLES: Record<string, CaptionStyle> = {
color: "#FFFFFF",
activeColor: "#FFFFFF",
uppercase: false,
- wordsPerChunk: 4,
+ wordsPerChunk: 3,
position: "lower-third",
marginBottom: 420,
},
diff --git a/tests/test_ai_fallback.py b/tests/test_ai_fallback.py
new file mode 100644
index 0000000..3ec4506
--- /dev/null
+++ b/tests/test_ai_fallback.py
@@ -0,0 +1,154 @@
+import json
+import os
+import subprocess
+import sys
+import unittest
+from unittest import mock
+
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+BACKEND_ROOT = os.path.join(ROOT, "backend")
+if BACKEND_ROOT not in sys.path:
+ sys.path.insert(0, BACKEND_ROOT)
+
+from services import claude_suggest as cs
+from services import content_generator as cg
+from services import thumbnail_ai as tai
+
+
+class AIFallbackTests(unittest.TestCase):
+ def test_suggest_with_claude_retries_with_codex_after_runtime_failure(self):
+ segments = [
+ {"start": 0.0, "end": 12.0, "speaker": "SPEAKER_00", "text": "Warmup context."},
+ {"start": 12.0, "end": 40.0, "speaker": "SPEAKER_00", "text": "The best grid connection is no grid connection."},
+ ]
+ progress = []
+ codex_payload = json.dumps({
+ "clips": [
+ {
+ "title": "The best grid connection is no grid connection",
+ "start_second": 12.0,
+ "end_second": 38.0,
+ "segments": [{"start": 12.0, "end": 38.0}],
+ "duration": 26,
+ "content_type": "hot_take",
+ "scores": {"standalone": 5, "hook": 5, "relevance": 4, "quotability": 4},
+ "quote": "The best grid connection is no grid connection",
+ "why": "Strong contrarian line with clean standalone context.",
+ }
+ ]
+ })
+
+ with mock.patch.object(
+ cs,
+ "_find_ai_cli_candidates",
+ return_value=[("/tmp/claude", "claude"), ("/tmp/codex", "codex")],
+ ), mock.patch.object(
+ cs,
+ "_run_ai_command",
+ side_effect=[
+ subprocess.CompletedProcess(args=["claude"], returncode=1, stdout="", stderr="claude down"),
+ subprocess.CompletedProcess(args=["codex"], returncode=0, stdout=codex_payload, stderr=""),
+ ],
+ ):
+ clips = cs.suggest_with_claude(
+ segments=segments,
+ top_n=1,
+ progress_callback=lambda _pct, msg: progress.append(msg),
+ )
+
+ self.assertIsNotNone(clips)
+ self.assertEqual(len(clips), 1)
+ self.assertEqual(clips[0]["_ai_engine"], "codex")
+ self.assertIn("Retrying with Codex...", progress)
+ self.assertIn("Codex suggested 1 clips", progress)
+
+ def test_generate_clip_content_retries_with_codex(self):
+ clip = {
+ "title": "Power demand is exploding",
+ "start_second": 10.0,
+ "end_second": 36.0,
+ "content_type": "market_landscape",
+ }
+ transcript_segments = [
+ {"start": 9.0, "speaker": "SPEAKER_00", "text": "Power demand is exploding."},
+ {"start": 18.0, "speaker": "SPEAKER_00", "text": "Data centers will need a lot more energy."},
+ ]
+ progress = []
+ codex_text = """TITLES (8 options, 40-60 chars, keyword-first, follow title spec):
+1. Power Demand Is Exploding Fast
+2. Data Centers Need Much More Energy
+TOP PICK: 1 — strongest hook
+
+DESCRIPTION:
+Power demand is exploding.
+Guest explains why data centers need more energy.
+
+TAGS:
+power demand, data centers, energy, ai infrastructure
+
+HASHTAGS:
+#power #energy #datacenters #ai #infrastructure"""
+
+ with mock.patch.object(
+ cg,
+ "_find_ai_cli_candidates",
+ return_value=[("/tmp/claude", "claude"), ("/tmp/codex", "codex")],
+ ), mock.patch.object(
+ cg,
+ "_run_ai_command",
+ side_effect=[
+ subprocess.CompletedProcess(args=["claude"], returncode=1, stdout="", stderr="claude down"),
+ subprocess.CompletedProcess(args=["codex"], returncode=0, stdout=codex_text, stderr=""),
+ ],
+ ):
+ result = cg.generate_clip_content(
+ clip=clip,
+ transcript_segments=transcript_segments,
+ progress_callback=lambda _pct, msg: progress.append(msg),
+ )
+
+ self.assertIsNotNone(result)
+ self.assertEqual(result["engine"], "codex")
+ self.assertTrue(result["titles"])
+ self.assertIn("Retrying content generation with Codex...", progress)
+
+ def test_thumbnail_layout_retries_with_codex(self):
+ codex_layout = """
+Some wrapper text
+{
+ "line1": "POWER DEMAND",
+ "line2": "IS EXPLODING",
+ "box_y": "78%",
+ "photo_object_position": "center 18%",
+ "line1_font_size": "96px",
+ "line2_font_size": "90px"
+}
+"""
+
+ with mock.patch.object(
+ cs,
+ "_find_ai_cli_candidates",
+ return_value=[("/tmp/claude", "claude"), ("/tmp/codex", "codex")],
+ ), mock.patch.object(
+ cs,
+ "_run_ai_command",
+ side_effect=[
+ subprocess.CompletedProcess(args=["claude"], returncode=1, stdout="", stderr="claude down"),
+ subprocess.CompletedProcess(args=["codex"], returncode=0, stdout=codex_layout, stderr=""),
+ ],
+ ):
+ layout = tai.ask_claude_for_layout(
+ title="Power demand is exploding",
+ frame_path="/tmp/frame.png",
+ frame_info={"face_x_pct": 50, "face_y_pct": 40, "face_w_pct": 20, "face_h_pct": 25},
+ config={"enabled": True},
+ )
+
+ self.assertEqual(layout["line1"], "POWER DEMAND")
+ self.assertEqual(layout["line2"], "IS EXPLODING")
+ self.assertEqual(layout["box_y"], "78%")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_thumbnail_html.py b/tests/test_thumbnail_html.py
new file mode 100644
index 0000000..fc0d82c
--- /dev/null
+++ b/tests/test_thumbnail_html.py
@@ -0,0 +1,125 @@
+import os
+import subprocess
+import sys
+import unittest
+from unittest import mock
+
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+BACKEND_ROOT = os.path.join(ROOT, "backend")
+if BACKEND_ROOT not in sys.path:
+ sys.path.insert(0, BACKEND_ROOT)
+
+from services import thumbnail_html as th
+
+
+class ThumbnailHtmlTests(unittest.TestCase):
+ def test_prepare_thumbnail_lines_compacts_long_sentence_title(self):
+ line1, line2 = th._prepare_thumbnail_lines(
+ "We build 10 megawatt data centers in 200 days — everyone is shocked",
+ )
+
+ self.assertIn("10MW", f"{line1} {line2}")
+ self.assertNotIn("everyone", f"{line1} {line2}".lower())
+ self.assertLessEqual(len(line1), 24)
+ self.assertLessEqual(len(line2), 24)
+ self.assertLessEqual(len((line1 + " " + line2).split()), 8)
+
+ def test_prepare_thumbnail_lines_rewrites_overlong_ai_lines(self):
+ line1, line2 = th._prepare_thumbnail_lines(
+ title="ignored",
+ line1="WE BUILD 10 MEGAWATT DATA",
+ line2="CENTERS IN 200 DAYS — EVERYONE IS SHOCKED",
+ )
+
+ self.assertLessEqual(len(line1), 24)
+ self.assertLessEqual(len(line2), 24)
+ self.assertNotIn("EVERYONE", f"{line1} {line2}")
+
+ def test_build_remotion_screenshot_command(self):
+ script_suffix = os.path.join("backend", "scripts", "remotion_screenshot.cjs")
+ renderer_suffix = os.path.join("node_modules", "@remotion", "renderer", "package.json")
+
+ def exists(path):
+ return path.endswith(script_suffix) or path.endswith(renderer_suffix)
+
+ with mock.patch.object(th.os.path, "exists", side_effect=exists), \
+ mock.patch.object(th.shutil, "which", side_effect=lambda name: { "node": "/usr/local/bin/node" }.get(name)):
+ cmd = th._build_remotion_screenshot_command(
+ script_path="/repo/backend/scripts/remotion_screenshot.cjs",
+ html_path="/tmp/thumb.html",
+ output_path="/tmp/thumb.png",
+ width=1080,
+ height=1920,
+ wait_ms=1500,
+ )
+
+ self.assertEqual(
+ cmd,
+ [
+ "/usr/local/bin/node",
+ "/repo/backend/scripts/remotion_screenshot.cjs",
+ "/tmp/thumb.html",
+ "/tmp/thumb.png",
+ "1080",
+ "1920",
+ "1500",
+ ],
+ )
+
+ def test_playwright_cli_candidates_prefers_local_then_global_then_npx(self):
+ local_suffix = os.path.join("node_modules", ".bin", "playwright")
+
+ with mock.patch.object(th.os.path, "exists", side_effect=lambda p: p.endswith(local_suffix)), \
+ mock.patch.object(th.shutil, "which", side_effect=lambda name: {"playwright": "/usr/local/bin/playwright", "npx": "/usr/local/bin/npx"}.get(name)):
+ candidates = th._playwright_cli_candidates()
+
+ self.assertTrue(candidates[0][0].endswith(local_suffix))
+ self.assertEqual(candidates[1:], [
+ ["/usr/local/bin/playwright"],
+ ["/usr/local/bin/npx", "--no-install", "playwright"],
+ ["/usr/local/bin/npx", "playwright"],
+ ])
+
+ def test_build_playwright_screenshot_command(self):
+ cmd = th._build_playwright_screenshot_command(
+ cli_cmd=["/usr/local/bin/npx", "--no-install", "playwright"],
+ html_path="/tmp/thumb.html",
+ output_path="/tmp/thumb.png",
+ width=1080,
+ height=1920,
+ wait_ms=1200,
+ )
+
+ self.assertEqual(
+ cmd,
+ [
+ "/usr/local/bin/npx",
+ "--no-install",
+ "playwright",
+ "screenshot",
+ "--viewport-size",
+ "1080, 1920",
+ "--wait-for-timeout",
+ "1200",
+ "file:///tmp/thumb.html",
+ "/tmp/thumb.png",
+ ],
+ )
+
+ def test_generate_thumbnail_prefers_remotion_before_playwright(self):
+ with mock.patch.object(th, "_build_html", return_value="<html></html>"), \
+ mock.patch.object(th, "_build_remotion_screenshot_command", return_value=["node", "script", "a", "b", "1080", "1920", "1500"]), \
+ mock.patch.object(th, "_playwright_cli_candidates", return_value=[["playwright"]]), \
+ mock.patch.object(th, "_build_playwright_screenshot_command", return_value=["playwright", "screenshot"]), \
+ mock.patch.object(th.subprocess, "run", side_effect=[
+ subprocess.CompletedProcess(args=["node"], returncode=0, stdout="", stderr=""),
+ ]) as run_mock:
+ output = th.generate_thumbnail("line1", "line2", "/tmp/thumb.png")
+
+ self.assertEqual(output, "/tmp/thumb.png")
+ self.assertEqual(run_mock.call_args_list[0].args[0], ["node", "script", "a", "b", "1080", "1920", "1500"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_video_processor.py b/tests/test_video_processor.py
new file mode 100644
index 0000000..c2fa0c6
--- /dev/null
+++ b/tests/test_video_processor.py
@@ -0,0 +1,524 @@
+import os
+import sys
+import unittest
+from unittest import mock
+
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+BACKEND_ROOT = os.path.join(ROOT, "backend")
+if BACKEND_ROOT not in sys.path:
+ sys.path.insert(0, BACKEND_ROOT)
+
+from services import video_processor as vp
+
+
+class VideoProcessorTests(unittest.TestCase):
+ def test_assign_face_tracks_keeps_ids_stable_across_frames(self):
+ tracked = vp._assign_face_tracks(
+ detections=[
+ (0.0, [{"cx": 320, "fw": 180}, {"cx": 1420, "fw": 200}]),
+ (0.1, [{"cx": 338, "fw": 176}, {"cx": 1402, "fw": 198}]),
+ (0.2, [{"cx": 356, "fw": 172}, {"cx": 1388, "fw": 194}]),
+ ],
+ width=1920,
+ )
+
+ left_ids = [faces[0]["track_id"] for _, faces in tracked]
+ right_ids = [faces[1]["track_id"] for _, faces in tracked]
+
+ self.assertEqual(left_ids, [left_ids[0], left_ids[0], left_ids[0]])
+ self.assertEqual(right_ids, [right_ids[0], right_ids[0], right_ids[0]])
+ self.assertNotEqual(left_ids[0], right_ids[0])
+
+ def test_assign_face_tracks_matches_center_solo_to_similar_size_split_face(self):
+ tracked = vp._assign_face_tracks(
+ detections=[
+ (0.0, [{"cx": 1054, "cy": 643, "fw": 370}]),
+ (0.8, [{"cx": 553, "cy": 655, "fw": 376}, {"cx": 1551, "cy": 458, "fw": 266}]),
+ ],
+ width=1920,
+ )
+
+ solo_id = tracked[0][1][0]["track_id"]
+ split_left_id = tracked[1][1][0]["track_id"]
+ split_right_id = tracked[1][1][1]["track_id"]
+
+ self.assertEqual(solo_id, split_left_id)
+ self.assertNotEqual(solo_id, split_right_id)
+
+ def test_choose_segment_tracks_prefers_speaker_anchor_over_reaction_track(self):
+ tracked_detections = [
+ (0.0, [{"cx": 320, "fw": 200, "track_id": 0}]),
+ (0.4, [{"cx": 330, "fw": 205, "track_id": 0}]),
+ (0.8, [{"cx": 340, "fw": 198, "track_id": 0}]),
+ (1.2, [{"cx": 320, "fw": 200, "track_id": 0}, {"cx": 1480, "fw": 240, "track_id": 1}]),
+ (1.6, [{"cx": 1490, "fw": 238, "track_id": 1}]),
+ (2.0, [{"cx": 1475, "fw": 235, "track_id": 1}]),
+ ]
+
+ segment_tracks, anchors, sides = vp._choose_segment_tracks(
+ segments=[(0.0, 2.5, "SPEAKER_03")],
+ tracked_detections=tracked_detections,
+ speaker_side={"SPEAKER_03": "right"},
+ speaker_anchor_x={"SPEAKER_03": 1500},
+ width=1920,
+ )
+
+ self.assertEqual(segment_tracks[0][3], 1)
+ self.assertGreater(anchors["SPEAKER_03"], 1200)
+ self.assertEqual(sides["SPEAKER_03"], "right")
+
+ def test_choose_segment_tracks_overrides_wrong_global_side_hint_with_local_evidence(self):
+ tracked_detections = [
+ (0.0, [{"cx": 1040, "fw": 370, "track_id": 0}]),
+ (0.5, [{"cx": 1025, "fw": 365, "track_id": 0}]),
+ (1.0, [{"cx": 553, "fw": 376, "track_id": 0}, {"cx": 1551, "fw": 266, "track_id": 1}]),
+ (1.5, [{"cx": 560, "fw": 382, "track_id": 0}, {"cx": 1596, "fw": 297, "track_id": 1}]),
+ ]
+
+ segment_tracks, anchors, sides = vp._choose_segment_tracks(
+ segments=[(0.0, 2.0, "SPEAKER_03")],
+ tracked_detections=tracked_detections,
+ speaker_side={"SPEAKER_03": "right"},
+ speaker_anchor_x={"SPEAKER_03": 1518},
+ width=1920,
+ )
+
+ self.assertEqual(segment_tracks[0][3], 0)
+ self.assertLess(anchors["SPEAKER_03"], 1200)
+ self.assertEqual(sides["SPEAKER_03"], "left")
+
+ def test_choose_track_segment_targets_picks_one_stable_target_per_turn(self):
+ targets = vp._choose_track_segment_targets(
+ segment_tracks=[
+ (0.0, 3.0, "SPEAKER_00", 0, True),
+ (3.0, 6.0, "SPEAKER_01", 1, True),
+ ],
+ tracked_detections=[
+ (0.2, [{"cx": 320, "fw": 200, "track_id": 0}]),
+ (1.0, [{"cx": 336, "fw": 205, "track_id": 0}]),
+ (2.0, [{"cx": 344, "fw": 198, "track_id": 0}]),
+ (3.2, [{"cx": 1480, "fw": 200, "track_id": 1}]),
+ (4.1, [{"cx": 1492, "fw": 205, "track_id": 1}]),
+ (5.0, [{"cx": 1500, "fw": 198, "track_id": 1}]),
+ ],
+ speaker_anchor_x={},
+ width=1920,
+ crop_w=607,
+ )
+
+ self.assertEqual(len(targets), 2)
+ self.assertEqual(targets[0][0:2], (0.0, 3.0))
+ self.assertEqual(targets[1][0:2], (3.0, 6.0))
+ self.assertLess(targets[0][2], 60)
+ self.assertGreater(targets[1][2], 1100)
+ self.assertEqual(targets[0][3], "SPEAKER_00")
+ self.assertEqual(targets[1][3], "SPEAKER_01")
+
+ def test_choose_track_segment_targets_splits_same_turn_on_big_reframed_shift(self):
+ targets = vp._choose_track_segment_targets(
+ segment_tracks=[
+ (0.0, 6.0, "SPEAKER_00", 0, True),
+ ],
+ tracked_detections=[
+ (0.2, [{"cx": 330, "fw": 200, "track_id": 0}]),
+ (1.1, [{"cx": 338, "fw": 205, "track_id": 0}]),
+ (2.0, [{"cx": 344, "fw": 198, "track_id": 0}]),
+ (3.2, [{"cx": 980, "fw": 200, "track_id": 0}]),
+ (3.5, [{"cx": 992, "fw": 205, "track_id": 0}]),
+ (4.2, [{"cx": 1004, "fw": 198, "track_id": 0}]),
+ ],
+ speaker_anchor_x={},
+ width=1920,
+ crop_w=607,
+ )
+
+ self.assertEqual(len(targets), 2)
+ self.assertEqual(targets[0][0:2], (0.0, 3.2))
+ self.assertEqual(targets[1][0:2], (3.2, 6.0))
+ self.assertLess(targets[0][2], 80)
+ self.assertGreater(targets[1][2], 650)
+ self.assertEqual(targets[0][3], "SPEAKER_00")
+ self.assertEqual(targets[1][3], "SPEAKER_00")
+
+ def test_build_track_turn_keyframes_uses_short_reframe_then_holds(self):
+ keyframes = vp._build_track_turn_keyframes(
+ segment_targets=[
+ (0.0, 3.0, 0),
+ (3.0, 6.0, 1147),
+ ],
+ default_x=0,
+ )
+
+ self.assertEqual(keyframes[0], (0.0, 0))
+ self.assertIn((3.0, 0), keyframes)
+ self.assertIn((3.3, 1147), keyframes)
+ self.assertEqual(len(keyframes), 3)
+
+ def test_build_track_turn_keyframes_reframes_faster_on_same_speaker_layout_shift(self):
+ keyframes = vp._build_track_turn_keyframes(
+ segment_targets=[
+ (0.0, 3.0, 140, "SPEAKER_00"),
+ (3.0, 6.0, 1147, "SPEAKER_00"),
+ ],
+ default_x=140,
+ )
+
+ self.assertIn((3.0, 140), keyframes)
+ self.assertIn((3.18, 1147), keyframes)
+
+ def test_build_track_turn_keyframes_snaps_out_of_bad_far_left_hold(self):
+ keyframes = vp._build_track_turn_keyframes(
+ segment_targets=[
+ (0.0, 3.0, 0, "SPEAKER_00"),
+ (3.0, 6.0, 620, "SPEAKER_00"),
+ ],
+ default_x=0,
+ )
+
+ self.assertIn((2.99, 0), keyframes)
+ self.assertIn((3.0, 620), keyframes)
+ self.assertNotIn((3.14, 620), keyframes)
+
+ def test_update_tripod_camera_holds_inside_safe_zone(self):
+ current = vp._update_tripod_camera(
+ current_center_x=960.0,
+ target_center_x=1000.0,
+ crop_w=607,
+ video_width=1920,
+ dt=0.1,
+ )
+
+ self.assertAlmostEqual(current, 960.0)
+
+ def test_update_tripod_camera_moves_gradually_when_target_is_far(self):
+ current = vp._update_tripod_camera(
+ current_center_x=960.0,
+ target_center_x=1320.0,
+ crop_w=607,
+ video_width=1920,
+ dt=0.1,
+ )
+
+ self.assertGreater(current, 960.0)
+ self.assertLess(current, 1320.0)
+
+ def test_update_tripod_camera_recenters_on_moderate_offset(self):
+ current = vp._update_tripod_camera(
+ current_center_x=960.0,
+ target_center_x=1060.0,
+ crop_w=607,
+ video_width=1920,
+ dt=0.1,
+ )
+
+ self.assertGreater(current, 960.0)
+ self.assertLess(current, 1060.0)
+
+ def test_choose_camera_speaker_ignores_short_interjection(self):
+ speaker, pending, count, switched = vp._choose_camera_speaker(
+ transcript_speaker="SPEAKER_01",
+ transcript_duration=1.4,
+ active_speaker="SPEAKER_00",
+ pending_speaker=None,
+ pending_count=0,
+ )
+
+ self.assertEqual(speaker, "SPEAKER_00")
+ self.assertIsNone(pending)
+ self.assertEqual(count, 0)
+ self.assertFalse(switched)
+
+ def test_choose_camera_speaker_requires_confirmation_for_real_turn_change(self):
+ speaker, pending, count, switched = vp._choose_camera_speaker(
+ transcript_speaker="SPEAKER_01",
+ transcript_duration=4.0,
+ active_speaker="SPEAKER_00",
+ pending_speaker=None,
+ pending_count=0,
+ )
+ self.assertEqual(speaker, "SPEAKER_00")
+ self.assertEqual(pending, "SPEAKER_01")
+ self.assertEqual(count, 1)
+ self.assertFalse(switched)
+
+ speaker, pending, count, switched = vp._choose_camera_speaker(
+ transcript_speaker="SPEAKER_01",
+ transcript_duration=4.0,
+ active_speaker=speaker,
+ pending_speaker=pending,
+ pending_count=count,
+ )
+ self.assertEqual(speaker, "SPEAKER_00")
+ self.assertEqual(pending, "SPEAKER_01")
+ self.assertEqual(count, 2)
+ self.assertFalse(switched)
+
+ speaker, pending, count, switched = vp._choose_camera_speaker(
+ transcript_speaker="SPEAKER_01",
+ transcript_duration=4.0,
+ active_speaker=speaker,
+ pending_speaker=pending,
+ pending_count=count,
+ )
+ self.assertEqual(speaker, "SPEAKER_01")
+ self.assertIsNone(pending)
+ self.assertEqual(count, 0)
+ self.assertTrue(switched)
+
+ def test_pick_tracking_face_holds_on_same_speaker_when_single_face_contradicts_anchor(self):
+ face, reliable = vp._pick_tracking_face(
+ faces=[{"cx": 1180, "fw": 180}],
+ speaker="SPEAKER_00",
+ speaker_side={"SPEAKER_00": "left"},
+ width=1920,
+ last_target_x=280,
+ last_speaker="SPEAKER_00",
+ speaker_anchor_x={"SPEAKER_00": 260},
+ has_any_split=True,
+ )
+
+ self.assertIsNone(face)
+ self.assertFalse(reliable)
+
+ def test_pick_tracking_face_accepts_single_face_on_real_speaker_change(self):
+ face, reliable = vp._pick_tracking_face(
+ faces=[{"cx": 1180, "fw": 180}],
+ speaker="SPEAKER_01",
+ speaker_side={"SPEAKER_01": "right"},
+ width=1920,
+ last_target_x=280,
+ last_speaker="SPEAKER_00",
+ speaker_anchor_x={"SPEAKER_01": 1160},
+ has_any_split=True,
+ )
+
+ self.assertEqual(face["cx"], 1180)
+ self.assertTrue(reliable)
+
+ def test_pick_tracking_face_uses_side_hint_in_split_screen(self):
+ face, reliable = vp._pick_tracking_face(
+ faces=[{"cx": 320, "fw": 160}, {"cx": 1420, "fw": 180}],
+ speaker="SPEAKER_01",
+ speaker_side={"SPEAKER_01": "right"},
+ width=1920,
+ last_target_x=300,
+ last_speaker="SPEAKER_00",
+ speaker_anchor_x={"SPEAKER_01": 1400},
+ has_any_split=True,
+ )
+
+ self.assertEqual(face["cx"], 1420)
+ self.assertTrue(reliable)
+
+ def test_choose_segment_targets_prefers_reliable_anchor_over_ambiguous_reaction_shot(self):
+ segments = [
+ (0.0, 3.0, "SPEAKER_00"),
+ (3.0, 6.0, "SPEAKER_01"),
+ ]
+ timed_crops = [
+ (0.5, 0, 20, "SPEAKER_00", True),
+ (1.5, 0, 20, "SPEAKER_00", True),
+ (3.2, 1147, 40, "SPEAKER_01", True),
+ (3.6, 1147, 40, "SPEAKER_01", True),
+ (4.7, 0, 20, "SPEAKER_01", False),
+ (5.2, 0, 20, "SPEAKER_01", False),
+ ]
+
+ targets = vp._choose_segment_targets(
+ segments=segments,
+ timed_crops=timed_crops,
+ speakers=["SPEAKER_00", "SPEAKER_01"],
+ default_x=0,
+ default_y=20,
+ max_crop_x=1374,
+ preferred_margin=54,
+ )
+
+ self.assertEqual(targets[0], (0.0, 3.0, 0, 20))
+ self.assertEqual(targets[1], (3.0, 6.0, 1147, 40))
+
+ def test_choose_segment_targets_penalizes_edge_hugging_opening_frame(self):
+ segments = [
+ (0.0, 3.0, "SPEAKER_00"),
+ ]
+ timed_crops = [
+ (0.1, 0, 20, "SPEAKER_00", True),
+ (0.5, 96, 24, "SPEAKER_00", True),
+ (1.0, 110, 25, "SPEAKER_00", True),
+ ]
+
+ targets = vp._choose_segment_targets(
+ segments=segments,
+ timed_crops=timed_crops,
+ speakers=["SPEAKER_00"],
+ default_x=0,
+ default_y=20,
+ max_crop_x=1374,
+ preferred_margin=54,
+ )
+
+ self.assertEqual(targets[0], (0.0, 3.0, 96, 24))
+
+ def test_choose_segment_targets_allows_local_segment_to_override_bad_anchor(self):
+ segments = [
+ (0.0, 3.0, "SPEAKER_00"),
+ (3.0, 6.0, "SPEAKER_01"),
+ ]
+ timed_crops = [
+ (0.2, 0, 20, "SPEAKER_00", True),
+ (1.0, 0, 20, "SPEAKER_00", True),
+ (4.2, 620, 32, "SPEAKER_01", False),
+ (4.8, 640, 34, "SPEAKER_01", False),
+ (6.2, 1147, 40, "SPEAKER_01", True),
+ (6.6, 1147, 40, "SPEAKER_01", True),
+ ]
+
+ targets = vp._choose_segment_targets(
+ segments=segments,
+ timed_crops=timed_crops,
+ speakers=["SPEAKER_00", "SPEAKER_01"],
+ default_x=0,
+ default_y=20,
+ max_crop_x=1374,
+ preferred_margin=54,
+ )
+
+ self.assertEqual(targets[1], (3.0, 6.0, 620, 32))
+
+ def test_build_transition_keyframes_eases_between_speaker_turns(self):
+ x_keyframes, y_keyframes = vp._build_transition_keyframes(
+ segment_targets=[
+ (0.0, 3.0, 0, 20),
+ (3.0, 6.0, 1147, 40),
+ ],
+ default_x=0,
+ default_y=20,
+ adj_w=546,
+ adjusted_h=972,
+ )
+
+ self.assertEqual(x_keyframes[0], (0.0, 0))
+ self.assertIn((3.0, 0), x_keyframes)
+ self.assertIn((3.38, 1147), x_keyframes)
+ self.assertIn((3.0, 20), y_keyframes)
+ self.assertIn((3.38, 40), y_keyframes)
+
+ def test_build_cam_expr_uses_quick_reframe_for_moderate_jump(self):
+ expr = vp._build_cam_expr(
+ keyframes=[(0.0, 100), (1.0, 240)],
+ duration=1.0,
+ is_split=False,
+ )
+
+ self.assertIn("100+((240-100)*((6*(((t-0.000)/0.180))*(((t-0.000)/0.180))*(((t-0.000)/0.180))*(((t-0.000)/0.180))*(((t-0.000)/0.180)))-(15*(((t-0.000)/0.180))*(((t-0.000)/0.180))*(((t-0.000)/0.180))*(((t-0.000)/0.180)))+(10*(((t-0.000)/0.180))*(((t-0.000)/0.180))*(((t-0.000)/0.180)))))", expr)
+ self.assertIn("if(between(t\\,0.180\\,1.000)\\,240\\,", expr)
+
+ def test_build_cam_expr_uses_blurred_cut_for_very_large_jump(self):
+ expr = vp._build_cam_expr(
+ keyframes=[(0.0, 100), (0.18, 420), (1.0, 420)],
+ duration=1.0,
+ is_split=False,
+ )
+
+ self.assertIn("if(between(t\\,0.000\\,0.180)\\,420\\,", expr)
+ self.assertNotIn("100+((420-100)", expr)
+
+ def test_build_motion_blur_filter_targets_only_short_reframes(self):
+ blur = vp._build_motion_blur_filter(
+ keyframes=[(0.0, 140), (0.18, 620), (1.0, 620), (1.6, 650)],
+ )
+
+ self.assertEqual(
+ blur,
+ ",gblur=sigma=5.4:steps=2:enable='between(t\\,0.000\\,0.230)'",
+ )
+
+ def test_build_motion_zoom_filter_targets_only_short_reframes(self):
+ zoom = vp._build_motion_zoom_filter(
+ keyframes=[(0.0, 140), (0.18, 620), (1.0, 620), (1.6, 650)],
+ target_w=1080,
+ target_h=1920,
+ )
+
+ self.assertIn("scale=w='iw*(1+0.0180*", zoom)
+ self.assertIn("between(t\\,0.000\\,0.180)", zoom)
+ self.assertIn("crop=1080:1920:(iw-1080)/2:(ih-1920)/2", zoom)
+ self.assertNotIn("between(t\\,1.000\\,1.600)", zoom)
+
+ def test_crop_to_vertical_prefers_sticky_tracker_before_face_map(self):
+ transcript_words = [
+ {"word": "hi", "start": 0.0, "end": 0.4, "speaker": "SPEAKER_00"},
+ {"word": "yo", "start": 0.5, "end": 0.9, "speaker": "SPEAKER_01"},
+ ]
+
+ with mock.patch.object(vp, "get_dimensions", return_value=(1920, 1080)), \
+ mock.patch.object(vp, "_build_speaker_aware_crop") as speaker_crop, \
+ mock.patch.object(vp, "_use_face_map") as face_map_crop, \
+ mock.patch.object(vp, "_track_and_crop", return_value="ok.mp4") as track_crop, \
+ mock.patch.object(vp, "_run_ffmpeg_with_fallback", return_value="ok.mp4") as runner:
+ result = vp.crop_to_vertical(
+ input_path="in.mp4",
+ output_path="out.mp4",
+ strategy="speaker",
+ transcript_words=transcript_words,
+ clip_start=0.0,
+ face_map={"clusters": [{"center_x": 100, "crop_x": 0}], "video_width": 1920},
+ )
+
+ self.assertEqual(result, "ok.mp4")
+ speaker_crop.assert_not_called()
+ face_map_crop.assert_not_called()
+ track_crop.assert_called_once()
+ runner.assert_not_called()
+
+ def test_crop_to_vertical_prefers_sticky_tracker_for_single_speaker_clip(self):
+ transcript_words = [
+ {"word": "we", "start": 0.0, "end": 0.2, "speaker": "SPEAKER_03"},
+ {"word": "deploy", "start": 0.2, "end": 0.5, "speaker": "SPEAKER_03"},
+ ]
+
+ with mock.patch.object(vp, "get_dimensions", return_value=(1920, 1080)), \
+ mock.patch.object(vp, "_build_speaker_aware_crop") as speaker_crop, \
+ mock.patch.object(vp, "_use_face_map") as face_map_crop, \
+ mock.patch.object(vp, "_track_and_crop", return_value="ok.mp4") as track_crop, \
+ mock.patch.object(vp, "_run_ffmpeg_with_fallback", return_value="ok.mp4") as runner:
+ result = vp.crop_to_vertical(
+ input_path="in.mp4",
+ output_path="out.mp4",
+ strategy="face",
+ transcript_words=transcript_words,
+ clip_start=1196.0,
+ face_map={"clusters": [{"center_x": 100, "crop_x": 0}], "video_width": 1920},
+ )
+
+ self.assertEqual(result, "ok.mp4")
+ speaker_crop.assert_not_called()
+ face_map_crop.assert_not_called()
+ track_crop.assert_called_once()
+ runner.assert_not_called()
+
+ def test_crop_to_vertical_uses_face_map_when_no_speaker_labels(self):
+ with mock.patch.object(vp, "get_dimensions", return_value=(1920, 1080)), \
+ mock.patch.object(vp, "_build_speaker_aware_crop") as speaker_crop, \
+ mock.patch.object(vp, "_use_face_map", return_value="456") as face_map_crop, \
+ mock.patch.object(vp, "_track_and_crop") as track_crop, \
+ mock.patch.object(vp, "_run_ffmpeg_with_fallback", return_value="ok.mp4") as runner:
+ result = vp.crop_to_vertical(
+ input_path="in.mp4",
+ output_path="out.mp4",
+ strategy="face",
+ transcript_words=[{"word": "hi", "start": 0.0, "end": 0.4, "speaker": None}],
+ clip_start=0.0,
+ face_map={"clusters": [{"center_x": 100, "crop_x": 0}], "video_width": 1920},
+ )
+
+ self.assertEqual(result, "ok.mp4")
+ speaker_crop.assert_not_called()
+ face_map_crop.assert_called_once()
+ track_crop.assert_not_called()
+ runner.assert_called_once()
+
+
+if __name__ == "__main__":
+ unittest.main()