Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

Harden batch rendering and speaker tracking

Nika Siradzed87ea2229b3b
diff --git a/backend/scripts/remotion_screenshot.cjs b/backend/scripts/remotion_screenshot.cjs
index db219f3..e6f2f9b 100644
--- a/backend/scripts/remotion_screenshot.cjs
+++ b/backend/scripts/remotion_screenshot.cjs
@@ -47,6 +47,9 @@ const waitForAssets = async (page) => {
(async () => {
const browser = await openBrowser("chrome", {logLevel: "error"});
+ const closeBrowser = () => { try { browser.close({silent: true}); } catch {} };
+ process.on("SIGINT", () => { closeBrowser(); process.exit(1); });
+ process.on("SIGTERM", () => { closeBrowser(); process.exit(1); });
try {
const page = await browser.newPage({
context: () => null,
diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.py
index acba080..f373543 100644
--- a/backend/services/clip_generator.py
+++ b/backend/services/clip_generator.py
@@ -159,7 +159,7 @@ def _build_tight_segments(
return segments
-_remotion_available = None # True/False/None — auto-detected on first render
+_remotion_available = None # True/False/None — environment availability, not per-clip success
def _render_with_remotion(
@@ -177,7 +177,8 @@ def _render_with_remotion(
import subprocess
import json
- # Quick bail if Remotion already failed this session
+ # Quick bail only if the environment is known unavailable for this session.
+ # Transient per-clip render failures should not poison the rest of the batch.
if _remotion_available is False:
return False
@@ -280,15 +281,15 @@ def _render_with_remotion(
# Redirect stderr to devnull to suppress Chrome/FFmpeg noise
# (avoids buffer deadlock and terminal spam)
- _devnull = open(os.devnull, 'w')
- result = subprocess.run(
- cmd,
- stdout=subprocess.PIPE,
- stderr=_devnull,
- text=True,
- timeout=600,
- cwd=project_root,
- )
+ with open(os.devnull, "w") as _devnull:
+ result = subprocess.run(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=_devnull,
+ text=True,
+ timeout=600,
+ cwd=project_root,
+ )
if result.returncode == 0 and os.path.exists(output_path):
_remotion_available = True
@@ -301,17 +302,14 @@ def _render_with_remotion(
if lines:
print(f" Remotion: {lines[-1][:120]}", flush=True)
- # Failed — disable for rest of session
- if "Error" in stdout:
- _remotion_available = False
+ print(" Remotion: falling back to ASS for this clip", flush=True)
return False
except subprocess.TimeoutExpired:
- print(" Remotion: timed out, using ASS for this session", flush=True)
- _remotion_available = False
+ print(" Remotion: timed out, using ASS for this clip", flush=True)
return False
except Exception as e:
- _remotion_available = False
+ print(" Remotion: render error, using ASS for this clip", flush=True)
return False
finally:
try:
diff --git a/backend/services/video_processor.py b/backend/services/video_processor.py
index 8b9c60b..e2b69fc 100644
--- a/backend/services/video_processor.py
+++ b/backend/services/video_processor.py
@@ -481,30 +481,54 @@ def _build_motion_blur_filter(
min_jump: int = 60,
max_window_duration: float = 0.5,
max_windows: int = 16,
- sigma: float = 5.4,
- steps: int = 2,
- pad_before: float = 0.03,
- pad_after: float = 0.05,
+ core_sigma: float = 3.8,
+ core_steps: int = 2,
+ core_pad_before: float = 0.02,
+ core_pad_after: float = 0.05,
+ outer_sigma: float = 1.6,
+ outer_steps: int = 1,
+ outer_pad_before: float = 0.05,
+ outer_pad_after: float = 0.08,
) -> str:
"""
- Add a stronger full-frame blur only while the camera is actively reframing.
+ Add a layered full-frame blur while the camera is actively reframing.
- The goal is to hide the crop move, not just soften it slightly.
+ A soft outer blur eases the transition in and out, while a tighter core
+ blur covers the actual crop move. This reads less like an abrupt on/off
+ effect than a single blur window.
"""
- windows = _expand_motion_windows(_motion_windows_from_keyframes(
+ base_windows = _motion_windows_from_keyframes(
keyframes=keyframes,
min_jump=min_jump,
max_window_duration=max_window_duration,
max_windows=max_windows,
- ), pad_before=pad_before, pad_after=pad_after)
- if not windows:
+ )
+ if not base_windows:
return ""
- enable_expr = "+".join(
+ outer_windows = _expand_motion_windows(
+ base_windows,
+ pad_before=outer_pad_before,
+ pad_after=outer_pad_after,
+ )
+ core_windows = _expand_motion_windows(
+ base_windows,
+ pad_before=core_pad_before,
+ pad_after=core_pad_after,
+ )
+
+ outer_enable = "+".join(
+ f"between(t\\,{t0:.3f}\\,{t1:.3f})"
+ for t0, t1 in outer_windows
+ )
+ core_enable = "+".join(
f"between(t\\,{t0:.3f}\\,{t1:.3f})"
- for t0, t1 in windows
+ for t0, t1 in core_windows
+ )
+ return (
+ f",gblur=sigma={outer_sigma:.1f}:steps={outer_steps}:enable='{outer_enable}'"
+ f",gblur=sigma={core_sigma:.1f}:steps={core_steps}:enable='{core_enable}'"
)
- return f",gblur=sigma={sigma:.1f}:steps={steps}:enable='{enable_expr}'"
def _build_motion_zoom_filter(
@@ -682,6 +706,8 @@ def _choose_segment_tracks(
learned_anchor_x = dict(speaker_anchor_x or {})
learned_side = dict(speaker_side or {})
last_track_for_speaker = {}
+ unique_speakers = {speaker for _, _, speaker in segments if speaker is not None}
+ allow_opposite_side_inference = len(unique_speakers) == 2
for start_t, end_t, speaker in segments:
candidates = {}
@@ -715,7 +741,7 @@ def _choose_segment_tracks(
side_hint_strength = 1.1
anchor_hint_penalty = 1.8
- if hint_side is None:
+ if hint_side is None and allow_opposite_side_inference:
other_sides = {sp: side for sp, side in learned_side.items() if sp != speaker}
if len(other_sides) == 1:
only_side = next(iter(other_sides.values()))
diff --git a/remotion/render.mjs b/remotion/render.mjs
index e136bfa..0dfa6ce 100644
--- a/remotion/render.mjs
+++ b/remotion/render.mjs
@@ -153,6 +153,11 @@ async function main() {
await new Promise((resolve) => assetServer.listen(0, "127.0.0.1", resolve));
const assetPort = assetServer.address().port;
+ // Ensure the asset server is closed on any exit path
+ const closeAssetServer = () => { try { assetServer.close(); } catch {} };
+ process.on("SIGINT", () => { closeAssetServer(); process.exit(1); });
+ process.on("SIGTERM", () => { closeAssetServer(); process.exit(1); });
+
const videoSrc = `http://127.0.0.1:${assetPort}/clip.mp4`;
const logoSrc = opts.logo ? `http://127.0.0.1:${assetPort}/logo.png` : undefined;
@@ -197,61 +202,63 @@ async function main() {
`Remotion: ${words.length} words, ${styleName}, ${renderW}x${renderH}, ${durationInFrames}f`
);
- // Select composition
- const composition = await selectComposition({
- serveUrl: bundleLocation,
- id: "CaptionedClip",
- inputProps,
- });
-
- // Render captions as transparent WebM overlay (no video = 10x faster)
- const cpus = os.cpus().length;
- const concurrency = Math.max(2, Math.min(cpus, 8));
- // Store overlay in /tmp to survive temp dir cleanup
- const overlayId = path.basename(opts.output, ".mp4");
- const captionOverlay = path.join(os.tmpdir(), `remotion_overlay_${overlayId}.mov`);
+ try {
+ // Select composition
+ const composition = await selectComposition({
+ serveUrl: bundleLocation,
+ id: "CaptionedClip",
+ inputProps,
+ });
- let lastPct = -1;
- await renderMedia({
- composition: {
- ...composition,
- durationInFrames,
- fps,
- width: renderW,
- height: renderH,
- },
- serveUrl: bundleLocation,
- codec: "prores",
- proResProfile: "4444",
- pixelFormat: "yuva444p10le",
- imageFormat: "png",
- outputLocation: captionOverlay,
- inputProps,
- concurrency,
- onProgress: ({ progress }) => {
- const pct = Math.round(progress * 100);
- if (pct > lastPct + 9) {
- lastPct = pct;
- process.stderr.write(` captions: ${pct}%\n`);
- }
- },
- });
+ // Render captions as transparent WebM overlay (no video = 10x faster)
+ const cpus = os.cpus().length;
+ const concurrency = Math.max(2, Math.min(cpus, 8));
+ // Store overlay in /tmp to survive temp dir cleanup
+ const overlayId = path.basename(opts.output, ".mp4");
+ const captionOverlay = path.join(os.tmpdir(), `remotion_overlay_${overlayId}.mov`);
- assetServer.close();
+ let lastPct = -1;
+ await renderMedia({
+ composition: {
+ ...composition,
+ durationInFrames,
+ fps,
+ width: renderW,
+ height: renderH,
+ },
+ serveUrl: bundleLocation,
+ codec: "prores",
+ proResProfile: "4444",
+ pixelFormat: "yuva444p10le",
+ imageFormat: "png",
+ outputLocation: captionOverlay,
+ inputProps,
+ concurrency,
+ onProgress: ({ progress }) => {
+ const pct = Math.round(progress * 100);
+ if (pct > lastPct + 9) {
+ lastPct = pct;
+ process.stderr.write(` captions: ${pct}%\n`);
+ }
+ },
+ });
- // Composite: overlay transparent captions (ProRes 4444 with alpha) onto video
- const { execSync } = await import("child_process");
- process.stderr.write(" compositing...\n");
- execSync(
- `ffmpeg -y -hide_banner -loglevel warning -i "${path.resolve(opts.video)}" -i "${captionOverlay}" ` +
- `-filter_complex "[0:v][1:v]overlay=0:0:shortest=1" ` +
- `-c:v libx264 -crf 18 -preset fast -map 0:a -c:a copy "${opts.output}"`,
- { stdio: ["pipe", "pipe", "pipe"], timeout: 300000 }
- );
+ // Composite: overlay transparent captions (ProRes 4444 with alpha) onto video
+ const { execSync } = await import("child_process");
+ process.stderr.write(" compositing...\n");
+ execSync(
+ `ffmpeg -y -hide_banner -loglevel warning -i "${path.resolve(opts.video)}" -i "${captionOverlay}" ` +
+ `-filter_complex "[0:v][1:v]overlay=0:0:shortest=1" ` +
+ `-c:v libx264 -crf 18 -preset fast -map 0:a -c:a copy "${opts.output}"`,
+ { stdio: ["pipe", "pipe", "pipe"], timeout: 300000 }
+ );
- // Clean up overlay
- try { fs.unlinkSync(captionOverlay); } catch {}
- console.log(`Done: ${opts.output}`);
+ // Clean up overlay
+ try { fs.unlinkSync(captionOverlay); } catch {}
+ console.log(`Done: ${opts.output}`);
+ } finally {
+ closeAssetServer();
+ }
}
main().catch((err) => {
diff --git a/tests/test_clip_generator.py b/tests/test_clip_generator.py
new file mode 100644
index 0000000..330894c
--- /dev/null
+++ b/tests/test_clip_generator.py
@@ -0,0 +1,103 @@
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+from unittest import mock
+
+
+ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+BACKEND_ROOT = os.path.join(ROOT, "backend")
+if BACKEND_ROOT not in sys.path:
+ sys.path.insert(0, BACKEND_ROOT)
+
+from services import clip_generator as cg
+
+
+class ClipGeneratorTests(unittest.TestCase):
+ def setUp(self):
+ self._orig_remotion_available = cg._remotion_available
+ cg._remotion_available = None
+
+ def tearDown(self):
+ cg._remotion_available = self._orig_remotion_available
+
+ def _fake_exists(self, real_exists):
+ def _exists(path):
+ if path.endswith(os.path.join("remotion", "render.mjs")):
+ return True
+ if path.endswith(os.path.join("remotion-bundle", "index.html")):
+ return True
+ return real_exists(path)
+ return _exists
+
+ def test_remotion_runtime_failure_does_not_disable_future_clips(self):
+ real_exists = os.path.exists
+ fail_result = subprocess.CompletedProcess(
+ args=["node"],
+ returncode=1,
+ stdout="Error: transient render failure",
+ stderr="",
+ )
+
+ with tempfile.TemporaryDirectory() as td:
+ video_path = os.path.join(td, "video.mp4")
+ output_path = os.path.join(td, "captioned.mp4")
+ with open(video_path, "wb"):
+ pass
+
+ with mock.patch.object(cg.os.path, "exists", side_effect=self._fake_exists(real_exists)), \
+ mock.patch.object(cg.shutil, "which", return_value="/usr/bin/node"), \
+ mock.patch("subprocess.run", return_value=fail_result) as mock_run:
+ first = cg._render_with_remotion(
+ video_path=video_path,
+ words=[{"word": "hello", "start": 0.0, "end": 0.5}],
+ caption_style="branded",
+ output_path=output_path,
+ )
+ second = cg._render_with_remotion(
+ video_path=video_path,
+ words=[{"word": "world", "start": 0.5, "end": 1.0}],
+ caption_style="branded",
+ output_path=output_path,
+ )
+
+ self.assertFalse(first)
+ self.assertFalse(second)
+ self.assertIsNone(cg._remotion_available)
+ self.assertGreaterEqual(mock_run.call_count, 4)
+
+ def test_remotion_timeout_does_not_disable_future_clips(self):
+ real_exists = os.path.exists
+
+ with tempfile.TemporaryDirectory() as td:
+ video_path = os.path.join(td, "video.mp4")
+ output_path = os.path.join(td, "captioned.mp4")
+ with open(video_path, "wb"):
+ pass
+
+ timeout_exc = subprocess.TimeoutExpired(cmd=["node"], timeout=600)
+ with mock.patch.object(cg.os.path, "exists", side_effect=self._fake_exists(real_exists)), \
+ mock.patch.object(cg.shutil, "which", return_value="/usr/bin/node"), \
+ mock.patch("subprocess.run", side_effect=timeout_exc) as mock_run:
+ first = cg._render_with_remotion(
+ video_path=video_path,
+ words=[{"word": "hello", "start": 0.0, "end": 0.5}],
+ caption_style="branded",
+ output_path=output_path,
+ )
+ second = cg._render_with_remotion(
+ video_path=video_path,
+ words=[{"word": "world", "start": 0.5, "end": 1.0}],
+ caption_style="branded",
+ output_path=output_path,
+ )
+
+ self.assertFalse(first)
+ self.assertFalse(second)
+ self.assertIsNone(cg._remotion_available)
+ self.assertGreaterEqual(mock_run.call_count, 4)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/tests/test_video_processor.py b/tests/test_video_processor.py
index d8d0d9b..8752c8a 100644
--- a/tests/test_video_processor.py
+++ b/tests/test_video_processor.py
@@ -124,6 +124,31 @@ class VideoProcessorTests(unittest.TestCase):
self.assertEqual(segment_tracks[1][3], 1)
self.assertEqual(sides["SPEAKER_01"], "right")
+ def test_choose_segment_tracks_does_not_infer_opposite_side_for_third_speaker(self):
+ tracked_detections = [
+ (0.0, [{"cx": 520, "fw": 350, "track_id": 0}]),
+ (0.4, [{"cx": 540, "fw": 346, "track_id": 0}]),
+ (1.4, [{"cx": 560, "fw": 220, "track_id": 2}]),
+ (1.8, [{"cx": 572, "fw": 222, "track_id": 2}, {"cx": 1495, "fw": 350, "track_id": 3}]),
+ (2.2, [{"cx": 580, "fw": 226, "track_id": 2}, {"cx": 1484, "fw": 352, "track_id": 3}]),
+ ]
+
+ segment_tracks, anchors, sides = vp._choose_segment_tracks(
+ segments=[
+ (0.0, 0.8, "SPEAKER_00"),
+ (0.8, 1.2, "SPEAKER_01"),
+ (1.2, 2.4, "SPEAKER_02"),
+ ],
+ tracked_detections=tracked_detections,
+ speaker_side={},
+ speaker_anchor_x={},
+ width=1920,
+ )
+
+ self.assertEqual(segment_tracks[2][3], 2)
+ self.assertLess(anchors["SPEAKER_02"], 960)
+ self.assertEqual(sides["SPEAKER_02"], "left")
+
def test_choose_track_segment_targets_picks_one_stable_target_per_turn(self):
targets = vp._choose_track_segment_targets(
segment_tracks=[
@@ -467,7 +492,7 @@ class VideoProcessorTests(unittest.TestCase):
self.assertEqual(
blur,
- ",gblur=sigma=5.4:steps=2:enable='between(t\\,0.000\\,0.230)'",
+ ",gblur=sigma=1.6:steps=1:enable='between(t\\,0.000\\,0.260)',gblur=sigma=3.8:steps=2:enable='between(t\\,0.000\\,0.230)'",
)
def test_build_motion_zoom_filter_targets_only_short_reframes(self):