Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.
Python ai-toolsauto-captionscaptionscliffmpegmcpmcp-serveropusclip-alternativepodcast-clipspodcastsshortform-videovertical-videowhisper
Harden batch rendering and speaker tracking
Nika Siradzed87ea2229b3b
diff --git a/backend/scripts/remotion_screenshot.cjs b/backend/scripts/remotion_screenshot.cjsindex db219f3..e6f2f9b 100644--- a/backend/scripts/remotion_screenshot.cjs+++ b/backend/scripts/remotion_screenshot.cjs@@ -47,6 +47,9 @@ const waitForAssets = async (page) => {(async () => {const browser = await openBrowser("chrome", {logLevel: "error"});+ const closeBrowser = () => { try { browser.close({silent: true}); } catch {} };+ process.on("SIGINT", () => { closeBrowser(); process.exit(1); });+ process.on("SIGTERM", () => { closeBrowser(); process.exit(1); });try {const page = await browser.newPage({context: () => null,diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.pyindex acba080..f373543 100644--- a/backend/services/clip_generator.py+++ b/backend/services/clip_generator.py@@ -159,7 +159,7 @@ def _build_tight_segments(return segments-_remotion_available = None # True/False/None — auto-detected on first render+_remotion_available = None # True/False/None — environment availability, not per-clip successdef _render_with_remotion(@@ -177,7 +177,8 @@ def _render_with_remotion(import subprocessimport json- # Quick bail if Remotion already failed this session+ # Quick bail only if the environment is known unavailable for this session.+ # Transient per-clip render failures should not poison the rest of the batch.if _remotion_available is False:return False@@ -280,15 +281,15 @@ def _render_with_remotion(# Redirect stderr to devnull to suppress Chrome/FFmpeg noise# (avoids buffer deadlock and terminal spam)- _devnull = open(os.devnull, 'w')- result = subprocess.run(- cmd,- stdout=subprocess.PIPE,- stderr=_devnull,- text=True,- timeout=600,- cwd=project_root,- )+ with open(os.devnull, "w") as _devnull:+ result = subprocess.run(+ cmd,+ stdout=subprocess.PIPE,+ stderr=_devnull,+ text=True,+ timeout=600,+ cwd=project_root,+ )if result.returncode == 0 and os.path.exists(output_path):_remotion_available = True@@ -301,17 +302,14 @@ def _render_with_remotion(if lines:print(f" Remotion: {lines[-1][:120]}", flush=True)- # Failed — disable for rest of session- if "Error" in stdout:- _remotion_available = False+ print(" Remotion: falling back to ASS for this clip", flush=True)return Falseexcept subprocess.TimeoutExpired:- print(" Remotion: timed out, using ASS for this session", flush=True)- _remotion_available = False+ print(" Remotion: timed out, using ASS for this clip", flush=True)return Falseexcept Exception as e:- _remotion_available = False+ print(" Remotion: render error, using ASS for this clip", flush=True)return Falsefinally:try:diff --git a/backend/services/video_processor.py b/backend/services/video_processor.pyindex 8b9c60b..e2b69fc 100644--- a/backend/services/video_processor.py+++ b/backend/services/video_processor.py@@ -481,30 +481,54 @@ def _build_motion_blur_filter(min_jump: int = 60,max_window_duration: float = 0.5,max_windows: int = 16,- sigma: float = 5.4,- steps: int = 2,- pad_before: float = 0.03,- pad_after: float = 0.05,+ core_sigma: float = 3.8,+ core_steps: int = 2,+ core_pad_before: float = 0.02,+ core_pad_after: float = 0.05,+ outer_sigma: float = 1.6,+ outer_steps: int = 1,+ outer_pad_before: float = 0.05,+ outer_pad_after: float = 0.08,) -> str:"""- Add a stronger full-frame blur only while the camera is actively reframing.+ Add a layered full-frame blur while the camera is actively reframing.- The goal is to hide the crop move, not just soften it slightly.+ A soft outer blur eases the transition in and out, while a tighter core+ blur covers the actual crop move. This reads less like an abrupt on/off+ effect than a single blur window."""- windows = _expand_motion_windows(_motion_windows_from_keyframes(+ base_windows = _motion_windows_from_keyframes(keyframes=keyframes,min_jump=min_jump,max_window_duration=max_window_duration,max_windows=max_windows,- ), pad_before=pad_before, pad_after=pad_after)- if not windows:+ )+ if not base_windows:return ""- enable_expr = "+".join(+ outer_windows = _expand_motion_windows(+ base_windows,+ pad_before=outer_pad_before,+ pad_after=outer_pad_after,+ )+ core_windows = _expand_motion_windows(+ base_windows,+ pad_before=core_pad_before,+ pad_after=core_pad_after,+ )++ outer_enable = "+".join(+ f"between(t\\,{t0:.3f}\\,{t1:.3f})"+ for t0, t1 in outer_windows+ )+ core_enable = "+".join(f"between(t\\,{t0:.3f}\\,{t1:.3f})"- for t0, t1 in windows+ for t0, t1 in core_windows+ )+ return (+ f",gblur=sigma={outer_sigma:.1f}:steps={outer_steps}:enable='{outer_enable}'"+ f",gblur=sigma={core_sigma:.1f}:steps={core_steps}:enable='{core_enable}'")- return f",gblur=sigma={sigma:.1f}:steps={steps}:enable='{enable_expr}'"def _build_motion_zoom_filter(@@ -682,6 +706,8 @@ def _choose_segment_tracks(learned_anchor_x = dict(speaker_anchor_x or {})learned_side = dict(speaker_side or {})last_track_for_speaker = {}+ unique_speakers = {speaker for _, _, speaker in segments if speaker is not None}+ allow_opposite_side_inference = len(unique_speakers) == 2for start_t, end_t, speaker in segments:candidates = {}@@ -715,7 +741,7 @@ def _choose_segment_tracks(side_hint_strength = 1.1anchor_hint_penalty = 1.8- if hint_side is None:+ if hint_side is None and allow_opposite_side_inference:other_sides = {sp: side for sp, side in learned_side.items() if sp != speaker}if len(other_sides) == 1:only_side = next(iter(other_sides.values()))diff --git a/remotion/render.mjs b/remotion/render.mjsindex e136bfa..0dfa6ce 100644--- a/remotion/render.mjs+++ b/remotion/render.mjs@@ -153,6 +153,11 @@ async function main() {await new Promise((resolve) => assetServer.listen(0, "127.0.0.1", resolve));const assetPort = assetServer.address().port;+ // Ensure the asset server is closed on any exit path+ const closeAssetServer = () => { try { assetServer.close(); } catch {} };+ process.on("SIGINT", () => { closeAssetServer(); process.exit(1); });+ process.on("SIGTERM", () => { closeAssetServer(); process.exit(1); });+const videoSrc = `http://127.0.0.1:${assetPort}/clip.mp4`;const logoSrc = opts.logo ? `http://127.0.0.1:${assetPort}/logo.png` : undefined;@@ -197,61 +202,63 @@ async function main() {`Remotion: ${words.length} words, ${styleName}, ${renderW}x${renderH}, ${durationInFrames}f`);- // Select composition- const composition = await selectComposition({- serveUrl: bundleLocation,- id: "CaptionedClip",- inputProps,- });-- // Render captions as transparent WebM overlay (no video = 10x faster)- const cpus = os.cpus().length;- const concurrency = Math.max(2, Math.min(cpus, 8));- // Store overlay in /tmp to survive temp dir cleanup- const overlayId = path.basename(opts.output, ".mp4");- const captionOverlay = path.join(os.tmpdir(), `remotion_overlay_${overlayId}.mov`);+ try {+ // Select composition+ const composition = await selectComposition({+ serveUrl: bundleLocation,+ id: "CaptionedClip",+ inputProps,+ });- let lastPct = -1;- await renderMedia({- composition: {- ...composition,- durationInFrames,- fps,- width: renderW,- height: renderH,- },- serveUrl: bundleLocation,- codec: "prores",- proResProfile: "4444",- pixelFormat: "yuva444p10le",- imageFormat: "png",- outputLocation: captionOverlay,- inputProps,- concurrency,- onProgress: ({ progress }) => {- const pct = Math.round(progress * 100);- if (pct > lastPct + 9) {- lastPct = pct;- process.stderr.write(` captions: ${pct}%\n`);- }- },- });+ // Render captions as transparent WebM overlay (no video = 10x faster)+ const cpus = os.cpus().length;+ const concurrency = Math.max(2, Math.min(cpus, 8));+ // Store overlay in /tmp to survive temp dir cleanup+ const overlayId = path.basename(opts.output, ".mp4");+ const captionOverlay = path.join(os.tmpdir(), `remotion_overlay_${overlayId}.mov`);- assetServer.close();+ let lastPct = -1;+ await renderMedia({+ composition: {+ ...composition,+ durationInFrames,+ fps,+ width: renderW,+ height: renderH,+ },+ serveUrl: bundleLocation,+ codec: "prores",+ proResProfile: "4444",+ pixelFormat: "yuva444p10le",+ imageFormat: "png",+ outputLocation: captionOverlay,+ inputProps,+ concurrency,+ onProgress: ({ progress }) => {+ const pct = Math.round(progress * 100);+ if (pct > lastPct + 9) {+ lastPct = pct;+ process.stderr.write(` captions: ${pct}%\n`);+ }+ },+ });- // Composite: overlay transparent captions (ProRes 4444 with alpha) onto video- const { execSync } = await import("child_process");- process.stderr.write(" compositing...\n");- execSync(- `ffmpeg -y -hide_banner -loglevel warning -i "${path.resolve(opts.video)}" -i "${captionOverlay}" ` +- `-filter_complex "[0:v][1:v]overlay=0:0:shortest=1" ` +- `-c:v libx264 -crf 18 -preset fast -map 0:a -c:a copy "${opts.output}"`,- { stdio: ["pipe", "pipe", "pipe"], timeout: 300000 }- );+ // Composite: overlay transparent captions (ProRes 4444 with alpha) onto video+ const { execSync } = await import("child_process");+ process.stderr.write(" compositing...\n");+ execSync(+ `ffmpeg -y -hide_banner -loglevel warning -i "${path.resolve(opts.video)}" -i "${captionOverlay}" ` ++ `-filter_complex "[0:v][1:v]overlay=0:0:shortest=1" ` ++ `-c:v libx264 -crf 18 -preset fast -map 0:a -c:a copy "${opts.output}"`,+ { stdio: ["pipe", "pipe", "pipe"], timeout: 300000 }+ );- // Clean up overlay- try { fs.unlinkSync(captionOverlay); } catch {}- console.log(`Done: ${opts.output}`);+ // Clean up overlay+ try { fs.unlinkSync(captionOverlay); } catch {}+ console.log(`Done: ${opts.output}`);+ } finally {+ closeAssetServer();+ }}main().catch((err) => {diff --git a/tests/test_clip_generator.py b/tests/test_clip_generator.pynew file mode 100644index 0000000..330894c--- /dev/null+++ b/tests/test_clip_generator.py@@ -0,0 +1,103 @@+import os+import subprocess+import sys+import tempfile+import unittest+from unittest import mock+++ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))+BACKEND_ROOT = os.path.join(ROOT, "backend")+if BACKEND_ROOT not in sys.path:+ sys.path.insert(0, BACKEND_ROOT)++from services import clip_generator as cg+++class ClipGeneratorTests(unittest.TestCase):+ def setUp(self):+ self._orig_remotion_available = cg._remotion_available+ cg._remotion_available = None++ def tearDown(self):+ cg._remotion_available = self._orig_remotion_available++ def _fake_exists(self, real_exists):+ def _exists(path):+ if path.endswith(os.path.join("remotion", "render.mjs")):+ return True+ if path.endswith(os.path.join("remotion-bundle", "index.html")):+ return True+ return real_exists(path)+ return _exists++ def test_remotion_runtime_failure_does_not_disable_future_clips(self):+ real_exists = os.path.exists+ fail_result = subprocess.CompletedProcess(+ args=["node"],+ returncode=1,+ stdout="Error: transient render failure",+ stderr="",+ )++ with tempfile.TemporaryDirectory() as td:+ video_path = os.path.join(td, "video.mp4")+ output_path = os.path.join(td, "captioned.mp4")+ with open(video_path, "wb"):+ pass++ with mock.patch.object(cg.os.path, "exists", side_effect=self._fake_exists(real_exists)), \+ mock.patch.object(cg.shutil, "which", return_value="/usr/bin/node"), \+ mock.patch("subprocess.run", return_value=fail_result) as mock_run:+ first = cg._render_with_remotion(+ video_path=video_path,+ words=[{"word": "hello", "start": 0.0, "end": 0.5}],+ caption_style="branded",+ output_path=output_path,+ )+ second = cg._render_with_remotion(+ video_path=video_path,+ words=[{"word": "world", "start": 0.5, "end": 1.0}],+ caption_style="branded",+ output_path=output_path,+ )++ self.assertFalse(first)+ self.assertFalse(second)+ self.assertIsNone(cg._remotion_available)+ self.assertGreaterEqual(mock_run.call_count, 4)++ def test_remotion_timeout_does_not_disable_future_clips(self):+ real_exists = os.path.exists++ with tempfile.TemporaryDirectory() as td:+ video_path = os.path.join(td, "video.mp4")+ output_path = os.path.join(td, "captioned.mp4")+ with open(video_path, "wb"):+ pass++ timeout_exc = subprocess.TimeoutExpired(cmd=["node"], timeout=600)+ with mock.patch.object(cg.os.path, "exists", side_effect=self._fake_exists(real_exists)), \+ mock.patch.object(cg.shutil, "which", return_value="/usr/bin/node"), \+ mock.patch("subprocess.run", side_effect=timeout_exc) as mock_run:+ first = cg._render_with_remotion(+ video_path=video_path,+ words=[{"word": "hello", "start": 0.0, "end": 0.5}],+ caption_style="branded",+ output_path=output_path,+ )+ second = cg._render_with_remotion(+ video_path=video_path,+ words=[{"word": "world", "start": 0.5, "end": 1.0}],+ caption_style="branded",+ output_path=output_path,+ )++ self.assertFalse(first)+ self.assertFalse(second)+ self.assertIsNone(cg._remotion_available)+ self.assertGreaterEqual(mock_run.call_count, 4)+++if __name__ == "__main__":+ unittest.main()diff --git a/tests/test_video_processor.py b/tests/test_video_processor.pyindex d8d0d9b..8752c8a 100644--- a/tests/test_video_processor.py+++ b/tests/test_video_processor.py@@ -124,6 +124,31 @@ class VideoProcessorTests(unittest.TestCase):self.assertEqual(segment_tracks[1][3], 1)self.assertEqual(sides["SPEAKER_01"], "right")+ def test_choose_segment_tracks_does_not_infer_opposite_side_for_third_speaker(self):+ tracked_detections = [+ (0.0, [{"cx": 520, "fw": 350, "track_id": 0}]),+ (0.4, [{"cx": 540, "fw": 346, "track_id": 0}]),+ (1.4, [{"cx": 560, "fw": 220, "track_id": 2}]),+ (1.8, [{"cx": 572, "fw": 222, "track_id": 2}, {"cx": 1495, "fw": 350, "track_id": 3}]),+ (2.2, [{"cx": 580, "fw": 226, "track_id": 2}, {"cx": 1484, "fw": 352, "track_id": 3}]),+ ]++ segment_tracks, anchors, sides = vp._choose_segment_tracks(+ segments=[+ (0.0, 0.8, "SPEAKER_00"),+ (0.8, 1.2, "SPEAKER_01"),+ (1.2, 2.4, "SPEAKER_02"),+ ],+ tracked_detections=tracked_detections,+ speaker_side={},+ speaker_anchor_x={},+ width=1920,+ )++ self.assertEqual(segment_tracks[2][3], 2)+ self.assertLess(anchors["SPEAKER_02"], 960)+ self.assertEqual(sides["SPEAKER_02"], "left")+def test_choose_track_segment_targets_picks_one_stable_target_per_turn(self):targets = vp._choose_track_segment_targets(segment_tracks=[@@ -467,7 +492,7 @@ class VideoProcessorTests(unittest.TestCase):self.assertEqual(blur,- ",gblur=sigma=5.4:steps=2:enable='between(t\\,0.000\\,0.230)'",+ ",gblur=sigma=1.6:steps=1:enable='between(t\\,0.000\\,0.260)',gblur=sigma=3.8:steps=2:enable='between(t\\,0.000\\,0.230)'",)def test_build_motion_zoom_filter_targets_only_short_reframes(self):