Skip to content

Open-source AI podcast clipper. Generate vertical clips with face tracking and burned-in captions. CLI, MCP server, web app.

Bound the compositor's frame cache and stream render progress (#204)

Nika Siradzeb393cebab1c3
diff --git a/backend/services/clip_generator.py b/backend/services/clip_generator.py
index d2ec2f5..90d4492 100644
--- a/backend/services/clip_generator.py
+++ b/backend/services/clip_generator.py
@@ -9,6 +9,7 @@ import os
import sys
import tempfile
import shutil
+import signal
import subprocess
import re
import threading
@@ -473,6 +474,96 @@ def _build_tight_segments(
_remotion_available = None # True/False/None — environment availability, not per-clip success
+def _stop_process_tree(proc: subprocess.Popen) -> None:
+ """Stop a render and everything it spawned.
+
+ proc.kill() reaches only Node. Chromium and the compositor are its
+ children, and a render that has timed out is exactly the one whose
+ children are still busy, holding the CPU the next job needs. On POSIX the
+ child is started in its own session so the whole group can be signalled:
+ TERM first, which render.mjs handles by closing its server and deleting
+ the half-written overlay, then KILL for whatever ignored it.
+ """
+ if hasattr(os, "killpg"):
+ try:
+ pgid = os.getpgid(proc.pid)
+ os.killpg(pgid, signal.SIGTERM)
+ try:
+ proc.wait(timeout=5)
+ return
+ except subprocess.TimeoutExpired:
+ pass
+ os.killpg(pgid, signal.SIGKILL)
+ proc.wait()
+ return
+ except OSError:
+ pass
+ proc.kill()
+ proc.wait()
+
+
+def _run_streaming(
+ cmd: list[str],
+ *,
+ timeout: float,
+ cwd: str,
+ env: dict,
+) -> subprocess.CompletedProcess:
+ """Run a render and relay its progress as it happens.
+
+ subprocess.run holds both pipes until the child exits, so for the whole of
+ a Remotion render this process printed nothing. The worker upstream reads
+ silence as a wedged engine and stops it, and a render that had finally
+ started completing was being killed at the ten-minute mark for working.
+
+ Both streams are drained on threads so neither pipe can fill and stall the
+ child, and every line is kept: the caller still gets a CompletedProcess to
+ report from exactly as before. Progress lines go straight to stderr with
+ the same flush the rest of this file uses, which is what the worker watches.
+ """
+ proc = subprocess.Popen(
+ cmd,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE,
+ text=True,
+ cwd=cwd,
+ env=env,
+ start_new_session=(os.name == "posix"),
+ )
+ captured: dict[str, list[str]] = {"out": [], "err": []}
+
+ def pump(stream, label: str) -> None:
+ for line in iter(stream.readline, ""):
+ captured[label].append(line)
+ text = line.rstrip()
+ if text:
+ print(f" Remotion: {text[:200]}", file=sys.stderr, flush=True)
+ stream.close()
+
+ threads = [
+ threading.Thread(target=pump, args=(proc.stdout, "out"), daemon=True),
+ threading.Thread(target=pump, args=(proc.stderr, "err"), daemon=True),
+ ]
+ for t in threads:
+ t.start()
+ try:
+ proc.wait(timeout=timeout)
+ except subprocess.TimeoutExpired:
+ _stop_process_tree(proc)
+ for t in threads:
+ t.join(timeout=5)
+ raise subprocess.TimeoutExpired(
+ cmd, timeout,
+ output="".join(captured["out"]),
+ stderr="".join(captured["err"]),
+ )
+ for t in threads:
+ t.join(timeout=5)
+ return subprocess.CompletedProcess(
+ cmd, proc.returncode, "".join(captured["out"]), "".join(captured["err"]),
+ )
+
+
def _report_remotion_failure(
stage: str,
stdout: str | bytes | None,
@@ -712,11 +803,8 @@ def _render_with_remotion(
# is genuinely just slow times out inside Remotion with an actionable
# message, rather than being hard-killed here first with nothing but
# "TimeoutExpired" to say why.
- result = subprocess.run(
+ result = _run_streaming(
cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.PIPE,
- text=True,
timeout=2 * 50 * 60 + 300,
cwd=project_root,
env=remotion_env,
diff --git a/remotion/render.mjs b/remotion/render.mjs
index c965b8d..1bf4dee 100644
--- a/remotion/render.mjs
+++ b/remotion/render.mjs
@@ -82,6 +82,22 @@ const timeoutFor = (durationSec) => {
return Math.round(Math.min(50 * 60_000, Math.max(120_000, 90_000 + durationSec * 15_000)));
};
+/*
+ * The compositor caches decoded <OffthreadVideo> frames, and left to itself
+ * sizes that cache at half of the memory "available on the system": the host's,
+ * not the container's cgroup limit, so a worker capped at 3GB was handed a
+ * ~4GB cache target and the kernel killed the compositor at 2.7GB resident,
+ * mid-render, on every clip that drew a card with the speaker still in frame.
+ * A bound the container can actually honour makes peak memory a constant
+ * rather than a function of whichever box the render happens to land on.
+ * Overridable in MB for a box whose real headroom has been measured.
+ */
+const videoCacheBytes = () => {
+ const mb = parseInt(process.env.PODCLI_REMOTION_VIDEO_CACHE_MB ?? "", 10);
+ return (Number.isFinite(mb) && mb > 0 ? mb : 512) * 1024 * 1024;
+};
+const offthreadVideoCacheSizeInBytes = videoCacheBytes();
+
async function main() {
const opts = parseArgs();
@@ -320,6 +336,7 @@ async function main() {
id: "CaptionedClip",
inputProps,
timeoutInMilliseconds: timeoutFor(durationSec),
+ offthreadVideoCacheSizeInBytes,
browserExecutable,
chromeMode,
});
@@ -363,6 +380,7 @@ async function main() {
inputProps,
concurrency,
timeoutInMilliseconds: timeoutFor(durationSec),
+ offthreadVideoCacheSizeInBytes,
onProgress: ({ progress }) => {
const pct = Math.round(progress * 100);
if (pct > lastPct + 9) {
diff --git a/tests/test_clip_generator.py b/tests/test_clip_generator.py
index bc1e593..e3fea25 100644
--- a/tests/test_clip_generator.py
+++ b/tests/test_clip_generator.py
@@ -36,6 +36,23 @@ class ClipGeneratorTests(unittest.TestCase):
return real_exists(path)
return _exists
+ def _fake_popen(self, returncode=0, stdout="", stderr="", timeout=None):
+ """A Popen stand-in for _run_streaming: readable pipes, wait(), kill().
+
+ Fresh per call, because the streaming reader drains and closes the pipes.
+ A timeout makes the first wait() raise the way a real child would, and
+ the second, after kill(), return.
+ """
+ proc = mock.Mock()
+ proc.stdout = io.StringIO(stdout)
+ proc.stderr = io.StringIO(stderr)
+ proc.returncode = returncode
+ if timeout is not None:
+ proc.wait.side_effect = [subprocess.TimeoutExpired(cmd=["node"], timeout=timeout), None]
+ else:
+ proc.wait.return_value = returncode
+ return proc
+
def test_kept_caption_overlay_path_matches_remotion_contract(self):
with tempfile.TemporaryDirectory() as td:
output_path = os.path.join(td, "captioned.mp4")
@@ -46,7 +63,6 @@ class ClipGeneratorTests(unittest.TestCase):
def _render_args(self, **kwargs):
"""The argv one Remotion render was invoked with."""
real_exists = os.path.exists
- ok = subprocess.CompletedProcess(args=["node"], returncode=0, stdout="", stderr="")
with tempfile.TemporaryDirectory() as td:
video_path = os.path.join(td, "video.mp4")
@@ -58,7 +74,7 @@ class ClipGeneratorTests(unittest.TestCase):
with mock.patch.object(cg.os.path, "exists", side_effect=self._fake_exists(real_exists)), \
mock.patch.object(cg.shutil, "which", return_value="/usr/bin/node"), \
- mock.patch("subprocess.run", return_value=ok) as mock_run:
+ mock.patch("subprocess.Popen", side_effect=lambda *a, **kw: self._fake_popen()) as mock_run:
cg._render_with_remotion(
video_path=video_path,
words=[{"word": "hello", "start": 0.0, "end": 0.5}],
@@ -97,7 +113,7 @@ class ClipGeneratorTests(unittest.TestCase):
if "--words" in args:
with open(args[args.index("--words") + 1]) as fh:
seen.update(json.load(fh))
- return subprocess.CompletedProcess(args=args, returncode=0, stdout="", stderr="")
+ return self._fake_popen()
with tempfile.TemporaryDirectory() as td:
video_path = os.path.join(td, "video.mp4")
@@ -108,7 +124,7 @@ class ClipGeneratorTests(unittest.TestCase):
with mock.patch.object(cg.os.path, "exists", side_effect=self._fake_exists(real_exists)), \
mock.patch.object(cg.shutil, "which", return_value="/usr/bin/node"), \
- mock.patch("subprocess.run", side_effect=capture):
+ mock.patch("subprocess.Popen", side_effect=capture):
cg._render_with_remotion(
video_path=video_path,
words=[{"word": "hello", "start": 0.0, "end": 0.5}],
@@ -144,8 +160,6 @@ class ClipGeneratorTests(unittest.TestCase):
report success for a clip missing everything it was asked for.
"""
real_exists = os.path.exists
- failed = subprocess.CompletedProcess(
- args=["node"], returncode=1, stdout="", stderr="boom")
with tempfile.TemporaryDirectory() as td:
video_path = os.path.join(td, "video.mp4")
@@ -155,7 +169,8 @@ class ClipGeneratorTests(unittest.TestCase):
with mock.patch.object(cg.os.path, "exists",
side_effect=self._fake_exists(real_exists)), \
mock.patch.object(cg.shutil, "which", return_value="/usr/bin/node"), \
- mock.patch("subprocess.run", return_value=failed):
+ mock.patch("subprocess.Popen",
+ side_effect=lambda *a, **kw: self._fake_popen(returncode=1, stderr="boom")):
ok, _ = cg._render_with_remotion(
video_path=video_path,
words=[],
@@ -216,12 +231,6 @@ class ClipGeneratorTests(unittest.TestCase):
def test_remotion_runtime_failure_does_not_disable_future_clips(self):
real_exists = os.path.exists
- fail_result = subprocess.CompletedProcess(
- args=["node"],
- returncode=1,
- stdout="Error: transient render failure",
- stderr="",
- )
with tempfile.TemporaryDirectory() as td:
video_path = os.path.join(td, "video.mp4")
@@ -231,7 +240,9 @@ class ClipGeneratorTests(unittest.TestCase):
with mock.patch.object(cg.os.path, "exists", side_effect=self._fake_exists(real_exists)), \
mock.patch.object(cg.shutil, "which", return_value="/usr/bin/node"), \
- mock.patch("subprocess.run", return_value=fail_result) as mock_run:
+ mock.patch("subprocess.Popen",
+ side_effect=lambda *a, **kw: self._fake_popen(
+ returncode=1, stdout="Error: transient render failure")) as mock_run:
first = cg._render_with_remotion(
video_path=video_path,
words=[{"word": "hello", "start": 0.0, "end": 0.5}],
@@ -259,10 +270,12 @@ class ClipGeneratorTests(unittest.TestCase):
with open(video_path, "wb"):
pass
- timeout_exc = subprocess.TimeoutExpired(cmd=["node"], timeout=600)
with mock.patch.object(cg.os.path, "exists", side_effect=self._fake_exists(real_exists)), \
mock.patch.object(cg.shutil, "which", return_value="/usr/bin/node"), \
- mock.patch("subprocess.run", side_effect=timeout_exc) as mock_run:
+ mock.patch.object(cg.os, "getpgid", return_value=4242, create=True), \
+ mock.patch.object(cg.os, "killpg", create=True) as mock_killpg, \
+ mock.patch("subprocess.Popen",
+ side_effect=lambda *a, **kw: self._fake_popen(timeout=600)) as mock_run:
first = cg._render_with_remotion(
video_path=video_path,
words=[{"word": "hello", "start": 0.0, "end": 0.5}],
@@ -280,6 +293,10 @@ class ClipGeneratorTests(unittest.TestCase):
self.assertEqual(second, (False, None))
self.assertIsNone(cg._remotion_available)
self.assertGreaterEqual(mock_run.call_count, 4)
+ # The whole group, not just node: Chromium and the compositor are its
+ # children and are what a timed-out render leaves busy.
+ self.assertTrue(mock_killpg.called, "a timed-out render must signal its process group")
+ self.assertEqual(mock_killpg.call_args_list[0].args, (4242, cg.signal.SIGTERM))
def test_remotion_failure_report_takes_bytes_as_well_as_text(self):
# TimeoutExpired carries what the child had written and does not honour