Skip to content

A local video studio for your coding agent. Describe a video in one sentence: your agent directs, your machine renders. Claude Code, Codex, Cursor, Devin and more. No API keys, no uploads.

tests: three flaky browser-timing tests on the release PR's full tier

Favio Vázquezb86e29845bef
diff --git a/skills/showtime/tests/test_pace.py b/skills/showtime/tests/test_pace.py
index ad630b7..5fc3cbe 100644
--- a/skills/showtime/tests/test_pace.py
+++ b/skills/showtime/tests/test_pace.py
@@ -42,6 +42,7 @@ from __future__ import annotations
import _isolate # noqa: F401 (run from a scratch folder: never write into the repository)
import json
+import os
import re
import shutil
import socket
@@ -362,37 +363,50 @@ process.exit(0);
def test_check_stopped_browser(self):
"""A browser that really stops answering (SIGSTOP, SHOWTIME_TEST_STOP_BROWSER): stopped before the first
page opens, check gives it up at the deadline, kills it and finishes in a new browser; stopped after the
- image lab page opened, the watchdog ends check with an error naming its last step. Both return in seconds
- (the open deadline is 3 s here) and leave no stopped browser behind."""
+ image lab page opened, the watchdog ends check with an error naming its last step. Each returns in under 120 s
+ (about 25 s on a quiet Mac) and leaves no stopped browser of its own behind.
+ The open deadline is 20 s here, not 3 s: a stopped browser never answers, so any deadline finds it and a
+ longer one only costs that wait; the healthy browser that replaces it must open its page inside the same
+ deadline, and on a busy macOS CI runner a fresh browser took over 3 s just to create its context and tab
+ (the 0.4.2 release PR). The page's own load gets 3/5 of it (PAGE_LOAD_MS: 1.8 s at 3 s, 12 s here), and the
+ watchdog pings the healthy browser with the same deadline. One value on every machine, so a local run is
+ the CI run."""
proj = self.tmp / "stopped"
write(proj / "showtime.json", json.dumps({"width": 320, "height": 180, "fps": 10, "duration": 1, "background": "#202830"}))
write(proj / "index.html", HEAD + "</head><body style=\"margin:0;background:#234\"><h1 style=\"color:#fff;"
"font:40px sans-serif\">Hi</h1><script>ST.onSeek(() => {});</script></body></html>")
args = ("check", proj, "--json", "--no-determinism", "--no-timeline", "--no-history", "--samples", "2")
- env = dict(ENV, SHOWTIME_TEST_OPEN_TIMEOUT="3")
- before = {l for l in subprocess.run(["ps", "-axo", "stat=,command="], stdout=subprocess.PIPE, encoding="utf-8",
- errors="replace").stdout.splitlines() if l.strip().startswith("T")}
- cp = showtime(*args, env=dict(env, SHOWTIME_TEST_STOP_BROWSER="open"), timeout=180)
- self.assertEqual(cp.returncode, 0, cp.stderr[-2000:])
- self.assertTrue(json.loads(cp.stdout)["ok"], cp.stdout[-1500:])
- self.assertEqual(cp.stderr.count("trying once more in a new browser"), 1, cp.stderr[-2000:])
- self.assertIn("stuck at: ", cp.stderr)
- cp = showtime(*args, env=dict(env, SHOWTIME_TEST_STOP_BROWSER="ready:2"), timeout=180)
- self.assertEqual(cp.returncode, 1, cp.stderr[-2000:])
- self.assertIn("the browser stopped answering", cp.stderr)
- self.assertIn("check's last step: opened the image lab page", cp.stderr)
- # the run's browsers hold a temp profile (--user-data-dir) of their own; the command line never names the
- # project, so a stopped (state T) Chrome after the run is any stopped Chrome that was not stopped before.
- # The stuck browser is killed (SIGKILL) as check ends; on a busy machine ps can still list it for a moment,
- # so it is asked again for up to 10 s before that counts as left behind.
+ own = own_tmp(self.tmp / "stopped-tmp")
+ env = dict(ENV, SHOWTIME_TEST_OPEN_TIMEOUT="20", TMPDIR=own, TEMP=own, TMP=own)
+ t0 = time.time()
+ cp1, groups = run_watching(args, dict(env, SHOWTIME_TEST_STOP_BROWSER="open"), own, 180)
+ took1 = time.time() - t0
+ self.assertEqual(cp1.returncode, 0, cp1.stderr[-2000:])
+ self.assertTrue(json.loads(cp1.stdout)["ok"], cp1.stdout[-1500:])
+ self.assertEqual(cp1.stderr.count("trying once more in a new browser"), 1, cp1.stderr[-2000:])
+ # the browser is stopped before its first page opens, so the step is always the first one
+ self.assertIn("stuck at: creating the browser context", cp1.stderr)
+ # the 20 s deadline, a new browser and a short check: never the 180 s the subprocess may take
+ self.assertLess(took1, 120, cp1.stderr[-1500:])
+ t0 = time.time()
+ cp2, g2 = run_watching(args, dict(env, SHOWTIME_TEST_STOP_BROWSER="ready:2"), own, 180)
+ took2 = time.time() - t0
+ groups |= g2
+ self.assertEqual(cp2.returncode, 1, cp2.stderr[-2000:])
+ self.assertIn("the browser stopped answering", cp2.stderr)
+ self.assertIn("check's last step: opened the image lab page", cp2.stderr)
+ # two pages opened, then the watchdog: a ping every 5 s, each with the 20 s deadline
+ self.assertLess(took2, 120, cp2.stderr[-1500:])
+ # The stuck browser is killed (SIGKILL to its process group, which a stopped process cannot ignore) as check
+ # ends; on a busy machine ps can still list it for a moment, so it is asked again for up to 10 s before that
+ # counts as left behind. Only this test's own browsers count (own_tmp, run_watching).
for _ in range(20):
- ps = subprocess.run(["ps", "-axo", "stat=,command="], stdout=subprocess.PIPE, encoding="utf-8", errors="replace").stdout
- stopped = [l for l in ps.splitlines() if l.strip().startswith("T") and "--user-data-dir" in l and "chrom" in l.lower()]
- left = [l for l in stopped if l not in before]
+ left = own_stopped(own, groups)
if not left:
break
time.sleep(0.5)
- self.assertEqual(left, [], "a stopped browser was left behind")
+ self.assertEqual(left, [], "a stopped browser was left behind: %s\nfirst check's stderr:\n%s\nsecond check's "
+ "stderr:\n%s" % (describe(left), cp1.stderr[-1500:], cp2.stderr[-1500:]))
class TestSizeReadWaitsForTheFrame(unittest.TestCase):
@@ -522,10 +536,57 @@ class hanging_server:
self.sock.close()
-def stopped_gpu_processes():
- """Chrome GPU processes in state T (stopped) on this machine."""
- ps = subprocess.run(["ps", "-axo", "pid=,stat=,command="], stdout=subprocess.PIPE, encoding="utf-8", errors="replace").stdout
- return {l.split()[0] for l in ps.splitlines() if len(l.split()) > 2 and l.split()[1].startswith("T") and "--type=gpu-process" in l}
+# Stopped browsers left behind: only this test's own count. Another run on the same machine (a second agent, a
+# parallel test file) stops browsers too, so a machine-wide scan of stopped Chromes failed this file for processes it
+# never started. Each run gets a TMPDIR of its own (own_tmp), which every browser it launches names in its profile
+# (--user-data-dir); while the command runs, the process groups of those browsers are noted too (run_watching), as a
+# browser's helper processes keep its group but need not name the profile.
+def own_tmp(base):
+ """A TMPDIR for one command's browsers (resolved: the profile path in their command lines is this one)."""
+ d = Path(os.path.realpath(str(base)))
+ d.mkdir(parents=True, exist_ok=True)
+ return str(d)
+
+
+def ps_rows():
+ """Every process: (pid, ppid, pgid, stat, etime, command)."""
+ out = subprocess.run(["ps", "-axo", "pid=,ppid=,pgid=,stat=,etime=,command="], stdout=subprocess.PIPE,
+ encoding="utf-8", errors="replace").stdout
+ return [tuple(f) for f in (l.split(None, 5) for l in out.splitlines()) if len(f) == 6]
+
+
+def run_watching(args, env, tag, timeout):
+ """showtime(*args), noting every 0.5 s while it runs the process groups of the processes that name tag.
+ -> (CompletedProcess, {pgid})"""
+ p = subprocess.Popen([sys.executable, str(LAUNCHER)] + [str(a) for a in args], env=env, stdout=subprocess.PIPE,
+ stderr=subprocess.PIPE, encoding="utf-8", errors="replace")
+ groups, t0 = set(), time.time()
+ while True:
+ try:
+ out, err = p.communicate(timeout=0.5)
+ break
+ except subprocess.TimeoutExpired:
+ groups |= {r[2] for r in ps_rows() if tag in r[5]}
+ if time.time() - t0 > timeout:
+ p.kill()
+ p.communicate()
+ raise
+ return subprocess.CompletedProcess(p.args, p.returncode, out, err), groups
+
+
+def own_stopped(tag, groups, kind=None):
+ """This run's processes in state T (stopped): they name tag or are in one of its browsers' groups; kind: a
+ substring of the command line (e.g. --type=gpu-process). -> [row]"""
+ return [r for r in ps_rows() if r[3].startswith("T") and (tag in r[5] or r[2] in groups) and (kind is None or kind in r[5])]
+
+
+def describe(rows):
+ """pid, ppid, etime and profile of each process, for an assertion message (the command lines are too long)."""
+ def udd(cmd):
+ m = re.search(r"--user-data-dir=(\S+)", cmd)
+ return m.group(1) if m else "(no --user-data-dir)"
+ return "; ".join("pid %s ppid %s pgid %s stat %s etime %s %s %s" % (r[0], r[1], r[2], r[3], r[4],
+ (re.search(r"--type=\S+", r[5]) or ["main"])[0], udd(r[5])) for r in rows)
@unittest.skipIf(sys.platform == "win32", "SIGSTOP is POSIX")
@@ -546,7 +607,8 @@ class TestFrameNeverComes(unittest.TestCase):
"const gl = document.getElementById('c').getContext('webgl', { preserveDrawingBuffer: true });\n"
"ST.onSeek((t) => { gl.clearColor(t / 2, 0.3, 1 - t / 2, 1); gl.clear(gl.COLOR_BUFFER_BIT); });"
"</script></body></html>")
- cls.stopped_before = stopped_gpu_processes()
+ cls.own = own_tmp(cls.tmp / "tmp")
+ cls.env = dict(ENV, TMPDIR=cls.own, TEMP=cls.own, TMP=cls.own)
@classmethod
def tearDownClass(cls):
@@ -557,23 +619,29 @@ class TestFrameNeverComes(unittest.TestCase):
if "stopped the GPU process before screenshot" not in cp.stderr:
self.skipTest("this browser has no GPU process to stop: " + cp.stderr[-300:])
- def assertNoStoppedGpu(self):
+ def sh(self, *args, timeout, **extra):
+ """showtime(*args) in this class's own TMPDIR, noting its browsers' process groups (for assertNoStoppedGpu)."""
+ cp, self.groups = run_watching(args, dict(self.env, **extra), self.own, timeout)
+ return cp
+
+ def assertNoStoppedGpu(self, cp):
# the stuck browser is killed as the command ends; on a busy machine ps can still list its stopped GPU process
- # for a moment, so it is asked again for up to 10 s before that counts as left behind
+ # for a moment, so it is asked again for up to 10 s before that counts as left behind. Only this run's own.
for _ in range(20):
time.sleep(0.5)
- left = stopped_gpu_processes() - self.stopped_before
+ left = own_stopped(self.own, self.groups, "--type=gpu-process")
if not left:
break
- self.assertEqual(left, set(), "a stopped GPU process was left behind")
+ self.assertEqual(left, [], "a stopped GPU process was left behind: %s\nthe command's stderr:\n%s"
+ % (describe(left), cp.stderr[-2000:]))
def test_render_draws_the_frame_again(self):
"""The screenshot fails once the GPU process and the renderers have done no work for 30 s (x the page's
pace), the frame is drawn again in a new browser (the stuck one killed), and the render finishes with every
frame and the retry in render.json."""
t0 = time.time()
- cp = showtime("render", self.proj, "-o", self.tmp / "render.mp4", "--no-check", "--workers", "1", "--no-audio",
- "--poster", "none", "--json", env=dict(ENV, SHOWTIME_TEST_STOP_GPU="shot:6"), timeout=400)
+ cp = self.sh("render", self.proj, "-o", self.tmp / "render.mp4", "--no-check", "--workers", "1", "--no-audio",
+ "--poster", "none", "--json", SHOWTIME_TEST_STOP_GPU="shot:6", timeout=400)
took = time.time() - t0
self.injected(cp)
self.assertEqual(cp.returncode, 0, cp.stderr[-2000:])
@@ -583,18 +651,18 @@ class TestFrameNeverComes(unittest.TestCase):
self.assertRegex(cp.stderr, r"timed out after \d+s: screenshot at 0\.500s " + STUCK_FRAME + r"; retrying")
# 30 s x the pace (5 at most) and a new browser: never the 10 minutes it was
self.assertLess(took, 300, cp.stderr[-1500:])
- self.assertNoStoppedGpu()
+ self.assertNoStoppedGpu(cp)
def test_check_ends_with_its_step(self):
"""check's screenshots had no deadline (its watchdog pings the browser process, which answers while the
frame never comes): check now ends with an error naming the screenshot and its last step."""
- cp = showtime("check", self.proj, "--json", "--no-determinism", "--no-timeline", "--no-history", "--samples", "2",
- env=dict(ENV, SHOWTIME_TEST_STOP_GPU="shot:1"), timeout=400)
+ cp = self.sh("check", self.proj, "--json", "--no-determinism", "--no-timeline", "--no-history", "--samples", "2",
+ SHOWTIME_TEST_STOP_GPU="shot:1", timeout=400)
self.injected(cp)
self.assertEqual(cp.returncode, 1, cp.stderr[-2000:])
self.assertRegex(cp.stderr, r"the browser stopped drawing: timed out after \d+s: screenshot " + STUCK_FRAME +
r"; check's last step: \S")
- self.assertNoStoppedGpu()
+ self.assertNoStoppedGpu(cp)
def test_command_timeout_names_the_step(self):
"""SHOWTIME_TIMEOUT (the caller's limit) ends a render that is still waiting with an error naming where each of
@@ -602,9 +670,8 @@ class TestFrameNeverComes(unittest.TestCase):
gets 90 s) so the 50 s limit comes first on a slow runner too."""
out = self.tmp / "limit.mp4"
t0 = time.time()
- cp = showtime("render", self.proj, "-o", out, "--no-check", "--workers", "1", "--no-audio", "--poster", "none",
- env=dict(ENV, SHOWTIME_TEST_STOP_GPU="shot:6", SHOWTIME_TIMEOUT="50", SHOWTIME_TEST_WAIT_SCALE="3"),
- timeout=200)
+ cp = self.sh("render", self.proj, "-o", out, "--no-check", "--workers", "1", "--no-audio", "--poster", "none",
+ SHOWTIME_TEST_STOP_GPU="shot:6", SHOWTIME_TIMEOUT="50", SHOWTIME_TEST_WAIT_SCALE="3", timeout=200)
took = time.time() - t0
self.injected(cp)
self.assertEqual(cp.returncode, 1, cp.stderr[-2000:])
@@ -615,7 +682,7 @@ class TestFrameNeverComes(unittest.TestCase):
self.assertLess(took, 90)
log = (self.tmp / "limit.work" / "logs" / "render.log").read_text(encoding="utf-8")
self.assertIn("error: did not finish within 50 s (SHOWTIME_TIMEOUT); stuck at: ", log)
- self.assertNoStoppedGpu()
+ self.assertNoStoppedGpu(cp)
if __name__ == "__main__":
diff --git a/skills/showtime/tests/test_review_notes.py b/skills/showtime/tests/test_review_notes.py
index 39d610a..b7d753e 100644
--- a/skills/showtime/tests/test_review_notes.py
+++ b/skills/showtime/tests/test_review_notes.py
@@ -412,6 +412,13 @@ try {
// edit, mark done, delete in the list
const id1 = n1.id;
await p.click(`#list [data-id="${id1}"] [data-act="edit"]`);
+ // the page focuses the edit box with the caret at the end one task after the click (setTimeout 0). fill is two
+ // steps (select all + focus, then insert the text): on a busy runner that late caret move landed between them
+ // and the text was appended to the old one ("Logo too small hereLogo much too small"), so wait for it first.
+ // Only the focus tells the states apart: setting the box's text already put the caret at the end. The caret
+ // part is kept so the check says all that a person sees (a box ready to type on the end of the note).
+ ok('edit box focused, caret at end', await until(() => p.evaluate((id) => { const ta = document.querySelector(`#list [data-id="${id}"] textarea`);
+ return !!ta && document.activeElement === ta && ta.selectionStart === ta.value.length && ta.selectionEnd === ta.value.length; }, id1)));
await p.fill(`#list [data-id="${id1}"] textarea`, 'Logo much too small');
// a re-render mid-edit (a late save reply on a busy machine) keeps the text, caret and focus
await p.evaluate(() => window.reviewPage.render());
diff --git a/skills/showtime/tests/test_studio.py b/skills/showtime/tests/test_studio.py
index 3fe116d..5b2ba68 100644
--- a/skills/showtime/tests/test_studio.py
+++ b/skills/showtime/tests/test_studio.py
@@ -183,15 +183,21 @@ try {
await p.click('.variant[data-v="bed-b"] [data-act="pick"]');
ok('sound pick', await until(() => fb().state.picks.music === 'bed-b'));
- // audio A/B keeps the playhead
+ // audio A/B keeps the playhead: B starts where A was, not at 0. Waited for, never a fixed sleep: on a busy
+ // runner bed A (preload none) took over 1 s to load and start, so a fixed 1.2 s found it at 0.02-0.17 s.
await p.click('.variant[data-v="bed-a"] .vplay');
- await sleep(1200);
- const tA = await p.evaluate(() => window.StudioBoard.state().audio.music.els['bed-a'].currentTime);
+ const aRan = await until(() => p.evaluate(() => { const a = window.StudioBoard.state().audio.music.els['bed-a']; return !a.paused && a.currentTime > 0.5; }), 30000);
+ // B's and A's positions as B starts playing, both read in the page by B's 'playing' event. The board reads A's
+ // position, pauses A and seeks B to it, so A still holds the position B was given: B must be within 0.1 s of it
+ // on either side (a B that restarts at 0 or skips ahead fails). A reading taken whenever the test gets to it
+ // would also pass a B that started at 0 and has played on since.
+ await p.evaluate(() => { const G = window.StudioBoard.state().audio.music, b = G.els['bed-b']; window.__abB = null;
+ b.addEventListener('playing', () => { if (window.__abB === null) window.__abB = { b: b.currentTime, a: G.els['bed-a'].currentTime }; }, { once: true }); });
await p.keyboard.press('b');
- await sleep(350);
- const ab = await p.evaluate(() => { const G = window.StudioBoard.state().audio.music; return { active: G.active, t: G.els['bed-b'].currentTime, aPaused: G.els['bed-a'].paused }; });
- ok('A/B shared playhead', tA > 0.2 && ab.active === 'bed-b' && ab.t >= tA - 0.05 && ab.aPaused);
- R.ab = { tA, tB: ab.t };
+ const ab = await until(() => p.evaluate(() => { const G = window.StudioBoard.state().audio.music;
+ return window.__abB !== null && { active: G.active, tA: window.__abB.a, tB: window.__abB.b, aPaused: G.els['bed-a'].paused }; }), 30000) || {};
+ ok('A/B shared playhead', aRan && ab.tA > 0.5 && ab.active === 'bed-b' && Math.abs(ab.tB - ab.tA) < 0.1 && ab.aPaused);
+ R.ab = { tA: ab.tA, tB: ab.tB };
await p.keyboard.press('b');
// phone sees the desktop's clicks over SSE