Skip to content

World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.

backlot: fix all five dogfood findings (F-01..F-05)

calesthio1d60f0da147a
diff --git a/backlot/server.py b/backlot/server.py
index b957c2ac..854cc938 100644
--- a/backlot/server.py
+++ b/backlot/server.py
@@ -239,7 +239,11 @@ def create_app() -> FastAPI:
width = min(THUMB_WIDTHS, key=lambda x: abs(x - w))
cached = await asyncio.to_thread(_thumbnail_for, target, width)
if cached is None:
- return FileResponse(target) # not an image we can thumb — serve as-is
+ # Never fall back to raw video bytes for an <img> consumer (F-03);
+ # non-thumbable images are safe to serve as-is.
+ if target.suffix.lower() in {".mp4", ".webm", ".mov"}:
+ raise HTTPException(status_code=404, detail="no poster frame available")
+ return FileResponse(target)
return FileResponse(cached, media_type="image/jpeg")
# ---- Media (range requests handled by FileResponse) ---------------
diff --git a/backlot/state.py b/backlot/state.py
index 94c9caf8..ee891878 100644
--- a/backlot/state.py
+++ b/backlot/state.py
@@ -31,6 +31,11 @@ FALLBACK_STAGES = [
# How long (seconds) without filesystem activity before a board reads "idle".
LIVE_WINDOW_SECONDS = 5 * 60
+# An in_progress stage with no filesystem activity for this long is flagged
+# as possibly stalled (F-05: a wedged agent must be visible, not silent —
+# heartbeat checkpoints and tool events both reset the clock).
+STALL_WINDOW_SECONDS = 10 * 60
+
def _read_json(path: Path) -> Optional[dict]:
"""Read a JSON file, returning None on any failure."""
@@ -84,13 +89,22 @@ def _load_pipeline_meta(pipeline_type: Optional[str]) -> dict[str, Any]:
def _resolve_artifact(project_dir: Path, value: Any) -> Optional[dict]:
- """Checkpoint artifacts may be inline dicts or path strings — resolve both."""
+ """Checkpoint artifacts may be inline dicts or path strings — resolve both.
+
+ Path references are only followed INSIDE the project directory: a
+ checkpoint must not be able to pull arbitrary JSON from elsewhere on
+ disk onto the board (F-04).
+ """
if isinstance(value, dict):
return value
if isinstance(value, str) and value:
p = Path(value)
if not p.is_absolute():
p = project_dir / value
+ try:
+ p.resolve().relative_to(Path(project_dir).resolve())
+ except (ValueError, OSError):
+ return None
return _read_json(p)
return None
@@ -553,6 +567,16 @@ def load_board_state(project_dir: Path) -> dict[str, Any]:
last_activity = _last_activity(project_dir)
now = time.time()
+ # Stall detection: an in_progress stage that stopped writing anything.
+ for stage_entry in stages:
+ if (
+ stage_entry["status"] == "in_progress"
+ and last_activity
+ and (now - last_activity) > STALL_WINDOW_SECONDS
+ ):
+ stage_entry["stalled"] = True
+ stage_entry["stalled_minutes"] = int((now - last_activity) / 60)
+
state: dict[str, Any] = {
"project_id": project_id,
"title": marker.get("title") or meta_json.get("name") or project_id.replace("-", " ").title(),
diff --git a/backlot/ui/board.css b/backlot/ui/board.css
index df7de240..9ae3c2f8 100644
--- a/backlot/ui/board.css
+++ b/backlot/ui/board.css
@@ -524,3 +524,11 @@ body:not(.first) .lib-card, body:not(.first) .drawer { animation: none; }
/* stages that ran but aren't declared by the pipeline manifest */
.stage.undeclared .node { border-style: dashed; opacity: .85; }
.stage.undeclared .name { font-style: italic; }
+
+/* spend past 90% of budget */
+.cost .bar i.crit { background: var(--red); }
+
+/* in_progress stage with no filesystem activity for a while (F-05) */
+.stage.stalled .node { border-color: var(--red); color: var(--red); background: var(--red-dim); animation: none; }
+.stage.stalled .name { color: var(--red); }
+.stage.stalled .sub { color: var(--red); }
diff --git a/backlot/ui/board.js b/backlot/ui/board.js
index c6878c40..1f2bcc30 100644
--- a/backlot/ui/board.js
+++ b/backlot/ui/board.js
@@ -32,9 +32,13 @@ function renderSlate(s) {
const awaiting = s.stages.find((x) => x.status === "awaiting_human");
const inProgress = s.stages.find((x) => x.status === "in_progress");
+ const stalled = s.stages.find((x) => x.stalled);
let liveEl;
if (awaiting) {
liveEl = el("span", { class: "live" }, el("span", { class: "dot" }), "◈ AWAITING YOU");
+ } else if (stalled) {
+ liveEl = el("span", { class: "live", style: "color:var(--red)" },
+ el("span", { class: "dot", style: "background:var(--red);animation:none" }), "⚠ STALLED?");
} else if (s.live || inProgress) {
liveEl = el("span", { class: "live" }, el("span", { class: "dot" }), "LIVE");
} else {
@@ -52,7 +56,7 @@ function renderSlate(s) {
hasBudget ? el("span", {}, ` / ${fmtMoney(budget)}`) : ""));
if (hasBudget) {
cost.append(el("div", { class: "bar" }, el("i", {
- class: pct > 75 ? "warn" : "", style: `width:${pct}%`,
+ class: pct > 90 ? "crit" : pct > 75 ? "warn" : "", style: `width:${pct}%`,
})));
}
cost.append(el("div", { class: "label" }, "generation spend"));
@@ -77,6 +81,9 @@ function renderSlate(s) {
function stageSub(st) {
if (st.status === "awaiting_human") return "awaiting your approval\nreply in chat to continue";
+ if (st.status === "in_progress" && st.stalled) {
+ return `stalled? no activity for ${st.stalled_minutes}m\nask the agent for status`;
+ }
if (st.status === "in_progress" && st.partial_progress) {
const done = st.partial_progress.completed_scene_ids;
if (Array.isArray(done)) return `${done.length} scene${done.length === 1 ? "" : "s"} done`;
@@ -96,7 +103,7 @@ function renderRail(s) {
let pendingIndex = 1;
for (const st of s.stages) {
const cls = st.status === "completed" ? "done"
- : st.status === "in_progress" ? "active"
+ : st.status === "in_progress" ? (st.stalled ? "active stalled" : "active")
: st.status === "awaiting_human" ? "await"
: st.status === "failed" ? "failed" : "";
const icon = STAGE_ICONS[st.status] || String(pendingIndex);
@@ -717,8 +724,31 @@ function render() {
if (renders) app.append(renders);
}
+// Defensive normalization (F-02): the server contract guarantees these
+// fields, but a sparse/legacy payload must degrade, never crash the board.
+function normalize(s) {
+ s.pipeline = s.pipeline || { pipeline_type: "unknown", stages: [], known: false };
+ s.stages = Array.isArray(s.stages) ? s.stages : [];
+ s.artifacts = s.artifacts || {};
+ s.media = s.media || {};
+ s.media.renders = Array.isArray(s.media.renders) ? s.media.renders : [];
+ s.media.snapshots = Array.isArray(s.media.snapshots) ? s.media.snapshots : [];
+ s.media.music = Array.isArray(s.media.music) ? s.media.music : [];
+ s.events = Array.isArray(s.events) ? s.events : [];
+ if (s.storyboard && Array.isArray(s.storyboard.scenes)) {
+ for (const c of s.storyboard.scenes) {
+ c.takes = Array.isArray(c.takes) ? c.takes : [];
+ c.audio = Array.isArray(c.audio) ? c.audio : [];
+ c.required_assets = Array.isArray(c.required_assets) ? c.required_assets : [];
+ }
+ } else {
+ s.storyboard = null;
+ }
+ return s;
+}
+
async function refresh() {
- state = await getJSON(`/api/project/${encodeURIComponent(projectId)}/state`);
+ state = normalize(await getJSON(`/api/project/${encodeURIComponent(projectId)}/state`));
render();
}
diff --git a/scripts/backlot_visual_eval.py b/scripts/backlot_visual_eval.py
new file mode 100644
index 00000000..34bd0438
--- /dev/null
+++ b/scripts/backlot_visual_eval.py
@@ -0,0 +1,234 @@
+"""Deterministic visual eval for Backlot.
+
+Stages the fictional Backlot projects, captures canonical browser screenshots,
+optionally compares them to goldens, and can run a small Playwright interaction
+smoke against the staged board.
+
+Examples:
+ python scripts/backlot_visual_eval.py
+ python scripts/backlot_visual_eval.py --bless
+ python scripts/backlot_visual_eval.py --interactions
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+import time
+import urllib.request
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from PIL import Image, ImageChops
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+STAGE_DIR = REPO_ROOT / ".backlot" / "screenshot-stage"
+GOLDENS_DIR = REPO_ROOT / "internal" / "evals" / "goldens"
+CAPTURE_ROOT = REPO_ROOT / "internal" / "evals" / "captures"
+PORT = 4791
+
+SHOTS = [
+ ("library", "/?static=1", 1560, 500, 4200, [
+ (1370, 20, 1510, 62), # live/idle badge
+ (90, 106, 422, 380), # card border/status animation variance
+ (440, 106, 772, 380),
+ (790, 106, 1122, 380),
+ (1140, 106, 1472, 380),
+ ]),
+ ("board-live", "/p/signal-in-the-static?static=1", 1560, 1150, 4200, []),
+ ("script-gate", "/p/the-slow-orchard?static=1", 1560, 760, 3200, []),
+ ("storyboard", "/p/the-last-lighthouse?static=1", 1560, 1500, 4200, []),
+]
+
+
+def compare_images(
+ expected_path: Path,
+ actual_path: Path,
+ diff_path: Path,
+ *,
+ threshold: float = 0.015,
+ masks: list[tuple[int, int, int, int]] | None = None,
+) -> dict[str, Any]:
+ """Compare screenshots by changed-pixel ratio and write a red diff image."""
+ expected = Image.open(expected_path).convert("RGB")
+ actual = Image.open(actual_path).convert("RGB")
+ if expected.size != actual.size:
+ diff_path.parent.mkdir(parents=True, exist_ok=True)
+ actual.save(diff_path)
+ return {"passed": False, "changed_ratio": 1.0, "reason": f"size {expected.size} != {actual.size}"}
+
+ masks = masks or []
+ for box in masks:
+ patch = expected.crop(box)
+ actual.paste(patch, box)
+
+ delta = ImageChops.difference(expected, actual)
+ changed = 0
+ pixels = delta.load()
+ width, height = delta.size
+ diff = Image.new("RGB", delta.size, (0, 0, 0))
+ diff_px = diff.load()
+ for y in range(height):
+ for x in range(width):
+ if max(pixels[x, y]) > 8:
+ changed += 1
+ diff_px[x, y] = (255, 40, 40)
+ else:
+ diff_px[x, y] = actual.getpixel((x, y))
+ ratio = changed / float(width * height)
+ diff_path.parent.mkdir(parents=True, exist_ok=True)
+ diff.save(diff_path)
+ return {"passed": ratio <= threshold, "changed_ratio": round(ratio, 6), "threshold": threshold}
+
+
+def run_stage() -> None:
+ subprocess.run(
+ [sys.executable, "scripts/backlot_screenshot_stage.py", "--stage-only"],
+ cwd=REPO_ROOT,
+ check=True,
+ timeout=180,
+ )
+
+
+def start_server() -> subprocess.Popen:
+ env = dict(os.environ)
+ env["OPENMONTAGE_PROJECTS_DIR"] = str(STAGE_DIR)
+ server = subprocess.Popen(
+ [sys.executable, "-m", "backlot", "serve", "--port", str(PORT)],
+ cwd=REPO_ROOT,
+ env=env,
+ stdout=subprocess.DEVNULL,
+ stderr=subprocess.DEVNULL,
+ )
+ deadline = time.time() + 20
+ while time.time() < deadline:
+ try:
+ with urllib.request.urlopen(f"http://127.0.0.1:{PORT}/api/health", timeout=1):
+ return server
+ except Exception:
+ time.sleep(0.3)
+ server.terminate()
+ raise RuntimeError("Backlot server did not become healthy")
+
+
+def capture_screenshot(url: str, output: Path, width: int, height: int, wait_ms: int) -> None:
+ output.parent.mkdir(parents=True, exist_ok=True)
+ subprocess.run(
+ [
+ "npx",
+ "playwright",
+ "screenshot",
+ "--viewport-size",
+ f"{width},{height}",
+ "--wait-for-timeout",
+ str(wait_ms),
+ url,
+ str(output),
+ ],
+ cwd=REPO_ROOT,
+ check=True,
+ timeout=120,
+ shell=(os.name == "nt"),
+ )
+
+
+def capture_shots(capture_dir: Path) -> list[dict[str, Any]]:
+ results = []
+ for name, path, width, height, wait_ms, _masks in SHOTS:
+ out = capture_dir / f"{name}.png"
+ capture_screenshot(f"http://127.0.0.1:{PORT}{path}", out, width, height, wait_ms)
+ results.append({"name": name, "path": out})
+ return results
+
+
+def compare_or_bless(capture_dir: Path, *, bless: bool, threshold: float) -> list[dict[str, Any]]:
+ GOLDENS_DIR.mkdir(parents=True, exist_ok=True)
+ report = []
+ for name, _path, _width, _height, _wait_ms, masks in SHOTS:
+ actual = capture_dir / f"{name}.png"
+ golden = GOLDENS_DIR / f"{name}.png"
+ if bless or not golden.exists():
+ shutil.copyfile(actual, golden)
+ report.append({"name": name, "status": "blessed", "golden": str(golden)})
+ continue
+ diff = capture_dir / "diffs" / f"{name}.png"
+ result = compare_images(golden, actual, diff, threshold=threshold, masks=masks)
+ result.update({"name": name, "diff": str(diff)})
+ report.append(result)
+ return report
+
+
+def run_interactions(capture_dir: Path) -> dict[str, Any]:
+ """Run browser interaction smoke through Python Playwright."""
+ from playwright.sync_api import sync_playwright
+
+ screenshot = capture_dir / "interaction-smoke.png"
+ with sync_playwright() as pw:
+ browser = pw.chromium.launch(headless=True)
+ page = browser.new_page(viewport={"width": 1560, "height": 1000})
+ page.goto(f"http://127.0.0.1:{PORT}/p/the-last-lighthouse?static=1")
+ page.wait_for_selector(".stage")
+ page.locator(".stage").first.click()
+ page.wait_for_selector(".drawer")
+ drawer_text = page.locator(".drawer").inner_text()
+ if "research" not in drawer_text:
+ raise RuntimeError("stage drawer did not open")
+ page.locator(".script-card").first.click()
+ page.wait_for_selector(".modal-bg.open")
+ page.keyboard.press("Escape")
+ page.wait_for_function("() => !document.querySelector('.modal-bg')?.classList.contains('open')")
+ if page.locator(".takes").count() < 1:
+ raise RuntimeError("takes drawer not present on staged takes scene")
+ replay_button = page.locator(".rp-btn", has_text="REPLAY RUN")
+ if replay_button.count():
+ replay_button.first.click()
+ page.wait_for_selector('input[type="range"]')
+ page.locator('input[type="range"]').fill("500")
+ page.screenshot(path=str(screenshot), full_page=True)
+ browser.close()
+ return {"status": "passed", "screenshot": str(capture_dir / "interaction-smoke.png")}
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--bless", action="store_true", help="Write current captures as goldens")
+ parser.add_argument("--no-stage", action="store_true", help="Reuse existing .backlot/screenshot-stage")
+ parser.add_argument("--interactions", action="store_true", help="Run Playwright interaction smoke")
+ parser.add_argument("--threshold", type=float, default=0.015)
+ parser.add_argument("--out-dir", type=Path, default=None)
+ args = parser.parse_args(argv)
+
+ if not args.no_stage:
+ run_stage()
+
+ stamp = datetime.now().strftime("visual-%Y%m%d-%H%M%S")
+ capture_dir = args.out_dir or (CAPTURE_ROOT / stamp)
+ capture_dir.mkdir(parents=True, exist_ok=True)
+
+ server = start_server()
+ try:
+ capture_shots(capture_dir)
+ report = compare_or_bless(capture_dir, bless=args.bless, threshold=args.threshold)
+ interaction_report = run_interactions(capture_dir) if args.interactions else None
+ finally:
+ server.terminate()
+ try:
+ server.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ server.kill()
+
+ passed = all(item.get("passed", item.get("status") == "blessed") for item in report)
+ payload = {"capture_dir": str(capture_dir), "shots": report, "interactions": interaction_report}
+ report_path = capture_dir / "report.json"
+ report_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")
+ print(json.dumps(payload, indent=2))
+ return 0 if passed else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/scripts/backlot_watch_captures.py b/scripts/backlot_watch_captures.py
new file mode 100644
index 00000000..0885276b
--- /dev/null
+++ b/scripts/backlot_watch_captures.py
@@ -0,0 +1,195 @@
+"""Capture Backlot board screenshots whenever watched project state changes.
+
+This is the Half-B dogfood watcher from internal/evals/BACKLOT_EVAL_PLAN.md.
+It polls the Backlot API, fingerprints board-relevant state, and captures the
+library plus the changed project board through Playwright.
+
+Example:
+ python scripts/backlot_watch_captures.py --projects why-cities-glow rain-on-glass
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+REPO_ROOT = Path(__file__).resolve().parent.parent
+DEFAULT_BASE_URL = "http://127.0.0.1:4750"
+DEFAULT_CAPTURE_ROOT = REPO_ROOT / "internal" / "evals" / "captures"
+
+
+def capture_slug(project_id: str, stage: str | None, status: str | None) -> str:
+ """Stable, filesystem-safe screenshot name stem."""
+ raw = "-".join(part for part in (project_id, stage or "unknown", status or "unknown") if part)
+ raw = raw.replace("\\", "-").replace("/", "-").replace("..", "")
+ slug = re.sub(r"[^A-Za-z0-9._-]+", "-", raw).strip(".-")
+ slug = re.sub(r"-{2,}", "-", slug)
+ return slug or "capture"
+
+
+def state_fingerprint(state: dict[str, Any]) -> str:
+ """Hashable representation of board-visible state.
+
+ Intentionally ignores mtime-ish noise such as last_activity while keeping
+ the pieces that should trigger a capture: stage transitions, generating
+ flags, scene visual changes, costs, renders, and event count/tail.
+ """
+ scenes = []
+ storyboard = state.get("storyboard") or {}
+ for card in storyboard.get("scenes") or []:
+ visual = card.get("visual") or {}
+ scenes.append({
+ "id": card.get("id"),
+ "generating": bool(card.get("generating")),
+ "generating_tool": card.get("generating_tool"),
+ "visual": {
+ "path": visual.get("path"),
+ "exists": visual.get("exists"),
+ "type": visual.get("type"),
+ },
+ "takes": [take.get("path") for take in (card.get("takes") or [])],
+ "audio": [asset.get("path") for asset in (card.get("audio") or [])],
+ })
+
+ media = state.get("media") or {}
+ events = state.get("events") or []
+ visible = {
+ "stages": [
+ {
+ "name": stage.get("name"),
+ "status": stage.get("status"),
+ "gate_skipped": stage.get("gate_skipped"),
+ "versions": stage.get("versions"),
+ "partial_progress": stage.get("partial_progress"),
+ }
+ for stage in state.get("stages") or []
+ ],
+ "scenes": scenes,
+ "cost": state.get("cost"),
+ "renders": [r.get("path") for r in media.get("renders") or []],
+ "snapshots": [s.get("path") for s in media.get("snapshots") or []],
+ "event_count": len(events),
+ "event_tail": events[-3:],
+ }
+ return json.dumps(visible, sort_keys=True, default=str, separators=(",", ":"))
+
+
+def active_stage(state: dict[str, Any]) -> tuple[str | None, str | None]:
+ for stage in state.get("stages") or []:
+ if stage.get("status") in {"in_progress", "awaiting_human", "failed", "blocked"}:
+ return stage.get("name"), stage.get("status")
+ for stage in reversed(state.get("stages") or []):
+ if stage.get("status") == "completed":
+ return stage.get("name"), stage.get("status")
+ return None, None
+
+
+def fetch_json(base_url: str, path: str) -> dict[str, Any] | list[Any]:
+ with urllib.request.urlopen(f"{base_url.rstrip('/')}{path}", timeout=10) as response:
+ return json.loads(response.read().decode("utf-8"))
+
+
+def capture_url(url: str, output: Path, *, width: int = 1560, height: int = 1150, wait_ms: int = 1200) -> None:
+ output.parent.mkdir(parents=True, exist_ok=True)
+ subprocess.run(
+ [
+ "npx",
+ "playwright",
+ "screenshot",
+ "--viewport-size",
+ f"{width},{height}",
+ "--wait-for-timeout",
+ str(wait_ms),
+ url,
+ str(output),
+ ],
+ cwd=REPO_ROOT,
+ check=True,
+ timeout=120,
+ shell=(os.name == "nt"),
+ )
+
+
+def capture_project(base_url: str, capture_dir: Path, project_id: str, seq: int, state: dict[str, Any]) -> None:
+ stage, status = active_stage(state)
+ stem = f"{seq:03d}-{capture_slug(project_id, stage, status)}"
+ capture_url(f"{base_url.rstrip('/')}/?static=1", capture_dir / "library" / f"{stem}.png", height=620)
+ capture_url(
+ f"{base_url.rstrip('/')}/p/{project_id}?static=1",
+ capture_dir / project_id / f"{stem}.png",
+ )
+
+
+def watch(
+ projects: list[str],
+ *,
+ base_url: str,
+ capture_dir: Path,
+ interval_s: float,
+ once: bool = False,
+ no_screenshots: bool = False,
+) -> int:
+ fingerprints: dict[str, str] = {}
+ seq = 0
+ capture_dir.mkdir(parents=True, exist_ok=True)
+ print(f"[watch] base={base_url} captures={capture_dir}")
+ while True:
+ changed = False
+ for project_id in projects:
+ try:
+ state = fetch_json(base_url, f"/api/project/{project_id}/state")
+ except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:
+ print(f"[watch] {project_id}: state fetch failed: {exc}", file=sys.stderr)
+ continue
+ fp = state_fingerprint(state)
+ if fingerprints.get(project_id) == fp:
+ continue
+ fingerprints[project_id] = fp
+ changed = True
+ seq += 1
+ stage, status = active_stage(state)
+ print(f"[watch] change {project_id}: {stage or 'unknown'} -> {status or 'unknown'}")
+ if not no_screenshots:
+ capture_project(base_url, capture_dir, project_id, seq, state)
+ if once:
+ return 0
+ if not changed:
+ time.sleep(interval_s)
+
+
+def main(argv: list[str] | None = None) -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--projects", nargs="+", required=True, help="Project ids to watch")
+ parser.add_argument("--base-url", default=DEFAULT_BASE_URL)
+ parser.add_argument("--interval", type=float, default=20.0, help="Polling interval in seconds")
+ parser.add_argument("--out-dir", type=Path, default=None)
+ parser.add_argument("--once", action="store_true", help="Poll once and exit")
+ parser.add_argument("--no-screenshots", action="store_true", help="Exercise polling without Playwright")
+ args = parser.parse_args(argv)
+
+ out_dir = args.out_dir
+ if out_dir is None:
+ stamp = datetime.now().strftime("dogfood-%Y%m%d-%H%M%S")
+ out_dir = DEFAULT_CAPTURE_ROOT / stamp
+ return watch(
+ args.projects,
+ base_url=args.base_url,
+ capture_dir=out_dir,
+ interval_s=args.interval,
+ once=args.once,
+ no_screenshots=args.no_screenshots,
+ )
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tests/backlot/test_gate_scenarios.py b/tests/backlot/test_gate_scenarios.py
new file mode 100644
index 00000000..d00920c3
--- /dev/null
+++ b/tests/backlot/test_gate_scenarios.py
@@ -0,0 +1,99 @@
+"""Gate-integrity scenarios for Backlot and checkpoint hardening."""
+
+import json
+from pathlib import Path
+
+import pytest
+
+from backlot import state as state_mod
+from backlot.state import load_board_state
+from lib.checkpoint import CheckpointValidationError, write_checkpoint
+
+
+def _script_artifact() -> dict:
+ return {
+ "version": "1.0",
+ "title": "Gate Test",
+ "total_duration_seconds": 5,
+ "sections": [{"id": "s1", "text": "Hello.", "start_seconds": 0, "end_seconds": 5}],
+ }
+
+
+def _manifest_artifact() -> dict:
+ return {"version": "1.0", "assets": [], "total_cost_usd": 0.0}
+
+
+def _write(path: Path, data: dict) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(data), encoding="utf-8")
+
+
+def test_completed_gated_stage_without_approval_is_rejected(tmp_path):
+ with pytest.raises(CheckpointValidationError, match="GATE VIOLATION"):
+ write_checkpoint(
+ tmp_path,
+ "film",
+ "script",
+ "completed",
+ {"script": _script_artifact()},
+ pipeline_type="cinematic",
+ )
+
+
+def test_typo_pipeline_type_fails_closed(tmp_path):
+ with pytest.raises(CheckpointValidationError, match="Unknown pipeline_type"):
+ write_checkpoint(
+ tmp_path,
+ "film",
+ "script",
+ "completed",
+ {"script": _script_artifact()},
+ pipeline_type="cinemtaic",
+ human_approved=True,
+ )
+
+
+def test_handwritten_completed_checkpoint_surfaces_gate_skip(tmp_path, monkeypatch):
+ monkeypatch.setattr(state_mod, "PROJECTS_DIR", tmp_path)
+ project = tmp_path / "film"
+ _write(project / "checkpoint_script.json", {
+ "version": "1.0",
+ "project_id": "film",
+ "pipeline_type": "cinematic",
+ "stage": "script",
+ "status": "completed",
+ "timestamp": "2026-07-02T00:00:00Z",
+ "artifacts": {"script": _script_artifact()},
+ })
+
+ state = load_board_state(project)
+
+ script = next(stage for stage in state["stages"] if stage["name"] == "script")
+ assert script["gate_skipped"] is True
+
+
+def test_awaiting_then_approved_archives_history_without_gate_skip(tmp_path):
+ write_checkpoint(
+ tmp_path,
+ "film",
+ "assets",
+ "awaiting_human",
+ {"asset_manifest": _manifest_artifact()},
+ pipeline_type="cinematic",
+ )
+ write_checkpoint(
+ tmp_path,
+ "film",
+ "assets",
+ "completed",
+ {"asset_manifest": _manifest_artifact()},
+ pipeline_type="cinematic",
+ human_approved=True,
+ )
+
+ state = load_board_state(tmp_path / "film")
+
+ assets = next(stage for stage in state["stages"] if stage["name"] == "assets")
+ assert assets.get("gate_skipped") in (None, False)
+ assert assets["versions"] == 2
+ assert assets["history_entries"][0]["status"] == "awaiting_human"
diff --git a/tests/backlot/test_server.py b/tests/backlot/test_server.py
new file mode 100644
index 00000000..7dfed8ed
--- /dev/null
+++ b/tests/backlot/test_server.py
@@ -0,0 +1,205 @@
+"""Server/API tests for Backlot.
+
+These cover the deterministic eval surface in internal/evals/BACKLOT_EVAL_PLAN.md:
+API shape, path safety, media/thumb serving, range requests, and loose
+performance budgets.
+"""
+
+from __future__ import annotations
+
+import io
+import json
+import time
+from pathlib import Path
+
+import pytest
+from fastapi.testclient import TestClient
+from PIL import Image
+
+from backlot import server as server_mod
+from backlot import state as state_mod
+
+
+def projects_root(tmp_path, monkeypatch):
+ root = tmp_path / "projects"
+ root.mkdir()
+ monkeypatch.setattr(state_mod, "PROJECTS_DIR", root)
+ monkeypatch.setattr(server_mod, "PROJECTS_DIR", root)
+ monkeypatch.setattr(server_mod, "_summary_cache", {})
+ monkeypatch.setattr(server_mod, "_PROJECTS_ROOT_STR", __import__("os").path.normcase(str(root.resolve())))
+ monkeypatch.setattr(server_mod, "THUMB_CACHE_DIR", tmp_path / "thumbs")
+ return root
+
+
+def client(projects_root, monkeypatch):
+ async def no_watch():
+ return None
+
+ monkeypatch.setattr(server_mod, "_watch_projects", no_watch)
+ with TestClient(server_mod.create_app()) as c:
+ yield c
+
+
+def _write_json(path: Path, data: dict) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(json.dumps(data), encoding="utf-8")
+
+
+def _make_project(root: Path, project_id: str = "film") -> Path:
+ project = root / project_id
+ (project / "artifacts").mkdir(parents=True)
+ (project / "assets" / "images").mkdir(parents=True)
+ (project / "assets" / "video").mkdir(parents=True)
+ (project / "renders").mkdir(parents=True)
+ _write_json(
+ project / "project.json",
+ {
+ "project_id": project_id,
+ "title": "Film",
+ "pipeline_type": "cinematic",
+ "created_at": "2026-07-02T00:00:00Z",
+ },
+ )
+ _write_json(
+ project / "checkpoint_script.json",
+ {
+ "version": "1.0",
+ "project_id": project_id,
+ "pipeline_type": "cinematic",
+ "stage": "script",
+ "status": "awaiting_human",
+ "timestamp": "2026-07-02T00:01:00Z",
+ "artifacts": {},
+ },
+ )
+ return project
+
+
+def _write_png(path: Path, color: tuple[int, int, int] = (200, 40, 80)) -> None:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ img = Image.new("RGB", (24, 16), color)
+ buf = io.BytesIO()
+ img.save(buf, format="PNG")
+ path.write_bytes(buf.getvalue())
+
+
+class TestBacklotServerApi:
+ def test_health(self, client):
+ response = client.get("/api/health")
+ assert response.status_code == 200
+ assert response.json() == {"ok": True, "app": "backlot"}
+
+ def test_projects_shape_and_state(self, client, projects_root):
+ _make_project(projects_root, "film")
+
+ projects = client.get("/api/projects")
+ assert projects.status_code == 200
+ body = projects.json()
+ assert len(body) == 1
+ assert body[0]["project_id"] == "film"
+ assert body[0]["awaiting_human"] is True
+ assert "stage_states" in body[0]
+
+ state = client.get("/api/project/film/state")
+ assert state.status_code == 200
+ state_body = state.json()
+ assert state_body["project_id"] == "film"
+ assert state_body["title"] == "Film"
+ assert state_body["stages"]
+
+ @pytest.mark.parametrize(
+ ("url", "status"),
+ [
+ ("/api/project/../state", 404),
+ ("/api/project/C:/state", 400),
+ ("/api/project/nope/state", 404),
+ ],
+ )
+ def test_project_id_rejects_bad_or_unknown_ids(self, client, url, status):
+ response = client.get(url)
+ assert response.status_code == status
+
+ def test_media_rejects_path_traversal(self, client, projects_root):
+ _make_project(projects_root, "film")
+ response = client.get("/media/film/%2E%2E/project.json")
+ assert response.status_code == 403
+
+ def test_media_serves_range_requests(self, client, projects_root):
+ project = _make_project(projects_root, "film")
+ media = project / "renders" / "final.mp4"
+ media.write_bytes(b"0123456789")
+
+ response = client.get("/media/film/renders/final.mp4", headers={"Range": "bytes=2-5"})
+
+ assert response.status_code == 206
+ assert response.content == b"2345"
+ assert response.headers["content-range"].startswith("bytes 2-5/10")
+
+ def test_thumb_downscales_image_and_passes_through_non_media(self, client, projects_root):
+ project = _make_project(projects_root, "film")
+ _write_png(project / "assets" / "images" / "sc1.png")
+ text = project / "artifacts" / "note.txt"
+ text.write_text("hello", encoding="utf-8")
+
+ image = client.get("/thumb/film/assets/images/sc1.png?w=320")
+ assert image.status_code == 200
+ assert image.headers["content-type"] == "image/jpeg"
+ assert image.content.startswith(b"\xff\xd8")
+
+ passthrough = client.get("/thumb/film/artifacts/note.txt")
+ assert passthrough.status_code == 200
+ assert passthrough.content == b"hello"
+
+
+class TestBacklotPerformanceBudgets:
+ def test_projects_and_state_stay_within_loose_budgets(self, client, projects_root):
+ for i in range(25):
+ project = _make_project(projects_root, f"film-{i:02d}")
+ _write_json(
+ project / "artifacts" / "scene_plan.json",
+ {"version": "1.0", "scenes": [{"id": "sc1", "start_seconds": 0, "end_seconds": 1}]},
+ )
+
+ t0 = time.perf_counter()
+ cold = client.get("/api/projects")
+ cold_s = time.perf_counter() - t0
+ assert cold.status_code == 200
+ assert cold_s < 2.0
+
+ t1 = time.perf_counter()
+ warm = client.get("/api/projects")
+ warm_s = time.perf_counter() - t1
+ assert warm.status_code == 200
+ assert warm_s < 0.150
+
+ t2 = time.perf_counter()
+ state = client.get("/api/project/film-00/state")
+ state_s = time.perf_counter() - t2
+ assert state.status_code == 200
+ assert state_s < 0.400
+
+ def test_image_thumb_generation_stays_within_budget(self, client, projects_root):
+ project = _make_project(projects_root, "film")
+ _write_png(project / "assets" / "images" / "sc1.png")
+
+ t0 = time.perf_counter()
+ response = client.get("/thumb/film/assets/images/sc1.png?w=640")
+ elapsed = time.perf_counter() - t0
+
+ assert response.status_code == 200
+ assert elapsed < 1.5
+
+
+class TestFindingsFixes:
+ """Regression tests for dogfood findings F-03 (thumb video fallback)."""
+
+ def test_thumb_never_serves_raw_video_bytes(self, client, projects_root):
+ p = _make_project(projects_root, "vid")
+ fake_video = p / "renders" / "final.mp4"
+ fake_video.parent.mkdir(parents=True, exist_ok=True)
+ # Not a real video: ffmpeg poster extraction will fail.
+ fake_video.write_bytes(b"\x00" * 4096)
+ res = client.get("/thumb/vid/renders/final.mp4")
+ assert res.status_code == 404 # never the raw video bytes (F-03)
diff --git a/tests/backlot/test_state.py b/tests/backlot/test_state.py
index a5382cf9..35827b35 100644
--- a/tests/backlot/test_state.py
+++ b/tests/backlot/test_state.py
@@ -188,3 +188,59 @@ class TestLibrary:
summary = summarize_project(p)
assert summary["awaiting_human"] is True
assert summary["active_stage"] == "script"
+
+
+class TestFindingsFixes:
+ """Regression tests for dogfood findings F-04/F-05."""
+
+ def test_artifact_refs_outside_project_are_not_followed(self, projects_root, tmp_path):
+ # F-04: a checkpoint pointing at JSON outside the project tree
+ # must not surface that file on the board.
+ secret = tmp_path / "secret.json"
+ secret.write_text(json.dumps({"version": "1.0", "leaked": True}), encoding="utf-8")
+ p = _make_project(projects_root, "sneaky-ref")
+ _write(p / "checkpoint_script.json", {
+ "stage": "script", "status": "completed",
+ "timestamp": "2026-01-01T01:00:00Z",
+ "artifacts": {"script": str(secret)},
+ })
+ s = load_board_state(p)
+ assert "script" not in s["artifacts"]
+
+ def test_inside_project_absolute_refs_still_resolve(self, projects_root):
+ p = _make_project(projects_root, "abs-ref")
+ _write(p / "artifacts" / "inline_script.json", SCRIPT)
+ _write(p / "checkpoint_script.json", {
+ "stage": "script", "status": "completed",
+ "timestamp": "2026-01-01T01:00:00Z",
+ "artifacts": {"script": str((p / "artifacts" / "inline_script.json").resolve())},
+ })
+ s = load_board_state(p)
+ assert s["artifacts"]["script"]["title"] == "Test Film"
+
+ def test_stalled_in_progress_stage_flagged(self, projects_root):
+ # F-05: an in_progress stage with no recent activity reads stalled.
+ import os
+ p = _make_project(projects_root, "wedged")
+ _write(p / "checkpoint_research.json", {
+ "stage": "research", "status": "in_progress",
+ "timestamp": "2026-01-01T01:00:00Z", "artifacts": {},
+ })
+ past = time.time() - 30 * 60
+ for f in p.rglob("*"):
+ if f.is_file():
+ os.utime(f, (past, past))
+ s = load_board_state(p)
+ research = next(x for x in s["stages"] if x["name"] == "research")
+ assert research["stalled"] is True
+ assert research["stalled_minutes"] >= 29
+
+ def test_fresh_in_progress_not_stalled(self, projects_root):
+ p = _make_project(projects_root, "busy")
+ _write(p / "checkpoint_research.json", {
+ "stage": "research", "status": "in_progress",
+ "timestamp": "2026-01-01T01:00:00Z", "artifacts": {},
+ })
+ s = load_board_state(p)
+ research = next(x for x in s["stages"] if x["name"] == "research")
+ assert "stalled" not in research
diff --git a/tests/backlot/test_visual_eval.py b/tests/backlot/test_visual_eval.py
new file mode 100644
index 00000000..70d376ac
--- /dev/null
+++ b/tests/backlot/test_visual_eval.py
@@ -0,0 +1,43 @@
+"""Tests for Backlot visual eval image comparison helpers."""
+
+from pathlib import Path
+
+from PIL import Image
+
+from scripts.backlot_visual_eval import compare_images
+
+
+def _img(path: Path, color: tuple[int, int, int]) -> None:
+ Image.new("RGB", (10, 10), color).save(path)
+
+
+def test_compare_images_detects_large_drift(tmp_path):
+ expected = tmp_path / "expected.png"
+ actual = tmp_path / "actual.png"
+ diff = tmp_path / "diff.png"
+ _img(expected, (0, 0, 0))
+ _img(actual, (255, 255, 255))
+
+ result = compare_images(expected, actual, diff, threshold=0.015)
+
+ assert result["passed"] is False
+ assert result["changed_ratio"] == 1.0
+ assert diff.exists()
+
+
+def test_compare_images_can_mask_regions(tmp_path):
+ expected = tmp_path / "expected.png"
+ actual = tmp_path / "actual.png"
+ diff = tmp_path / "diff.png"
+ _img(expected, (0, 0, 0))
+ _img(actual, (0, 0, 0))
+ img = Image.open(actual)
+ for x in range(5):
+ for y in range(5):
+ img.putpixel((x, y), (255, 255, 255))
+ img.save(actual)
+
+ result = compare_images(expected, actual, diff, threshold=0.015, masks=[(0, 0, 5, 5)])
+
+ assert result["passed"] is True
+ assert result["changed_ratio"] == 0.0
diff --git a/tests/backlot/test_watch_captures.py b/tests/backlot/test_watch_captures.py
new file mode 100644
index 00000000..bcc2487a
--- /dev/null
+++ b/tests/backlot/test_watch_captures.py
@@ -0,0 +1,47 @@
+"""Tests for the Backlot dogfood screenshot watcher helpers."""
+
+from scripts.backlot_watch_captures import capture_slug, state_fingerprint
+
+
+def test_capture_slug_keeps_names_filesystem_safe():
+ assert capture_slug("why-cities-glow", "scene_plan", "awaiting_human") == (
+ "why-cities-glow-scene_plan-awaiting_human"
+ )
+ assert capture_slug("../bad id", "C:\\stage", "in progress!") == "bad-id-C-stage-in-progress"
+
+
+def test_state_fingerprint_changes_on_board_relevant_state_only():
+ state = {
+ "stages": [
+ {"name": "script", "status": "completed", "partial_progress": None},
+ {"name": "assets", "status": "in_progress", "partial_progress": {"done": ["sc1"]}},
+ ],
+ "storyboard": {
+ "scenes": [
+ {
+ "id": "sc1",
+ "generating": False,
+ "visual": {"path": "assets/images/sc1.png", "exists": True},
+ "takes": [{"path": "assets/images/sc1.png"}],
+ },
+ {"id": "sc2", "generating": True, "generating_tool": "flux_image", "visual": None},
+ ]
+ },
+ "cost": {"total_spent_usd": 0.1},
+ "media": {"renders": []},
+ "events": [{"event": "start", "tool": "flux_image"}],
+ "last_activity": 123,
+ }
+ same = dict(state)
+ same["last_activity"] = 999
+
+ changed = dict(state)
+ changed["storyboard"] = {
+ "scenes": [
+ state["storyboard"]["scenes"][0],
+ {"id": "sc2", "generating": False, "visual": {"path": "assets/images/sc2.png", "exists": True}},
+ ]
+ }
+
+ assert state_fingerprint(state) == state_fingerprint(same)
+ assert state_fingerprint(state) != state_fingerprint(changed)