World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.
Python agentagentic-aiaiclaudecopilotcursorelevenlabsffmpegfluximage-generationopen-sourceopenaipythonremotionstable-diffusiontext-to-speechtext-to-videovideo-generationvideo-production
backlot: fix all five dogfood findings (F-01..F-05)
calesthio1d60f0da147a
diff --git a/backlot/server.py b/backlot/server.pyindex b957c2ac..854cc938 100644--- a/backlot/server.py+++ b/backlot/server.py@@ -239,7 +239,11 @@ def create_app() -> FastAPI:width = min(THUMB_WIDTHS, key=lambda x: abs(x - w))cached = await asyncio.to_thread(_thumbnail_for, target, width)if cached is None:- return FileResponse(target) # not an image we can thumb — serve as-is+ # Never fall back to raw video bytes for an <img> consumer (F-03);+ # non-thumbable images are safe to serve as-is.+ if target.suffix.lower() in {".mp4", ".webm", ".mov"}:+ raise HTTPException(status_code=404, detail="no poster frame available")+ return FileResponse(target)return FileResponse(cached, media_type="image/jpeg")# ---- Media (range requests handled by FileResponse) ---------------diff --git a/backlot/state.py b/backlot/state.pyindex 94c9caf8..ee891878 100644--- a/backlot/state.py+++ b/backlot/state.py@@ -31,6 +31,11 @@ FALLBACK_STAGES = [# How long (seconds) without filesystem activity before a board reads "idle".LIVE_WINDOW_SECONDS = 5 * 60+# An in_progress stage with no filesystem activity for this long is flagged+# as possibly stalled (F-05: a wedged agent must be visible, not silent —+# heartbeat checkpoints and tool events both reset the clock).+STALL_WINDOW_SECONDS = 10 * 60+def _read_json(path: Path) -> Optional[dict]:"""Read a JSON file, returning None on any failure."""@@ -84,13 +89,22 @@ def _load_pipeline_meta(pipeline_type: Optional[str]) -> dict[str, Any]:def _resolve_artifact(project_dir: Path, value: Any) -> Optional[dict]:- """Checkpoint artifacts may be inline dicts or path strings — resolve both."""+ """Checkpoint artifacts may be inline dicts or path strings — resolve both.++ Path references are only followed INSIDE the project directory: a+ checkpoint must not be able to pull arbitrary JSON from elsewhere on+ disk onto the board (F-04).+ """if isinstance(value, dict):return valueif isinstance(value, str) and value:p = Path(value)if not p.is_absolute():p = project_dir / value+ try:+ p.resolve().relative_to(Path(project_dir).resolve())+ except (ValueError, OSError):+ return Nonereturn _read_json(p)return None@@ -553,6 +567,16 @@ def load_board_state(project_dir: Path) -> dict[str, Any]:last_activity = _last_activity(project_dir)now = time.time()+ # Stall detection: an in_progress stage that stopped writing anything.+ for stage_entry in stages:+ if (+ stage_entry["status"] == "in_progress"+ and last_activity+ and (now - last_activity) > STALL_WINDOW_SECONDS+ ):+ stage_entry["stalled"] = True+ stage_entry["stalled_minutes"] = int((now - last_activity) / 60)+state: dict[str, Any] = {"project_id": project_id,"title": marker.get("title") or meta_json.get("name") or project_id.replace("-", " ").title(),diff --git a/backlot/ui/board.css b/backlot/ui/board.cssindex df7de240..9ae3c2f8 100644--- a/backlot/ui/board.css+++ b/backlot/ui/board.css@@ -524,3 +524,11 @@ body:not(.first) .lib-card, body:not(.first) .drawer { animation: none; }/* stages that ran but aren't declared by the pipeline manifest */.stage.undeclared .node { border-style: dashed; opacity: .85; }.stage.undeclared .name { font-style: italic; }++/* spend past 90% of budget */+.cost .bar i.crit { background: var(--red); }++/* in_progress stage with no filesystem activity for a while (F-05) */+.stage.stalled .node { border-color: var(--red); color: var(--red); background: var(--red-dim); animation: none; }+.stage.stalled .name { color: var(--red); }+.stage.stalled .sub { color: var(--red); }diff --git a/backlot/ui/board.js b/backlot/ui/board.jsindex c6878c40..1f2bcc30 100644--- a/backlot/ui/board.js+++ b/backlot/ui/board.js@@ -32,9 +32,13 @@ function renderSlate(s) {const awaiting = s.stages.find((x) => x.status === "awaiting_human");const inProgress = s.stages.find((x) => x.status === "in_progress");+ const stalled = s.stages.find((x) => x.stalled);let liveEl;if (awaiting) {liveEl = el("span", { class: "live" }, el("span", { class: "dot" }), "◈ AWAITING YOU");+ } else if (stalled) {+ liveEl = el("span", { class: "live", style: "color:var(--red)" },+ el("span", { class: "dot", style: "background:var(--red);animation:none" }), "⚠ STALLED?");} else if (s.live || inProgress) {liveEl = el("span", { class: "live" }, el("span", { class: "dot" }), "LIVE");} else {@@ -52,7 +56,7 @@ function renderSlate(s) {hasBudget ? el("span", {}, ` / ${fmtMoney(budget)}`) : ""));if (hasBudget) {cost.append(el("div", { class: "bar" }, el("i", {- class: pct > 75 ? "warn" : "", style: `width:${pct}%`,+ class: pct > 90 ? "crit" : pct > 75 ? "warn" : "", style: `width:${pct}%`,})));}cost.append(el("div", { class: "label" }, "generation spend"));@@ -77,6 +81,9 @@ function renderSlate(s) {function stageSub(st) {if (st.status === "awaiting_human") return "awaiting your approval\nreply in chat to continue";+ if (st.status === "in_progress" && st.stalled) {+ return `stalled? no activity for ${st.stalled_minutes}m\nask the agent for status`;+ }if (st.status === "in_progress" && st.partial_progress) {const done = st.partial_progress.completed_scene_ids;if (Array.isArray(done)) return `${done.length} scene${done.length === 1 ? "" : "s"} done`;@@ -96,7 +103,7 @@ function renderRail(s) {let pendingIndex = 1;for (const st of s.stages) {const cls = st.status === "completed" ? "done"- : st.status === "in_progress" ? "active"+ : st.status === "in_progress" ? (st.stalled ? "active stalled" : "active"): st.status === "awaiting_human" ? "await": st.status === "failed" ? "failed" : "";const icon = STAGE_ICONS[st.status] || String(pendingIndex);@@ -717,8 +724,31 @@ function render() {if (renders) app.append(renders);}+// Defensive normalization (F-02): the server contract guarantees these+// fields, but a sparse/legacy payload must degrade, never crash the board.+function normalize(s) {+ s.pipeline = s.pipeline || { pipeline_type: "unknown", stages: [], known: false };+ s.stages = Array.isArray(s.stages) ? s.stages : [];+ s.artifacts = s.artifacts || {};+ s.media = s.media || {};+ s.media.renders = Array.isArray(s.media.renders) ? s.media.renders : [];+ s.media.snapshots = Array.isArray(s.media.snapshots) ? s.media.snapshots : [];+ s.media.music = Array.isArray(s.media.music) ? s.media.music : [];+ s.events = Array.isArray(s.events) ? s.events : [];+ if (s.storyboard && Array.isArray(s.storyboard.scenes)) {+ for (const c of s.storyboard.scenes) {+ c.takes = Array.isArray(c.takes) ? c.takes : [];+ c.audio = Array.isArray(c.audio) ? c.audio : [];+ c.required_assets = Array.isArray(c.required_assets) ? c.required_assets : [];+ }+ } else {+ s.storyboard = null;+ }+ return s;+}+async function refresh() {- state = await getJSON(`/api/project/${encodeURIComponent(projectId)}/state`);+ state = normalize(await getJSON(`/api/project/${encodeURIComponent(projectId)}/state`));render();}diff --git a/scripts/backlot_visual_eval.py b/scripts/backlot_visual_eval.pynew file mode 100644index 00000000..34bd0438--- /dev/null+++ b/scripts/backlot_visual_eval.py@@ -0,0 +1,234 @@+"""Deterministic visual eval for Backlot.++Stages the fictional Backlot projects, captures canonical browser screenshots,+optionally compares them to goldens, and can run a small Playwright interaction+smoke against the staged board.++Examples:+ python scripts/backlot_visual_eval.py+ python scripts/backlot_visual_eval.py --bless+ python scripts/backlot_visual_eval.py --interactions+"""++from __future__ import annotations++import argparse+import json+import os+import shutil+import subprocess+import sys+import time+import urllib.request+from datetime import datetime+from pathlib import Path+from typing import Any++from PIL import Image, ImageChops++REPO_ROOT = Path(__file__).resolve().parent.parent+STAGE_DIR = REPO_ROOT / ".backlot" / "screenshot-stage"+GOLDENS_DIR = REPO_ROOT / "internal" / "evals" / "goldens"+CAPTURE_ROOT = REPO_ROOT / "internal" / "evals" / "captures"+PORT = 4791++SHOTS = [+ ("library", "/?static=1", 1560, 500, 4200, [+ (1370, 20, 1510, 62), # live/idle badge+ (90, 106, 422, 380), # card border/status animation variance+ (440, 106, 772, 380),+ (790, 106, 1122, 380),+ (1140, 106, 1472, 380),+ ]),+ ("board-live", "/p/signal-in-the-static?static=1", 1560, 1150, 4200, []),+ ("script-gate", "/p/the-slow-orchard?static=1", 1560, 760, 3200, []),+ ("storyboard", "/p/the-last-lighthouse?static=1", 1560, 1500, 4200, []),+]+++def compare_images(+ expected_path: Path,+ actual_path: Path,+ diff_path: Path,+ *,+ threshold: float = 0.015,+ masks: list[tuple[int, int, int, int]] | None = None,+) -> dict[str, Any]:+ """Compare screenshots by changed-pixel ratio and write a red diff image."""+ expected = Image.open(expected_path).convert("RGB")+ actual = Image.open(actual_path).convert("RGB")+ if expected.size != actual.size:+ diff_path.parent.mkdir(parents=True, exist_ok=True)+ actual.save(diff_path)+ return {"passed": False, "changed_ratio": 1.0, "reason": f"size {expected.size} != {actual.size}"}++ masks = masks or []+ for box in masks:+ patch = expected.crop(box)+ actual.paste(patch, box)++ delta = ImageChops.difference(expected, actual)+ changed = 0+ pixels = delta.load()+ width, height = delta.size+ diff = Image.new("RGB", delta.size, (0, 0, 0))+ diff_px = diff.load()+ for y in range(height):+ for x in range(width):+ if max(pixels[x, y]) > 8:+ changed += 1+ diff_px[x, y] = (255, 40, 40)+ else:+ diff_px[x, y] = actual.getpixel((x, y))+ ratio = changed / float(width * height)+ diff_path.parent.mkdir(parents=True, exist_ok=True)+ diff.save(diff_path)+ return {"passed": ratio <= threshold, "changed_ratio": round(ratio, 6), "threshold": threshold}+++def run_stage() -> None:+ subprocess.run(+ [sys.executable, "scripts/backlot_screenshot_stage.py", "--stage-only"],+ cwd=REPO_ROOT,+ check=True,+ timeout=180,+ )+++def start_server() -> subprocess.Popen:+ env = dict(os.environ)+ env["OPENMONTAGE_PROJECTS_DIR"] = str(STAGE_DIR)+ server = subprocess.Popen(+ [sys.executable, "-m", "backlot", "serve", "--port", str(PORT)],+ cwd=REPO_ROOT,+ env=env,+ stdout=subprocess.DEVNULL,+ stderr=subprocess.DEVNULL,+ )+ deadline = time.time() + 20+ while time.time() < deadline:+ try:+ with urllib.request.urlopen(f"http://127.0.0.1:{PORT}/api/health", timeout=1):+ return server+ except Exception:+ time.sleep(0.3)+ server.terminate()+ raise RuntimeError("Backlot server did not become healthy")+++def capture_screenshot(url: str, output: Path, width: int, height: int, wait_ms: int) -> None:+ output.parent.mkdir(parents=True, exist_ok=True)+ subprocess.run(+ [+ "npx",+ "playwright",+ "screenshot",+ "--viewport-size",+ f"{width},{height}",+ "--wait-for-timeout",+ str(wait_ms),+ url,+ str(output),+ ],+ cwd=REPO_ROOT,+ check=True,+ timeout=120,+ shell=(os.name == "nt"),+ )+++def capture_shots(capture_dir: Path) -> list[dict[str, Any]]:+ results = []+ for name, path, width, height, wait_ms, _masks in SHOTS:+ out = capture_dir / f"{name}.png"+ capture_screenshot(f"http://127.0.0.1:{PORT}{path}", out, width, height, wait_ms)+ results.append({"name": name, "path": out})+ return results+++def compare_or_bless(capture_dir: Path, *, bless: bool, threshold: float) -> list[dict[str, Any]]:+ GOLDENS_DIR.mkdir(parents=True, exist_ok=True)+ report = []+ for name, _path, _width, _height, _wait_ms, masks in SHOTS:+ actual = capture_dir / f"{name}.png"+ golden = GOLDENS_DIR / f"{name}.png"+ if bless or not golden.exists():+ shutil.copyfile(actual, golden)+ report.append({"name": name, "status": "blessed", "golden": str(golden)})+ continue+ diff = capture_dir / "diffs" / f"{name}.png"+ result = compare_images(golden, actual, diff, threshold=threshold, masks=masks)+ result.update({"name": name, "diff": str(diff)})+ report.append(result)+ return report+++def run_interactions(capture_dir: Path) -> dict[str, Any]:+ """Run browser interaction smoke through Python Playwright."""+ from playwright.sync_api import sync_playwright++ screenshot = capture_dir / "interaction-smoke.png"+ with sync_playwright() as pw:+ browser = pw.chromium.launch(headless=True)+ page = browser.new_page(viewport={"width": 1560, "height": 1000})+ page.goto(f"http://127.0.0.1:{PORT}/p/the-last-lighthouse?static=1")+ page.wait_for_selector(".stage")+ page.locator(".stage").first.click()+ page.wait_for_selector(".drawer")+ drawer_text = page.locator(".drawer").inner_text()+ if "research" not in drawer_text:+ raise RuntimeError("stage drawer did not open")+ page.locator(".script-card").first.click()+ page.wait_for_selector(".modal-bg.open")+ page.keyboard.press("Escape")+ page.wait_for_function("() => !document.querySelector('.modal-bg')?.classList.contains('open')")+ if page.locator(".takes").count() < 1:+ raise RuntimeError("takes drawer not present on staged takes scene")+ replay_button = page.locator(".rp-btn", has_text="REPLAY RUN")+ if replay_button.count():+ replay_button.first.click()+ page.wait_for_selector('input[type="range"]')+ page.locator('input[type="range"]').fill("500")+ page.screenshot(path=str(screenshot), full_page=True)+ browser.close()+ return {"status": "passed", "screenshot": str(capture_dir / "interaction-smoke.png")}+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--bless", action="store_true", help="Write current captures as goldens")+ parser.add_argument("--no-stage", action="store_true", help="Reuse existing .backlot/screenshot-stage")+ parser.add_argument("--interactions", action="store_true", help="Run Playwright interaction smoke")+ parser.add_argument("--threshold", type=float, default=0.015)+ parser.add_argument("--out-dir", type=Path, default=None)+ args = parser.parse_args(argv)++ if not args.no_stage:+ run_stage()++ stamp = datetime.now().strftime("visual-%Y%m%d-%H%M%S")+ capture_dir = args.out_dir or (CAPTURE_ROOT / stamp)+ capture_dir.mkdir(parents=True, exist_ok=True)++ server = start_server()+ try:+ capture_shots(capture_dir)+ report = compare_or_bless(capture_dir, bless=args.bless, threshold=args.threshold)+ interaction_report = run_interactions(capture_dir) if args.interactions else None+ finally:+ server.terminate()+ try:+ server.wait(timeout=5)+ except subprocess.TimeoutExpired:+ server.kill()++ passed = all(item.get("passed", item.get("status") == "blessed") for item in report)+ payload = {"capture_dir": str(capture_dir), "shots": report, "interactions": interaction_report}+ report_path = capture_dir / "report.json"+ report_path.write_text(json.dumps(payload, indent=2), encoding="utf-8")+ print(json.dumps(payload, indent=2))+ return 0 if passed else 1+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/scripts/backlot_watch_captures.py b/scripts/backlot_watch_captures.pynew file mode 100644index 00000000..0885276b--- /dev/null+++ b/scripts/backlot_watch_captures.py@@ -0,0 +1,195 @@+"""Capture Backlot board screenshots whenever watched project state changes.++This is the Half-B dogfood watcher from internal/evals/BACKLOT_EVAL_PLAN.md.+It polls the Backlot API, fingerprints board-relevant state, and captures the+library plus the changed project board through Playwright.++Example:+ python scripts/backlot_watch_captures.py --projects why-cities-glow rain-on-glass+"""++from __future__ import annotations++import argparse+import json+import os+import re+import subprocess+import sys+import time+import urllib.error+import urllib.request+from datetime import datetime+from pathlib import Path+from typing import Any++REPO_ROOT = Path(__file__).resolve().parent.parent+DEFAULT_BASE_URL = "http://127.0.0.1:4750"+DEFAULT_CAPTURE_ROOT = REPO_ROOT / "internal" / "evals" / "captures"+++def capture_slug(project_id: str, stage: str | None, status: str | None) -> str:+ """Stable, filesystem-safe screenshot name stem."""+ raw = "-".join(part for part in (project_id, stage or "unknown", status or "unknown") if part)+ raw = raw.replace("\\", "-").replace("/", "-").replace("..", "")+ slug = re.sub(r"[^A-Za-z0-9._-]+", "-", raw).strip(".-")+ slug = re.sub(r"-{2,}", "-", slug)+ return slug or "capture"+++def state_fingerprint(state: dict[str, Any]) -> str:+ """Hashable representation of board-visible state.++ Intentionally ignores mtime-ish noise such as last_activity while keeping+ the pieces that should trigger a capture: stage transitions, generating+ flags, scene visual changes, costs, renders, and event count/tail.+ """+ scenes = []+ storyboard = state.get("storyboard") or {}+ for card in storyboard.get("scenes") or []:+ visual = card.get("visual") or {}+ scenes.append({+ "id": card.get("id"),+ "generating": bool(card.get("generating")),+ "generating_tool": card.get("generating_tool"),+ "visual": {+ "path": visual.get("path"),+ "exists": visual.get("exists"),+ "type": visual.get("type"),+ },+ "takes": [take.get("path") for take in (card.get("takes") or [])],+ "audio": [asset.get("path") for asset in (card.get("audio") or [])],+ })++ media = state.get("media") or {}+ events = state.get("events") or []+ visible = {+ "stages": [+ {+ "name": stage.get("name"),+ "status": stage.get("status"),+ "gate_skipped": stage.get("gate_skipped"),+ "versions": stage.get("versions"),+ "partial_progress": stage.get("partial_progress"),+ }+ for stage in state.get("stages") or []+ ],+ "scenes": scenes,+ "cost": state.get("cost"),+ "renders": [r.get("path") for r in media.get("renders") or []],+ "snapshots": [s.get("path") for s in media.get("snapshots") or []],+ "event_count": len(events),+ "event_tail": events[-3:],+ }+ return json.dumps(visible, sort_keys=True, default=str, separators=(",", ":"))+++def active_stage(state: dict[str, Any]) -> tuple[str | None, str | None]:+ for stage in state.get("stages") or []:+ if stage.get("status") in {"in_progress", "awaiting_human", "failed", "blocked"}:+ return stage.get("name"), stage.get("status")+ for stage in reversed(state.get("stages") or []):+ if stage.get("status") == "completed":+ return stage.get("name"), stage.get("status")+ return None, None+++def fetch_json(base_url: str, path: str) -> dict[str, Any] | list[Any]:+ with urllib.request.urlopen(f"{base_url.rstrip('/')}{path}", timeout=10) as response:+ return json.loads(response.read().decode("utf-8"))+++def capture_url(url: str, output: Path, *, width: int = 1560, height: int = 1150, wait_ms: int = 1200) -> None:+ output.parent.mkdir(parents=True, exist_ok=True)+ subprocess.run(+ [+ "npx",+ "playwright",+ "screenshot",+ "--viewport-size",+ f"{width},{height}",+ "--wait-for-timeout",+ str(wait_ms),+ url,+ str(output),+ ],+ cwd=REPO_ROOT,+ check=True,+ timeout=120,+ shell=(os.name == "nt"),+ )+++def capture_project(base_url: str, capture_dir: Path, project_id: str, seq: int, state: dict[str, Any]) -> None:+ stage, status = active_stage(state)+ stem = f"{seq:03d}-{capture_slug(project_id, stage, status)}"+ capture_url(f"{base_url.rstrip('/')}/?static=1", capture_dir / "library" / f"{stem}.png", height=620)+ capture_url(+ f"{base_url.rstrip('/')}/p/{project_id}?static=1",+ capture_dir / project_id / f"{stem}.png",+ )+++def watch(+ projects: list[str],+ *,+ base_url: str,+ capture_dir: Path,+ interval_s: float,+ once: bool = False,+ no_screenshots: bool = False,+) -> int:+ fingerprints: dict[str, str] = {}+ seq = 0+ capture_dir.mkdir(parents=True, exist_ok=True)+ print(f"[watch] base={base_url} captures={capture_dir}")+ while True:+ changed = False+ for project_id in projects:+ try:+ state = fetch_json(base_url, f"/api/project/{project_id}/state")+ except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as exc:+ print(f"[watch] {project_id}: state fetch failed: {exc}", file=sys.stderr)+ continue+ fp = state_fingerprint(state)+ if fingerprints.get(project_id) == fp:+ continue+ fingerprints[project_id] = fp+ changed = True+ seq += 1+ stage, status = active_stage(state)+ print(f"[watch] change {project_id}: {stage or 'unknown'} -> {status or 'unknown'}")+ if not no_screenshots:+ capture_project(base_url, capture_dir, project_id, seq, state)+ if once:+ return 0+ if not changed:+ time.sleep(interval_s)+++def main(argv: list[str] | None = None) -> int:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--projects", nargs="+", required=True, help="Project ids to watch")+ parser.add_argument("--base-url", default=DEFAULT_BASE_URL)+ parser.add_argument("--interval", type=float, default=20.0, help="Polling interval in seconds")+ parser.add_argument("--out-dir", type=Path, default=None)+ parser.add_argument("--once", action="store_true", help="Poll once and exit")+ parser.add_argument("--no-screenshots", action="store_true", help="Exercise polling without Playwright")+ args = parser.parse_args(argv)++ out_dir = args.out_dir+ if out_dir is None:+ stamp = datetime.now().strftime("dogfood-%Y%m%d-%H%M%S")+ out_dir = DEFAULT_CAPTURE_ROOT / stamp+ return watch(+ args.projects,+ base_url=args.base_url,+ capture_dir=out_dir,+ interval_s=args.interval,+ once=args.once,+ no_screenshots=args.no_screenshots,+ )+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/tests/backlot/test_gate_scenarios.py b/tests/backlot/test_gate_scenarios.pynew file mode 100644index 00000000..d00920c3--- /dev/null+++ b/tests/backlot/test_gate_scenarios.py@@ -0,0 +1,99 @@+"""Gate-integrity scenarios for Backlot and checkpoint hardening."""++import json+from pathlib import Path++import pytest++from backlot import state as state_mod+from backlot.state import load_board_state+from lib.checkpoint import CheckpointValidationError, write_checkpoint+++def _script_artifact() -> dict:+ return {+ "version": "1.0",+ "title": "Gate Test",+ "total_duration_seconds": 5,+ "sections": [{"id": "s1", "text": "Hello.", "start_seconds": 0, "end_seconds": 5}],+ }+++def _manifest_artifact() -> dict:+ return {"version": "1.0", "assets": [], "total_cost_usd": 0.0}+++def _write(path: Path, data: dict) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(json.dumps(data), encoding="utf-8")+++def test_completed_gated_stage_without_approval_is_rejected(tmp_path):+ with pytest.raises(CheckpointValidationError, match="GATE VIOLATION"):+ write_checkpoint(+ tmp_path,+ "film",+ "script",+ "completed",+ {"script": _script_artifact()},+ pipeline_type="cinematic",+ )+++def test_typo_pipeline_type_fails_closed(tmp_path):+ with pytest.raises(CheckpointValidationError, match="Unknown pipeline_type"):+ write_checkpoint(+ tmp_path,+ "film",+ "script",+ "completed",+ {"script": _script_artifact()},+ pipeline_type="cinemtaic",+ human_approved=True,+ )+++def test_handwritten_completed_checkpoint_surfaces_gate_skip(tmp_path, monkeypatch):+ monkeypatch.setattr(state_mod, "PROJECTS_DIR", tmp_path)+ project = tmp_path / "film"+ _write(project / "checkpoint_script.json", {+ "version": "1.0",+ "project_id": "film",+ "pipeline_type": "cinematic",+ "stage": "script",+ "status": "completed",+ "timestamp": "2026-07-02T00:00:00Z",+ "artifacts": {"script": _script_artifact()},+ })++ state = load_board_state(project)++ script = next(stage for stage in state["stages"] if stage["name"] == "script")+ assert script["gate_skipped"] is True+++def test_awaiting_then_approved_archives_history_without_gate_skip(tmp_path):+ write_checkpoint(+ tmp_path,+ "film",+ "assets",+ "awaiting_human",+ {"asset_manifest": _manifest_artifact()},+ pipeline_type="cinematic",+ )+ write_checkpoint(+ tmp_path,+ "film",+ "assets",+ "completed",+ {"asset_manifest": _manifest_artifact()},+ pipeline_type="cinematic",+ human_approved=True,+ )++ state = load_board_state(tmp_path / "film")++ assets = next(stage for stage in state["stages"] if stage["name"] == "assets")+ assert assets.get("gate_skipped") in (None, False)+ assert assets["versions"] == 2+ assert assets["history_entries"][0]["status"] == "awaiting_human"diff --git a/tests/backlot/test_server.py b/tests/backlot/test_server.pynew file mode 100644index 00000000..7dfed8ed--- /dev/null+++ b/tests/backlot/test_server.py@@ -0,0 +1,205 @@+"""Server/API tests for Backlot.++These cover the deterministic eval surface in internal/evals/BACKLOT_EVAL_PLAN.md:+API shape, path safety, media/thumb serving, range requests, and loose+performance budgets.+"""++from __future__ import annotations++import io+import json+import time+from pathlib import Path++import pytest+from fastapi.testclient import TestClient+from PIL import Image++from backlot import server as server_mod+from backlot import state as state_mod+++def projects_root(tmp_path, monkeypatch):+ root = tmp_path / "projects"+ root.mkdir()+ monkeypatch.setattr(state_mod, "PROJECTS_DIR", root)+ monkeypatch.setattr(server_mod, "PROJECTS_DIR", root)+ monkeypatch.setattr(server_mod, "_summary_cache", {})+ monkeypatch.setattr(server_mod, "_PROJECTS_ROOT_STR", __import__("os").path.normcase(str(root.resolve())))+ monkeypatch.setattr(server_mod, "THUMB_CACHE_DIR", tmp_path / "thumbs")+ return root+++def client(projects_root, monkeypatch):+ async def no_watch():+ return None++ monkeypatch.setattr(server_mod, "_watch_projects", no_watch)+ with TestClient(server_mod.create_app()) as c:+ yield c+++def _write_json(path: Path, data: dict) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(json.dumps(data), encoding="utf-8")+++def _make_project(root: Path, project_id: str = "film") -> Path:+ project = root / project_id+ (project / "artifacts").mkdir(parents=True)+ (project / "assets" / "images").mkdir(parents=True)+ (project / "assets" / "video").mkdir(parents=True)+ (project / "renders").mkdir(parents=True)+ _write_json(+ project / "project.json",+ {+ "project_id": project_id,+ "title": "Film",+ "pipeline_type": "cinematic",+ "created_at": "2026-07-02T00:00:00Z",+ },+ )+ _write_json(+ project / "checkpoint_script.json",+ {+ "version": "1.0",+ "project_id": project_id,+ "pipeline_type": "cinematic",+ "stage": "script",+ "status": "awaiting_human",+ "timestamp": "2026-07-02T00:01:00Z",+ "artifacts": {},+ },+ )+ return project+++def _write_png(path: Path, color: tuple[int, int, int] = (200, 40, 80)) -> None:+ path.parent.mkdir(parents=True, exist_ok=True)+ img = Image.new("RGB", (24, 16), color)+ buf = io.BytesIO()+ img.save(buf, format="PNG")+ path.write_bytes(buf.getvalue())+++class TestBacklotServerApi:+ def test_health(self, client):+ response = client.get("/api/health")+ assert response.status_code == 200+ assert response.json() == {"ok": True, "app": "backlot"}++ def test_projects_shape_and_state(self, client, projects_root):+ _make_project(projects_root, "film")++ projects = client.get("/api/projects")+ assert projects.status_code == 200+ body = projects.json()+ assert len(body) == 1+ assert body[0]["project_id"] == "film"+ assert body[0]["awaiting_human"] is True+ assert "stage_states" in body[0]++ state = client.get("/api/project/film/state")+ assert state.status_code == 200+ state_body = state.json()+ assert state_body["project_id"] == "film"+ assert state_body["title"] == "Film"+ assert state_body["stages"]++ @pytest.mark.parametrize(+ ("url", "status"),+ [+ ("/api/project/../state", 404),+ ("/api/project/C:/state", 400),+ ("/api/project/nope/state", 404),+ ],+ )+ def test_project_id_rejects_bad_or_unknown_ids(self, client, url, status):+ response = client.get(url)+ assert response.status_code == status++ def test_media_rejects_path_traversal(self, client, projects_root):+ _make_project(projects_root, "film")+ response = client.get("/media/film/%2E%2E/project.json")+ assert response.status_code == 403++ def test_media_serves_range_requests(self, client, projects_root):+ project = _make_project(projects_root, "film")+ media = project / "renders" / "final.mp4"+ media.write_bytes(b"0123456789")++ response = client.get("/media/film/renders/final.mp4", headers={"Range": "bytes=2-5"})++ assert response.status_code == 206+ assert response.content == b"2345"+ assert response.headers["content-range"].startswith("bytes 2-5/10")++ def test_thumb_downscales_image_and_passes_through_non_media(self, client, projects_root):+ project = _make_project(projects_root, "film")+ _write_png(project / "assets" / "images" / "sc1.png")+ text = project / "artifacts" / "note.txt"+ text.write_text("hello", encoding="utf-8")++ image = client.get("/thumb/film/assets/images/sc1.png?w=320")+ assert image.status_code == 200+ assert image.headers["content-type"] == "image/jpeg"+ assert image.content.startswith(b"\xff\xd8")++ passthrough = client.get("/thumb/film/artifacts/note.txt")+ assert passthrough.status_code == 200+ assert passthrough.content == b"hello"+++class TestBacklotPerformanceBudgets:+ def test_projects_and_state_stay_within_loose_budgets(self, client, projects_root):+ for i in range(25):+ project = _make_project(projects_root, f"film-{i:02d}")+ _write_json(+ project / "artifacts" / "scene_plan.json",+ {"version": "1.0", "scenes": [{"id": "sc1", "start_seconds": 0, "end_seconds": 1}]},+ )++ t0 = time.perf_counter()+ cold = client.get("/api/projects")+ cold_s = time.perf_counter() - t0+ assert cold.status_code == 200+ assert cold_s < 2.0++ t1 = time.perf_counter()+ warm = client.get("/api/projects")+ warm_s = time.perf_counter() - t1+ assert warm.status_code == 200+ assert warm_s < 0.150++ t2 = time.perf_counter()+ state = client.get("/api/project/film-00/state")+ state_s = time.perf_counter() - t2+ assert state.status_code == 200+ assert state_s < 0.400++ def test_image_thumb_generation_stays_within_budget(self, client, projects_root):+ project = _make_project(projects_root, "film")+ _write_png(project / "assets" / "images" / "sc1.png")++ t0 = time.perf_counter()+ response = client.get("/thumb/film/assets/images/sc1.png?w=640")+ elapsed = time.perf_counter() - t0++ assert response.status_code == 200+ assert elapsed < 1.5+++class TestFindingsFixes:+ """Regression tests for dogfood findings F-03 (thumb video fallback)."""++ def test_thumb_never_serves_raw_video_bytes(self, client, projects_root):+ p = _make_project(projects_root, "vid")+ fake_video = p / "renders" / "final.mp4"+ fake_video.parent.mkdir(parents=True, exist_ok=True)+ # Not a real video: ffmpeg poster extraction will fail.+ fake_video.write_bytes(b"\x00" * 4096)+ res = client.get("/thumb/vid/renders/final.mp4")+ assert res.status_code == 404 # never the raw video bytes (F-03)diff --git a/tests/backlot/test_state.py b/tests/backlot/test_state.pyindex a5382cf9..35827b35 100644--- a/tests/backlot/test_state.py+++ b/tests/backlot/test_state.py@@ -188,3 +188,59 @@ class TestLibrary:summary = summarize_project(p)assert summary["awaiting_human"] is Trueassert summary["active_stage"] == "script"+++class TestFindingsFixes:+ """Regression tests for dogfood findings F-04/F-05."""++ def test_artifact_refs_outside_project_are_not_followed(self, projects_root, tmp_path):+ # F-04: a checkpoint pointing at JSON outside the project tree+ # must not surface that file on the board.+ secret = tmp_path / "secret.json"+ secret.write_text(json.dumps({"version": "1.0", "leaked": True}), encoding="utf-8")+ p = _make_project(projects_root, "sneaky-ref")+ _write(p / "checkpoint_script.json", {+ "stage": "script", "status": "completed",+ "timestamp": "2026-01-01T01:00:00Z",+ "artifacts": {"script": str(secret)},+ })+ s = load_board_state(p)+ assert "script" not in s["artifacts"]++ def test_inside_project_absolute_refs_still_resolve(self, projects_root):+ p = _make_project(projects_root, "abs-ref")+ _write(p / "artifacts" / "inline_script.json", SCRIPT)+ _write(p / "checkpoint_script.json", {+ "stage": "script", "status": "completed",+ "timestamp": "2026-01-01T01:00:00Z",+ "artifacts": {"script": str((p / "artifacts" / "inline_script.json").resolve())},+ })+ s = load_board_state(p)+ assert s["artifacts"]["script"]["title"] == "Test Film"++ def test_stalled_in_progress_stage_flagged(self, projects_root):+ # F-05: an in_progress stage with no recent activity reads stalled.+ import os+ p = _make_project(projects_root, "wedged")+ _write(p / "checkpoint_research.json", {+ "stage": "research", "status": "in_progress",+ "timestamp": "2026-01-01T01:00:00Z", "artifacts": {},+ })+ past = time.time() - 30 * 60+ for f in p.rglob("*"):+ if f.is_file():+ os.utime(f, (past, past))+ s = load_board_state(p)+ research = next(x for x in s["stages"] if x["name"] == "research")+ assert research["stalled"] is True+ assert research["stalled_minutes"] >= 29++ def test_fresh_in_progress_not_stalled(self, projects_root):+ p = _make_project(projects_root, "busy")+ _write(p / "checkpoint_research.json", {+ "stage": "research", "status": "in_progress",+ "timestamp": "2026-01-01T01:00:00Z", "artifacts": {},+ })+ s = load_board_state(p)+ research = next(x for x in s["stages"] if x["name"] == "research")+ assert "stalled" not in researchdiff --git a/tests/backlot/test_visual_eval.py b/tests/backlot/test_visual_eval.pynew file mode 100644index 00000000..70d376ac--- /dev/null+++ b/tests/backlot/test_visual_eval.py@@ -0,0 +1,43 @@+"""Tests for Backlot visual eval image comparison helpers."""++from pathlib import Path++from PIL import Image++from scripts.backlot_visual_eval import compare_images+++def _img(path: Path, color: tuple[int, int, int]) -> None:+ Image.new("RGB", (10, 10), color).save(path)+++def test_compare_images_detects_large_drift(tmp_path):+ expected = tmp_path / "expected.png"+ actual = tmp_path / "actual.png"+ diff = tmp_path / "diff.png"+ _img(expected, (0, 0, 0))+ _img(actual, (255, 255, 255))++ result = compare_images(expected, actual, diff, threshold=0.015)++ assert result["passed"] is False+ assert result["changed_ratio"] == 1.0+ assert diff.exists()+++def test_compare_images_can_mask_regions(tmp_path):+ expected = tmp_path / "expected.png"+ actual = tmp_path / "actual.png"+ diff = tmp_path / "diff.png"+ _img(expected, (0, 0, 0))+ _img(actual, (0, 0, 0))+ img = Image.open(actual)+ for x in range(5):+ for y in range(5):+ img.putpixel((x, y), (255, 255, 255))+ img.save(actual)++ result = compare_images(expected, actual, diff, threshold=0.015, masks=[(0, 0, 5, 5)])++ assert result["passed"] is True+ assert result["changed_ratio"] == 0.0diff --git a/tests/backlot/test_watch_captures.py b/tests/backlot/test_watch_captures.pynew file mode 100644index 00000000..bcc2487a--- /dev/null+++ b/tests/backlot/test_watch_captures.py@@ -0,0 +1,47 @@+"""Tests for the Backlot dogfood screenshot watcher helpers."""++from scripts.backlot_watch_captures import capture_slug, state_fingerprint+++def test_capture_slug_keeps_names_filesystem_safe():+ assert capture_slug("why-cities-glow", "scene_plan", "awaiting_human") == (+ "why-cities-glow-scene_plan-awaiting_human"+ )+ assert capture_slug("../bad id", "C:\\stage", "in progress!") == "bad-id-C-stage-in-progress"+++def test_state_fingerprint_changes_on_board_relevant_state_only():+ state = {+ "stages": [+ {"name": "script", "status": "completed", "partial_progress": None},+ {"name": "assets", "status": "in_progress", "partial_progress": {"done": ["sc1"]}},+ ],+ "storyboard": {+ "scenes": [+ {+ "id": "sc1",+ "generating": False,+ "visual": {"path": "assets/images/sc1.png", "exists": True},+ "takes": [{"path": "assets/images/sc1.png"}],+ },+ {"id": "sc2", "generating": True, "generating_tool": "flux_image", "visual": None},+ ]+ },+ "cost": {"total_spent_usd": 0.1},+ "media": {"renders": []},+ "events": [{"event": "start", "tool": "flux_image"}],+ "last_activity": 123,+ }+ same = dict(state)+ same["last_activity"] = 999++ changed = dict(state)+ changed["storyboard"] = {+ "scenes": [+ state["storyboard"]["scenes"][0],+ {"id": "sc2", "generating": False, "visual": {"path": "assets/images/sc2.png", "exists": True}},+ ]+ }++ assert state_fingerprint(state) == state_fingerprint(same)+ assert state_fingerprint(state) != state_fingerprint(changed)