Open-source design agent: on-brand stills, decks, diagrams, charts and motion graphics from a chat thread, built on Deep Agents.
JavaScript agentagentsai-agentdeepagentsdesignhyperframeslanggraphmanaged-agentsmotion-graphicspaperslackvideo
"""Gates that need the sandbox: Chrome Headless Shell, ffmpeg/ffprobe, sharp and the brand fonts.
Each test runs the tools inside the locally built sandbox image with the sandbox's own layout:
workspace/tools/ at /workspace/tools, the sample brand at /workspace/brand, the gate fixtures at
/workspace/fixtures (all read-only), and a scratch directory at /workspace/out.
docker build -t design-agent-sandbox:local sandbox/
uv run pytest -q -m render tests/render/test_gates_render.py
"""
from __future__ import annotations
import json
import os
import shutil
import subprocess
from collections.abc import Callable
from pathlib import Path
from typing import Any
import pytest
ROOT = Path(__file__).resolve().parents[2]
IMAGE = os.environ.get("DESIGN_AGENT_SANDBOX_IMAGE", "design-agent-sandbox:local")
DOCKER = shutil.which("docker")
SIZES = [(1080, 1350), (1080, 1080), (1600, 900)]
GSAP = "/opt/design-agent/render/node_modules/gsap/dist/gsap.min.js"
def _image_available() -> bool:
if DOCKER is None:
return False
probe = subprocess.run([DOCKER, "image", "inspect", IMAGE], capture_output=True, check=False)
return probe.returncode == 0
pytestmark = [
pytest.mark.render,
pytest.mark.skipif(
not _image_available(), reason=f"docker or the {IMAGE} image is unavailable"
),
]
Sandbox = Callable[[str], subprocess.CompletedProcess[str]]
@pytest.fixture
def out(tmp_path: Path) -> Path:
return tmp_path
def sandbox_in(out: Path) -> Sandbox:
def run(script: str) -> subprocess.CompletedProcess[str]:
assert DOCKER is not None
mounts = [
f"{ROOT / 'workspace' / 'tools'}:/workspace/tools:ro",
f"{ROOT / 'examples' / 'brands' / 'example-studio'}:/workspace/brand:ro",
# The sample brand's fonts are staged, not baked, as the workspace stager does.
f"{ROOT / 'fonts'}:/workspace/fonts:ro",
f"{ROOT / 'tests' / 'fixtures' / 'gates'}:/workspace/fixtures:ro",
f"{out}:/workspace/out",
]
return subprocess.run(
[
DOCKER,
"run",
"--rm",
"--network=none",
f"--user={os.getuid()}:{os.getgid()}",
"--env=HOME=/tmp",
"--workdir=/workspace",
*[arg for mount in mounts for arg in ("--volume", mount)],
IMAGE,
"bash",
"-c",
script,
],
capture_output=True,
text=True,
timeout=600,
check=False,
)
return run
@pytest.fixture
def sandbox(out: Path) -> Sandbox:
return sandbox_in(out)
def last_json(proc: subprocess.CompletedProcess[str]) -> Any:
return json.loads(proc.stdout.strip().splitlines()[-1])
# --- stills ----------------------------------------------------------------------------------------
@pytest.fixture(scope="module")
def three_sizes(
tmp_path_factory: pytest.TempPathFactory,
) -> tuple[Path, subprocess.CompletedProcess[str]]:
out = tmp_path_factory.mktemp("stills")
sizes = " ".join(f"--size {w}x{h}" for w, h in SIZES)
proc = sandbox_in(out)(
f"node tools/html2png.mjs fixtures/brand/clean.html --out out --name clean {sizes} "
'--require-font "Geist" --require-font "Fragment Mono" --brand-fonts'
)
return out, proc
def test_html2png_renders_each_size_with_brand_fonts(
three_sizes: tuple[Path, subprocess.CompletedProcess[str]],
) -> None:
from PIL import Image
out, proc = three_sizes
assert proc.returncode == 0, proc.stderr
report = last_json(proc)
assert report["ok"] is True
assert report["fonts"] == {"Geist": True, "Fragment Mono": True}
assert [(o["path"], o["width"], o["height"]) for o in report["outputs"]] == [
(f"/workspace/out/clean-{w}x{h}.png", w, h) for w, h in SIZES
]
for (w, h), entry in zip(SIZES, report["outputs"], strict=True):
local = out / f"clean-{w}x{h}.png"
assert local.stat().st_size == entry["bytes"] > 0
with Image.open(local) as image:
assert image.size == (w, h)
colors = image.convert("RGB").getcolors(maxcolors=1 << 20) or []
assert len(colors) > 50, "a render with almost no distinct colors drew nothing"
def test_html2png_recomposes_rather_than_scales(
three_sizes: tuple[Path, subprocess.CompletedProcess[str]],
) -> None:
from PIL import Image
out, _ = three_sizes
# The signal block sits top-right in portrait and fills the right side in landscape.
with (
Image.open(out / "clean-1080x1350.png") as portrait,
Image.open(out / "clean-1600x900.png") as landscape,
):
signal = (255, 90, 54)
assert portrait.convert("RGB").getpixel((540, 700)) != signal
assert landscape.convert("RGB").getpixel((1300, 700)) == signal
def test_rendered_stills_meet_the_artifact_contract(
three_sizes: tuple[Path, subprocess.CompletedProcess[str]],
) -> None:
out, _ = three_sizes
expects = " ".join(f"--expect out/clean-{w}x{h}.png:{w}x{h}" for w, h in SIZES)
proc = sandbox_in(out)(f"node tools/artifact-contract.mjs {expects} --json")
assert proc.returncode == 0, proc.stdout + proc.stderr
assert last_json(proc)["ok"] is True
def test_html2png_fails_when_a_required_font_is_missing(sandbox: Sandbox, out: Path) -> None:
proc = sandbox(
'node tools/html2png.mjs fixtures/brand/clean.html --out out --name x --size 1080x1080 --require-font "Nonexistent Sans"'
)
assert proc.returncode == 1, proc.stderr
report = last_json(proc)
assert report == {**report, "ok": False, "outputs": [], "fonts": {"Nonexistent Sans": False}}
assert list(out.iterdir()) == []
def test_html2png_counts_a_web_font_under_its_css_family(sandbox: Sandbox, out: Path) -> None:
"""woff2 faces load only through @font-face, and CDP names them by the name inside the file
(here "Inter"), not the CSS family the page asked for."""
(out / "f").mkdir()
shutil.copy(ROOT / "fonts" / "Inter-Variable.ttf", out / "f" / "probe.ttf")
(out / "page.html").write_text(
'<style>@font-face{font-family:"Probe Face";src:url("f/probe.woff2") format("woff2")}'
'body{font-family:"Probe Face"}</style><body>Probe</body>'
)
proc = sandbox(
"woff2_compress out/f/probe.ttf >/dev/null && "
'node tools/html2png.mjs out/page.html --out out --name p --size 400x200 --require-font "Probe Face"'
)
assert proc.returncode == 0, proc.stdout + proc.stderr
assert last_json(proc)["fonts"] == {"Probe Face": True}
def test_html2png_writes_jpeg_and_webp(sandbox: Sandbox, out: Path) -> None:
proc = sandbox(
"node tools/html2png.mjs fixtures/brand/clean.html --out out --name j --size 1080x1080 --format jpeg && "
"node tools/html2png.mjs fixtures/brand/clean.html --out out --name w --size 1080x1080 --format webp --scale 2 && "
"node tools/artifact-contract.mjs --expect out/j-1080x1080.jpg:1080x1080 --expect out/w-1080x1080.webp:2160x2160"
)
assert proc.returncode == 0, proc.stdout + proc.stderr
def test_review_thumb_makes_a_contact_sheet(sandbox: Sandbox, out: Path) -> None:
from PIL import Image
sizes = " ".join(f"--size {w}x{h}" for w, h in SIZES)
proc = sandbox(
f"node tools/html2png.mjs fixtures/brand/clean.html --out out --name s {sizes} --scale 2 >/dev/null && "
"node tools/review-thumb.mjs --sheet out/s-1080x1350.png out/s-1080x1080.png out/s-1600x900.png"
)
assert proc.returncode == 0, proc.stderr
assert proc.stdout.strip().splitlines()[-1] == "/workspace/out/review-sheet.jpg"
with Image.open(out / "review-sheet.jpg") as sheet:
assert max(sheet.size) <= 1600
def test_a_contact_sheet_takes_the_frames_shape(sandbox: Sandbox, out: Path) -> None:
"""Four 16:9 frames make a wide 2x2 sheet, not four letterboxed squares."""
from PIL import Image
proc = sandbox(
"node tools/html2png.mjs fixtures/brand/clean.html --out out --name w --size 1600x900 >/dev/null && "
"node tools/review-thumb.mjs --sheet out/w-1600x900.png out/w-1600x900.png "
"out/w-1600x900.png out/w-1600x900.png"
)
assert proc.returncode == 0, proc.stderr
with Image.open(out / "review-sheet.jpg") as sheet:
width, height = sheet.size
assert width == 1600 and height < 1000, sheet.size
# --- PDF and copy ----------------------------------------------------------------------------------
def test_html2pdf_writes_an_audited_pdf(sandbox: Sandbox, out: Path) -> None:
proc = sandbox(
"node tools/html2pdf.mjs --html fixtures/brand/clean.html --out out/clean.pdf --size 8.5x11in --brand-fonts && "
"node tools/text-fidelity.mjs out/clean.pdf --json"
)
assert proc.returncode == 0, proc.stdout + proc.stderr
assert "__HTML2PDF_OK__" in proc.stdout
assert (out / "clean.pdf").read_bytes()[:5] == b"%PDF-"
audit = last_json(proc)
assert audit["ok"] is True
assert audit["runsJudged"] >= 1
def test_html2pdf_refuses_a_missing_font(sandbox: Sandbox, out: Path) -> None:
proc = sandbox(
'node tools/html2pdf.mjs --html fixtures/brand/clean.html --out out/x.pdf --require-font "Nonexistent Sans"'
)
assert proc.returncode == 1
assert "did not resolve" in proc.stderr
assert not (out / "x.pdf").exists()
@pytest.mark.parametrize(
("page", "code", "problem"),
[("brand/clean.html", 0, None), ("text/transformed.html", 1, "casing")],
)
def test_text_fidelity_compares_rendered_copy(
sandbox: Sandbox, page: str, code: int, problem: str | None
) -> None:
proc = sandbox(
f"node tools/text-fidelity.mjs --html fixtures/{page} --expect fixtures/text/clean-copy.txt --size 1080x1350 --json"
)
assert proc.returncode == code, proc.stderr
report = last_json(proc)
if problem:
assert {p["problem"] for p in report["problems"]} >= {problem}
assert {
"line": "Field notes, issue 3",
"problem": "casing",
"rendered": "FIELD NOTES, ISSUE 3",
} in report["problems"]
# --- film ------------------------------------------------------------------------------------------
def film_script(name: str, command: str) -> str:
return f"cp fixtures/video/{name}.html {GSAP} out/ && {command}"
def test_film_probe_records_a_model_the_analysis_passes(sandbox: Sandbox, out: Path) -> None:
proc = sandbox(
film_script(
"clean",
"node tools/film-probe.mjs out/clean.html --out out/model.json && node tools/film-analysis.mjs out/model.json --json",
)
)
assert proc.returncode == 0, proc.stderr
model = json.loads((out / "model.json").read_text())
assert model["duration"] == 3
assert model["stage"] == {"w": 1920, "h": 1080}
assert model["timeline"]["paused"] is True
assert min(f["ink"] for f in model["frames"]) > 0.00145
assert last_json(proc)["errors"] == 0
@pytest.mark.parametrize(("name", "code"), [("clean", 0), ("blank-gap", 1)])
def test_video_score_gates_the_rendered_film(sandbox: Sandbox, name: str, code: int) -> None:
proc = sandbox(film_script(name, f"node tools/video-score.mjs out/{name}.html --json"))
assert proc.returncode == code, proc.stderr
report = last_json(proc)
assert report["measured"] is True
assert report["pass"] is (code == 0)
errors = {f["code"] for f in report["film"]["findings"] if f["sev"] == "ERROR"}
assert ("frames.blank" in errors) is (name == "blank-gap")
def test_film_probe_without_a_timeline_cannot_run(sandbox: Sandbox) -> None:
proc = sandbox("node tools/film-probe.mjs fixtures/brand/clean.html")
assert proc.returncode == 2
assert "no GSAP timeline" in proc.stderr
# --- video and audio files -------------------------------------------------------------------------
TESTSRC = "ffmpeg -v error -f lavfi -i testsrc=size=1280x720:rate=30 -t 2 -pix_fmt yuv420p out/testsrc.mp4"
BLACK = "ffmpeg -v error -f lavfi -i color=c=black:size=1280x720:rate=30 -t 2 -pix_fmt yuv420p out/black.mp4"
def test_artifact_contract_measures_a_video(sandbox: Sandbox) -> None:
proc = sandbox(
f"{TESTSRC} && node tools/artifact-contract.mjs --expect out/testsrc.mp4:1280x720@2 --json"
)
assert proc.returncode == 0, proc.stdout + proc.stderr
(entry,) = last_json(proc)["outputs"]
assert (entry["width"], entry["height"], entry["codec"]) == (1280, 720, "h264")
assert abs(entry["duration"] - 2) <= 1 / 30
assert entry["firstFrameInk"] > 0.00145 and entry["lastFrameInk"] > 0.00145
@pytest.mark.parametrize(
("make", "spec", "problems"),
[
(
BLACK,
"out/black.mp4:1280x720@2",
["its first frame is blank", "its last frame is blank"],
),
(TESTSRC, "out/testsrc.mp4:1920x1080@2", ["is 1280x720, expected 1920x1080"]),
(TESTSRC, "out/testsrc.mp4:1280x720@3", ["runs 2s, expected 3s"]),
],
)
def test_artifact_contract_fails_a_wrong_video(
sandbox: Sandbox, make: str, spec: str, problems: list[str]
) -> None:
proc = sandbox(f"{make} && node tools/artifact-contract.mjs --expect {spec} --json")
assert proc.returncode == 1, proc.stderr
(entry,) = last_json(proc)["outputs"]
for problem in problems:
assert any(p.startswith(problem) for p in entry["problems"]), entry["problems"]
def test_artifact_contract_allows_declared_blank_ends(sandbox: Sandbox) -> None:
proc = sandbox(
f"{BLACK} && node tools/artifact-contract.mjs --expect out/black.mp4:1280x720@2 --allow-blank-ends"
)
assert proc.returncode == 0, proc.stdout + proc.stderr
TONE = "ffmpeg -v error -f lavfi -i sine=frequency=440:duration=5 -af loudnorm=I=-14:TP=-2:LRA=7 -ar 48000 out/tone.wav"
def test_audio_lint_passes_a_mastered_tone(sandbox: Sandbox) -> None:
proc = sandbox(f"{TONE} && node tools/audio-lint.mjs out/tone.wav --expect-duration 5 --json")
assert proc.returncode == 0, proc.stdout + proc.stderr
report = last_json(proc)
assert abs(report["lufs"] + 14) <= 1
assert report["truePeak"] <= -1
@pytest.mark.parametrize(
("make", "args", "finding"),
[
(
"ffmpeg -v error -f lavfi -i sine=frequency=440:duration=5 -af volume=-30dB -ar 48000 out/a.wav",
"out/a.wav",
"integrated loudness",
),
(
"ffmpeg -v error -f lavfi -i sine=frequency=440:duration=5 -af volume=20dB -ar 48000 out/a.wav",
"out/a.wav",
"true peak",
),
(
TONE.replace("out/tone.wav", "out/a.wav"),
"out/a.wav --expect-duration 6",
"audio runs 5",
),
(TESTSRC.replace("out/testsrc.mp4", "out/a.mp4"), "out/a.mp4", "no audio stream"),
],
)
def test_audio_lint_fails_out_of_range_audio(
sandbox: Sandbox, make: str, args: str, finding: str
) -> None:
proc = sandbox(f"{make} && node tools/audio-lint.mjs {args} --json")
assert proc.returncode == 1, proc.stderr
assert any(f.startswith(finding) for f in last_json(proc)["findings"])