Open-source design agent: on-brand stills, decks, diagrams, charts and motion graphics from a chat thread, built on Deep Agents.
JavaScript agentagentsai-agentdeepagentsdesignhyperframeslanggraphmanaged-agentsmotion-graphicspaperslackvideo
Print the run's cost after ask; evals keep the sample brand; ship no blank secrets
Amal Irgashev37854e73afd6
diff --git a/src/design_agent/cli.py b/src/design_agent/cli.pyindex bdfe589..d1ff13a 100644--- a/src/design_agent/cli.py+++ b/src/design_agent/cli.py@@ -80,6 +80,7 @@ async def _ask(from langchain_core.messages import AIMessagefrom design_agent.doctor import missing_model_key+ from design_agent.harness.meter import Meterfrom design_agent.runtime.local import build_local_runtime, final_textfrom design_agent.runtime.persistence import open_persistence, workspace_offrom design_agent.tools.mcp import load_mcp_tools, servers_for_runtime@@ -102,6 +103,7 @@ async def _ask()servers = [s for s in servers if s.name not in loaded.failed]loaded.tools = admit_paper_tools(loaded.tools, PaperPolicy.from_settings(settings))+ meter = Meter()async with open_persistence(settings, workspace_of(settings, root)) as (checkpointer, store):runtime = build_local_runtime(settings,@@ -145,6 +147,14 @@ async def _ask(await runtime.services.aclose()if runtime.binding.runner is not None:runtime.binding.runner.stop_all()+ spent = meter.summary(thread)+ meter.close()+ if spent.calls:+ click.echo(+ f"cost: about ${spent.usd:.2f} at list prices ({spent.calls} model calls, "+ f"{spent.hit_ratio:.0%} of input read from cache)",+ err=True,+ )folder = runtime.out_dir / threadif folder.is_dir():for path in sorted(folder.iterdir()):@@ -367,7 +377,11 @@ def eval_run(for task in chosen:click.echo(f"{model} · {task.id} …")result = await run_task(- task, root, model, out_root / model.replace(":", "_").replace("/", "_")+ task,+ root,+ model,+ out_root / model.replace(":", "_").replace("/", "_"),+ set_brand=tasks.brand,)click.echo(f" {result.status} in {result.seconds / 60:.1f} min, ~${result.usd:.2f} {result.note}"diff --git a/src/design_agent/deploy.py b/src/design_agent/deploy.pyindex 32e49d3..cb7e3fe 100644--- a/src/design_agent/deploy.py+++ b/src/design_agent/deploy.py@@ -94,10 +94,13 @@ class Preflight:def env_split(values: Mapping[str, str | None]) -> tuple[dict[str, str], list[str]]:- """The `.env` entries a deployment ships, and the names it holds back."""+ """The `.env` entries a deployment ships, and the names it holds back. Blank entries are+ neither: they are unset, and shipping them would only list empty secrets."""shipped: dict[str, str] = {}held: list[str] = []for key, value in values.items():+ if value is not None and not value.strip():+ continueif value is None or key in NEVER_SHIPPED:held.append(key)elif key in ENV_KEYS or key.startswith(ENV_PREFIXES):diff --git a/src/design_agent/evals/runner.py b/src/design_agent/evals/runner.pyindex 35041e7..512b952 100644--- a/src/design_agent/evals/runner.py+++ b/src/design_agent/evals/runner.py@@ -75,13 +75,19 @@ def _repeated(trace: Trace) -> int:return repeats-async def run_task(task: Task, project_root: Path, model: str, out_root: Path) -> TaskResult:+async def run_task(+ task: Task, project_root: Path, model: str, out_root: Path, *, set_brand: str | None = None+) -> TaskResult:+ """Run one task. Its brand is its own (beside tasks.json), else the set's (from the project+ root), never the operator's: the prompts name the sample brand's product and copy."""missing = [name for name in task.needs if not os.environ.get(name)]if missing:return TaskResult(task.id, model, "skipped", note=f"needs {', '.join(missing)}")env = {**task.env, "DESIGN_AGENT_MODEL": model}if task.brand:env["DESIGN_AGENT_BRAND_PATH"] = str(project_root / EVAL_ROOT / task.brand)+ elif set_brand:+ env["DESIGN_AGENT_BRAND_PATH"] = str(project_root / set_brand)with _environ(env):settings = Settings()thread = f"eval-{task.id}-{int(time.time())}"diff --git a/tests/unit/test_deploy.py b/tests/unit/test_deploy.pyindex 24a768d..190b9fa 100644--- a/tests/unit/test_deploy.py+++ b/tests/unit/test_deploy.py@@ -23,9 +23,13 @@ def test_env_ships_agent_keys_and_holds_everything_else() -> None:"LANGSMITH_API_KEY": "ls","MY_OTHER_PROJECT_KEY": "x","EMPTY": None,+ "OPENAI_API_KEY": "",+ "DESIGN_AGENT_HELPER_MODEL": " ",})- assert set(shipped) == {"ANTHROPIC_API_KEY", "DESIGN_AGENT_MODEL", "PAPER_MODE"}+ assert set(shipped) == {"ANTHROPIC_API_KEY", "DESIGN_AGENT_MODEL", "PAPER_MODE"}, (+ "blanks are unset"+ )assert held == ["DATABASE_URL","EMPTY",@@ -86,6 +90,8 @@ def test_preflight_refuses_the_docker_base_and_a_missing_model_key() -> None:for key in ("ANTHROPIC_API_KEY", "DESIGN_AGENT_MODEL"):monkeypatch.delenv(key, raising=False)+ # An operator's checkout may already declare its published snapshot.+ monkeypatch.setattr(deploy, "declared_base", lambda _root: "docker_image")report = deploy.preflight(ROOT, Settings(_env_file=None)) # type: ignore[call-arg]assert any("sandbox publish" in p for p in report.problems)assert any("ANTHROPIC_API_KEY" in p for p in report.problems) or (ROOT / ".env").is_file()diff --git a/tests/unit/test_evals.py b/tests/unit/test_evals.pyindex 41c3195..ed744f4 100644--- a/tests/unit/test_evals.py+++ b/tests/unit/test_evals.py@@ -11,6 +11,7 @@ import pytestfrom langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessagefrom PIL import Image+from design_agent.config import Settingsfrom design_agent.evals import graders, runnerfrom design_agent.evals.tasks import Task, load_tasksfrom design_agent.evals.trace import Trace@@ -179,3 +180,20 @@ async def test_the_runner_grades_a_scripted_run(report = runner.write_report([result], TASKS, tmp_path / "out")assert "| honest-delivery |" in (report / "README.md").read_text()assert runner.summarize([result])["anthropic:claude-opus-5-5"]["success_rate"] == 1.0+++async def test_tasks_without_a_brand_run_on_the_sets_brand_not_the_operators(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path+) -> None:+ seen: dict[str, Path] = {}++ def stop(settings: Settings, _root: Path) -> None:+ seen["brand"] = settings.brand_path+ raise RuntimeError("stop before any model call")++ monkeypatch.setattr(runner, "select_binding", stop)+ with pytest.raises(RuntimeError):+ await runner.run_task(+ task("exact-copy"), ROOT, "anthropic:claude-opus-5-5", tmp_path, set_brand=TASKS.brand+ )+ assert seen["brand"] == ROOT / TASKS.branddiff --git a/tests/unit/test_sandbox_publish.py b/tests/unit/test_sandbox_publish.pyindex 1352283..e66b010 100644--- a/tests/unit/test_sandbox_publish.py+++ b/tests/unit/test_sandbox_publish.py@@ -2,6 +2,7 @@from __future__ import annotations+import reimport shutilfrom pathlib import Pathfrom types import SimpleNamespace@@ -53,9 +54,14 @@ class FakeClient:@pytest.fixturedef project(tmp_path: Path) -> Path:+ """A checkout on the Docker base, whatever this one declares (an operator's may name its+ published snapshot)."""(tmp_path / "sandbox").mkdir()- for name in ("__init__.py", "setup.sh"):- shutil.copy(ROOT / "sandbox" / name, tmp_path / "sandbox" / name)+ shutil.copy(ROOT / "sandbox" / "setup.sh", tmp_path / "sandbox" / "setup.sh")+ declaration = (ROOT / "sandbox" / "__init__.py").read_text()+ declaration = re.sub(r'^SNAPSHOT_NAME = "[^"]*"\n', "", declaration, flags=re.MULTILINE)+ declaration = declaration.replace("snapshot_name=SNAPSHOT_NAME,", "docker_image=BASE_IMAGE,")+ (tmp_path / "sandbox" / "__init__.py").write_text(declaration)return tmp_path