Skip to content

Open-source design agent: on-brand stills, decks, diagrams, charts and motion graphics from a chat thread, built on Deep Agents.

Print the run's cost after ask; evals keep the sample brand; ship no blank secrets

Amal Irgashev37854e73afd6
diff --git a/src/design_agent/cli.py b/src/design_agent/cli.py
index bdfe589..d1ff13a 100644
--- a/src/design_agent/cli.py
+++ b/src/design_agent/cli.py
@@ -80,6 +80,7 @@ async def _ask(
from langchain_core.messages import AIMessage
from design_agent.doctor import missing_model_key
+ from design_agent.harness.meter import Meter
from design_agent.runtime.local import build_local_runtime, final_text
from design_agent.runtime.persistence import open_persistence, workspace_of
from design_agent.tools.mcp import load_mcp_tools, servers_for_runtime
@@ -102,6 +103,7 @@ async def _ask(
)
servers = [s for s in servers if s.name not in loaded.failed]
loaded.tools = admit_paper_tools(loaded.tools, PaperPolicy.from_settings(settings))
+ meter = Meter()
async with open_persistence(settings, workspace_of(settings, root)) as (checkpointer, store):
runtime = build_local_runtime(
settings,
@@ -145,6 +147,14 @@ async def _ask(
await runtime.services.aclose()
if runtime.binding.runner is not None:
runtime.binding.runner.stop_all()
+ spent = meter.summary(thread)
+ meter.close()
+ if spent.calls:
+ click.echo(
+ f"cost: about ${spent.usd:.2f} at list prices ({spent.calls} model calls, "
+ f"{spent.hit_ratio:.0%} of input read from cache)",
+ err=True,
+ )
folder = runtime.out_dir / thread
if folder.is_dir():
for path in sorted(folder.iterdir()):
@@ -367,7 +377,11 @@ def eval_run(
for task in chosen:
click.echo(f"{model} · {task.id} …")
result = await run_task(
- task, root, model, out_root / model.replace(":", "_").replace("/", "_")
+ task,
+ root,
+ model,
+ out_root / model.replace(":", "_").replace("/", "_"),
+ set_brand=tasks.brand,
)
click.echo(
f" {result.status} in {result.seconds / 60:.1f} min, ~${result.usd:.2f} {result.note}"
diff --git a/src/design_agent/deploy.py b/src/design_agent/deploy.py
index 32e49d3..cb7e3fe 100644
--- a/src/design_agent/deploy.py
+++ b/src/design_agent/deploy.py
@@ -94,10 +94,13 @@ class Preflight:
def env_split(values: Mapping[str, str | None]) -> tuple[dict[str, str], list[str]]:
- """The `.env` entries a deployment ships, and the names it holds back."""
+ """The `.env` entries a deployment ships, and the names it holds back. Blank entries are
+ neither: they are unset, and shipping them would only list empty secrets."""
shipped: dict[str, str] = {}
held: list[str] = []
for key, value in values.items():
+ if value is not None and not value.strip():
+ continue
if value is None or key in NEVER_SHIPPED:
held.append(key)
elif key in ENV_KEYS or key.startswith(ENV_PREFIXES):
diff --git a/src/design_agent/evals/runner.py b/src/design_agent/evals/runner.py
index 35041e7..512b952 100644
--- a/src/design_agent/evals/runner.py
+++ b/src/design_agent/evals/runner.py
@@ -75,13 +75,19 @@ def _repeated(trace: Trace) -> int:
return repeats
-async def run_task(task: Task, project_root: Path, model: str, out_root: Path) -> TaskResult:
+async def run_task(
+ task: Task, project_root: Path, model: str, out_root: Path, *, set_brand: str | None = None
+) -> TaskResult:
+ """Run one task. Its brand is its own (beside tasks.json), else the set's (from the project
+ root), never the operator's: the prompts name the sample brand's product and copy."""
missing = [name for name in task.needs if not os.environ.get(name)]
if missing:
return TaskResult(task.id, model, "skipped", note=f"needs {', '.join(missing)}")
env = {**task.env, "DESIGN_AGENT_MODEL": model}
if task.brand:
env["DESIGN_AGENT_BRAND_PATH"] = str(project_root / EVAL_ROOT / task.brand)
+ elif set_brand:
+ env["DESIGN_AGENT_BRAND_PATH"] = str(project_root / set_brand)
with _environ(env):
settings = Settings()
thread = f"eval-{task.id}-{int(time.time())}"
diff --git a/tests/unit/test_deploy.py b/tests/unit/test_deploy.py
index 24a768d..190b9fa 100644
--- a/tests/unit/test_deploy.py
+++ b/tests/unit/test_deploy.py
@@ -23,9 +23,13 @@ def test_env_ships_agent_keys_and_holds_everything_else() -> None:
"LANGSMITH_API_KEY": "ls",
"MY_OTHER_PROJECT_KEY": "x",
"EMPTY": None,
+ "OPENAI_API_KEY": "",
+ "DESIGN_AGENT_HELPER_MODEL": " ",
}
)
- assert set(shipped) == {"ANTHROPIC_API_KEY", "DESIGN_AGENT_MODEL", "PAPER_MODE"}
+ assert set(shipped) == {"ANTHROPIC_API_KEY", "DESIGN_AGENT_MODEL", "PAPER_MODE"}, (
+ "blanks are unset"
+ )
assert held == [
"DATABASE_URL",
"EMPTY",
@@ -86,6 +90,8 @@ def test_preflight_refuses_the_docker_base_and_a_missing_model_key(
) -> None:
for key in ("ANTHROPIC_API_KEY", "DESIGN_AGENT_MODEL"):
monkeypatch.delenv(key, raising=False)
+ # An operator's checkout may already declare its published snapshot.
+ monkeypatch.setattr(deploy, "declared_base", lambda _root: "docker_image")
report = deploy.preflight(ROOT, Settings(_env_file=None)) # type: ignore[call-arg]
assert any("sandbox publish" in p for p in report.problems)
assert any("ANTHROPIC_API_KEY" in p for p in report.problems) or (ROOT / ".env").is_file()
diff --git a/tests/unit/test_evals.py b/tests/unit/test_evals.py
index 41c3195..ed744f4 100644
--- a/tests/unit/test_evals.py
+++ b/tests/unit/test_evals.py
@@ -11,6 +11,7 @@ import pytest
from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage
from PIL import Image
+from design_agent.config import Settings
from design_agent.evals import graders, runner
from design_agent.evals.tasks import Task, load_tasks
from design_agent.evals.trace import Trace
@@ -179,3 +180,20 @@ async def test_the_runner_grades_a_scripted_run(
report = runner.write_report([result], TASKS, tmp_path / "out")
assert "| honest-delivery |" in (report / "README.md").read_text()
assert runner.summarize([result])["anthropic:claude-opus-5-5"]["success_rate"] == 1.0
+
+
+async def test_tasks_without_a_brand_run_on_the_sets_brand_not_the_operators(
+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path
+) -> None:
+ seen: dict[str, Path] = {}
+
+ def stop(settings: Settings, _root: Path) -> None:
+ seen["brand"] = settings.brand_path
+ raise RuntimeError("stop before any model call")
+
+ monkeypatch.setattr(runner, "select_binding", stop)
+ with pytest.raises(RuntimeError):
+ await runner.run_task(
+ task("exact-copy"), ROOT, "anthropic:claude-opus-5-5", tmp_path, set_brand=TASKS.brand
+ )
+ assert seen["brand"] == ROOT / TASKS.brand
diff --git a/tests/unit/test_sandbox_publish.py b/tests/unit/test_sandbox_publish.py
index 1352283..e66b010 100644
--- a/tests/unit/test_sandbox_publish.py
+++ b/tests/unit/test_sandbox_publish.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import re
import shutil
from pathlib import Path
from types import SimpleNamespace
@@ -53,9 +54,14 @@ class FakeClient:
@pytest.fixture
def project(tmp_path: Path) -> Path:
+ """A checkout on the Docker base, whatever this one declares (an operator's may name its
+ published snapshot)."""
(tmp_path / "sandbox").mkdir()
- for name in ("__init__.py", "setup.sh"):
- shutil.copy(ROOT / "sandbox" / name, tmp_path / "sandbox" / name)
+ shutil.copy(ROOT / "sandbox" / "setup.sh", tmp_path / "sandbox" / "setup.sh")
+ declaration = (ROOT / "sandbox" / "__init__.py").read_text()
+ declaration = re.sub(r'^SNAPSHOT_NAME = "[^"]*"\n', "", declaration, flags=re.MULTILINE)
+ declaration = declaration.replace("snapshot_name=SNAPSHOT_NAME,", "docker_image=BASE_IMAGE,")
+ (tmp_path / "sandbox" / "__init__.py").write_text(declaration)
return tmp_path