Open-source design agent: on-brand stills, decks, diagrams, charts and motion graphics from a chat thread, built on Deep Agents.
JavaScript agentagentsai-agentdeepagentsdesignhyperframeslanggraphmanaged-agentsmotion-graphicspaperslackvideo
"""The footer under the lead's replies while it has work in the background (middleware/footer.py)."""
from __future__ import annotations
import asyncio
import time
from collections.abc import Callable, Iterator
from types import SimpleNamespace
from typing import Any
import pytest
from langchain_core.messages import AIMessage
from langgraph.store.memory import InMemoryStore
from design_agent.harness import events as ev
from design_agent.harness import tasks
from design_agent.middleware import footer
from design_agent.middleware.footer import Footers, ReplyFooterMiddleware, footer_line
from design_agent.middleware.meter import METER
from design_agent.middleware.tracing import TraceLinks, TraceRoot
from design_agent.surfaces.slack.target import SlackSurface
from design_agent.testing.fake_slack import BOT, THREAD_TS, FakeResolver, FakeSlack, binding
THREAD = "thread-footer"
TRACE = "https://smith.example.com/r/1"
def test_the_line_reads_as_the_mock() -> None:
live = footer_line(
TRACE, "live", usd=12.4, running=2, progress="render 45%, ~6 min left", minutes=12
)
assert (
live
== f"<{TRACE}|Trace> • 2 agents running • render 45%, ~6 min left • 12 min • ~$12.40 so far"
)
assert (
footer_line(TRACE, "done", usd=18.2, agents=3, minutes=14)
== f"<{TRACE}|Trace> • 3 agents • 14 min • ~$18.20 total"
)
assert footer_line("", "snapshot", usd=3.1) == "~$3.10 so far"
assert "1 agent running" in footer_line("", "live", usd=0, running=1)
# A job watched in the sandbox, with no agent of its own, says nothing about agents.
assert footer_line("", "live", usd=0.06, progress="teaser 44%") == "teaser 44% • ~$0.06 so far"
assert footer_line("", "done", usd=0, agents=1, minutes=0) == "1 agent • 1 min • ~$0.00 total"
@pytest.fixture
def fast(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]:
monkeypatch.setattr(footer, "REPLY_CHECKS", (0.0,))
monkeypatch.setattr(footer, "REFRESH_SECONDS", 0.01)
monkeypatch.setattr(footer, "SETTLE_SECONDS", 0.05)
yield
tasks.REGISTRY.pop("t-f00001", None)
METER.forget(THREAD)
class _Links(TraceLinks):
def __init__(self) -> None:
super().__init__("https://smith.example.com")
async def url(self, root: TraceRoot | None) -> str:
return TRACE
def _helper(started: float) -> tasks.TaskRecord:
record = tasks.TaskRecord(
id="t-f00001", thread_id=THREAD, effort="low", label="render", wave=None, started_at=started
)
tasks.REGISTRY[record.id] = record
return record
def _spend(usd: float) -> None:
METER.record(ev.Event(ev.USAGE, THREAD, "lead", data={"usd": usd}))
async def _run_ends(
fake: FakeSlack, footers: Footers, started: float, *, store: Any = None, replied: bool = True
) -> None:
ended = time.time()
if replied: # the platform posts the run's reply into the thread
fake.thread.append({"ts": f"{ended + 0.5:.6f}", "user": BOT, "text": "the reply"})
footers.after_run(
THREAD,
binding(fake),
root=None,
started=started,
usd_at_start=0.0,
ended=ended,
store=store,
)
await asyncio.gather(*list(footers._pending))
def _posts(fake: FakeSlack) -> list[str]:
return [fields["text"] for method, fields in fake.calls if method == "chat.postMessage"]
def _draws(fake: FakeSlack) -> list[tuple[str, str]]:
return [
(fields["ts"], fields["text"]) for method, fields in fake.calls if method == "chat.update"
]
async def _until(check: Callable[[], bool], seconds: float = 2.0) -> None:
deadline = time.monotonic() + seconds
while not check():
assert time.monotonic() < deadline, "timed out"
await asyncio.sleep(0.01)
async def test_a_reply_with_agents_running_gets_a_live_footer_that_settles_to_totals(
fast: None,
) -> None:
fake, footers = FakeSlack(), Footers(traces=_Links())
started = time.time()
helper = _helper(started)
_spend(2.0)
await _run_ends(fake, footers, started)
assert _posts(fake) == [f"<{TRACE}|Trace> • 1 agent running • ~$2.00 so far"]
(posted,) = fake.posted.values()
assert posted.thread_ts == THREAD_TS and posted.blocks[0]["type"] == "context"
_spend(1.5)
await _until(lambda: any("$3.50 so far" in text for _, text in _draws(fake)))
helper.status, helper.delivered = "done", True
await _until(lambda: bool(_draws(fake)) and "total" in _draws(fake)[-1][1])
assert _draws(fake)[-1] == (posted.ts, f"<{TRACE}|Trace> • 1 agent • 1 min • ~$3.50 total")
await footers.aclose()
async def test_a_newer_reply_takes_the_footer_over_and_the_last_one_closes_the_job(
fast: None,
) -> None:
fake, footers = FakeSlack(), Footers(traces=_Links())
started = time.time()
helper = _helper(started)
_spend(1.0)
await _run_ends(fake, footers, started)
first = list(fake.posted)[0]
# The person asks something while the agent works: the new reply gets the live footer, and
# the older one keeps what the job had cost by then.
await _run_ends(fake, footers, time.time())
assert len(_posts(fake)) == 2 and "running" in _posts(fake)[1]
assert (first, f"<{TRACE}|Trace> • ~$1.00 so far") in _draws(fake)
# The wake-up that reports the finish is the job's last reply: its footer settles under it.
helper.status, helper.delivered = "done", True
await _run_ends(fake, footers, time.time())
assert _posts(fake)[2] == f"<{TRACE}|Trace> • ~$1.00 so far"
await _until(lambda: bool(_draws(fake)) and "total" in _draws(fake)[-1][1])
assert _draws(fake)[-1] == (
list(fake.posted)[-1],
f"<{TRACE}|Trace> • 1 agent • 1 min • ~$1.00 total",
)
await footers.aclose()
async def test_the_job_stays_open_while_a_helpers_news_is_on_its_way(fast: None) -> None:
"""A helper the lead has not heard about yet keeps the job open: the totals go under the
reply that delivers its work, not the one before."""
fake, footers = FakeSlack(), Footers()
started = time.time()
helper = _helper(started)
await _run_ends(fake, footers, started)
helper.status = "done" # ended, and the wake-up that tells the lead is on its way
await asyncio.sleep(footer.SETTLE_SECONDS * 4)
assert not any("total" in text for _, text in _draws(fake))
helper.delivered = True
await _until(lambda: bool(_draws(fake)) and "total" in _draws(fake)[-1][1])
await footers.aclose()
async def test_after_a_restart_the_job_keeps_its_cost_and_the_dead_footer_is_retired(
fast: None,
) -> None:
"""A restart kills the process holding the footer: the next one counts on from what the job
had cost, and the dead footer stops saying an agent is running."""
store, fake = InMemoryStore(), FakeSlack()
started = time.time()
_helper(started)
_spend(2.0)
before = Footers()
await _run_ends(fake, before, started, store=store)
dead = list(fake.posted)[0]
assert _posts(fake) == ["1 agent running • ~$2.00 so far"]
# The process dies with its footers and its meter; a new one takes over the thread.
await before.aclose()
METER.forget(THREAD)
_spend(0.5)
after = Footers()
await _run_ends(fake, after, time.time(), store=store)
assert _draws(fake)[0] == (dead, "~$2.50 so far")
assert _posts(fake)[1] == "1 agent running • ~$2.50 so far"
await after.aclose()
async def test_a_turn_with_nothing_in_the_background_gets_no_footer_and_reads_nothing(
fast: None,
) -> None:
fake = FakeSlack()
_spend(0.84)
await _run_ends(fake, Footers(), time.time())
assert fake.calls == []
async def test_a_run_that_posted_no_reply_gets_no_footer(fast: None) -> None:
fake = FakeSlack()
started = time.time()
_helper(started)
await _run_ends(fake, Footers(), started, replied=False)
assert "chat.postMessage" not in fake.methods() and "conversations.replies" in fake.methods()
async def test_the_trace_link_names_the_root_run_on_the_configured_host() -> None:
looked_up: list[str] = []
def project_ids(name: str) -> tuple[str, str]:
looked_up.append(name)
return "0000aaaa", "1111bbbb"
links = TraceLinks("https://smith.example.com/", project_ids=project_ids)
root = TraceRoot(run_id="r-1", trace_id="r-1", project="design-agent")
url = await links.url(root)
assert (
url
== "https://smith.example.com/o/0000aaaa/projects/p/1111bbbb/r/r-1?trace_id=r-1&poll=true"
)
assert await links.url(root) == url and looked_up == ["design-agent"]
assert await links.url(None) == ""
assert await TraceLinks("").url(root) == ""
class _Noted(Footers):
def __init__(self) -> None:
super().__init__()
self.after: list[str] = []
def after_run(self, thread_id: str, *_args: Any, **_kwargs: Any) -> None:
self.after.append(thread_id)
async def test_only_a_reply_of_the_jobs_own_gets_a_footer() -> None:
"""A run that ended in code before this saw it start (a follow-up already worked in) and a
`[quiet]` turn wrote no reply of the job's: the live footer stays where it is."""
noted = _Noted()
middleware = ReplyFooterMiddleware(SlackSurface(FakeResolver(FakeSlack())), footers=noted)
runtime = SimpleNamespace(config={"configurable": {"thread_id": THREAD}})
answered = {"messages": [AIMessage(content="Here it is.")]}
await middleware.aafter_agent(answered, runtime)
assert noted.after == []
await middleware.abefore_agent({"messages": []}, runtime)
await middleware.aafter_agent({"messages": [AIMessage(content="[quiet]")]}, runtime)
assert noted.after == []
await middleware.abefore_agent({"messages": []}, runtime)
await middleware.aafter_agent(answered, runtime)
assert noted.after == [THREAD]