Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python
31.6 KB
Raw Blame
import json
import sqlite3
from pathlib import Path
from typing import Any, Optional

import pytest

from agent.providers.base import StreamEvent
from costs.pricing import MODEL_PRICING
from costs.token_usage import TokenUsage
from agent.tools.types import ToolCall, ToolExecutionResult
from fs_logging.agent_runs import (
    RUN_ID_PATTERN,
    AgentRunRecorder,
    get_agent_runs_db_path,
    get_agent_runs_directory,
)
from llm import Llm


@pytest.fixture
def logs_path(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:
    monkeypatch.setenv("LOGS_PATH", str(tmp_path))
    return tmp_path


def _make_recorder(enabled: bool = True, **overrides: Any) -> AgentRunRecorder:
    kwargs: dict[str, Any] = dict(
        generation_id="gen_test_00000000",
        variant_index=0,
        entry_point="websocket",
        stack="html_tailwind",
        input_mode="image",
        generation_type="create",
        enabled=enabled,
    )
    kwargs.update(overrides)
    return AgentRunRecorder(**kwargs)


def _read_events(recorder: AgentRunRecorder) -> list[dict[str, Any]]:
    events_path = Path(recorder.run_dir) / "events.jsonl"
    with open(events_path) as f:
        return [json.loads(line) for line in f if line.strip()]


def _db_row(query: str, params: tuple[Any, ...] = ()) -> Optional[tuple[Any, ...]]:
    conn = sqlite3.connect(get_agent_runs_db_path())
    try:
        cursor = conn.execute(query, params)
        return cursor.fetchone()
    finally:
        conn.close()


@pytest.mark.asyncio
async def test_disabled_recorder_writes_nothing(logs_path: Path) -> None:
    recorder = _make_recorder(enabled=False)
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [{"role": "user", "content": "hi"}])
    recorder.record_llm_request("openai", "gpt-5.5", {"input": []})
    recorder.record_llm_response("text", [], TokenUsage(input=10, output=5, total=15))
    await recorder.record_run_end("completed", final_html="<html></html>")

    assert not (logs_path / "run_logs").exists()


def test_run_start_creates_events_and_db_row(logs_path: Path) -> None:
    recorder = _make_recorder()
    assert RUN_ID_PATTERN.match(recorder.run_id)
    recorder.record_run_start(
        Llm.CLAUDE_OPUS_4_8_HIGH, [{"role": "user", "content": "make a page"}]
    )

    events = _read_events(recorder)
    assert [e["type"] for e in events] == ["run_start"]
    assert events[0]["model"] == Llm.CLAUDE_OPUS_4_8_HIGH.value
    assert events[0]["provider"] == "anthropic"
    assert events[0]["prompt_messages"] == [{"role": "user", "content": "make a page"}]

    row = _db_row(
        "SELECT generation_id, variant_index, entry_point, provider, status "
        "FROM runs WHERE run_id = ?",
        (recorder.run_id,),
    )
    assert row == ("gen_test_00000000", 0, "websocket", "anthropic", "running")


@pytest.mark.asyncio
async def test_full_run_records_events_costs_and_totals(logs_path: Path) -> None:
    recorder = _make_recorder()
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [{"role": "user", "content": "go"}])

    # Step 1: thinking + assistant deltas, one tool call
    recorder.record_llm_request("openai", "gpt-5.5", {"input": ["go"]})
    recorder.record_stream_event(
        StreamEvent(type="thinking_delta", text="hmm "), "thinking-0-abc"
    )
    recorder.record_stream_event(
        StreamEvent(type="thinking_delta", text="ok"), "thinking-0-abc"
    )
    recorder.record_stream_event(
        StreamEvent(
            type="tool_call_delta",
            tool_call_id="call_1",
            tool_name="create_file",
            tool_arguments='{"path": "index.html"',
        ),
        "call_1",
    )
    recorder.record_stream_event(
        StreamEvent(
            type="tool_call_delta",
            tool_call_id="call_1",
            tool_name="create_file",
            tool_arguments='{"path": "index.html", "content": "<html>"}',
        ),
        "call_1",
    )
    usage_1 = TokenUsage(input=1000, output=500, cache_read=200, total=1700)
    tool_call = ToolCall(
        id="call_1",
        name="create_file",
        arguments={"path": "index.html", "content": "<html>"},
    )
    recorder.record_llm_response("", [tool_call], usage_1)

    recorder.record_tool_start("call_1", tool_call)
    recorder.record_tool_end(
        "call_1",
        tool_call,
        ToolExecutionResult(
            ok=True,
            result={"path": "index.html"},
            summary={"path": "index.html"},
            updated_content="<html></html>",
        ),
    )
    recorder.record_set_code(14, "tool_result")

    # Step 2: final assistant turn, no tools
    recorder.record_llm_request("openai", "gpt-5.5", {"input": ["continue"]})
    recorder.record_stream_event(
        StreamEvent(type="assistant_delta", text="done"), "assistant-0-def"
    )
    usage_2 = TokenUsage(input=2000, output=100, total=2100)
    recorder.record_llm_response("done", [], usage_2)

    await recorder.record_run_end("completed", final_html="<html>final</html>")

    events = _read_events(recorder)
    assert [e["type"] for e in events] == [
        "run_start",
        "llm_call_start",
        "stream_delta",
        "stream_delta",
        "stream_delta",
        "stream_delta",
        "llm_call_end",
        "tool_call_start",
        "tool_call_end",
        "set_code",
        "llm_call_start",
        "stream_delta",
        "llm_call_end",
        "run_end",
    ]
    assert [e["seq"] for e in events] == list(range(1, len(events) + 1))

    # Cumulative tool-arg snapshots are recorded as incremental suffixes.
    tool_deltas = [
        e for e in events if e["type"] == "stream_delta" and e["kind"] == "tool_call"
    ]
    assert tool_deltas[0]["args_delta"] == '{"path": "index.html"'
    assert tool_deltas[1]["args_delta"] == ', "content": "<html>"}'

    llm_end_1 = events[6]
    assert llm_end_1["thinking_text"] == "hmm ok"
    assert llm_end_1["thinking_ms"] >= 0
    assert llm_end_1["tool_calls"] == [
        {
            "id": "call_1",
            "name": "create_file",
            "arguments": {"path": "index.html", "content": "<html>"},
        }
    ]
    pricing = MODEL_PRICING["gpt-5.5"]
    assert llm_end_1["cost_usd"] == pytest.approx(usage_1.cost(pricing))

    tool_end = events[8]
    assert tool_end["ok"] is True
    assert tool_end["result"] == {"path": "index.html"}
    assert tool_end["updated_content_len"] == len("<html></html>")
    assert tool_end["duration_ms"] >= 0

    run_end = events[-1]
    expected_total = usage_1.cost(pricing) + usage_2.cost(pricing)
    assert run_end["status"] == "completed"
    assert run_end["num_llm_calls"] == 2
    assert run_end["num_tool_calls"] == 1
    assert run_end["usage"]["input"] == 3000
    assert run_end["usage"]["total"] == 3800
    assert run_end["total_cost_usd"] == pytest.approx(expected_total)
    assert run_end["has_unpriced_calls"] is False
    assert run_end["final_html_len"] == len("<html>final</html>")

    row = _db_row(
        "SELECT status, num_llm_calls, num_tool_calls, input_tokens, total_tokens, "
        "total_cost_usd, has_unpriced_calls, total_duration_ms, llm_time_ms, "
        "tool_time_ms, api_model_name FROM runs WHERE run_id = ?",
        (recorder.run_id,),
    )
    assert row is not None
    assert row[0] == "completed"
    assert row[1] == 2
    assert row[2] == 1
    assert row[3] == 3000
    assert row[4] == 3800
    assert row[5] == pytest.approx(expected_total)
    assert row[6] == 0
    assert row[7] >= 0 and row[8] >= 0 and row[9] >= 0
    assert row[10] == "gpt-5.5"

    call_rows = []
    conn = sqlite3.connect(get_agent_runs_db_path())
    try:
        call_rows = conn.execute(
            "SELECT step, input_tokens, cost_usd FROM llm_calls "
            "WHERE run_id = ? ORDER BY step",
            (recorder.run_id,),
        ).fetchall()
    finally:
        conn.close()
    assert len(call_rows) == 2
    assert call_rows[0][1] == 1000
    assert call_rows[0][2] == pytest.approx(usage_1.cost(pricing))

    run_json = json.loads((Path(recorder.run_dir) / "run.json").read_text())
    assert run_json["final_html"] == "<html>final</html>"
    assert len(run_json["llm_calls"]) == 2
    assert len(run_json["tool_calls"]) == 1
    assert (Path(recorder.run_dir) / "final.html").read_text() == "<html>final</html>"


@pytest.mark.asyncio
async def test_unpriced_model_flags_and_null_cost(logs_path: Path) -> None:
    recorder = _make_recorder()
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
    recorder.record_llm_request("openai", "unknown-model-x", {})
    recorder.record_llm_response("hi", [], TokenUsage(input=10, output=5, total=15))
    await recorder.record_run_end("completed")

    events = _read_events(recorder)
    llm_end = next(e for e in events if e["type"] == "llm_call_end")
    assert llm_end["cost_usd"] is None
    run_end = events[-1]
    assert run_end["total_cost_usd"] is None
    assert run_end["has_unpriced_calls"] is True

    row = _db_row(
        "SELECT total_cost_usd, has_unpriced_calls FROM runs WHERE run_id = ?",
        (recorder.run_id,),
    )
    assert row == (None, 1)


@pytest.mark.asyncio
async def test_gemini_none_usage_is_tolerated(logs_path: Path) -> None:
    recorder = _make_recorder()
    recorder.record_run_start(Llm.GEMINI_3_5_FLASH_HIGH, [])
    recorder.record_llm_request("gemini", "gemini-3.5-flash", {})
    recorder.record_llm_response("hi", [], None)
    await recorder.record_run_end("completed")

    events = _read_events(recorder)
    llm_end = next(e for e in events if e["type"] == "llm_call_end")
    assert llm_end["usage"] is None
    assert llm_end["cost_usd"] is None
    assert events[-1]["has_unpriced_calls"] is True


@pytest.mark.asyncio
async def test_failed_run_and_idempotent_run_end(logs_path: Path) -> None:
    recorder = _make_recorder()
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
    await recorder.record_run_end("failed", error="boom")
    # A second call (e.g. failure path after cancel) must not double-write.
    await recorder.record_run_end("completed", final_html="<html></html>")

    events = _read_events(recorder)
    assert [e["type"] for e in events] == ["run_start", "run_end"]
    assert events[-1]["status"] == "failed"
    assert events[-1]["error"] == "boom"

    row = _db_row(
        "SELECT status, error FROM runs WHERE run_id = ?", (recorder.run_id,)
    )
    assert row == ("failed", "boom")


@pytest.mark.asyncio
async def test_asset_snapshot_copies_local_and_rewrites(
    logs_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    local_asset_dir = logs_path / "local_assets"
    local_asset_dir.mkdir()
    (local_asset_dir / "crop_abc.png").write_bytes(b"png-bytes")
    monkeypatch.setattr(
        "fs_logging.agent_runs.LOCAL_ASSET_DIR", str(local_asset_dir)
    )

    html = (
        '<html><img src="http://127.0.0.1:7001/local-assets/crop_abc.png">'
        '<img src="/local-assets/missing.png">'
        '<div style="background: url(/local-assets/crop_abc.png)"></div></html>'
    )
    recorder = _make_recorder()
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
    await recorder.record_run_end("completed", final_html=html)

    run_dir = Path(recorder.run_dir)
    assert (run_dir / "final.html").read_text() == html
    assert (run_dir / "assets" / "crop_abc.png").read_bytes() == b"png-bytes"

    rewritten = (run_dir / "final_selfcontained.html").read_text()
    assert "assets/crop_abc.png" in rewritten
    assert "http://127.0.0.1:7001/local-assets/" not in rewritten
    assert "url(assets/crop_abc.png)" in rewritten

    manifest = json.loads((run_dir / "assets_manifest.json").read_text())
    statuses = {entry["url"]: entry["status"] for entry in manifest}
    assert statuses["http://127.0.0.1:7001/local-assets/crop_abc.png"] == "copied"
    assert statuses["/local-assets/missing.png"] == "missing"


@pytest.mark.asyncio
async def test_asset_snapshot_downloads_remote_images(
    logs_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    class FakeResponse:
        content = b"remote-image-bytes"
        headers = {"content-type": "image/png"}

        def raise_for_status(self) -> None:
            return None

    class FakeClient:
        def __init__(self, *args: Any, **kwargs: Any) -> None:
            self.requested: list[str] = []

        async def __aenter__(self) -> "FakeClient":
            return self

        async def __aexit__(self, *args: Any) -> None:
            return None

        async def get(self, url: str) -> FakeResponse:
            self.requested.append(url)
            if "broken" in url:
                raise RuntimeError("download failed")
            return FakeResponse()

    import httpx

    monkeypatch.setattr(httpx, "AsyncClient", FakeClient)

    html = (
        '<html><img src="https://replicate.delivery/xyz/generated">'
        '<img src="https://example.com/broken.png"></html>'
    )
    recorder = _make_recorder()
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
    await recorder.record_run_end("completed", final_html=html)

    run_dir = Path(recorder.run_dir)
    manifest = json.loads((run_dir / "assets_manifest.json").read_text())
    by_url = {entry["url"]: entry for entry in manifest}
    ok_entry = by_url["https://replicate.delivery/xyz/generated"]
    assert ok_entry["status"] == "downloaded"
    saved = run_dir / ok_entry["file"]
    assert saved.read_bytes() == b"remote-image-bytes"
    assert by_url["https://example.com/broken.png"]["status"] == "failed"

    rewritten = (run_dir / "final_selfcontained.html").read_text()
    assert "https://replicate.delivery/xyz/generated" not in rewritten
    # Failed downloads keep their original URL.
    assert "https://example.com/broken.png" in rewritten


@pytest.mark.asyncio
async def test_asset_snapshot_catches_urls_in_js_strings(
    logs_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    class FakeResponse:
        content = b"js-loaded-image"
        headers = {"content-type": "image/jpeg"}

        def raise_for_status(self) -> None:
            return None

    class FakeClient:
        def __init__(self, *args: Any, **kwargs: Any) -> None:
            pass

        async def __aenter__(self) -> "FakeClient":
            return self

        async def __aexit__(self, *args: Any) -> None:
            return None

        async def get(self, url: str) -> FakeResponse:
            return FakeResponse()

    import httpx

    monkeypatch.setattr(httpx, "AsyncClient", FakeClient)

    # Image loaded programmatically — no src= attribute anywhere.
    html = (
        "<html><script>loadPhoto(el, "
        "'https://replicate.delivery/abc/output_1.jpeg');</script></html>"
    )
    recorder = _make_recorder()
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
    await recorder.record_run_end("completed", final_html=html)

    run_dir = Path(recorder.run_dir)
    manifest = json.loads((run_dir / "assets_manifest.json").read_text())
    assert manifest[0]["status"] == "downloaded"
    rewritten = (run_dir / "final_selfcontained.html").read_text()
    assert "replicate.delivery" not in rewritten
    assert "assets/" in rewritten


class _ToolThenDoneSession:
    """One create_file tool turn, then a final assistant turn."""

    def __init__(self, cost_usd: Optional[float] = None) -> None:
        self.turns = 0
        self.closed = False
        self.cost_usd = cost_usd

    def total_cost_usd(self) -> Optional[float]:
        return self.cost_usd

    async def stream_turn(self, on_event: Any) -> Any:
        from agent.providers.base import ProviderTurn, StreamEvent

        self.turns += 1
        if self.turns == 1:
            await on_event(StreamEvent(type="thinking_delta", text="planning"))
            return ProviderTurn(
                assistant_text="",
                tool_calls=[
                    ToolCall(
                        id="call_1",
                        name="create_file",
                        arguments={
                            "path": "index.html",
                            "content": "<html>from tool</html>",
                        },
                    )
                ],
            )
        return ProviderTurn(assistant_text="done", tool_calls=[])

    async def append_tool_results(self, turn: Any, executed_tool_calls: Any) -> None:
        return None

    async def close(self) -> None:
        self.closed = True


class _ExplodingSession:
    async def stream_turn(self, on_event: Any) -> Any:
        raise RuntimeError("provider exploded")

    async def append_tool_results(self, turn: Any, executed_tool_calls: Any) -> None:
        return None

    async def close(self) -> None:
        return None


def _make_engine(recorder: AgentRunRecorder) -> Any:
    from agent.engine import AgentEngine

    async def send_message(
        message_type: str,
        value: Optional[str],
        variant_index: int,
        data: Optional[dict[str, Any]],
        event_id: Optional[str],
    ) -> None:
        return None

    return AgentEngine(
        send_message=send_message,
        variant_index=0,
        openai_api_key="key",
        openai_base_url=None,
        anthropic_api_key=None,
        gemini_api_key=None,
        replicate_api_key=None,
        should_generate_images=False,
        recorder=recorder,
    )


@pytest.mark.asyncio
async def test_engine_records_full_run(
    logs_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    session = _ToolThenDoneSession()
    monkeypatch.setattr(
        "agent.engine.create_provider_session", lambda **kwargs: session
    )

    recorder = _make_recorder()
    engine = _make_engine(recorder)
    result = await engine.run(
        Llm.GPT_5_5_HIGH, [{"role": "user", "content": "build it"}]  # type: ignore[list-item]
    )

    assert result == "<html>from tool</html>"
    events = _read_events(recorder)
    types = [e["type"] for e in events]
    assert types[0] == "run_start"
    assert "stream_delta" in types
    assert "tool_call_start" in types and "tool_call_end" in types
    assert types[-1] == "run_end"
    assert events[-1]["status"] == "completed"
    assert events[-1]["num_tool_calls"] == 1

    tool_end = next(e for e in events if e["type"] == "tool_call_end")
    assert tool_end["name"] == "create_file"
    assert tool_end["ok"] is True

    row = _db_row("SELECT status FROM runs WHERE run_id = ?", (recorder.run_id,))
    assert row == ("completed",)
    assert (Path(recorder.run_dir) / "final.html").read_text() == (
        "<html>from tool</html>"
    )


@pytest.mark.asyncio
async def test_engine_aborts_over_budget_run(
    logs_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    from agent.engine import BudgetExceededError

    session = _ToolThenDoneSession(cost_usd=5.0)
    monkeypatch.setattr(
        "agent.engine.create_provider_session", lambda **kwargs: session
    )

    recorder = _make_recorder()
    engine = _make_engine(recorder)
    with pytest.raises(BudgetExceededError) as excinfo:
        await engine.run(Llm.GPT_5_5_HIGH, [])

    # User-facing message must not leak cost figures.
    assert "$" not in str(excinfo.value)
    events = _read_events(recorder)
    assert events[-1]["status"] == "failed"
    # Only the first turn ran; its tools were never executed.
    assert session.turns == 1


@pytest.mark.asyncio
async def test_engine_allows_final_turn_over_budget(
    logs_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    # A run finishing naturally is returned even when over budget: aborting
    # after the final turn would waste tokens already paid for.
    session = _ToolThenDoneSession(cost_usd=0.5)
    monkeypatch.setattr(
        "agent.engine.create_provider_session", lambda **kwargs: session
    )
    recorder = _make_recorder()
    engine = _make_engine(recorder)

    async def creep_cost(*args: Any, **kwargs: Any) -> Any:
        # Cost crosses the ceiling only on the final (no-tool) turn.
        result = await _ToolThenDoneSession.stream_turn(session, *args, **kwargs)
        if session.turns == 2:
            session.cost_usd = 5.0
        return result

    monkeypatch.setattr(session, "stream_turn", creep_cost)
    result = await engine.run(Llm.GPT_5_5_HIGH, [])
    assert result == "<html>from tool</html>"


@pytest.mark.asyncio
async def test_eval_runner_does_not_retry_budget_abort(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from agent.engine import BudgetExceededError
    from evals.runner import generate_code_and_time

    calls = {"count": 0}

    async def always_over_budget(**kwargs: Any) -> str:
        calls["count"] += 1
        raise BudgetExceededError()

    monkeypatch.setattr(
        "evals.runner.generate_code_for_image", always_over_budget
    )
    (
        filename,
        attempt,
        content,
        duration,
        error,
        retries_used,
    ) = await generate_code_and_time(
        image_url="data:image/png;base64,eA==",
        stack="html_tailwind",
        model=Llm.GPT_5_5_HIGH,
        original_input_filename="x.png",
        attempt_idx=0,
    )
    assert isinstance(error, BudgetExceededError)
    assert content is None
    assert retries_used == 0
    assert calls["count"] == 1  # no retries


@pytest.mark.asyncio
async def test_engine_fails_empty_output_runs(
    logs_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    from agent.engine import EmptyOutputError
    from agent.providers.base import ProviderTurn

    class _NoOutputSession:
        def total_cost_usd(self) -> Optional[float]:
            return None

        async def stream_turn(self, on_event: Any) -> Any:
            # The observed gemini-3.6-flash failure mode: a final turn with
            # no tool calls and no text at all.
            return ProviderTurn(assistant_text="", tool_calls=[])

        async def append_tool_results(self, turn: Any, executed: Any) -> None:
            return None

        async def close(self) -> None:
            return None

    monkeypatch.setattr(
        "agent.engine.create_provider_session", lambda **kwargs: _NoOutputSession()
    )
    recorder = _make_recorder()
    engine = _make_engine(recorder)
    with pytest.raises(EmptyOutputError):
        await engine.run(Llm.GPT_5_5_HIGH, [])

    events = _read_events(recorder)
    assert events[-1]["status"] == "failed"
    assert "without producing" in events[-1]["error"]


@pytest.mark.asyncio
async def test_engine_records_failed_run_and_reraises(
    logs_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    monkeypatch.setattr(
        "agent.engine.create_provider_session",
        lambda **kwargs: _ExplodingSession(),
    )

    recorder = _make_recorder()
    engine = _make_engine(recorder)
    with pytest.raises(RuntimeError, match="provider exploded"):
        await engine.run(Llm.GPT_5_5_HIGH, [])

    events = _read_events(recorder)
    assert events[-1]["type"] == "run_end"
    assert events[-1]["status"] == "failed"
    assert "provider exploded" in events[-1]["error"]

    row = _db_row(
        "SELECT status, error FROM runs WHERE run_id = ?", (recorder.run_id,)
    )
    assert row is not None
    assert row[0] == "failed"
    assert "provider exploded" in row[1]


async def _record_minimal_run(**overrides: Any) -> AgentRunRecorder:
    recorder = _make_recorder(**overrides)
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [{"role": "user", "content": "go"}])
    recorder.record_llm_request("openai", "gpt-5.5", {"input": ["go"]})
    recorder.record_llm_response(
        "done", [], TokenUsage(input=100, output=50, total=150)
    )
    await recorder.record_run_end("completed", final_html="<html>out</html>")
    return recorder


@pytest.mark.asyncio
async def test_list_and_detail_routes(logs_path: Path) -> None:
    from routes.agent_runs import get_agent_run, list_agent_runs

    recorder_a = await _record_minimal_run(variant_index=0)
    recorder_b = await _record_minimal_run(
        variant_index=1, generation_id="gen_test_11111111"
    )

    listing = await list_agent_runs()
    assert {r.run_id for r in listing.runs} == {recorder_a.run_id, recorder_b.run_id}
    assert listing.total_size_bytes > 0
    by_id = {r.run_id: r for r in listing.runs}
    assert by_id[recorder_a.run_id].generation_id == "gen_test_00000000"
    assert by_id[recorder_a.run_id].status == "completed"
    assert by_id[recorder_a.run_id].num_llm_calls == 1
    assert by_id[recorder_a.run_id].total_cost_usd is not None

    detail = await get_agent_run(recorder_a.run_id)
    types = [e["type"] for e in detail.events]
    assert "run_start" in types and "llm_call_end" in types and "run_end" in types
    assert "stream_delta" not in types

    recorder_c = _make_recorder()
    recorder_c.record_run_start(Llm.GPT_5_5_HIGH, [])
    recorder_c.record_llm_request("openai", "gpt-5.5", {})
    recorder_c.record_stream_event(
        StreamEvent(type="assistant_delta", text="hi"), "a-1"
    )
    recorder_c.record_llm_response("hi", [], None)
    await recorder_c.record_run_end("completed")
    detail_with_deltas = await get_agent_run(
        recorder_c.run_id, include_stream_deltas=True
    )
    assert "stream_delta" in [e["type"] for e in detail_with_deltas.events]


@pytest.mark.asyncio
async def test_detail_route_rejects_bad_ids(logs_path: Path) -> None:
    from fastapi import HTTPException

    from routes.agent_runs import get_agent_run, list_agent_runs

    listing = await list_agent_runs()
    assert listing.runs == []

    with pytest.raises(HTTPException) as excinfo:
        await get_agent_run("../secrets")
    assert excinfo.value.status_code == 400

    await _record_minimal_run()
    with pytest.raises(HTTPException) as excinfo:
        await get_agent_run("run_20260101_000000_deadbeef")
    assert excinfo.value.status_code == 404


@pytest.mark.asyncio
async def test_output_and_asset_routes(logs_path: Path) -> None:
    from fastapi import HTTPException

    from routes.agent_runs import get_agent_run_asset, get_agent_run_output

    recorder = await _record_minimal_run()
    (Path(recorder.run_dir) / "assets").mkdir(exist_ok=True)
    (Path(recorder.run_dir) / "assets" / "pic.png").write_bytes(b"img")

    response = await get_agent_run_output(recorder.run_id)
    assert bytes(response.body).decode() == "<html>out</html>"

    asset_response = await get_agent_run_asset(recorder.run_id, "pic.png")
    assert asset_response.path == str(Path(recorder.run_dir) / "assets" / "pic.png")

    with pytest.raises(HTTPException) as excinfo:
        await get_agent_run_asset(recorder.run_id, "nope.png")
    assert excinfo.value.status_code == 404


@pytest.mark.asyncio
async def test_prune_removes_old_runs_and_keeps_index(logs_path: Path) -> None:
    import os

    from routes.agent_runs import (
        PruneAgentRunsRequest,
        list_agent_runs,
        prune_agent_runs,
    )

    old_recorder = await _record_minimal_run()
    fresh_recorder = await _record_minimal_run(generation_id="gen_test_22222222")

    ninety_days_ago = 90 * 24 * 3600
    old_time = __import__("time").time() - ninety_days_ago
    os.utime(old_recorder.run_dir, (old_time, old_time))

    result = await prune_agent_runs(PruneAgentRunsRequest(max_age_days=7))
    assert result.deleted_count == 1
    assert result.freed_bytes > 0
    assert not Path(old_recorder.run_dir).exists()
    assert Path(fresh_recorder.run_dir).exists()
    assert Path(get_agent_runs_db_path()).exists()

    listing = await list_agent_runs()
    assert [r.run_id for r in listing.runs] == [fresh_recorder.run_id]

    # Orphan rows (directory removed out-of-band) get cleaned up too.
    import shutil as _shutil

    _shutil.rmtree(fresh_recorder.run_dir)
    await prune_agent_runs(PruneAgentRunsRequest(max_age_days=7))
    listing = await list_agent_runs()
    assert listing.runs == []


@pytest.mark.asyncio
async def test_prompt_reports_prune_skips_agent_runs(logs_path: Path) -> None:
    import os

    from routes.prompt_reports import (
        PrunePromptReportsRequest,
        prune_prompt_reports,
    )

    recorder = await _record_minimal_run()

    ninety_days_ago = 90 * 24 * 3600
    old_time = __import__("time").time() - ninety_days_ago
    agent_runs_dir = Path(get_agent_runs_directory())
    os.utime(agent_runs_dir, (old_time, old_time))
    os.utime(recorder.run_dir, (old_time, old_time))

    result = await prune_prompt_reports(PrunePromptReportsRequest(max_age_days=7))
    assert result.deleted_count == 0
    assert Path(recorder.run_dir).exists()
    assert Path(get_agent_runs_db_path()).exists()


@pytest.mark.asyncio
async def test_tool_replicate_outputs_saved_when_logging(
    logs_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
    class FakeResponse:
        content = b"replicate-image-bytes"
        headers = {"content-type": "image/png"}

        def raise_for_status(self) -> None:
            return None

    class FakeClient:
        def __init__(self, *args: Any, **kwargs: Any) -> None:
            pass

        async def __aenter__(self) -> "FakeClient":
            return self

        async def __aexit__(self, *args: Any) -> None:
            return None

        async def get(self, url: str) -> FakeResponse:
            if "broken" in url:
                raise RuntimeError("download failed")
            return FakeResponse()

    import httpx

    monkeypatch.setattr(httpx, "AsyncClient", FakeClient)

    recorder = _make_recorder()
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
    tool_call = ToolCall(id="call_img", name="generate_images", arguments={})
    recorder.record_tool_start("call_img", tool_call)
    recorder.record_tool_end(
        "call_img",
        tool_call,
        ToolExecutionResult(
            ok=True,
            result={
                "images": {
                    "a prompt": "https://replicate.delivery/xyz/one.png",
                    "another": "https://replicate.delivery/broken/two.png",
                }
            },
            summary={},
        ),
    )
    await recorder.record_run_end("completed")

    run_dir = Path(recorder.run_dir)
    saved = list((run_dir / "tool_assets").glob("*.png"))
    assert len(saved) == 1
    assert saved[0].read_bytes() == b"replicate-image-bytes"

    run_json = json.loads((run_dir / "run.json").read_text())
    statuses = {e["url"]: e["status"] for e in run_json["tool_assets"]}
    assert statuses["https://replicate.delivery/xyz/one.png"] == "saved"
    assert statuses["https://replicate.delivery/broken/two.png"] == "failed"


@pytest.mark.asyncio
async def test_tool_asset_downloads_disabled_without_logging(
    logs_path: Path,
) -> None:
    recorder = _make_recorder(enabled=False)
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
    tool_call = ToolCall(id="call_img", name="generate_images", arguments={})
    recorder.record_tool_end(
        "call_img",
        tool_call,
        ToolExecutionResult(
            ok=True,
            result={"images": {"p": "https://replicate.delivery/xyz/one.png"}},
            summary={},
        ),
    )
    await recorder.record_run_end("completed")
    assert not (logs_path / "run_logs").exists()


def test_tool_asset_scheduling_without_event_loop_is_safe(
    logs_path: Path,
) -> None:
    # Sync context (no running loop): scheduling must skip, not raise.
    recorder = _make_recorder()
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
    tool_call = ToolCall(id="call_img", name="generate_images", arguments={})
    recorder.record_tool_end(
        "call_img",
        tool_call,
        ToolExecutionResult(
            ok=True,
            result={"images": {"p": "https://replicate.delivery/xyz/one.png"}},
            summary={},
        ),
    )


def test_recorder_survives_internal_errors(logs_path: Path) -> None:
    recorder = _make_recorder()
    recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
    # Unserializable-ish payloads and bad state must not raise.
    recorder.record_llm_request("openai", "gpt-5.5", {"obj": object()})
    recorder.record_llm_response("x", [], TokenUsage())


def test_agent_runs_directory_under_logs_path(logs_path: Path) -> None:
    assert get_agent_runs_directory() == str(logs_path / "run_logs" / "agent_runs")