Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Capture expiring tool-output images and JS-embedded asset URLs

Abi Rajad1385abdf259
diff --git a/backend/fs_logging/agent_runs.py b/backend/fs_logging/agent_runs.py
index fa2c6d78..abb14446 100644
--- a/backend/fs_logging/agent_runs.py
+++ b/backend/fs_logging/agent_runs.py
@@ -22,6 +22,7 @@ Recording must never break generation: every public method no-ops when
disabled and swallows its own exceptions.
"""
+import asyncio
import base64
import hashlib
import json
@@ -65,6 +66,13 @@ _ALWAYS_SNAPSHOT_HOSTS = ("replicate.delivery",)
_ASSET_DOWNLOAD_TIMEOUT_SECONDS = 15.0
+# Replicate delivery URLs expire within ~an hour; when run logging is on,
+# tool outputs on these hosts are downloaded into the run folder as they are
+# produced — even ones the model later discards from the final HTML.
+_EXPIRING_ASSET_URL_RE = re.compile(
+ r"https?://[^\s\"'\\]*replicate\.delivery/[^\s\"'\\]+"
+)
+
def get_agent_runs_directory() -> str:
return os.path.join(get_run_logs_directory(), AGENT_RUNS_DIRNAME)
@@ -289,6 +297,9 @@ class AgentRunRecorder:
self._num_tool_calls = 0
self._llm_call_summaries: list[dict[str, Any]] = []
self._tool_call_summaries: list[dict[str, Any]] = []
+ self._tool_asset_urls: set[str] = set()
+ self._tool_asset_tasks: list["asyncio.Task[None]"] = []
+ self._tool_asset_manifest: list[dict[str, Any]] = []
# ------------------------------------------------------------------ paths
@@ -627,9 +638,79 @@ class AgentRunRecorder:
"multimodal_parts": multimodal_meta,
},
)
+ self._schedule_tool_asset_downloads(tool_event_id, result)
except Exception as exc:
print(f"[AGENT RUN] Failed to record tool end: {exc}")
+ def _schedule_tool_asset_downloads(
+ self, tool_event_id: str, result: "ToolExecutionResult"
+ ) -> None:
+ """Kick off background downloads of expiring tool-output URLs.
+
+ Fire-and-forget so generation latency is unaffected; the tasks are
+ gathered in ``record_run_end`` before the manifest is written. Only
+ runs when recording is enabled, so without prompt logging nothing is
+ ever fetched or stored.
+ """
+ try:
+ blob = json.dumps(to_serializable(result.result), ensure_ascii=False)
+ urls = [
+ url
+ for url in dict.fromkeys(_EXPIRING_ASSET_URL_RE.findall(blob))
+ if url not in self._tool_asset_urls
+ ]
+ if not urls:
+ return
+ loop = asyncio.get_running_loop()
+ for index, url in enumerate(urls):
+ self._tool_asset_urls.add(url)
+ entry: dict[str, Any] = {
+ "url": url,
+ "tool_event_id": tool_event_id,
+ "status": "pending",
+ }
+ self._tool_asset_manifest.append(entry)
+ self._tool_asset_tasks.append(
+ loop.create_task(
+ self._download_tool_asset(url, tool_event_id, index, entry)
+ )
+ )
+ except RuntimeError:
+ # No running event loop (sync callers/tests): skip quietly.
+ return
+ except Exception as exc:
+ print(f"[AGENT RUN] Failed to schedule tool asset downloads: {exc}")
+
+ async def _download_tool_asset(
+ self, url: str, tool_event_id: str, index: int, entry: dict[str, Any]
+ ) -> None:
+ try:
+ import httpx
+
+ tool_assets_dir = os.path.join(self.run_dir, "tool_assets")
+ os.makedirs(tool_assets_dir, exist_ok=True)
+ async with httpx.AsyncClient(
+ timeout=_ASSET_DOWNLOAD_TIMEOUT_SECONDS, follow_redirects=True
+ ) as client:
+ response = await client.get(url)
+ response.raise_for_status()
+ content = response.content
+ safe_event = re.sub(r"[^A-Za-z0-9_-]", "_", tool_event_id)[:60]
+ ext = _guess_extension(url, response.headers.get("content-type"))
+ filename = f"{safe_event}_{index}{ext}"
+ with open(os.path.join(tool_assets_dir, filename), "wb") as f:
+ f.write(content)
+ entry.update(
+ {
+ "file": f"tool_assets/{filename}",
+ "bytes": len(content),
+ "sha256": hashlib.sha256(content).hexdigest()[:12],
+ "status": "saved",
+ }
+ )
+ except Exception as exc:
+ entry.update({"status": "failed", "error": str(exc)})
+
def record_set_code(self, content_len: int, source: str) -> None:
if not self.enabled:
return
@@ -656,6 +737,10 @@ class AgentRunRecorder:
total_cost_usd = (
self._priced_cost_usd if self._num_priced_calls > 0 else None
)
+ if self._tool_asset_tasks:
+ await asyncio.gather(
+ *self._tool_asset_tasks, return_exceptions=True
+ )
if final_html:
await self._snapshot_output(final_html)
self._append_event(
@@ -705,6 +790,7 @@ class AgentRunRecorder:
"has_unpriced_calls": self._has_unpriced_calls,
"llm_calls": self._llm_call_summaries,
"tool_calls": self._tool_call_summaries,
+ "tool_assets": self._tool_asset_manifest,
"final_html": final_html,
}
os.makedirs(self.run_dir, exist_ok=True)
@@ -826,6 +912,12 @@ class AgentRunRecorder:
host.endswith(h) for h in _ALWAYS_SNAPSHOT_HOSTS
):
remote_urls.append(url)
+ # Expiring-host URLs can also appear outside markup attributes —
+ # e.g. as JS string literals (`image.src = '...'`) — so sweep the
+ # whole document for them regardless of context.
+ for url in _EXPIRING_ASSET_URL_RE.findall(final_html):
+ if url not in remote_urls:
+ remote_urls.append(url)
if remote_urls:
import httpx
diff --git a/backend/tests/test_agent_runs.py b/backend/tests/test_agent_runs.py
index 6ca0d11c..7f704dd9 100644
--- a/backend/tests/test_agent_runs.py
+++ b/backend/tests/test_agent_runs.py
@@ -386,6 +386,51 @@ async def test_asset_snapshot_downloads_remote_images(
assert "https://example.com/broken.png" in rewritten
+async def test_asset_snapshot_catches_urls_in_js_strings(
+ logs_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ class FakeResponse:
+ content = b"js-loaded-image"
+ headers = {"content-type": "image/jpeg"}
+
+ def raise_for_status(self) -> None:
+ return None
+
+ class FakeClient:
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ pass
+
+ async def __aenter__(self) -> "FakeClient":
+ return self
+
+ async def __aexit__(self, *args: Any) -> None:
+ return None
+
+ async def get(self, url: str) -> FakeResponse:
+ return FakeResponse()
+
+ import httpx
+
+ monkeypatch.setattr(httpx, "AsyncClient", FakeClient)
+
+ # Image loaded programmatically — no src= attribute anywhere.
+ html = (
+ "<html><script>loadPhoto(el, "
+ "'https://replicate.delivery/abc/output_1.jpeg');</script></html>"
+ )
+ recorder = _make_recorder()
+ recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
+ await recorder.record_run_end("completed", final_html=html)
+
+ run_dir = Path(recorder.run_dir)
+ manifest = json.loads((run_dir / "assets_manifest.json").read_text())
+ assert manifest[0]["status"] == "downloaded"
+ rewritten = (run_dir / "final_selfcontained.html").read_text()
+ assert "replicate.delivery" not in rewritten
+ assert "assets/" in rewritten
+
+
class _ToolThenDoneSession:
"""One create_file tool turn, then a final assistant turn."""
@@ -667,6 +712,105 @@ async def test_prompt_reports_prune_skips_agent_runs(logs_path: Path) -> None:
assert Path(get_agent_runs_db_path()).exists()
+async def test_tool_replicate_outputs_saved_when_logging(
+ logs_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ class FakeResponse:
+ content = b"replicate-image-bytes"
+ headers = {"content-type": "image/png"}
+
+ def raise_for_status(self) -> None:
+ return None
+
+ class FakeClient:
+ def __init__(self, *args: Any, **kwargs: Any) -> None:
+ pass
+
+ async def __aenter__(self) -> "FakeClient":
+ return self
+
+ async def __aexit__(self, *args: Any) -> None:
+ return None
+
+ async def get(self, url: str) -> FakeResponse:
+ if "broken" in url:
+ raise RuntimeError("download failed")
+ return FakeResponse()
+
+ import httpx
+
+ monkeypatch.setattr(httpx, "AsyncClient", FakeClient)
+
+ recorder = _make_recorder()
+ recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
+ tool_call = ToolCall(id="call_img", name="generate_images", arguments={})
+ recorder.record_tool_start("call_img", tool_call)
+ recorder.record_tool_end(
+ "call_img",
+ tool_call,
+ ToolExecutionResult(
+ ok=True,
+ result={
+ "images": {
+ "a prompt": "https://replicate.delivery/xyz/one.png",
+ "another": "https://replicate.delivery/broken/two.png",
+ }
+ },
+ summary={},
+ ),
+ )
+ await recorder.record_run_end("completed")
+
+ run_dir = Path(recorder.run_dir)
+ saved = list((run_dir / "tool_assets").glob("*.png"))
+ assert len(saved) == 1
+ assert saved[0].read_bytes() == b"replicate-image-bytes"
+
+ run_json = json.loads((run_dir / "run.json").read_text())
+ statuses = {e["url"]: e["status"] for e in run_json["tool_assets"]}
+ assert statuses["https://replicate.delivery/xyz/one.png"] == "saved"
+ assert statuses["https://replicate.delivery/broken/two.png"] == "failed"
+
+
+async def test_tool_asset_downloads_disabled_without_logging(
+ logs_path: Path,
+) -> None:
+ recorder = _make_recorder(enabled=False)
+ recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
+ tool_call = ToolCall(id="call_img", name="generate_images", arguments={})
+ recorder.record_tool_end(
+ "call_img",
+ tool_call,
+ ToolExecutionResult(
+ ok=True,
+ result={"images": {"p": "https://replicate.delivery/xyz/one.png"}},
+ summary={},
+ ),
+ )
+ await recorder.record_run_end("completed")
+ assert not (logs_path / "run_logs").exists()
+
+
+def test_tool_asset_scheduling_without_event_loop_is_safe(
+ logs_path: Path,
+) -> None:
+ # Sync context (no running loop): scheduling must skip, not raise.
+ recorder = _make_recorder()
+ recorder.record_run_start(Llm.GPT_5_5_HIGH, [])
+ tool_call = ToolCall(id="call_img", name="generate_images", arguments={})
+ recorder.record_tool_end(
+ "call_img",
+ tool_call,
+ ToolExecutionResult(
+ ok=True,
+ result={"images": {"p": "https://replicate.delivery/xyz/one.png"}},
+ summary={},
+ ),
+ )
+
+
def test_recorder_survives_internal_errors(logs_path: Path) -> None:
recorder = _make_recorder()
recorder.record_run_start(Llm.GPT_5_5_HIGH, [])