Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Finalize extracted assets directly; stop double-saving them

Abi Rajae06ff2e663d2
diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.py
index dc301e41..1fb8fffc 100644
--- a/backend/agent/tools/definitions.py
+++ b/backend/agent/tools/definitions.py
@@ -233,8 +233,10 @@ def canonical_tool_definitions(
description=(
"Extract one or more visual assets from the input screenshot or "
"reference images using Gemini. Pass a list of text descriptions "
- "for the assets to extract. Returns extracted image data URLs in "
- "the same order."
+ "for the assets to extract. Returns each asset (in the same order) "
+ "with a permanent, embeddable public_url. These assets are already "
+ "saved — do NOT call save_assets on them (save_assets is only for "
+ "user-uploaded images)."
),
parameters=_extract_assets_schema(),
)
diff --git a/backend/agent/tools/extract_assets.py b/backend/agent/tools/extract_assets.py
index 3ba66b3e..d6be5e4e 100644
--- a/backend/agent/tools/extract_assets.py
+++ b/backend/agent/tools/extract_assets.py
@@ -2,10 +2,7 @@ import base64
from typing import Any, Dict, List, Optional, cast
from asset_extraction import extract_assets_from_images
-from uploaded_assets.store import (
- persist_data_url_as_temporary_asset,
- promote_temporary_asset_id,
-)
+from uploaded_assets.store import persist_data_url_as_asset
from agent.state import ensure_str
from agent.tools.summaries import summarize_text
@@ -105,7 +102,6 @@ async def run_extract_assets(
data_url = ensure_str(asset.get("data_url"))
public_url: str | None = None
content_type: str | None = None
- asset_id: str | None = None
image_part_index: int | None = None
image_display_name: str | None = None
status = ensure_str(asset.get("status")) or "missing"
@@ -123,22 +119,21 @@ async def run_extract_assets(
image_part_index = len(multimodal_parts) - 1
image_display_name = display_name
- temporary_asset = persist_data_url_as_temporary_asset(
+ # Extraction is the commitment — the model asked for this crop,
+ # so finalize it straight to a served asset (no temp staging).
+ saved_asset = await persist_data_url_as_asset(
data_url,
asset_base_url,
+ user_id=user_id,
)
- if temporary_asset:
- saved_asset = await promote_temporary_asset_id(
- temporary_asset.asset_id,
- user_id=user_id,
- )
- if saved_asset:
- public_url = saved_asset.public_url
- content_type = saved_asset.content_type
- asset_id = saved_asset.asset_id
- status = "ok"
- else:
- status = "error"
+ if saved_asset:
+ public_url = saved_asset.public_url
+ content_type = saved_asset.content_type
+ # No asset_id is surfaced on purpose: extracted crops are
+ # already finalized, and a save_assets-style id makes models
+ # redundantly call save_assets on them (which now fails — the
+ # crops were never temp-staged). The model embeds public_url.
+ status = "ok"
else:
status = "error"
@@ -146,7 +141,6 @@ async def run_extract_assets(
{
"description": asset.get("description"),
"public_url": public_url,
- "asset_id": asset_id,
"content_type": content_type,
"status": status,
"image_part_index": image_part_index,
@@ -177,7 +171,6 @@ async def run_extract_assets(
),
"status": ensure_str(asset.get("status")),
"public_url": public_url,
- "asset_id": ensure_str(asset.get("asset_id")),
"content_type": ensure_str(asset.get("content_type")),
"image_part_index": asset.get("image_part_index"),
"image_display_name": asset.get("image_display_name"),
diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.py
index bddfe96e..e7483ec5 100644
--- a/backend/agent/tools/runtime.py
+++ b/backend/agent/tools/runtime.py
@@ -79,7 +79,6 @@ class AgentToolRuntime:
return await run_screenshot_preview(
tool_call.arguments,
file_state=self.file_state,
- asset_base_url=self.asset_base_url,
)
if tool_call.name == "save_assets":
return await run_save_assets(tool_call.arguments, user_id=self.user_id)
diff --git a/backend/agent/tools/screenshot_preview.py b/backend/agent/tools/screenshot_preview.py
index e2802108..dd109636 100644
--- a/backend/agent/tools/screenshot_preview.py
+++ b/backend/agent/tools/screenshot_preview.py
@@ -5,10 +5,6 @@ from preview_screenshot import capture_preview_screenshot
from agent.state import AgentFileState
from agent.tools.types import ToolExecutionResult, ToolMultimodalPart
-from uploaded_assets.store import (
- persist_data_url_as_temporary_asset,
- promote_temporary_asset_id,
-)
PREVIEW_VIEWPORTS = ("desktop", "mobile")
@@ -18,12 +14,13 @@ async def run_screenshot_preview(
_args: Dict[str, Any],
*,
file_state: AgentFileState,
- asset_base_url: str,
) -> ToolExecutionResult:
"""Render the current HTML in headless Chromium and return screenshots.
- PNGs are attached as multimodal parts so the model can visually verify
- its work; public URLs are kept in the summary for the UI.
+ These previews are for *seeing*, not keeping: the model views them as
+ attached image bytes (multimodal parts) to verify its work and never
+ embeds them in its output, so they are NOT persisted as assets. A data
+ URL is inlined into the summary purely so the UI can show the same preview.
"""
if not file_state.content:
return ToolExecutionResult(
@@ -45,15 +42,6 @@ async def run_screenshot_preview(
image_part_index = len(multimodal_parts)
encoded_image = base64.b64encode(image_bytes).decode("ascii")
data_url = f"data:image/png;base64,{encoded_image}"
- temporary_asset = persist_data_url_as_temporary_asset(
- data_url,
- asset_base_url,
- )
- if not temporary_asset:
- raise Exception(f"Could not persist {viewport} screenshot")
- saved_asset = await promote_temporary_asset_id(temporary_asset.asset_id)
- if not saved_asset:
- raise Exception(f"Could not publish {viewport} screenshot")
screenshots.append(
{
"viewport": viewport,
@@ -61,7 +49,8 @@ async def run_screenshot_preview(
"image_part_index": image_part_index,
"image_display_name": display_name,
"image_bytes": len(image_bytes),
- "image_url": saved_asset.public_url,
+ # Inlined for the UI thumbnail only — never stored as an asset.
+ "image_url": data_url,
"status": "ok",
}
)
diff --git a/backend/tests/test_agent_tool_runtime.py b/backend/tests/test_agent_tool_runtime.py
index a0518036..32cf51a9 100644
--- a/backend/tests/test_agent_tool_runtime.py
+++ b/backend/tests/test_agent_tool_runtime.py
@@ -202,6 +202,13 @@ async def test_extract_assets_returns_mocked_gemini_assets(
assert result.summary["assets"][0]["public_url"].startswith(
"http://127.0.0.1:7001/local-assets/"
)
+ # Extracted crops finalize straight to the served store — no temp staging.
+ assert not temp_dir.exists()
+ assert len(list(asset_dir.iterdir())) == 2
+ # No save_assets-style id is surfaced (already finalized), so the model
+ # can't redundantly promote extracted crops.
+ assert "asset_id" not in result.result["assets"][0]
+ assert "asset_id" not in result.summary["assets"][0]
assert result.result["assets"][0]["image_part_index"] == 0
assert result.result["assets"][0]["image_display_name"] == "asset_0.png"
assert result.result["assets"][1]["image_part_index"] == 1
@@ -296,17 +303,20 @@ async def test_screenshot_preview_returns_image_part(
]
assert result.summary["status"] == "ok"
screenshots = cast(list[dict[str, Any]], result.summary["screenshots"])
- assert screenshots[0]["image_url"].startswith(
- "http://127.0.0.1:7001/local-assets/"
- )
- assert screenshots[1]["image_url"].startswith(
- "http://127.0.0.1:7001/local-assets/"
- )
- assert screenshots[0]["image_url"] != screenshots[1]["image_url"]
+ # Previews are inlined as data URLs for the UI, not persisted as assets.
+ desktop_url = cast(str, screenshots[0]["image_url"])
+ mobile_url = cast(str, screenshots[1]["image_url"])
+ assert desktop_url.startswith("data:image/png;base64,")
+ assert mobile_url.startswith("data:image/png;base64,")
+ assert desktop_url != mobile_url
+ assert base64.b64decode(desktop_url.split(",", 1)[1]) == b"desktop-png-bytes"
+ assert base64.b64decode(mobile_url.split(",", 1)[1]) == b"mobile-png-bytes"
# UI URLs and image payloads must not leak into the model-facing result payload.
assert "image_url" not in result.result["details"]["screenshots"][0]
assert "image_data_url" not in result.result["details"]["screenshots"][0]
- assert len(list(asset_dir.iterdir())) == 2
+ # Previews are not assets: nothing is written to the temp or served stores.
+ assert not asset_dir.exists()
+ assert not temp_dir.exists()
assert result.multimodal_parts is not None
assert [part.display_name for part in result.multimodal_parts] == [
"preview_desktop.png",
diff --git a/backend/tests/test_uploaded_assets.py b/backend/tests/test_uploaded_assets.py
index d92cdb2a..77551a98 100644
--- a/backend/tests/test_uploaded_assets.py
+++ b/backend/tests/test_uploaded_assets.py
@@ -5,6 +5,7 @@ import pytest
from uploaded_assets import (
append_uploaded_asset_ids_to_prompt,
+ persist_data_url_as_asset,
persist_data_url_as_temporary_asset,
promote_temporary_asset_id,
)
@@ -61,6 +62,43 @@ async def test_promote_temporary_asset_id_returns_permanent_url(
assert len(list(asset_dir.iterdir())) == 1
+async def test_persist_data_url_as_asset_finalizes_without_temp_staging(
+ monkeypatch,
+ tmp_path: Path,
+) -> None:
+ temp_dir = tmp_path / "tmp-assets"
+ asset_dir = tmp_path / "local-assets"
+ monkeypatch.setattr("uploaded_assets.store.TEMP_ASSET_DIR", str(temp_dir))
+ monkeypatch.setattr("uploaded_assets.store.LOCAL_ASSET_DIR", str(asset_dir))
+
+ saved = await persist_data_url_as_asset(
+ _data_url(b"crop-bytes"),
+ "http://127.0.0.1:7001",
+ )
+
+ assert saved is not None
+ assert saved.public_url.startswith("http://127.0.0.1:7001/local-assets/asset_")
+ assert saved.content_type == "image/png"
+ # Goes straight to the served store; the temp staging dir is never created.
+ assert not temp_dir.exists()
+ assert len(list(asset_dir.iterdir())) == 1
+
+ # Identical bytes are content-addressed: re-persisting dedupes to one file.
+ again = await persist_data_url_as_asset(
+ _data_url(b"crop-bytes"),
+ "http://127.0.0.1:7001",
+ )
+ assert again is not None
+ assert again.public_url == saved.public_url
+ assert len(list(asset_dir.iterdir())) == 1
+
+
+async def test_persist_data_url_as_asset_rejects_non_image() -> None:
+ assert await persist_data_url_as_asset("not-a-data-url", "http://127.0.0.1:7001") is None
+
+
def test_append_uploaded_asset_ids_to_prompt_keeps_image_and_adds_id(
monkeypatch,
tmp_path: Path,
diff --git a/backend/uploaded_assets/__init__.py b/backend/uploaded_assets/__init__.py
index 0e08e567..25a260b4 100644
--- a/backend/uploaded_assets/__init__.py
+++ b/backend/uploaded_assets/__init__.py
@@ -7,6 +7,7 @@ from uploaded_assets.store import (
TemporaryAsset,
configure_uploaded_asset_routes,
infer_local_asset_base_url,
+ persist_data_url_as_asset,
persist_data_url_as_temporary_asset,
promote_temporary_asset_id,
)
@@ -18,6 +19,7 @@ __all__ = [
"append_uploaded_asset_ids_to_prompt",
"configure_uploaded_asset_routes",
"infer_local_asset_base_url",
+ "persist_data_url_as_asset",
"persist_data_url_as_temporary_asset",
"promote_temporary_asset_id",
]
diff --git a/backend/uploaded_assets/store.py b/backend/uploaded_assets/store.py
index 90d43e07..19078ea1 100644
--- a/backend/uploaded_assets/store.py
+++ b/backend/uploaded_assets/store.py
@@ -2,7 +2,6 @@ import base64
import hashlib
import json
import os
-import shutil
import tempfile
from dataclasses import dataclass
from typing import Any, cast
@@ -148,10 +147,12 @@ def _temporary_asset_path(asset_id: str) -> tuple[str, str, str, str] | None:
return None
-def persist_data_url_as_temporary_asset(
- data_url: str,
- asset_base_url: str,
-) -> TemporaryAsset | None:
+def _decode_image_data_url(data_url: str) -> tuple[bytes, str, str] | None:
+ """Decode a ``data:image/*`` URL into ``(bytes, content_type, extension)``.
+
+ Returns ``None`` for non-image data URLs, unsupported image types,
+ undecodable base64, or payloads over ``MAX_UPLOADED_ASSET_BYTES``.
+ """
if not data_url.startswith("data:image/") or "," not in data_url:
return None
@@ -169,6 +170,57 @@ def persist_data_url_as_temporary_asset(
if len(image_bytes) > MAX_UPLOADED_ASSET_BYTES:
return None
+ return image_bytes, content_type, extension
+
+
+def _finalize_asset_bytes(
+ image_bytes: bytes,
+ extension: str,
+ content_type: str,
+ asset_base_url: str,
+ user_id: str | None,
+) -> SavedAsset:
+ """Write asset bytes to the served ``LOCAL_ASSET_DIR`` and return a SavedAsset.
+
+ The single "finalize" step that turns bytes into a durable, served asset.
+ Content-addressed by digest, so identical bytes dedupe to one file.
+ ``user_id`` is unused locally but is the hook hosted backends key durable
+ per-user storage off of.
+ """
+ _ = user_id
+ digest = _digest_for_bytes(image_bytes)
+ asset_id = _asset_id_for_digest(digest)
+ permanent_filename = f"asset_{digest}{extension}"
+
+ os.makedirs(LOCAL_ASSET_DIR, exist_ok=True)
+ destination_path = os.path.join(LOCAL_ASSET_DIR, permanent_filename)
+ if not os.path.exists(destination_path):
+ with open(destination_path, "wb") as file:
+ file.write(image_bytes)
+
+ return SavedAsset(
+ asset_id=asset_id,
+ public_url=_asset_url(asset_base_url, "local-assets", permanent_filename),
+ content_type=content_type,
+ )
+
+
+def persist_data_url_as_temporary_asset(
+ data_url: str,
+ asset_base_url: str,
+) -> TemporaryAsset | None:
+ """Stage an uploaded data URL as a temporary asset, pending a later promote.
+
+ Used for user uploads, where only some images turn out to be assets the
+ model keeps (via ``save_assets``); the rest stay reference-only. Callers
+ that already know the image is an asset should use
+ ``persist_data_url_as_asset`` instead and skip this staging hop.
+ """
+ decoded = _decode_image_data_url(data_url)
+ if decoded is None:
+ return None
+ image_bytes, content_type, extension = decoded
+
os.makedirs(TEMP_ASSET_DIR, exist_ok=True)
digest = _digest_for_bytes(image_bytes)
asset_id = _asset_id_for_digest(digest)
@@ -185,29 +237,40 @@ def persist_data_url_as_temporary_asset(
)
+async def persist_data_url_as_asset(
+ data_url: str,
+ asset_base_url: str,
+ user_id: str | None = None,
+) -> SavedAsset | None:
+ """Persist a data URL straight to a durable, served asset.
+
+ Skips the temp-staging hop used for uploads: callers that already know the
+ image is an asset (e.g. ``extract_assets``) commit immediately. Async to
+ match ``promote_temporary_asset_id`` so hosted backends can finalize to
+ remote storage without changing callers.
+ """
+ decoded = _decode_image_data_url(data_url)
+ if decoded is None:
+ return None
+ image_bytes, content_type, extension = decoded
+ return _finalize_asset_bytes(
+ image_bytes, extension, content_type, asset_base_url, user_id
+ )
+
+
async def promote_temporary_asset_id(
asset_id: str,
user_id: str | None = None,
) -> SavedAsset | None:
- _ = user_id
+ """Finalize a previously-staged temporary asset into a served asset."""
temporary_asset = _temporary_asset_path(asset_id)
if not temporary_asset:
return None
source_path, temporary_filename, content_type, asset_base_url = temporary_asset
_, extension = os.path.splitext(temporary_filename)
- digest = _digest_for_asset_id(asset_id)
- if not digest:
- return None
- permanent_filename = f"asset_{digest}{extension}"
-
- os.makedirs(LOCAL_ASSET_DIR, exist_ok=True)
- destination_path = os.path.join(LOCAL_ASSET_DIR, permanent_filename)
- if not os.path.exists(destination_path):
- shutil.copyfile(source_path, destination_path)
-
- return SavedAsset(
- asset_id=asset_id,
- public_url=_asset_url(asset_base_url, "local-assets", permanent_filename),
- content_type=content_type,
+ with open(source_path, "rb") as file:
+ image_bytes = file.read()
+ return _finalize_asset_bytes(
+ image_bytes, extension, content_type, asset_base_url, user_id
)