Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Finalize extracted assets directly; stop double-saving them
Abi Rajae06ff2e663d2
diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.pyindex dc301e41..1fb8fffc 100644--- a/backend/agent/tools/definitions.py+++ b/backend/agent/tools/definitions.py@@ -233,8 +233,10 @@ def canonical_tool_definitions(description=("Extract one or more visual assets from the input screenshot or ""reference images using Gemini. Pass a list of text descriptions "- "for the assets to extract. Returns extracted image data URLs in "- "the same order."+ "for the assets to extract. Returns each asset (in the same order) "+ "with a permanent, embeddable public_url. These assets are already "+ "saved — do NOT call save_assets on them (save_assets is only for "+ "user-uploaded images)."),parameters=_extract_assets_schema(),)diff --git a/backend/agent/tools/extract_assets.py b/backend/agent/tools/extract_assets.pyindex 3ba66b3e..d6be5e4e 100644--- a/backend/agent/tools/extract_assets.py+++ b/backend/agent/tools/extract_assets.py@@ -2,10 +2,7 @@ import base64from typing import Any, Dict, List, Optional, castfrom asset_extraction import extract_assets_from_images-from uploaded_assets.store import (- persist_data_url_as_temporary_asset,- promote_temporary_asset_id,-)+from uploaded_assets.store import persist_data_url_as_assetfrom agent.state import ensure_strfrom agent.tools.summaries import summarize_text@@ -105,7 +102,6 @@ async def run_extract_assets(data_url = ensure_str(asset.get("data_url"))public_url: str | None = Nonecontent_type: str | None = None- asset_id: str | None = Noneimage_part_index: int | None = Noneimage_display_name: str | None = Nonestatus = ensure_str(asset.get("status")) or "missing"@@ -123,22 +119,21 @@ async def run_extract_assets(image_part_index = len(multimodal_parts) - 1image_display_name = display_name- temporary_asset = persist_data_url_as_temporary_asset(+ # Extraction is the commitment — the model asked for this crop,+ # so finalize it straight to a served asset (no temp staging).+ saved_asset = await persist_data_url_as_asset(data_url,asset_base_url,+ user_id=user_id,)- if temporary_asset:- saved_asset = await promote_temporary_asset_id(- temporary_asset.asset_id,- user_id=user_id,- )- if saved_asset:- public_url = saved_asset.public_url- content_type = saved_asset.content_type- asset_id = saved_asset.asset_id- status = "ok"- else:- status = "error"+ if saved_asset:+ public_url = saved_asset.public_url+ content_type = saved_asset.content_type+ # No asset_id is surfaced on purpose: extracted crops are+ # already finalized, and a save_assets-style id makes models+ # redundantly call save_assets on them (which now fails — the+ # crops were never temp-staged). The model embeds public_url.+ status = "ok"else:status = "error"@@ -146,7 +141,6 @@ async def run_extract_assets({"description": asset.get("description"),"public_url": public_url,- "asset_id": asset_id,"content_type": content_type,"status": status,"image_part_index": image_part_index,@@ -177,7 +171,6 @@ async def run_extract_assets(),"status": ensure_str(asset.get("status")),"public_url": public_url,- "asset_id": ensure_str(asset.get("asset_id")),"content_type": ensure_str(asset.get("content_type")),"image_part_index": asset.get("image_part_index"),"image_display_name": asset.get("image_display_name"),diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.pyindex bddfe96e..e7483ec5 100644--- a/backend/agent/tools/runtime.py+++ b/backend/agent/tools/runtime.py@@ -79,7 +79,6 @@ class AgentToolRuntime:return await run_screenshot_preview(tool_call.arguments,file_state=self.file_state,- asset_base_url=self.asset_base_url,)if tool_call.name == "save_assets":return await run_save_assets(tool_call.arguments, user_id=self.user_id)diff --git a/backend/agent/tools/screenshot_preview.py b/backend/agent/tools/screenshot_preview.pyindex e2802108..dd109636 100644--- a/backend/agent/tools/screenshot_preview.py+++ b/backend/agent/tools/screenshot_preview.py@@ -5,10 +5,6 @@ from preview_screenshot import capture_preview_screenshotfrom agent.state import AgentFileStatefrom agent.tools.types import ToolExecutionResult, ToolMultimodalPart-from uploaded_assets.store import (- persist_data_url_as_temporary_asset,- promote_temporary_asset_id,-)PREVIEW_VIEWPORTS = ("desktop", "mobile")@@ -18,12 +14,13 @@ async def run_screenshot_preview(_args: Dict[str, Any],*,file_state: AgentFileState,- asset_base_url: str,) -> ToolExecutionResult:"""Render the current HTML in headless Chromium and return screenshots.- PNGs are attached as multimodal parts so the model can visually verify- its work; public URLs are kept in the summary for the UI.+ These previews are for *seeing*, not keeping: the model views them as+ attached image bytes (multimodal parts) to verify its work and never+ embeds them in its output, so they are NOT persisted as assets. A data+ URL is inlined into the summary purely so the UI can show the same preview."""if not file_state.content:return ToolExecutionResult(@@ -45,15 +42,6 @@ async def run_screenshot_preview(image_part_index = len(multimodal_parts)encoded_image = base64.b64encode(image_bytes).decode("ascii")data_url = f"data:image/png;base64,{encoded_image}"- temporary_asset = persist_data_url_as_temporary_asset(- data_url,- asset_base_url,- )- if not temporary_asset:- raise Exception(f"Could not persist {viewport} screenshot")- saved_asset = await promote_temporary_asset_id(temporary_asset.asset_id)- if not saved_asset:- raise Exception(f"Could not publish {viewport} screenshot")screenshots.append({"viewport": viewport,@@ -61,7 +49,8 @@ async def run_screenshot_preview("image_part_index": image_part_index,"image_display_name": display_name,"image_bytes": len(image_bytes),- "image_url": saved_asset.public_url,+ # Inlined for the UI thumbnail only — never stored as an asset.+ "image_url": data_url,"status": "ok",})diff --git a/backend/tests/test_agent_tool_runtime.py b/backend/tests/test_agent_tool_runtime.pyindex a0518036..32cf51a9 100644--- a/backend/tests/test_agent_tool_runtime.py+++ b/backend/tests/test_agent_tool_runtime.py@@ -202,6 +202,13 @@ async def test_extract_assets_returns_mocked_gemini_assets(assert result.summary["assets"][0]["public_url"].startswith("http://127.0.0.1:7001/local-assets/")+ # Extracted crops finalize straight to the served store — no temp staging.+ assert not temp_dir.exists()+ assert len(list(asset_dir.iterdir())) == 2+ # No save_assets-style id is surfaced (already finalized), so the model+ # can't redundantly promote extracted crops.+ assert "asset_id" not in result.result["assets"][0]+ assert "asset_id" not in result.summary["assets"][0]assert result.result["assets"][0]["image_part_index"] == 0assert result.result["assets"][0]["image_display_name"] == "asset_0.png"assert result.result["assets"][1]["image_part_index"] == 1@@ -296,17 +303,20 @@ async def test_screenshot_preview_returns_image_part(]assert result.summary["status"] == "ok"screenshots = cast(list[dict[str, Any]], result.summary["screenshots"])- assert screenshots[0]["image_url"].startswith(- "http://127.0.0.1:7001/local-assets/"- )- assert screenshots[1]["image_url"].startswith(- "http://127.0.0.1:7001/local-assets/"- )- assert screenshots[0]["image_url"] != screenshots[1]["image_url"]+ # Previews are inlined as data URLs for the UI, not persisted as assets.+ desktop_url = cast(str, screenshots[0]["image_url"])+ mobile_url = cast(str, screenshots[1]["image_url"])+ assert desktop_url.startswith("data:image/png;base64,")+ assert mobile_url.startswith("data:image/png;base64,")+ assert desktop_url != mobile_url+ assert base64.b64decode(desktop_url.split(",", 1)[1]) == b"desktop-png-bytes"+ assert base64.b64decode(mobile_url.split(",", 1)[1]) == b"mobile-png-bytes"# UI URLs and image payloads must not leak into the model-facing result payload.assert "image_url" not in result.result["details"]["screenshots"][0]assert "image_data_url" not in result.result["details"]["screenshots"][0]- assert len(list(asset_dir.iterdir())) == 2+ # Previews are not assets: nothing is written to the temp or served stores.+ assert not asset_dir.exists()+ assert not temp_dir.exists()assert result.multimodal_parts is not Noneassert [part.display_name for part in result.multimodal_parts] == ["preview_desktop.png",diff --git a/backend/tests/test_uploaded_assets.py b/backend/tests/test_uploaded_assets.pyindex d92cdb2a..77551a98 100644--- a/backend/tests/test_uploaded_assets.py+++ b/backend/tests/test_uploaded_assets.py@@ -5,6 +5,7 @@ import pytestfrom uploaded_assets import (append_uploaded_asset_ids_to_prompt,+ persist_data_url_as_asset,persist_data_url_as_temporary_asset,promote_temporary_asset_id,)@@ -61,6 +62,43 @@ async def test_promote_temporary_asset_id_returns_permanent_url(assert len(list(asset_dir.iterdir())) == 1+async def test_persist_data_url_as_asset_finalizes_without_temp_staging(+ monkeypatch,+ tmp_path: Path,+) -> None:+ temp_dir = tmp_path / "tmp-assets"+ asset_dir = tmp_path / "local-assets"+ monkeypatch.setattr("uploaded_assets.store.TEMP_ASSET_DIR", str(temp_dir))+ monkeypatch.setattr("uploaded_assets.store.LOCAL_ASSET_DIR", str(asset_dir))++ saved = await persist_data_url_as_asset(+ _data_url(b"crop-bytes"),+ "http://127.0.0.1:7001",+ )++ assert saved is not None+ assert saved.public_url.startswith("http://127.0.0.1:7001/local-assets/asset_")+ assert saved.content_type == "image/png"+ # Goes straight to the served store; the temp staging dir is never created.+ assert not temp_dir.exists()+ assert len(list(asset_dir.iterdir())) == 1++ # Identical bytes are content-addressed: re-persisting dedupes to one file.+ again = await persist_data_url_as_asset(+ _data_url(b"crop-bytes"),+ "http://127.0.0.1:7001",+ )+ assert again is not None+ assert again.public_url == saved.public_url+ assert len(list(asset_dir.iterdir())) == 1+++async def test_persist_data_url_as_asset_rejects_non_image() -> None:+ assert await persist_data_url_as_asset("not-a-data-url", "http://127.0.0.1:7001") is None++def test_append_uploaded_asset_ids_to_prompt_keeps_image_and_adds_id(monkeypatch,tmp_path: Path,diff --git a/backend/uploaded_assets/__init__.py b/backend/uploaded_assets/__init__.pyindex 0e08e567..25a260b4 100644--- a/backend/uploaded_assets/__init__.py+++ b/backend/uploaded_assets/__init__.py@@ -7,6 +7,7 @@ from uploaded_assets.store import (TemporaryAsset,configure_uploaded_asset_routes,infer_local_asset_base_url,+ persist_data_url_as_asset,persist_data_url_as_temporary_asset,promote_temporary_asset_id,)@@ -18,6 +19,7 @@ __all__ = ["append_uploaded_asset_ids_to_prompt","configure_uploaded_asset_routes","infer_local_asset_base_url",+ "persist_data_url_as_asset","persist_data_url_as_temporary_asset","promote_temporary_asset_id",]diff --git a/backend/uploaded_assets/store.py b/backend/uploaded_assets/store.pyindex 90d43e07..19078ea1 100644--- a/backend/uploaded_assets/store.py+++ b/backend/uploaded_assets/store.py@@ -2,7 +2,6 @@ import base64import hashlibimport jsonimport os-import shutilimport tempfilefrom dataclasses import dataclassfrom typing import Any, cast@@ -148,10 +147,12 @@ def _temporary_asset_path(asset_id: str) -> tuple[str, str, str, str] | None:return None-def persist_data_url_as_temporary_asset(- data_url: str,- asset_base_url: str,-) -> TemporaryAsset | None:+def _decode_image_data_url(data_url: str) -> tuple[bytes, str, str] | None:+ """Decode a ``data:image/*`` URL into ``(bytes, content_type, extension)``.++ Returns ``None`` for non-image data URLs, unsupported image types,+ undecodable base64, or payloads over ``MAX_UPLOADED_ASSET_BYTES``.+ """if not data_url.startswith("data:image/") or "," not in data_url:return None@@ -169,6 +170,57 @@ def persist_data_url_as_temporary_asset(if len(image_bytes) > MAX_UPLOADED_ASSET_BYTES:return None+ return image_bytes, content_type, extension+++def _finalize_asset_bytes(+ image_bytes: bytes,+ extension: str,+ content_type: str,+ asset_base_url: str,+ user_id: str | None,+) -> SavedAsset:+ """Write asset bytes to the served ``LOCAL_ASSET_DIR`` and return a SavedAsset.++ The single "finalize" step that turns bytes into a durable, served asset.+ Content-addressed by digest, so identical bytes dedupe to one file.+ ``user_id`` is unused locally but is the hook hosted backends key durable+ per-user storage off of.+ """+ _ = user_id+ digest = _digest_for_bytes(image_bytes)+ asset_id = _asset_id_for_digest(digest)+ permanent_filename = f"asset_{digest}{extension}"++ os.makedirs(LOCAL_ASSET_DIR, exist_ok=True)+ destination_path = os.path.join(LOCAL_ASSET_DIR, permanent_filename)+ if not os.path.exists(destination_path):+ with open(destination_path, "wb") as file:+ file.write(image_bytes)++ return SavedAsset(+ asset_id=asset_id,+ public_url=_asset_url(asset_base_url, "local-assets", permanent_filename),+ content_type=content_type,+ )+++def persist_data_url_as_temporary_asset(+ data_url: str,+ asset_base_url: str,+) -> TemporaryAsset | None:+ """Stage an uploaded data URL as a temporary asset, pending a later promote.++ Used for user uploads, where only some images turn out to be assets the+ model keeps (via ``save_assets``); the rest stay reference-only. Callers+ that already know the image is an asset should use+ ``persist_data_url_as_asset`` instead and skip this staging hop.+ """+ decoded = _decode_image_data_url(data_url)+ if decoded is None:+ return None+ image_bytes, content_type, extension = decoded+os.makedirs(TEMP_ASSET_DIR, exist_ok=True)digest = _digest_for_bytes(image_bytes)asset_id = _asset_id_for_digest(digest)@@ -185,29 +237,40 @@ def persist_data_url_as_temporary_asset()+async def persist_data_url_as_asset(+ data_url: str,+ asset_base_url: str,+ user_id: str | None = None,+) -> SavedAsset | None:+ """Persist a data URL straight to a durable, served asset.++ Skips the temp-staging hop used for uploads: callers that already know the+ image is an asset (e.g. ``extract_assets``) commit immediately. Async to+ match ``promote_temporary_asset_id`` so hosted backends can finalize to+ remote storage without changing callers.+ """+ decoded = _decode_image_data_url(data_url)+ if decoded is None:+ return None+ image_bytes, content_type, extension = decoded+ return _finalize_asset_bytes(+ image_bytes, extension, content_type, asset_base_url, user_id+ )++async def promote_temporary_asset_id(asset_id: str,user_id: str | None = None,) -> SavedAsset | None:- _ = user_id+ """Finalize a previously-staged temporary asset into a served asset."""temporary_asset = _temporary_asset_path(asset_id)if not temporary_asset:return Nonesource_path, temporary_filename, content_type, asset_base_url = temporary_asset_, extension = os.path.splitext(temporary_filename)- digest = _digest_for_asset_id(asset_id)- if not digest:- return None- permanent_filename = f"asset_{digest}{extension}"-- os.makedirs(LOCAL_ASSET_DIR, exist_ok=True)- destination_path = os.path.join(LOCAL_ASSET_DIR, permanent_filename)- if not os.path.exists(destination_path):- shutil.copyfile(source_path, destination_path)-- return SavedAsset(- asset_id=asset_id,- public_url=_asset_url(asset_base_url, "local-assets", permanent_filename),- content_type=content_type,+ with open(source_path, "rb") as file:+ image_bytes = file.read()+ return _finalize_asset_bytes(+ image_bytes, extension, content_type, asset_base_url, user_id)