Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Add extract_assets agent tool for reusing screenshot assets

Abi Rajad1fe15893c2a
diff --git a/backend/agent/engine.py b/backend/agent/engine.py
index 83ae8be3..3dd027a6 100644
--- a/backend/agent/engine.py
+++ b/backend/agent/engine.py
@@ -1,6 +1,6 @@
import asyncio
import uuid
-from typing import Any, Awaitable, Callable, Dict, List, Optional
+from typing import Any, Awaitable, Callable, Dict, List, Optional, cast
from openai.types.chat import ChatCompletionMessageParam
@@ -32,6 +32,7 @@ class AgentEngine:
anthropic_api_key: Optional[str],
gemini_api_key: Optional[str],
should_generate_images: bool,
+ asset_base_url: str = "",
initial_file_state: Optional[Dict[str, str]] = None,
option_codes: Optional[List[str]] = None,
):
@@ -53,10 +54,32 @@ class AgentEngine:
should_generate_images=should_generate_images,
openai_api_key=openai_api_key,
openai_base_url=openai_base_url,
+ gemini_api_key=gemini_api_key,
+ asset_base_url=asset_base_url,
option_codes=option_codes,
)
self._tool_preview_lengths: Dict[str, int] = {}
+ @staticmethod
+ def _extract_input_images(
+ prompt_messages: List[ChatCompletionMessageParam],
+ ) -> List[str]:
+ images: List[str] = []
+ for message in prompt_messages:
+ content = message.get("content")
+ if not isinstance(content, list):
+ continue
+ for part in content:
+ if not isinstance(part, dict) or part.get("type") != "image_url":
+ continue
+ image_url = part.get("image_url")
+ if not isinstance(image_url, dict):
+ continue
+ url = cast(object, image_url.get("url"))
+ if isinstance(url, str) and url:
+ images.append(url)
+ return images
+
def _next_event_id(self, prefix: str) -> str:
return f"{prefix}-{self.variant_index}-{uuid.uuid4().hex[:8]}"
@@ -226,6 +249,7 @@ class AgentEngine:
raise Exception("Agent exceeded max tool turns")
async def run(self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]) -> str:
+ self.tool_runtime.input_images = self._extract_input_images(prompt_messages)
seed_file_state_from_messages(self.file_state, prompt_messages)
session = create_provider_session(
diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.py
index b72aa1cf..6b5606d3 100644
--- a/backend/agent/tools/definitions.py
+++ b/backend/agent/tools/definitions.py
@@ -133,6 +133,25 @@ def _edit_image_schema() -> Dict[str, Any]:
}
+def _extract_assets_schema() -> Dict[str, Any]:
+ return {
+ "type": "object",
+ "properties": {
+ "asset_descriptions": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "description": (
+ "Short text description of one visual asset to extract "
+ "from the input image."
+ ),
+ },
+ },
+ },
+ "required": ["asset_descriptions"],
+ }
+
+
def _retrieve_option_schema() -> Dict[str, Any]:
return {
"type": "object",
@@ -207,6 +226,16 @@ def canonical_tool_definitions(
)
tools.extend(
[
+ CanonicalToolDefinition(
+ name="extract_assets",
+ description=(
+ "Extract one or more visual assets from the input screenshot or "
+ "reference images using Gemini. Pass a list of text descriptions "
+ "for the assets to extract. Returns extracted image data URLs in "
+ "the same order."
+ ),
+ parameters=_extract_assets_schema(),
+ ),
SAVE_ASSETS_TOOL_DEFINITION,
CanonicalToolDefinition(
name="retrieve_option",
diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.py
index 49198c3f..e70d6d1e 100644
--- a/backend/agent/tools/runtime.py
+++ b/backend/agent/tools/runtime.py
@@ -7,18 +7,28 @@ import os
from urllib.parse import unquote, urlparse
from typing import Any, Dict, List, Optional, Tuple, Union
+from asset_extraction import extract_assets_from_images
from codegen.utils import extract_html_content
from config import LOCAL_ASSET_DIR, REPLICATE_API_KEY
from image_generation.generation import process_tasks
from image_generation.replicate import edit_image, remove_background
+from uploaded_assets.store import (
+ persist_data_url_as_temporary_asset,
+ promote_temporary_asset_id,
+)
from uploaded_assets.tools import run_save_assets
from agent.state import AgentFileState, ensure_str
-from agent.tools.types import ToolCall, ToolExecutionResult
+from agent.tools.types import ToolCall, ToolExecutionResult, ToolMultimodalPart
from agent.tools.summaries import summarize_text
LOCAL_ASSET_HOSTS = {"127.0.0.1", "localhost", "::1"}
+IMAGE_EXTENSION_BY_MIME_TYPE = {
+ "image/jpeg": ".jpg",
+ "image/png": ".png",
+ "image/webp": ".webp",
+}
def _local_asset_url_to_data_url(image_url: str) -> str:
@@ -46,6 +56,30 @@ def _local_asset_url_to_data_url(image_url: str) -> str:
return f"data:{content_type};base64,{encoded}"
+def _image_data_url_to_multimodal_part(
+ data_url: str,
+ display_name: str,
+) -> ToolMultimodalPart | None:
+ if not data_url.startswith("data:image/") or "," not in data_url:
+ return None
+
+ header, encoded = data_url.split(",", 1)
+ mime_type = header.removeprefix("data:").split(";", 1)[0].lower()
+ if mime_type not in {"image/png", "image/jpeg", "image/webp"}:
+ return None
+
+ try:
+ data = base64.b64decode(encoded, validate=True)
+ except ValueError:
+ return None
+
+ return ToolMultimodalPart(
+ display_name=display_name,
+ mime_type=mime_type,
+ data=data,
+ )
+
+
class AgentToolRuntime:
def __init__(
self,
@@ -53,6 +87,9 @@ class AgentToolRuntime:
should_generate_images: bool,
openai_api_key: Optional[str],
openai_base_url: Optional[str],
+ gemini_api_key: Optional[str] = None,
+ input_images: Optional[List[str]] = None,
+ asset_base_url: str = "",
user_id: Optional[str] = None,
option_codes: Optional[List[str]] = None,
):
@@ -60,6 +97,9 @@ class AgentToolRuntime:
self.should_generate_images = should_generate_images
self.openai_api_key = openai_api_key
self.openai_base_url = openai_base_url
+ self.gemini_api_key = gemini_api_key
+ self.input_images = input_images or []
+ self.asset_base_url = asset_base_url
self.user_id = user_id
self.option_codes = option_codes or []
@@ -85,6 +125,8 @@ class AgentToolRuntime:
return await self._remove_background(tool_call.arguments)
if tool_call.name == "edit_image":
return await self._edit_image(tool_call.arguments)
+ if tool_call.name == "extract_assets":
+ return await self._extract_assets(tool_call.arguments)
if tool_call.name == "save_assets":
return await run_save_assets(tool_call.arguments, user_id=self.user_id)
if tool_call.name == "retrieve_option":
@@ -464,6 +506,151 @@ class AgentToolRuntime:
}
return ToolExecutionResult(ok=True, result=result, summary=summary)
+ async def _extract_assets(self, args: Dict[str, Any]) -> ToolExecutionResult:
+ if not self.gemini_api_key:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "Asset extraction requires GEMINI_API_KEY."},
+ summary={"error": "Missing Gemini API key"},
+ )
+
+ if not self.input_images:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "No input images are available for asset extraction."},
+ summary={"error": "No input images"},
+ )
+
+ raw_descriptions = args.get("asset_descriptions") or args.get("descriptions")
+ if not isinstance(raw_descriptions, list) or not raw_descriptions:
+ return ToolExecutionResult(
+ ok=False,
+ result={
+ "error": "extract_assets requires a non-empty asset_descriptions list"
+ },
+ summary={"error": "Missing asset_descriptions"},
+ )
+
+ descriptions = [
+ description.strip()
+ for description in raw_descriptions
+ if isinstance(description, str) and description.strip()
+ ]
+ if not descriptions:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "No valid asset descriptions provided"},
+ summary={"error": "No valid asset_descriptions"},
+ )
+
+ result = await extract_assets_from_images(
+ image_data_urls=self.input_images,
+ asset_descriptions=descriptions,
+ gemini_api_key=self.gemini_api_key,
+ )
+ assets = result.get("assets", [])
+ promoted_assets: List[Dict[str, Any]] = []
+ multimodal_parts: List[ToolMultimodalPart] = []
+ if isinstance(assets, list):
+ for index, asset in enumerate(assets):
+ if not isinstance(asset, dict):
+ continue
+
+ data_url = ensure_str(asset.get("data_url"))
+ public_url: str | None = None
+ content_type: str | None = None
+ asset_id: str | None = None
+ image_part_index: int | None = None
+ image_display_name: str | None = None
+ status = ensure_str(asset.get("status")) or "missing"
+
+ if data_url:
+ mime_type = data_url.split(";", 1)[0].removeprefix("data:")
+ extension = IMAGE_EXTENSION_BY_MIME_TYPE.get(mime_type, ".png")
+ display_name = f"asset_{index}{extension}"
+ multimodal_part = _image_data_url_to_multimodal_part(
+ data_url,
+ display_name,
+ )
+ if multimodal_part:
+ multimodal_parts.append(multimodal_part)
+ image_part_index = len(multimodal_parts) - 1
+ image_display_name = display_name
+
+ temporary_asset = persist_data_url_as_temporary_asset(
+ data_url,
+ self.asset_base_url,
+ )
+ if temporary_asset:
+ saved_asset = await promote_temporary_asset_id(
+ temporary_asset.asset_id,
+ user_id=self.user_id,
+ )
+ if saved_asset:
+ public_url = saved_asset.public_url
+ content_type = saved_asset.content_type
+ asset_id = saved_asset.asset_id
+ status = "ok"
+ else:
+ status = "error"
+ else:
+ status = "error"
+
+ promoted_assets.append(
+ {
+ "description": asset.get("description"),
+ "public_url": public_url,
+ "asset_id": asset_id,
+ "content_type": content_type,
+ "status": status,
+ "image_part_index": image_part_index,
+ "image_display_name": image_display_name,
+ "box_2d": asset.get("box_2d"),
+ "image_index": asset.get("image_index"),
+ "label": asset.get("label"),
+ }
+ )
+
+ result = {
+ **result,
+ "assets": promoted_assets,
+ }
+ if any(asset.get("status") != "ok" for asset in promoted_assets):
+ result["error"] = (
+ result.get("error")
+ or "Could not persist every extracted asset to a public URL."
+ )
+
+ summary_assets = []
+ for asset in promoted_assets:
+ public_url = ensure_str(asset.get("public_url"))
+ summary_assets.append(
+ {
+ "description": summarize_text(
+ ensure_str(asset.get("description")), 120
+ ),
+ "status": ensure_str(asset.get("status")),
+ "public_url": public_url,
+ "asset_id": ensure_str(asset.get("asset_id")),
+ "content_type": ensure_str(asset.get("content_type")),
+ "image_part_index": asset.get("image_part_index"),
+ "image_display_name": asset.get("image_display_name"),
+ "box_2d": asset.get("box_2d"),
+ "image_index": asset.get("image_index"),
+ "label": ensure_str(asset.get("label")),
+ }
+ )
+
+ return ToolExecutionResult(
+ ok=not result.get("error"),
+ result=result,
+ summary={
+ "assets": summary_assets,
+ "error": result.get("error"),
+ },
+ multimodal_parts=multimodal_parts,
+ )
+
def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:
raw_option_number = args.get("option_number")
raw_index = args.get("index")
diff --git a/backend/agent/tools/summaries.py b/backend/agent/tools/summaries.py
index 2af8decf..9984743d 100644
--- a/backend/agent/tools/summaries.py
+++ b/backend/agent/tools/summaries.py
@@ -77,6 +77,15 @@ def summarize_tool_input(tool_call: ToolCall, file_state: AgentFileState) -> Dic
}
return {"prompt": ensure_str(args.get("prompt")), "image_urls": []}
+ if tool_call.name == "extract_assets":
+ descriptions = args.get("asset_descriptions") or args.get("descriptions") or []
+ if isinstance(descriptions, list):
+ return {
+ "count": len(descriptions),
+ "asset_descriptions": [ensure_str(item) for item in descriptions],
+ }
+ return {"asset_descriptions": []}
+
if tool_call.name == "save_assets":
return summarize_save_assets_input(args)
diff --git a/backend/asset_extraction.py b/backend/asset_extraction.py
new file mode 100644
index 00000000..047c4246
--- /dev/null
+++ b/backend/asset_extraction.py
@@ -0,0 +1,197 @@
+import base64
+import io
+import json
+import re
+from dataclasses import dataclass
+from typing import Any, Dict, List, cast
+
+from google import genai
+from google.genai import types
+from PIL import Image
+
+
+ASSET_EXTRACTION_GEMINI_MODEL = "gemini-3-flash-preview"
+
+
+@dataclass(frozen=True)
+class SourceImage:
+ part: types.Part
+ image: Image.Image
+ mime_type: str
+
+
+def _data_url_to_source_image(data_url: str) -> SourceImage | None:
+ if not data_url.startswith("data:image/") or "," not in data_url:
+ return None
+
+ header, encoded = data_url.split(",", 1)
+ mime_type = header.removeprefix("data:").split(";", 1)[0].lower()
+ try:
+ image_bytes = base64.b64decode(encoded, validate=True)
+ image = Image.open(io.BytesIO(image_bytes))
+ image.load()
+ except Exception:
+ return None
+
+ return SourceImage(
+ part=types.Part.from_bytes(
+ data=image_bytes,
+ mime_type=mime_type,
+ media_resolution=types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH,
+ ),
+ image=image,
+ mime_type=mime_type,
+ )
+
+
+def _extract_json_object(text: str) -> Dict[str, Any] | None:
+ stripped = text.strip()
+ if stripped.startswith("```"):
+ stripped = re.sub(r"^```(?:json)?\s*", "", stripped)
+ stripped = re.sub(r"\s*```$", "", stripped)
+
+ try:
+ parsed: Any = json.loads(stripped)
+ except json.JSONDecodeError:
+ match = re.search(r"\{.*\}", stripped, re.DOTALL)
+ if not match:
+ return None
+ try:
+ parsed = json.loads(match.group(0))
+ except json.JSONDecodeError:
+ return None
+
+ if not isinstance(parsed, dict):
+ return None
+ return cast(Dict[str, Any], parsed)
+
+
+def _coerce_box(value: Any) -> list[int] | None:
+ if not isinstance(value, list) or len(value) != 4:
+ return None
+
+ box: list[int] = []
+ for coordinate in cast(list[object], value):
+ if not isinstance(coordinate, (int, float, str)):
+ return None
+ try:
+ box.append(int(round(float(coordinate))))
+ except (TypeError, ValueError):
+ return None
+ return box
+
+
+def _crop_box_to_data_url(image: Image.Image, box_2d: list[int]) -> str | None:
+ width, height = image.size
+ y0, x0, y1, x1 = [max(0, min(1000, value)) for value in box_2d]
+
+ left = int(x0 / 1000 * width)
+ top = int(y0 / 1000 * height)
+ right = int(x1 / 1000 * width)
+ bottom = int(y1 / 1000 * height)
+
+ if right <= left or bottom <= top:
+ return None
+
+ cropped = image.crop((left, top, right, bottom))
+ output = io.BytesIO()
+ cropped.save(output, format="PNG")
+ encoded = base64.b64encode(output.getvalue()).decode("ascii")
+ return f"data:image/png;base64,{encoded}"
+
+
+async def _locate_asset_box(
+ client: genai.Client,
+ source_images: List[SourceImage],
+ description: str,
+) -> Dict[str, Any] | None:
+ image_list_text = "\n".join(
+ f"- image_index {index}" for index in range(1, len(source_images) + 1)
+ )
+ prompt = f"""Find the bounding box for this visual asset:
+
+{description}
+
+Images available:
+{image_list_text}
+
+Return only JSON with this exact shape:
+{{"image_index": 1, "box_2d": [y0, x0, y1, x1], "label": "short label"}}
+
+Rules:
+- image_index is 1-based and identifies which input image contains the asset.
+- box_2d uses normalized coordinates from 0 to 1000.
+- box_2d order must be [y0, x0, y1, x1].
+- Crop tightly around the requested asset.
+- If the asset is not visible, return {{"image_index": null, "box_2d": null, "label": "not found"}}.
+- Return JSON only. No markdown.
+"""
+
+ response = await client.aio.models.generate_content(
+ model=ASSET_EXTRACTION_GEMINI_MODEL,
+ contents=[
+ types.Content(
+ role="user",
+ parts=[source.part for source in source_images] + [types.Part(text=prompt)],
+ )
+ ],
+ config=types.GenerateContentConfig(
+ temperature=0,
+ response_mime_type="application/json",
+ thinking_config=types.ThinkingConfig(
+ thinking_level=cast(Any, "minimal")
+ ),
+ ),
+ )
+
+ return _extract_json_object(response.text or "")
+
+
+async def extract_assets_from_images(
+ image_data_urls: List[str],
+ asset_descriptions: List[str],
+ gemini_api_key: str,
+) -> Dict[str, Any]:
+ source_images = [
+ source
+ for image in image_data_urls
+ if (source := _data_url_to_source_image(image)) is not None
+ ]
+ if not source_images:
+ return {
+ "assets": [],
+ "error": "No valid input images were available for asset extraction.",
+ }
+
+ client = genai.Client(api_key=gemini_api_key)
+ assets: List[Dict[str, Any]] = []
+
+ for description in asset_descriptions:
+ detection = await _locate_asset_box(client, source_images, description)
+ image_index = detection.get("image_index") if detection else None
+ box = _coerce_box(detection.get("box_2d")) if detection else None
+
+ data_url = None
+ status = "missing"
+ if isinstance(image_index, int) and box:
+ source_index = image_index - 1
+ if 0 <= source_index < len(source_images):
+ data_url = _crop_box_to_data_url(source_images[source_index].image, box)
+ status = "ok" if data_url else "error"
+
+ assets.append(
+ {
+ "description": description,
+ "data_url": data_url,
+ "status": status,
+ "box_2d": box,
+ "image_index": image_index,
+ "label": detection.get("label") if detection else None,
+ }
+ )
+
+ result: Dict[str, Any] = {"assets": assets}
+ if any(asset["status"] != "ok" for asset in assets):
+ result["error"] = "Gemini did not return usable bounding boxes for every requested asset."
+
+ return result
diff --git a/backend/prompts/create/image.py b/backend/prompts/create/image.py
index 39bfa886..a84fbaa2 100644
--- a/backend/prompts/create/image.py
+++ b/backend/prompts/create/image.py
@@ -23,8 +23,15 @@ Generate code for a web page that looks exactly like the provided screenshot(s).
## Replication instructions
-- Make sure the app looks exactly like the screenshot.
+- Make sure the web page looks exactly like the screenshot.
- Use the exact text from the screenshot.
+- Our goal is to make the web page look as close to the screenshot as possible. This means we need to extract the exact image assets where possible and generate images for the assets that are not extractable.
+- Extracting assets can be done with the extract_assets tool. After extracting assets, make sure to inspect the extracted image closely to ensure that it is what we want.
+- When available, use the edit_image tool to edit the assets when needed. A good example of this might be if the extracted asset is very low resolution, or if the extracted asset has unwanted elements.
+- If an asset in the original screenshot is not extractable (for example, occluded by other objects or is the background), when available, use generate_images to create image URLs from prompts (you may pass multiple prompts).
+- The image generation AI is not capable of generating images with a transparent background. NEVER USE this tool to extract the entire screenshot and embed it on the page. Our goal here is to create nicely coded pages. We should only use extracted assets for images, not for layout, etc.
+- Use the remove_background where needed on assets.
+
- {image_policy}
## Multiple screenshots
diff --git a/backend/prompts/system_prompt.py b/backend/prompts/system_prompt.py
index cb7dedb1..66d34f2e 100644
--- a/backend/prompts/system_prompt.py
+++ b/backend/prompts/system_prompt.py
@@ -15,7 +15,8 @@ You are a coding agent that's an expert at building front-ends.
- For a brand new app, call create_file exactly once with the full HTML.
- For updates, call edit_file using exact string replacements. Do NOT regenerate the entire file.
- Do not output raw HTML in chat. Any code changes must go through tools.
-- When available, use generate_images to create image URLs from prompts (you may pass multiple prompts). The image generation AI is not capable of generating images with a transparent background.
+- Use extract_assets to extract existing visual assets from the input screenshot.
+- If an asset in the original screenshot is not extractable (for example, occluded by other objects or is the background), when available, use generate_images to create image URLs from prompts (you may pass multiple prompts). The image generation AI is not capable of generating images with a transparent background. NEVER USE this tool to extract the entire screenshot and embed it on the page. Our goal here is to create nicely coded pages. We should only use extracted assets for images, not for layout, etc.
- Use remove_background to remove backgrounds from provided image URLs when needed (you may pass multiple image URLs).
- Use retrieve_option to fetch the full HTML for a specific option (1-based option_number) when a user references another option.
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index ba9ebfc9..745993dd 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -241,6 +241,7 @@ class ExtractedParams:
history: List[PromptHistoryMessage]
file_state: Dict[str, str] | None
option_codes: List[str]
+ asset_base_url: str = ""
design_system: str | None = None
@@ -356,6 +357,7 @@ class ParameterExtractionStage:
history=history,
file_state=file_state,
option_codes=option_codes,
+ asset_base_url=self.asset_base_url,
design_system=design_system,
)
@@ -525,6 +527,7 @@ class AgenticGenerationStage:
gemini_api_key: str | None,
should_generate_images: bool,
file_state: Dict[str, str] | None,
+ asset_base_url: str,
option_codes: List[str] | None,
):
self.send_message = send_message
@@ -534,6 +537,7 @@ class AgenticGenerationStage:
self.gemini_api_key = gemini_api_key
self.should_generate_images = should_generate_images
self.file_state = file_state
+ self.asset_base_url = asset_base_url
self.option_codes = option_codes or []
async def process_variants(
@@ -590,6 +594,7 @@ class AgenticGenerationStage:
anthropic_api_key=self.anthropic_api_key,
gemini_api_key=self.gemini_api_key,
should_generate_images=self.should_generate_images,
+ asset_base_url=self.asset_base_url,
initial_file_state=self.file_state,
option_codes=self.option_codes,
)
@@ -771,6 +776,7 @@ class CodeGenerationMiddleware(Middleware):
gemini_api_key=context.extracted_params.gemini_api_key,
should_generate_images=context.extracted_params.should_generate_images,
file_state=context.extracted_params.file_state,
+ asset_base_url=context.extracted_params.asset_base_url,
option_codes=context.extracted_params.option_codes,
)
diff --git a/backend/tests/test_agent_tool_runtime.py b/backend/tests/test_agent_tool_runtime.py
index 825de183..0281772f 100644
--- a/backend/tests/test_agent_tool_runtime.py
+++ b/backend/tests/test_agent_tool_runtime.py
@@ -110,3 +110,108 @@ async def test_save_assets_promotes_temporary_asset_id(
)
assert temp_asset.asset_id not in images[0]["public_url"]
assert len(list(asset_dir.iterdir())) == 1
+
+
+async def test_extract_assets_requires_gemini_api_key() -> None:
+ runtime = AgentToolRuntime(
+ file_state=AgentFileState(),
+ should_generate_images=False,
+ openai_api_key=None,
+ openai_base_url=None,
+ input_images=[_data_url(b"source-image")],
+ )
+
+ result = await runtime.execute(
+ ToolCall(
+ id="call-1",
+ name="extract_assets",
+ arguments={"asset_descriptions": ["logo"]},
+ )
+ )
+
+ assert result.ok is False
+ assert result.summary["error"] == "Missing Gemini API key"
+
+
+async def test_extract_assets_returns_mocked_gemini_assets(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ temp_dir = tmp_path / "tmp-assets"
+ asset_dir = tmp_path / "local-assets"
+ monkeypatch.setattr("uploaded_assets.store.TEMP_ASSET_DIR", str(temp_dir))
+ monkeypatch.setattr("uploaded_assets.store.LOCAL_ASSET_DIR", str(asset_dir))
+
+ async def fake_extract_assets_from_images(
+ image_data_urls: list[str],
+ asset_descriptions: list[str],
+ gemini_api_key: str,
+ ) -> dict[str, object]:
+ assert image_data_urls == [_data_url(b"source-image")]
+ assert asset_descriptions == ["logo", "avatar"]
+ assert gemini_api_key == "gemini-key"
+ return {
+ "assets": [
+ {
+ "description": "logo",
+ "data_url": _data_url(b"logo-image"),
+ "status": "ok",
+ "box_2d": [0, 0, 500, 500],
+ "image_index": 1,
+ "label": "logo",
+ },
+ {
+ "description": "avatar",
+ "data_url": _data_url(b"avatar-image"),
+ "status": "ok",
+ },
+ ]
+ }
+
+ monkeypatch.setattr(
+ "agent.tools.runtime.extract_assets_from_images",
+ fake_extract_assets_from_images,
+ )
+
+ runtime = AgentToolRuntime(
+ file_state=AgentFileState(),
+ should_generate_images=False,
+ openai_api_key=None,
+ openai_base_url=None,
+ gemini_api_key="gemini-key",
+ input_images=[_data_url(b"source-image")],
+ asset_base_url="http://127.0.0.1:7001",
+ )
+
+ result = await runtime.execute(
+ ToolCall(
+ id="call-1",
+ name="extract_assets",
+ arguments={"asset_descriptions": ["logo", "avatar"]},
+ )
+ )
+
+ assert result.ok is True
+ assert result.result["assets"][0]["description"] == "logo"
+ assert result.result["assets"][0]["public_url"].startswith(
+ "http://127.0.0.1:7001/local-assets/"
+ )
+ assert result.summary["assets"][0]["public_url"].startswith(
+ "http://127.0.0.1:7001/local-assets/"
+ )
+ assert result.result["assets"][0]["image_part_index"] == 0
+ assert result.result["assets"][0]["image_display_name"] == "asset_0.png"
+ assert result.result["assets"][1]["image_part_index"] == 1
+ assert result.result["assets"][1]["image_display_name"] == "asset_1.png"
+ assert result.summary["assets"][0]["box_2d"] == [0, 0, 500, 500]
+ assert result.summary["assets"][0]["image_index"] == 1
+ assert result.multimodal_parts is not None
+ assert [part.display_name for part in result.multimodal_parts] == [
+ "asset_0.png",
+ "asset_1.png",
+ ]
+ assert result.multimodal_parts[0].mime_type == "image/png"
+ assert result.multimodal_parts[0].data == b"logo-image"
+
diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.py
index 43852d82..4dd84347 100644
--- a/backend/tests/test_agent_tools.py
+++ b/backend/tests/test_agent_tools.py
@@ -47,6 +47,34 @@ def test_save_assets_tool_input_summary_uses_asset_ids() -> None:
}
+def test_canonical_tool_definitions_include_extract_assets() -> None:
+ tools = canonical_tool_definitions(True)
+ tool_names = [tool.name for tool in tools]
+ assert "extract_assets" in tool_names
+ extract_assets_tool = next(tool for tool in tools if tool.name == "extract_assets")
+ assert extract_assets_tool.parameters["required"] == ["asset_descriptions"]
+ assert (
+ extract_assets_tool.parameters["properties"]["asset_descriptions"]["type"]
+ == "array"
+ )
+
+
+def test_extract_assets_tool_input_summary_uses_asset_descriptions() -> None:
+ summary = summarize_tool_input(
+ ToolCall(
+ id="call-1",
+ name="extract_assets",
+ arguments={"asset_descriptions": ["logo", "avatar"]},
+ ),
+ AgentFileState(),
+ )
+
+ assert summary == {
+ "count": 2,
+ "asset_descriptions": ["logo", "avatar"],
+ }
+
+
def test_canonical_tool_definitions_include_edit_image() -> None:
tools = canonical_tool_definitions(True)
tool_names = [tool.name for tool in tools]
diff --git a/backend/tests/test_asset_extraction.py b/backend/tests/test_asset_extraction.py
new file mode 100644
index 00000000..05428962
--- /dev/null
+++ b/backend/tests/test_asset_extraction.py
@@ -0,0 +1,83 @@
+import base64
+import io
+
+import pytest
+from google.genai import types
+from PIL import Image
+
+from asset_extraction import extract_assets_from_images
+
+
+def _png_data_url(width: int = 10, height: int = 10) -> str:
+ image = Image.new("RGB", (width, height), "white")
+ # Put a black square in the top-left quadrant so the crop is inspectable.
+ for x in range(0, width // 2):
+ for y in range(0, height // 2):
+ image.putpixel((x, y), (0, 0, 0))
+
+ output = io.BytesIO()
+ image.save(output, format="PNG")
+ encoded = base64.b64encode(output.getvalue()).decode("ascii")
+ return f"data:image/png;base64,{encoded}"
+
+
+def _decode_png_data_url(data_url: str) -> Image.Image:
+ _, encoded = data_url.split(",", 1)
+ image = Image.open(io.BytesIO(base64.b64decode(encoded)))
+ image.load()
+ return image
+
+
+async def test_extract_assets_from_images_uses_gemini_boxes_and_crops_original(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ calls: list[dict[str, object]] = []
+
+ class FakeModels:
+ async def generate_content(self, **kwargs):
+ calls.append(kwargs)
+ return types.GenerateContentResponse(
+ candidates=[
+ types.Candidate(
+ content=types.Content(
+ role="model",
+ parts=[
+ types.Part(
+ text=(
+ '{"image_index": 1, '
+ '"box_2d": [0, 0, 500, 500], '
+ '"label": "logo"}'
+ )
+ )
+ ],
+ )
+ )
+ ]
+ )
+
+ class FakeAio:
+ models = FakeModels()
+
+ class FakeClient:
+ aio = FakeAio()
+
+ def __init__(self, api_key: str) -> None:
+ assert api_key == "gemini-key"
+
+ monkeypatch.setattr("asset_extraction.genai.Client", FakeClient)
+
+ result = await extract_assets_from_images(
+ image_data_urls=[_png_data_url()],
+ asset_descriptions=["logo", "avatar"],
+ gemini_api_key="gemini-key",
+ )
+
+ assert len(calls) == 2
+ assert calls[0]["model"] == "gemini-3-flash-preview"
+ assets = result["assets"]
+ assert assets[0]["description"] == "logo"
+ assert assets[0]["status"] == "ok"
+ assert assets[0]["box_2d"] == [0, 0, 500, 500]
+ cropped = _decode_png_data_url(assets[0]["data_url"])
+ assert cropped.size == (5, 5)
diff --git a/frontend/src/components/agent/AgentActivity.tsx b/frontend/src/components/agent/AgentActivity.tsx
index 48d23ce0..162f5fd2 100644
--- a/frontend/src/components/agent/AgentActivity.tsx
+++ b/frontend/src/components/agent/AgentActivity.tsx
@@ -17,6 +17,7 @@ import {
BsScissors,
BsFiles,
BsBookmarkCheck,
+ BsBoundingBox,
} from "react-icons/bs";
import ReactMarkdown from "react-markdown";
import { Light as SyntaxHighlighterBase } from "react-syntax-highlighter";
@@ -86,6 +87,42 @@ function formatVariantWallClockDuration(
}
+function getArrayField(value: unknown, field: string): unknown[] | null {
+ if (!value || typeof value !== "object") return null;
+ const fieldValue = (value as Record<string, unknown>)[field];
+ return Array.isArray(fieldValue) ? fieldValue : null;
+}
+
+function getRecord(value: unknown): Record<string, unknown> | null {
+ return value && typeof value === "object"
+ ? (value as Record<string, unknown>)
+ : null;
+}
+
+function getExtractedAssetPreviewUrl(asset: unknown): string | null {
+ const assetRecord = getRecord(asset);
+ if (!assetRecord) return null;
+ if (typeof assetRecord.public_url === "string" && assetRecord.public_url) {
+ return assetRecord.public_url;
+ }
+ if (typeof assetRecord.data_url === "string" && assetRecord.data_url) {
+ return assetRecord.data_url;
+ }
+ return null;
+}
+
+function isSuccessfulExtractedAsset(asset: unknown): boolean {
+ const assetRecord = getRecord(asset);
+ if (!assetRecord) return false;
+ const status =
+ typeof assetRecord.status === "string" ? assetRecord.status : null;
+ return Boolean(
+ getExtractedAssetPreviewUrl(asset) &&
+ status !== "missing" &&
+ status !== "error"
+ );
+}
+
function getEventIcon(type: AgentEventType, toolName?: string) {
if (type === "thinking") {
return <BsLightbulb className="text-yellow-500" />;
@@ -114,6 +151,9 @@ function getEventIcon(type: AgentEventType, toolName?: string) {
if (toolName === "save_assets") {
return <BsBookmarkCheck className="text-emerald-500" />;
}
+ if (toolName === "extract_assets") {
+ return <BsBoundingBox className="text-orange-500" />;
+ }
return <BsFileEarmarkPlus className="text-gray-500" />;
}
@@ -179,6 +219,32 @@ function getEventTitle(event: AgentEvent): string {
}
return saveCount > 1 ? `Saved ${saveCount} assets` : "Saved asset";
}
+ if (event.toolName === "extract_assets") {
+ const extractInputDescriptions = getArrayField(event.input, "asset_descriptions");
+ const extractOutputAssets = getArrayField(event.output, "assets");
+ const requestedCount = extractInputDescriptions?.length || 0;
+ const successfulCount =
+ extractOutputAssets?.filter(isSuccessfulExtractedAsset).length || 0;
+ const extractCount =
+ extractOutputAssets?.length || extractInputDescriptions?.length || 0;
+ if (event.status === "running") {
+ return extractCount > 1
+ ? `Extracting ${extractCount} assets`
+ : "Extracting asset";
+ }
+ if (
+ extractOutputAssets &&
+ requestedCount > 0 &&
+ successfulCount < requestedCount
+ ) {
+ return successfulCount > 0
+ ? `Extracted ${successfulCount} of ${requestedCount} assets`
+ : "Could not extract assets";
+ }
+ return extractCount > 1
+ ? `Extracted ${extractCount} assets`
+ : "Extracted asset";
+ }
return event.status === "running" ? "Running tool" : "Tool completed";
}
return "Activity";
@@ -213,6 +279,14 @@ function renderToolDetails(event: AgentEvent, variantCode?: string) {
output && Array.isArray(output.images) ? (output.images as Array<any>) : null;
const edits =
output && Array.isArray(output.edits) ? (output.edits as Array<any>) : null;
+ const extractedAssets =
+ output && Array.isArray(output.assets) ? (output.assets as Array<unknown>) : null;
+ const successfulExtractedAssets =
+ extractedAssets?.filter(isSuccessfulExtractedAsset) || null;
+ const visibleExtractedAssets =
+ hasError && successfulExtractedAssets
+ ? successfulExtractedAssets
+ : extractedAssets;
return (
<div className="text-sm text-gray-700 dark:text-gray-200">
@@ -527,6 +601,126 @@ function renderToolDetails(event: AgentEvent, variantCode?: string) {
</div>
)}
+ {event.toolName === "extract_assets" &&
+ (!hasError || Boolean(visibleExtractedAssets?.length)) && (
+ <div>
+ {!hasError &&
+ event.status === "running" &&
+ input?.asset_descriptions &&
+ Array.isArray(input.asset_descriptions) && (
+ <div className="divide-y divide-gray-100 dark:divide-gray-800">
+ {input.asset_descriptions.map((description: string, index: number) => (
+ <div key={`${description}-${index}`} className="py-2">
+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1">
+ Asset {index + 1}
+ </div>
+ <div className="text-xs text-gray-600 dark:text-gray-300">
+ {description}
+ </div>
+ </div>
+ ))}
+ </div>
+ )}
+ {event.status !== "running" && visibleExtractedAssets && (
+ <div className="divide-y divide-gray-100 dark:divide-gray-800">
+ {visibleExtractedAssets.map((asset, index) => {
+ const assetRecord =
+ asset && typeof asset === "object"
+ ? (asset as Record<string, unknown>)
+ : {};
+ const description =
+ typeof assetRecord.description === "string"
+ ? assetRecord.description
+ : `Asset ${index + 1}`;
+ const publicUrl =
+ typeof assetRecord.public_url === "string"
+ ? assetRecord.public_url
+ : null;
+ const previewUrl =
+ publicUrl ||
+ (typeof assetRecord.data_url === "string"
+ ? assetRecord.data_url
+ : null);
+ const boxText = Array.isArray(assetRecord.box_2d)
+ ? assetRecord.box_2d.join(", ")
+ : "No box";
+ const statusLabel =
+ typeof assetRecord.status === "string"
+ ? assetRecord.status
+ : previewUrl
+ ? "ok"
+ : "missing";
+ return (
+ <div key={`${description}-${index}`} className="flex gap-3 py-2">
+ <div className="w-1/2 shrink-0">
+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1">
+ Extracted crop
+ </div>
+ {previewUrl ? (
+ <img
+ src={previewUrl}
+ alt={description}
+ className="max-h-48 w-full rounded object-contain bg-gray-50 dark:bg-gray-800"
+ loading="lazy"
+ />
+ ) : (
+ <div className="aspect-square rounded bg-gray-100 dark:bg-gray-800 flex items-center justify-center text-xs text-gray-400">
+ Missing
+ </div>
+ )}
+ </div>
+ <div className="w-1/2 self-center space-y-2">
+ <div>
+ <div className="text-xs text-gray-500 dark:text-gray-400">
+ Requested asset
+ </div>
+ <div className="mt-1 text-xs text-gray-600 dark:text-gray-300">
+ {description}
+ </div>
+ </div>
+ {publicUrl && (
+ <div>
+ <div className="text-xs text-gray-500 dark:text-gray-400">
+ Public URL
+ </div>
+ <div className="mt-1 break-all rounded bg-gray-50 p-2 font-mono text-xs text-gray-600 dark:bg-gray-800 dark:text-gray-300">
+ {publicUrl}
+ </div>
+ </div>
+ )}
+ <div className="grid grid-cols-2 gap-2 text-xs">
+ <div>
+ <div className="text-gray-500 dark:text-gray-400">Status</div>
+ <div className="mt-1 font-mono text-gray-600 dark:text-gray-300">
+ {statusLabel}
+ </div>
+ </div>
+ <div>
+ <div className="text-gray-500 dark:text-gray-400">
+ Source image
+ </div>
+ <div className="mt-1 font-mono text-gray-600 dark:text-gray-300">
+ {String(assetRecord.image_index ?? "-")}
+ </div>
+ </div>
+ </div>
+ <div>
+ <div className="text-xs text-gray-500 dark:text-gray-400">
+ Bounding box
+ </div>
+ <div className="mt-1 break-all rounded bg-gray-50 p-2 font-mono text-xs text-gray-600 dark:bg-gray-800 dark:text-gray-300">
+ [{boxText}]
+ </div>
+ </div>
+ </div>
+ </div>
+ );
+ })}
+ </div>
+ )}
+ </div>
+ )}
+
{!event.toolName && !hasError && (
<>
{event.input && (