Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Gate and split extract assets tool

Abi Raja86bb6b06c3e4
diff --git a/backend/agent/providers/factory.py b/backend/agent/providers/factory.py
index 48f326da..fda2ad32 100644
--- a/backend/agent/providers/factory.py
+++ b/backend/agent/providers/factory.py
@@ -27,6 +27,8 @@ def create_provider_session(
image_generation_enabled=should_generate_images,
# The edit_image tool calls Replicate, so don't offer it without a key.
image_editing_enabled=bool(REPLICATE_API_KEY),
+ # The extract_assets tool calls Gemini, so don't offer it without a key.
+ asset_extraction_enabled=bool(gemini_api_key),
)
if model in OPENAI_MODELS:
diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.py
index fccf2c6b..17b37d8c 100644
--- a/backend/agent/tools/definitions.py
+++ b/backend/agent/tools/definitions.py
@@ -161,6 +161,7 @@ def _retrieve_option_schema() -> Dict[str, Any]:
def canonical_tool_definitions(
image_generation_enabled: bool = True,
image_editing_enabled: bool = True,
+ asset_extraction_enabled: bool = True,
) -> List[CanonicalToolDefinition]:
tools: List[CanonicalToolDefinition] = [
CanonicalToolDefinition(
@@ -217,8 +218,8 @@ def canonical_tool_definitions(
parameters=_edit_image_schema(),
)
)
- tools.extend(
- [
+ if asset_extraction_enabled:
+ tools.append(
CanonicalToolDefinition(
name="extract_assets",
description=(
@@ -228,7 +229,10 @@ def canonical_tool_definitions(
"the same order."
),
parameters=_extract_assets_schema(),
- ),
+ )
+ )
+ tools.extend(
+ [
SAVE_ASSETS_TOOL_DEFINITION,
CanonicalToolDefinition(
name="retrieve_option",
diff --git a/backend/agent/tools/extract_assets.py b/backend/agent/tools/extract_assets.py
new file mode 100644
index 00000000..3ba66b3e
--- /dev/null
+++ b/backend/agent/tools/extract_assets.py
@@ -0,0 +1,198 @@
+import base64
+from typing import Any, Dict, List, Optional, cast
+
+from asset_extraction import extract_assets_from_images
+from uploaded_assets.store import (
+ persist_data_url_as_temporary_asset,
+ promote_temporary_asset_id,
+)
+
+from agent.state import ensure_str
+from agent.tools.summaries import summarize_text
+from agent.tools.types import ToolExecutionResult, ToolMultimodalPart
+
+
+IMAGE_EXTENSION_BY_MIME_TYPE = {
+ "image/jpeg": ".jpg",
+ "image/png": ".png",
+ "image/webp": ".webp",
+}
+
+
+def _image_data_url_to_multimodal_part(
+ data_url: str,
+ display_name: str,
+) -> ToolMultimodalPart | None:
+ if not data_url.startswith("data:image/") or "," not in data_url:
+ return None
+
+ header, encoded = data_url.split(",", 1)
+ mime_type = header.removeprefix("data:").split(";", 1)[0].lower()
+ if mime_type not in {"image/png", "image/jpeg", "image/webp"}:
+ return None
+
+ try:
+ data = base64.b64decode(encoded, validate=True)
+ except ValueError:
+ return None
+
+ return ToolMultimodalPart(
+ display_name=display_name,
+ mime_type=mime_type,
+ data=data,
+ )
+
+
+async def run_extract_assets(
+ args: Dict[str, Any],
+ *,
+ gemini_api_key: Optional[str],
+ input_images: List[str],
+ asset_base_url: str,
+ user_id: Optional[str],
+) -> ToolExecutionResult:
+ if not gemini_api_key:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "Asset extraction requires GEMINI_API_KEY."},
+ summary={"error": "Missing Gemini API key"},
+ )
+
+ if not input_images:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "No input images are available for asset extraction."},
+ summary={"error": "No input images"},
+ )
+
+ raw_descriptions = args.get("asset_descriptions") or args.get("descriptions")
+ if not isinstance(raw_descriptions, list) or not raw_descriptions:
+ return ToolExecutionResult(
+ ok=False,
+ result={
+ "error": "extract_assets requires a non-empty asset_descriptions list"
+ },
+ summary={"error": "Missing asset_descriptions"},
+ )
+
+ raw_description_items = cast(list[object], raw_descriptions)
+ descriptions: List[str] = [
+ description.strip()
+ for description in raw_description_items
+ if isinstance(description, str) and description.strip()
+ ]
+ if not descriptions:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "No valid asset descriptions provided"},
+ summary={"error": "No valid asset_descriptions"},
+ )
+
+ extraction_result: Dict[str, Any] = await extract_assets_from_images(
+ image_data_urls=input_images,
+ asset_descriptions=descriptions,
+ gemini_api_key=gemini_api_key,
+ )
+ assets = extraction_result.get("assets", [])
+ promoted_assets: List[Dict[str, Any]] = []
+ multimodal_parts: List[ToolMultimodalPart] = []
+ if isinstance(assets, list):
+ for index, raw_asset in enumerate(cast(list[object], assets)):
+ if not isinstance(raw_asset, dict):
+ continue
+ asset = cast(Dict[str, Any], raw_asset)
+
+ data_url = ensure_str(asset.get("data_url"))
+ public_url: str | None = None
+ content_type: str | None = None
+ asset_id: str | None = None
+ image_part_index: int | None = None
+ image_display_name: str | None = None
+ status = ensure_str(asset.get("status")) or "missing"
+
+ if data_url:
+ mime_type = data_url.split(";", 1)[0].removeprefix("data:")
+ extension = IMAGE_EXTENSION_BY_MIME_TYPE.get(mime_type, ".png")
+ display_name = f"asset_{index}{extension}"
+ multimodal_part = _image_data_url_to_multimodal_part(
+ data_url,
+ display_name,
+ )
+ if multimodal_part:
+ multimodal_parts.append(multimodal_part)
+ image_part_index = len(multimodal_parts) - 1
+ image_display_name = display_name
+
+ temporary_asset = persist_data_url_as_temporary_asset(
+ data_url,
+ asset_base_url,
+ )
+ if temporary_asset:
+ saved_asset = await promote_temporary_asset_id(
+ temporary_asset.asset_id,
+ user_id=user_id,
+ )
+ if saved_asset:
+ public_url = saved_asset.public_url
+ content_type = saved_asset.content_type
+ asset_id = saved_asset.asset_id
+ status = "ok"
+ else:
+ status = "error"
+ else:
+ status = "error"
+
+ promoted_assets.append(
+ {
+ "description": asset.get("description"),
+ "public_url": public_url,
+ "asset_id": asset_id,
+ "content_type": content_type,
+ "status": status,
+ "image_part_index": image_part_index,
+ "image_display_name": image_display_name,
+ "box_2d": asset.get("box_2d"),
+ "image_index": asset.get("image_index"),
+ "label": asset.get("label"),
+ }
+ )
+
+ result: Dict[str, Any] = {
+ **extraction_result,
+ "assets": promoted_assets,
+ }
+ if any(asset.get("status") != "ok" for asset in promoted_assets):
+ result["error"] = (
+ result.get("error")
+ or "Could not persist every extracted asset to a public URL."
+ )
+
+ summary_assets: List[Dict[str, Any]] = []
+ for asset in promoted_assets:
+ public_url = ensure_str(asset.get("public_url"))
+ summary_assets.append(
+ {
+ "description": summarize_text(
+ ensure_str(asset.get("description")), 120
+ ),
+ "status": ensure_str(asset.get("status")),
+ "public_url": public_url,
+ "asset_id": ensure_str(asset.get("asset_id")),
+ "content_type": ensure_str(asset.get("content_type")),
+ "image_part_index": asset.get("image_part_index"),
+ "image_display_name": asset.get("image_display_name"),
+ "box_2d": asset.get("box_2d"),
+ "image_index": asset.get("image_index"),
+ "label": ensure_str(asset.get("label")),
+ }
+ )
+
+ return ToolExecutionResult(
+ ok=not result.get("error"),
+ result=result,
+ summary={
+ "assets": summary_assets,
+ "error": result.get("error"),
+ },
+ multimodal_parts=multimodal_parts,
+ )
diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.py
index c415fec3..f17846b4 100644
--- a/backend/agent/tools/runtime.py
+++ b/backend/agent/tools/runtime.py
@@ -7,9 +7,9 @@ import os
from urllib.parse import unquote, urlparse
from typing import Any, Dict, List, Optional, Tuple, Union, cast
-from asset_extraction import extract_assets_from_images
from codegen.utils import extract_html_content
from config import LOCAL_ASSET_DIR, REPLICATE_API_KEY
+from agent.tools.extract_assets import run_extract_assets
from image_generation.generation import process_tasks
from image_generation.replicate import (
P_IMAGE_EDIT_ASPECT_RATIOS,
@@ -17,23 +17,14 @@ from image_generation.replicate import (
edit_image,
remove_background,
)
-from uploaded_assets.store import (
- persist_data_url_as_temporary_asset,
- promote_temporary_asset_id,
-)
from uploaded_assets.tools import run_save_assets
from agent.state import AgentFileState, ensure_str
-from agent.tools.types import ToolCall, ToolExecutionResult, ToolMultimodalPart
+from agent.tools.types import ToolCall, ToolExecutionResult
from agent.tools.summaries import summarize_text
LOCAL_ASSET_HOSTS = {"127.0.0.1", "localhost", "::1"}
-IMAGE_EXTENSION_BY_MIME_TYPE = {
- "image/jpeg": ".jpg",
- "image/png": ".png",
- "image/webp": ".webp",
-}
def _local_asset_url_to_data_url(image_url: str) -> str:
@@ -61,30 +52,6 @@ def _local_asset_url_to_data_url(image_url: str) -> str:
return f"data:{content_type};base64,{encoded}"
-def _image_data_url_to_multimodal_part(
- data_url: str,
- display_name: str,
-) -> ToolMultimodalPart | None:
- if not data_url.startswith("data:image/") or "," not in data_url:
- return None
-
- header, encoded = data_url.split(",", 1)
- mime_type = header.removeprefix("data:").split(";", 1)[0].lower()
- if mime_type not in {"image/png", "image/jpeg", "image/webp"}:
- return None
-
- try:
- data = base64.b64decode(encoded, validate=True)
- except ValueError:
- return None
-
- return ToolMultimodalPart(
- display_name=display_name,
- mime_type=mime_type,
- data=data,
- )
-
-
class AgentToolRuntime:
def __init__(
self,
@@ -131,7 +98,13 @@ class AgentToolRuntime:
if tool_call.name == "edit_image":
return await self._edit_image(tool_call.arguments)
if tool_call.name == "extract_assets":
- return await self._extract_assets(tool_call.arguments)
+ return await run_extract_assets(
+ tool_call.arguments,
+ gemini_api_key=self.gemini_api_key,
+ input_images=self.input_images,
+ asset_base_url=self.asset_base_url,
+ user_id=self.user_id,
+ )
if tool_call.name == "save_assets":
return await run_save_assets(tool_call.arguments, user_id=self.user_id)
if tool_call.name == "retrieve_option":
@@ -502,151 +475,6 @@ class AgentToolRuntime:
}
return ToolExecutionResult(ok=True, result=result, summary=summary)
- async def _extract_assets(self, args: Dict[str, Any]) -> ToolExecutionResult:
- if not self.gemini_api_key:
- return ToolExecutionResult(
- ok=False,
- result={"error": "Asset extraction requires GEMINI_API_KEY."},
- summary={"error": "Missing Gemini API key"},
- )
-
- if not self.input_images:
- return ToolExecutionResult(
- ok=False,
- result={"error": "No input images are available for asset extraction."},
- summary={"error": "No input images"},
- )
-
- raw_descriptions = args.get("asset_descriptions") or args.get("descriptions")
- if not isinstance(raw_descriptions, list) or not raw_descriptions:
- return ToolExecutionResult(
- ok=False,
- result={
- "error": "extract_assets requires a non-empty asset_descriptions list"
- },
- summary={"error": "Missing asset_descriptions"},
- )
-
- descriptions = [
- description.strip()
- for description in raw_descriptions
- if isinstance(description, str) and description.strip()
- ]
- if not descriptions:
- return ToolExecutionResult(
- ok=False,
- result={"error": "No valid asset descriptions provided"},
- summary={"error": "No valid asset_descriptions"},
- )
-
- result = await extract_assets_from_images(
- image_data_urls=self.input_images,
- asset_descriptions=descriptions,
- gemini_api_key=self.gemini_api_key,
- )
- assets = result.get("assets", [])
- promoted_assets: List[Dict[str, Any]] = []
- multimodal_parts: List[ToolMultimodalPart] = []
- if isinstance(assets, list):
- for index, asset in enumerate(assets):
- if not isinstance(asset, dict):
- continue
-
- data_url = ensure_str(asset.get("data_url"))
- public_url: str | None = None
- content_type: str | None = None
- asset_id: str | None = None
- image_part_index: int | None = None
- image_display_name: str | None = None
- status = ensure_str(asset.get("status")) or "missing"
-
- if data_url:
- mime_type = data_url.split(";", 1)[0].removeprefix("data:")
- extension = IMAGE_EXTENSION_BY_MIME_TYPE.get(mime_type, ".png")
- display_name = f"asset_{index}{extension}"
- multimodal_part = _image_data_url_to_multimodal_part(
- data_url,
- display_name,
- )
- if multimodal_part:
- multimodal_parts.append(multimodal_part)
- image_part_index = len(multimodal_parts) - 1
- image_display_name = display_name
-
- temporary_asset = persist_data_url_as_temporary_asset(
- data_url,
- self.asset_base_url,
- )
- if temporary_asset:
- saved_asset = await promote_temporary_asset_id(
- temporary_asset.asset_id,
- user_id=self.user_id,
- )
- if saved_asset:
- public_url = saved_asset.public_url
- content_type = saved_asset.content_type
- asset_id = saved_asset.asset_id
- status = "ok"
- else:
- status = "error"
- else:
- status = "error"
-
- promoted_assets.append(
- {
- "description": asset.get("description"),
- "public_url": public_url,
- "asset_id": asset_id,
- "content_type": content_type,
- "status": status,
- "image_part_index": image_part_index,
- "image_display_name": image_display_name,
- "box_2d": asset.get("box_2d"),
- "image_index": asset.get("image_index"),
- "label": asset.get("label"),
- }
- )
-
- result = {
- **result,
- "assets": promoted_assets,
- }
- if any(asset.get("status") != "ok" for asset in promoted_assets):
- result["error"] = (
- result.get("error")
- or "Could not persist every extracted asset to a public URL."
- )
-
- summary_assets = []
- for asset in promoted_assets:
- public_url = ensure_str(asset.get("public_url"))
- summary_assets.append(
- {
- "description": summarize_text(
- ensure_str(asset.get("description")), 120
- ),
- "status": ensure_str(asset.get("status")),
- "public_url": public_url,
- "asset_id": ensure_str(asset.get("asset_id")),
- "content_type": ensure_str(asset.get("content_type")),
- "image_part_index": asset.get("image_part_index"),
- "image_display_name": asset.get("image_display_name"),
- "box_2d": asset.get("box_2d"),
- "image_index": asset.get("image_index"),
- "label": ensure_str(asset.get("label")),
- }
- )
-
- return ToolExecutionResult(
- ok=not result.get("error"),
- result=result,
- summary={
- "assets": summary_assets,
- "error": result.get("error"),
- },
- multimodal_parts=multimodal_parts,
- )
-
def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:
raw_option_number = args.get("option_number")
raw_index = args.get("index")
diff --git a/backend/tests/test_agent_tool_runtime.py b/backend/tests/test_agent_tool_runtime.py
index 0281772f..85328ff2 100644
--- a/backend/tests/test_agent_tool_runtime.py
+++ b/backend/tests/test_agent_tool_runtime.py
@@ -171,7 +171,7 @@ async def test_extract_assets_returns_mocked_gemini_assets(
}
monkeypatch.setattr(
- "agent.tools.runtime.extract_assets_from_images",
+ "agent.tools.extract_assets.extract_assets_from_images",
fake_extract_assets_from_images,
)
@@ -214,4 +214,3 @@ async def test_extract_assets_returns_mocked_gemini_assets(
]
assert result.multimodal_parts[0].mime_type == "image/png"
assert result.multimodal_parts[0].data == b"logo-image"
-
diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.py
index b17985b4..9c665507 100644
--- a/backend/tests/test_agent_tools.py
+++ b/backend/tests/test_agent_tools.py
@@ -1,7 +1,11 @@
+from typing import Any, cast
+
from agent.state import AgentFileState
+from agent.providers.factory import create_provider_session
from agent.tools import canonical_tool_definitions, summarize_tool_input
from agent.tools.types import ToolCall
from image_generation.replicate import P_IMAGE_EDIT_ASPECT_RATIOS
+from llm import Llm
def test_canonical_tool_definitions_include_generate_images_when_enabled() -> None:
@@ -49,7 +53,7 @@ def test_save_assets_tool_input_summary_uses_asset_ids() -> None:
def test_canonical_tool_definitions_include_extract_assets() -> None:
- tools = canonical_tool_definitions(True)
+ tools = canonical_tool_definitions(True, asset_extraction_enabled=True)
tool_names = [tool.name for tool in tools]
assert "extract_assets" in tool_names
extract_assets_tool = next(tool for tool in tools if tool.name == "extract_assets")
@@ -60,6 +64,46 @@ def test_canonical_tool_definitions_include_extract_assets() -> None:
)
+def test_canonical_tool_definitions_exclude_extract_assets_when_disabled() -> None:
+ tool_names = [
+ tool.name
+ for tool in canonical_tool_definitions(True, asset_extraction_enabled=False)
+ ]
+ assert "extract_assets" not in tool_names
+
+
+def test_provider_session_excludes_extract_assets_without_gemini_key() -> None:
+ session = create_provider_session(
+ model=Llm.GPT_5_2_CODEX_HIGH,
+ prompt_messages=[{"role": "user", "content": "Build a page."}],
+ should_generate_images=True,
+ openai_api_key="openai-key",
+ openai_base_url=None,
+ anthropic_api_key=None,
+ gemini_api_key=None,
+ )
+
+ tools = cast(list[dict[str, Any]], getattr(session, "_tools"))
+ tool_names = [tool["name"] for tool in tools]
+ assert "extract_assets" not in tool_names
+
+
+def test_provider_session_includes_extract_assets_with_gemini_key() -> None:
+ session = create_provider_session(
+ model=Llm.GPT_5_2_CODEX_HIGH,
+ prompt_messages=[{"role": "user", "content": "Build a page."}],
+ should_generate_images=True,
+ openai_api_key="openai-key",
+ openai_base_url=None,
+ anthropic_api_key=None,
+ gemini_api_key="gemini-key",
+ )
+
+ tools = cast(list[dict[str, Any]], getattr(session, "_tools"))
+ tool_names = [tool["name"] for tool in tools]
+ assert "extract_assets" in tool_names
+
+
def test_extract_assets_tool_input_summary_uses_asset_descriptions() -> None:
summary = summarize_tool_input(
ToolCall(