Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Gate and split extract assets tool
Abi Raja86bb6b06c3e4
diff --git a/backend/agent/providers/factory.py b/backend/agent/providers/factory.pyindex 48f326da..fda2ad32 100644--- a/backend/agent/providers/factory.py+++ b/backend/agent/providers/factory.py@@ -27,6 +27,8 @@ def create_provider_session(image_generation_enabled=should_generate_images,# The edit_image tool calls Replicate, so don't offer it without a key.image_editing_enabled=bool(REPLICATE_API_KEY),+ # The extract_assets tool calls Gemini, so don't offer it without a key.+ asset_extraction_enabled=bool(gemini_api_key),)if model in OPENAI_MODELS:diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.pyindex fccf2c6b..17b37d8c 100644--- a/backend/agent/tools/definitions.py+++ b/backend/agent/tools/definitions.py@@ -161,6 +161,7 @@ def _retrieve_option_schema() -> Dict[str, Any]:def canonical_tool_definitions(image_generation_enabled: bool = True,image_editing_enabled: bool = True,+ asset_extraction_enabled: bool = True,) -> List[CanonicalToolDefinition]:tools: List[CanonicalToolDefinition] = [CanonicalToolDefinition(@@ -217,8 +218,8 @@ def canonical_tool_definitions(parameters=_edit_image_schema(),))- tools.extend(- [+ if asset_extraction_enabled:+ tools.append(CanonicalToolDefinition(name="extract_assets",description=(@@ -228,7 +229,10 @@ def canonical_tool_definitions("the same order."),parameters=_extract_assets_schema(),- ),+ )+ )+ tools.extend(+ [SAVE_ASSETS_TOOL_DEFINITION,CanonicalToolDefinition(name="retrieve_option",diff --git a/backend/agent/tools/extract_assets.py b/backend/agent/tools/extract_assets.pynew file mode 100644index 00000000..3ba66b3e--- /dev/null+++ b/backend/agent/tools/extract_assets.py@@ -0,0 +1,198 @@+import base64+from typing import Any, Dict, List, Optional, cast++from asset_extraction import extract_assets_from_images+from uploaded_assets.store import (+ persist_data_url_as_temporary_asset,+ promote_temporary_asset_id,+)++from agent.state import ensure_str+from agent.tools.summaries import summarize_text+from agent.tools.types import ToolExecutionResult, ToolMultimodalPart+++IMAGE_EXTENSION_BY_MIME_TYPE = {+ "image/jpeg": ".jpg",+ "image/png": ".png",+ "image/webp": ".webp",+}+++def _image_data_url_to_multimodal_part(+ data_url: str,+ display_name: str,+) -> ToolMultimodalPart | None:+ if not data_url.startswith("data:image/") or "," not in data_url:+ return None++ header, encoded = data_url.split(",", 1)+ mime_type = header.removeprefix("data:").split(";", 1)[0].lower()+ if mime_type not in {"image/png", "image/jpeg", "image/webp"}:+ return None++ try:+ data = base64.b64decode(encoded, validate=True)+ except ValueError:+ return None++ return ToolMultimodalPart(+ display_name=display_name,+ mime_type=mime_type,+ data=data,+ )+++async def run_extract_assets(+ args: Dict[str, Any],+ *,+ gemini_api_key: Optional[str],+ input_images: List[str],+ asset_base_url: str,+ user_id: Optional[str],+) -> ToolExecutionResult:+ if not gemini_api_key:+ return ToolExecutionResult(+ ok=False,+ result={"error": "Asset extraction requires GEMINI_API_KEY."},+ summary={"error": "Missing Gemini API key"},+ )++ if not input_images:+ return ToolExecutionResult(+ ok=False,+ result={"error": "No input images are available for asset extraction."},+ summary={"error": "No input images"},+ )++ raw_descriptions = args.get("asset_descriptions") or args.get("descriptions")+ if not isinstance(raw_descriptions, list) or not raw_descriptions:+ return ToolExecutionResult(+ ok=False,+ result={+ "error": "extract_assets requires a non-empty asset_descriptions list"+ },+ summary={"error": "Missing asset_descriptions"},+ )++ raw_description_items = cast(list[object], raw_descriptions)+ descriptions: List[str] = [+ description.strip()+ for description in raw_description_items+ if isinstance(description, str) and description.strip()+ ]+ if not descriptions:+ return ToolExecutionResult(+ ok=False,+ result={"error": "No valid asset descriptions provided"},+ summary={"error": "No valid asset_descriptions"},+ )++ extraction_result: Dict[str, Any] = await extract_assets_from_images(+ image_data_urls=input_images,+ asset_descriptions=descriptions,+ gemini_api_key=gemini_api_key,+ )+ assets = extraction_result.get("assets", [])+ promoted_assets: List[Dict[str, Any]] = []+ multimodal_parts: List[ToolMultimodalPart] = []+ if isinstance(assets, list):+ for index, raw_asset in enumerate(cast(list[object], assets)):+ if not isinstance(raw_asset, dict):+ continue+ asset = cast(Dict[str, Any], raw_asset)++ data_url = ensure_str(asset.get("data_url"))+ public_url: str | None = None+ content_type: str | None = None+ asset_id: str | None = None+ image_part_index: int | None = None+ image_display_name: str | None = None+ status = ensure_str(asset.get("status")) or "missing"++ if data_url:+ mime_type = data_url.split(";", 1)[0].removeprefix("data:")+ extension = IMAGE_EXTENSION_BY_MIME_TYPE.get(mime_type, ".png")+ display_name = f"asset_{index}{extension}"+ multimodal_part = _image_data_url_to_multimodal_part(+ data_url,+ display_name,+ )+ if multimodal_part:+ multimodal_parts.append(multimodal_part)+ image_part_index = len(multimodal_parts) - 1+ image_display_name = display_name++ temporary_asset = persist_data_url_as_temporary_asset(+ data_url,+ asset_base_url,+ )+ if temporary_asset:+ saved_asset = await promote_temporary_asset_id(+ temporary_asset.asset_id,+ user_id=user_id,+ )+ if saved_asset:+ public_url = saved_asset.public_url+ content_type = saved_asset.content_type+ asset_id = saved_asset.asset_id+ status = "ok"+ else:+ status = "error"+ else:+ status = "error"++ promoted_assets.append(+ {+ "description": asset.get("description"),+ "public_url": public_url,+ "asset_id": asset_id,+ "content_type": content_type,+ "status": status,+ "image_part_index": image_part_index,+ "image_display_name": image_display_name,+ "box_2d": asset.get("box_2d"),+ "image_index": asset.get("image_index"),+ "label": asset.get("label"),+ }+ )++ result: Dict[str, Any] = {+ **extraction_result,+ "assets": promoted_assets,+ }+ if any(asset.get("status") != "ok" for asset in promoted_assets):+ result["error"] = (+ result.get("error")+ or "Could not persist every extracted asset to a public URL."+ )++ summary_assets: List[Dict[str, Any]] = []+ for asset in promoted_assets:+ public_url = ensure_str(asset.get("public_url"))+ summary_assets.append(+ {+ "description": summarize_text(+ ensure_str(asset.get("description")), 120+ ),+ "status": ensure_str(asset.get("status")),+ "public_url": public_url,+ "asset_id": ensure_str(asset.get("asset_id")),+ "content_type": ensure_str(asset.get("content_type")),+ "image_part_index": asset.get("image_part_index"),+ "image_display_name": asset.get("image_display_name"),+ "box_2d": asset.get("box_2d"),+ "image_index": asset.get("image_index"),+ "label": ensure_str(asset.get("label")),+ }+ )++ return ToolExecutionResult(+ ok=not result.get("error"),+ result=result,+ summary={+ "assets": summary_assets,+ "error": result.get("error"),+ },+ multimodal_parts=multimodal_parts,+ )diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.pyindex c415fec3..f17846b4 100644--- a/backend/agent/tools/runtime.py+++ b/backend/agent/tools/runtime.py@@ -7,9 +7,9 @@ import osfrom urllib.parse import unquote, urlparsefrom typing import Any, Dict, List, Optional, Tuple, Union, cast-from asset_extraction import extract_assets_from_imagesfrom codegen.utils import extract_html_contentfrom config import LOCAL_ASSET_DIR, REPLICATE_API_KEY+from agent.tools.extract_assets import run_extract_assetsfrom image_generation.generation import process_tasksfrom image_generation.replicate import (P_IMAGE_EDIT_ASPECT_RATIOS,@@ -17,23 +17,14 @@ from image_generation.replicate import (edit_image,remove_background,)-from uploaded_assets.store import (- persist_data_url_as_temporary_asset,- promote_temporary_asset_id,-)from uploaded_assets.tools import run_save_assetsfrom agent.state import AgentFileState, ensure_str-from agent.tools.types import ToolCall, ToolExecutionResult, ToolMultimodalPart+from agent.tools.types import ToolCall, ToolExecutionResultfrom agent.tools.summaries import summarize_textLOCAL_ASSET_HOSTS = {"127.0.0.1", "localhost", "::1"}-IMAGE_EXTENSION_BY_MIME_TYPE = {- "image/jpeg": ".jpg",- "image/png": ".png",- "image/webp": ".webp",-}def _local_asset_url_to_data_url(image_url: str) -> str:@@ -61,30 +52,6 @@ def _local_asset_url_to_data_url(image_url: str) -> str:return f"data:{content_type};base64,{encoded}"-def _image_data_url_to_multimodal_part(- data_url: str,- display_name: str,-) -> ToolMultimodalPart | None:- if not data_url.startswith("data:image/") or "," not in data_url:- return None-- header, encoded = data_url.split(",", 1)- mime_type = header.removeprefix("data:").split(";", 1)[0].lower()- if mime_type not in {"image/png", "image/jpeg", "image/webp"}:- return None-- try:- data = base64.b64decode(encoded, validate=True)- except ValueError:- return None-- return ToolMultimodalPart(- display_name=display_name,- mime_type=mime_type,- data=data,- )--class AgentToolRuntime:def __init__(self,@@ -131,7 +98,13 @@ class AgentToolRuntime:if tool_call.name == "edit_image":return await self._edit_image(tool_call.arguments)if tool_call.name == "extract_assets":- return await self._extract_assets(tool_call.arguments)+ return await run_extract_assets(+ tool_call.arguments,+ gemini_api_key=self.gemini_api_key,+ input_images=self.input_images,+ asset_base_url=self.asset_base_url,+ user_id=self.user_id,+ )if tool_call.name == "save_assets":return await run_save_assets(tool_call.arguments, user_id=self.user_id)if tool_call.name == "retrieve_option":@@ -502,151 +475,6 @@ class AgentToolRuntime:}return ToolExecutionResult(ok=True, result=result, summary=summary)- async def _extract_assets(self, args: Dict[str, Any]) -> ToolExecutionResult:- if not self.gemini_api_key:- return ToolExecutionResult(- ok=False,- result={"error": "Asset extraction requires GEMINI_API_KEY."},- summary={"error": "Missing Gemini API key"},- )-- if not self.input_images:- return ToolExecutionResult(- ok=False,- result={"error": "No input images are available for asset extraction."},- summary={"error": "No input images"},- )-- raw_descriptions = args.get("asset_descriptions") or args.get("descriptions")- if not isinstance(raw_descriptions, list) or not raw_descriptions:- return ToolExecutionResult(- ok=False,- result={- "error": "extract_assets requires a non-empty asset_descriptions list"- },- summary={"error": "Missing asset_descriptions"},- )-- descriptions = [- description.strip()- for description in raw_descriptions- if isinstance(description, str) and description.strip()- ]- if not descriptions:- return ToolExecutionResult(- ok=False,- result={"error": "No valid asset descriptions provided"},- summary={"error": "No valid asset_descriptions"},- )-- result = await extract_assets_from_images(- image_data_urls=self.input_images,- asset_descriptions=descriptions,- gemini_api_key=self.gemini_api_key,- )- assets = result.get("assets", [])- promoted_assets: List[Dict[str, Any]] = []- multimodal_parts: List[ToolMultimodalPart] = []- if isinstance(assets, list):- for index, asset in enumerate(assets):- if not isinstance(asset, dict):- continue-- data_url = ensure_str(asset.get("data_url"))- public_url: str | None = None- content_type: str | None = None- asset_id: str | None = None- image_part_index: int | None = None- image_display_name: str | None = None- status = ensure_str(asset.get("status")) or "missing"-- if data_url:- mime_type = data_url.split(";", 1)[0].removeprefix("data:")- extension = IMAGE_EXTENSION_BY_MIME_TYPE.get(mime_type, ".png")- display_name = f"asset_{index}{extension}"- multimodal_part = _image_data_url_to_multimodal_part(- data_url,- display_name,- )- if multimodal_part:- multimodal_parts.append(multimodal_part)- image_part_index = len(multimodal_parts) - 1- image_display_name = display_name-- temporary_asset = persist_data_url_as_temporary_asset(- data_url,- self.asset_base_url,- )- if temporary_asset:- saved_asset = await promote_temporary_asset_id(- temporary_asset.asset_id,- user_id=self.user_id,- )- if saved_asset:- public_url = saved_asset.public_url- content_type = saved_asset.content_type- asset_id = saved_asset.asset_id- status = "ok"- else:- status = "error"- else:- status = "error"-- promoted_assets.append(- {- "description": asset.get("description"),- "public_url": public_url,- "asset_id": asset_id,- "content_type": content_type,- "status": status,- "image_part_index": image_part_index,- "image_display_name": image_display_name,- "box_2d": asset.get("box_2d"),- "image_index": asset.get("image_index"),- "label": asset.get("label"),- }- )-- result = {- **result,- "assets": promoted_assets,- }- if any(asset.get("status") != "ok" for asset in promoted_assets):- result["error"] = (- result.get("error")- or "Could not persist every extracted asset to a public URL."- )-- summary_assets = []- for asset in promoted_assets:- public_url = ensure_str(asset.get("public_url"))- summary_assets.append(- {- "description": summarize_text(- ensure_str(asset.get("description")), 120- ),- "status": ensure_str(asset.get("status")),- "public_url": public_url,- "asset_id": ensure_str(asset.get("asset_id")),- "content_type": ensure_str(asset.get("content_type")),- "image_part_index": asset.get("image_part_index"),- "image_display_name": asset.get("image_display_name"),- "box_2d": asset.get("box_2d"),- "image_index": asset.get("image_index"),- "label": ensure_str(asset.get("label")),- }- )-- return ToolExecutionResult(- ok=not result.get("error"),- result=result,- summary={- "assets": summary_assets,- "error": result.get("error"),- },- multimodal_parts=multimodal_parts,- )-def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:raw_option_number = args.get("option_number")raw_index = args.get("index")diff --git a/backend/tests/test_agent_tool_runtime.py b/backend/tests/test_agent_tool_runtime.pyindex 0281772f..85328ff2 100644--- a/backend/tests/test_agent_tool_runtime.py+++ b/backend/tests/test_agent_tool_runtime.py@@ -171,7 +171,7 @@ async def test_extract_assets_returns_mocked_gemini_assets(}monkeypatch.setattr(- "agent.tools.runtime.extract_assets_from_images",+ "agent.tools.extract_assets.extract_assets_from_images",fake_extract_assets_from_images,)@@ -214,4 +214,3 @@ async def test_extract_assets_returns_mocked_gemini_assets(]assert result.multimodal_parts[0].mime_type == "image/png"assert result.multimodal_parts[0].data == b"logo-image"-diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.pyindex b17985b4..9c665507 100644--- a/backend/tests/test_agent_tools.py+++ b/backend/tests/test_agent_tools.py@@ -1,7 +1,11 @@+from typing import Any, cast+from agent.state import AgentFileState+from agent.providers.factory import create_provider_sessionfrom agent.tools import canonical_tool_definitions, summarize_tool_inputfrom agent.tools.types import ToolCallfrom image_generation.replicate import P_IMAGE_EDIT_ASPECT_RATIOS+from llm import Llmdef test_canonical_tool_definitions_include_generate_images_when_enabled() -> None:@@ -49,7 +53,7 @@ def test_save_assets_tool_input_summary_uses_asset_ids() -> None:def test_canonical_tool_definitions_include_extract_assets() -> None:- tools = canonical_tool_definitions(True)+ tools = canonical_tool_definitions(True, asset_extraction_enabled=True)tool_names = [tool.name for tool in tools]assert "extract_assets" in tool_namesextract_assets_tool = next(tool for tool in tools if tool.name == "extract_assets")@@ -60,6 +64,46 @@ def test_canonical_tool_definitions_include_extract_assets() -> None:)+def test_canonical_tool_definitions_exclude_extract_assets_when_disabled() -> None:+ tool_names = [+ tool.name+ for tool in canonical_tool_definitions(True, asset_extraction_enabled=False)+ ]+ assert "extract_assets" not in tool_names+++def test_provider_session_excludes_extract_assets_without_gemini_key() -> None:+ session = create_provider_session(+ model=Llm.GPT_5_2_CODEX_HIGH,+ prompt_messages=[{"role": "user", "content": "Build a page."}],+ should_generate_images=True,+ openai_api_key="openai-key",+ openai_base_url=None,+ anthropic_api_key=None,+ gemini_api_key=None,+ )++ tools = cast(list[dict[str, Any]], getattr(session, "_tools"))+ tool_names = [tool["name"] for tool in tools]+ assert "extract_assets" not in tool_names+++def test_provider_session_includes_extract_assets_with_gemini_key() -> None:+ session = create_provider_session(+ model=Llm.GPT_5_2_CODEX_HIGH,+ prompt_messages=[{"role": "user", "content": "Build a page."}],+ should_generate_images=True,+ openai_api_key="openai-key",+ openai_base_url=None,+ anthropic_api_key=None,+ gemini_api_key="gemini-key",+ )++ tools = cast(list[dict[str, Any]], getattr(session, "_tools"))+ tool_names = [tool["name"] for tool in tools]+ assert "extract_assets" in tool_names++def test_extract_assets_tool_input_summary_uses_asset_descriptions() -> None:summary = summarize_tool_input(ToolCall(