Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Make extract_assets degrade gracefully on provider + partial failures

Abi Rajad34be1159889
diff --git a/backend/agent/tools/extract_assets.py b/backend/agent/tools/extract_assets.py
index d6be5e4e..ff3e8b72 100644
--- a/backend/agent/tools/extract_assets.py
+++ b/backend/agent/tools/extract_assets.py
@@ -85,11 +85,23 @@ async def run_extract_assets(
summary={"error": "No valid asset_descriptions"},
)
- extraction_result: Dict[str, Any] = await extract_assets_from_images(
- image_data_urls=input_images,
- asset_descriptions=descriptions,
- gemini_api_key=gemini_api_key,
- )
+ try:
+ extraction_result: Dict[str, Any] = await extract_assets_from_images(
+ image_data_urls=input_images,
+ asset_descriptions=descriptions,
+ gemini_api_key=gemini_api_key,
+ )
+ except Exception as exc:
+ # A provider failure (quota, auth, timeout, transient network) must not
+ # abort the whole variant. Like the other external image tools, report it
+ # as a tool error so the agent can continue or fall back (e.g. redraw with
+ # generate_images) instead of propagating out of the tool runtime.
+ print(f"Asset extraction failed: {exc}")
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": f"Asset extraction failed: {exc}"},
+ summary={"error": "Asset extraction failed"},
+ )
assets = extraction_result.get("assets", [])
promoted_assets: List[Dict[str, Any]] = []
multimodal_parts: List[ToolMultimodalPart] = []
@@ -151,14 +163,34 @@ async def run_extract_assets(
}
)
+ # extraction_result carries a top-level "error" whenever Gemini couldn't box
+ # every requested asset. Don't let that fail the whole tool when we still got
+ # usable crops — re-derive tool status from what actually came back.
+ extraction_note = ensure_str(extraction_result.get("error")) or None
result: Dict[str, Any] = {
**extraction_result,
"assets": promoted_assets,
}
- if any(asset.get("status") != "ok" for asset in promoted_assets):
+ result.pop("error", None)
+
+ extracted = [asset for asset in promoted_assets if asset.get("status") == "ok"]
+ unresolved = [asset for asset in promoted_assets if asset.get("status") != "ok"]
+
+ # Partial success is still success: keep the crops Gemini did find (and their
+ # images) and tell the model which descriptions came back empty, so it can
+ # fall back for just those (e.g. generate_images) instead of discarding the
+ # lot. Only fail the tool when nothing could be extracted at all.
+ if unresolved:
+ result["unresolved_assets"] = [
+ {
+ "description": asset.get("description"),
+ "status": asset.get("status"),
+ }
+ for asset in unresolved
+ ]
+ if not extracted:
result["error"] = (
- result.get("error")
- or "Could not persist every extracted asset to a public URL."
+ extraction_note or "No assets could be extracted from the input images."
)
summary_assets: List[Dict[str, Any]] = []
@@ -181,10 +213,11 @@ async def run_extract_assets(
)
return ToolExecutionResult(
- ok=not result.get("error"),
+ ok=bool(extracted),
result=result,
summary={
"assets": summary_assets,
+ "unresolved_count": len(unresolved),
"error": result.get("error"),
},
multimodal_parts=multimodal_parts,
diff --git a/backend/tests/test_agent_tool_runtime.py b/backend/tests/test_agent_tool_runtime.py
index 32cf51a9..9e266248 100644
--- a/backend/tests/test_agent_tool_runtime.py
+++ b/backend/tests/test_agent_tool_runtime.py
@@ -135,6 +135,38 @@ async def test_extract_assets_requires_gemini_api_key() -> None:
assert result.summary["error"] == "Missing Gemini API key"
+async def test_extract_assets_returns_error_when_extraction_raises(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def boom(**_kwargs: Any) -> dict[str, object]:
+ raise RuntimeError("429 quota exceeded")
+
+ monkeypatch.setattr("agent.tools.extract_assets.extract_assets_from_images", boom)
+ runtime = AgentToolRuntime(
+ file_state=AgentFileState(),
+ should_generate_images=False,
+ openai_api_key=None,
+ openai_base_url=None,
+ gemini_api_key="gemini-key",
+ input_images=[_data_url(b"source-image")],
+ )
+
+ # A provider failure must surface as a tool error, not propagate out of the
+ # runtime and abort the whole variant.
+ result = await runtime.execute(
+ ToolCall(
+ id="call-1",
+ name="extract_assets",
+ arguments={"asset_descriptions": ["logo"]},
+ )
+ )
+
+ assert result.ok is False
+ assert result.summary["error"] == "Asset extraction failed"
+ assert "429 quota exceeded" in result.result["error"]
+
+
@pytest.mark.asyncio
async def test_extract_assets_returns_mocked_gemini_assets(
monkeypatch: pytest.MonkeyPatch,
@@ -224,6 +256,106 @@ async def test_extract_assets_returns_mocked_gemini_assets(
assert result.multimodal_parts[0].data == b"logo-image"
+async def test_extract_assets_partial_success_keeps_found_assets(
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+) -> None:
+ temp_dir = tmp_path / "tmp-assets"
+ asset_dir = tmp_path / "local-assets"
+ monkeypatch.setattr("uploaded_assets.store.TEMP_ASSET_DIR", str(temp_dir))
+ monkeypatch.setattr("uploaded_assets.store.LOCAL_ASSET_DIR", str(asset_dir))
+
+ async def fake_extract(**_kwargs: Any) -> dict[str, object]:
+ # One asset boxed + cropped, one with no bounding box — exactly what
+ # asset_extraction returns (incl. its top-level error) when Gemini finds
+ # some but not all of the requested assets.
+ return {
+ "assets": [
+ {"description": "logo", "data_url": _data_url(b"logo-bytes"), "status": "ok"},
+ {"description": "mascot", "data_url": None, "status": "missing"},
+ ],
+ "error": "Gemini did not return usable bounding boxes for every requested asset.",
+ }
+
+ monkeypatch.setattr(
+ "agent.tools.extract_assets.extract_assets_from_images", fake_extract
+ )
+ runtime = AgentToolRuntime(
+ file_state=AgentFileState(),
+ should_generate_images=False,
+ openai_api_key=None,
+ openai_base_url=None,
+ gemini_api_key="gemini-key",
+ input_images=[_data_url(b"source-image")],
+ asset_base_url="http://127.0.0.1:7001",
+ )
+
+ result = await runtime.execute(
+ ToolCall(
+ id="call-1",
+ name="extract_assets",
+ arguments={"asset_descriptions": ["logo", "mascot"]},
+ )
+ )
+
+ # One missing asset must NOT fail the whole tool — the found crop is kept.
+ assert result.ok is True
+ assert "error" not in result.result
+ assert result.summary["error"] is None
+ found = result.result["assets"][0]
+ assert found["status"] == "ok"
+ assert found["public_url"].startswith("http://127.0.0.1:7001/local-assets/")
+ assert result.result["assets"][1]["status"] == "missing"
+ # The model is told which descriptions came back empty so it can fall back.
+ assert result.result["unresolved_assets"] == [
+ {"description": "mascot", "status": "missing"}
+ ]
+ assert result.summary["unresolved_count"] == 1
+ # Only the found crop's image is attached.
+ assert result.multimodal_parts is not None
+ assert [p.display_name for p in result.multimodal_parts] == ["asset_0.png"]
+ assert result.multimodal_parts[0].data == b"logo-bytes"
+
+
+async def test_extract_assets_all_missing_returns_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ async def fake_extract(**_kwargs: Any) -> dict[str, object]:
+ return {
+ "assets": [
+ {"description": "logo", "data_url": None, "status": "missing"},
+ ],
+ "error": "Gemini did not return usable bounding boxes for every requested asset.",
+ }
+
+ monkeypatch.setattr(
+ "agent.tools.extract_assets.extract_assets_from_images", fake_extract
+ )
+ runtime = AgentToolRuntime(
+ file_state=AgentFileState(),
+ should_generate_images=False,
+ openai_api_key=None,
+ openai_base_url=None,
+ gemini_api_key="gemini-key",
+ input_images=[_data_url(b"source-image")],
+ )
+
+ result = await runtime.execute(
+ ToolCall(
+ id="call-1",
+ name="extract_assets",
+ arguments={"asset_descriptions": ["logo"]},
+ )
+ )
+
+ # Nothing extracted at all → the tool legitimately fails.
+ assert result.ok is False
+ assert "bounding boxes" in result.result["error"]
+ assert not result.multimodal_parts
+
+
@pytest.mark.asyncio
async def test_screenshot_preview_requires_file_content() -> None:
runtime = AgentToolRuntime(