Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Prevent asset extraction for video-only prompts

Flightdeck Agentf2960cd157f7
diff --git a/backend/agent/engine.py b/backend/agent/engine.py
index a809639a..9ce6e986 100644
--- a/backend/agent/engine.py
+++ b/backend/agent/engine.py
@@ -81,7 +81,15 @@ class AgentEngine:
if not isinstance(image_url, dict):
continue
url = cast(object, image_url.get("url"))
- if isinstance(url, str) and url:
+ # Video parts use the OpenAI-compatible `image_url` shape too,
+ # but extract_assets can only crop still-image data URLs. Keep
+ # non-image media out of the tool runtime so video-only prompts
+ # do not expose a tool that is guaranteed to fail.
+ if (
+ isinstance(url, str)
+ and url.startswith("data:image/")
+ and "," in url
+ ):
images.append(url)
return images
@@ -266,7 +274,13 @@ class AgentEngine:
anthropic_api_key=self.anthropic_api_key,
gemini_api_key=self.gemini_api_key,
replicate_api_key=self.replicate_api_key,
- should_extract_assets=self.should_extract_assets,
+ # Only advertise extraction when the request actually contains a
+ # still image the runtime can crop. In particular, Gemini videos
+ # share the image_url message shape but are not valid extractor
+ # inputs.
+ should_extract_assets=(
+ self.should_extract_assets and bool(self.tool_runtime.input_images)
+ ),
)
try:
return await self._run_with_session(session)
diff --git a/backend/tests/test_agent_engine.py b/backend/tests/test_agent_engine.py
new file mode 100644
index 00000000..7275201f
--- /dev/null
+++ b/backend/tests/test_agent_engine.py
@@ -0,0 +1,109 @@
+from typing import Any, cast
+
+import pytest
+from openai.types.chat import ChatCompletionMessageParam
+
+from agent.engine import AgentEngine
+from agent.providers.base import EventSink, ExecutedToolCall, ProviderTurn
+from llm import Llm
+
+
+class NoToolProviderSession:
+ def __init__(self) -> None:
+ self.closed = False
+
+ async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
+ return ProviderTurn(assistant_text="", tool_calls=[])
+
+ async def append_tool_results(
+ self,
+ turn: ProviderTurn,
+ executed_tool_calls: list[ExecutedToolCall],
+ ) -> None:
+ raise AssertionError("A no-tool turn must not append tool results")
+
+ async def close(self) -> None:
+ self.closed = True
+
+
+def _prompt_with_media(media_url: str) -> list[ChatCompletionMessageParam]:
+ return cast(
+ list[ChatCompletionMessageParam],
+ [
+ {"role": "system", "content": "System prompt"},
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {"url": media_url, "detail": "high"},
+ },
+ {"type": "text", "text": "Build this interface."},
+ ],
+ },
+ ],
+ )
+
+
+ ("media_url", "expected_images", "expected_extraction_enabled"),
+ [
+ (
+ "data:image/png;base64,aW1hZ2U=",
+ ["data:image/png;base64,aW1hZ2U="],
+ True,
+ ),
+ ("data:video/mp4;base64,dmlkZW8=", [], False),
+ ("data:video/webm;base64,dmlkZW8=", [], False),
+ ],
+)
+async def test_asset_extraction_is_only_offered_for_still_image_inputs(
+ monkeypatch: pytest.MonkeyPatch,
+ media_url: str,
+ expected_images: list[str],
+ expected_extraction_enabled: bool,
+) -> None:
+ factory_arguments: dict[str, Any] = {}
+ session = NoToolProviderSession()
+
+ def fake_create_provider_session(**kwargs: Any) -> NoToolProviderSession:
+ factory_arguments.update(kwargs)
+ return session
+
+ monkeypatch.setattr(
+ "agent.engine.create_provider_session", fake_create_provider_session
+ )
+
+ async def send_message(
+ message_type: str,
+ value: str | None,
+ variant_index: int,
+ data: dict[str, Any] | None,
+ event_id: str | None,
+ ) -> None:
+ return None
+
+ engine = AgentEngine(
+ send_message=send_message,
+ variant_index=0,
+ openai_api_key=None,
+ openai_base_url=None,
+ anthropic_api_key=None,
+ gemini_api_key="gemini-key",
+ replicate_api_key=None,
+ should_generate_images=True,
+ should_extract_assets=True,
+ )
+
+ await engine.run(
+ Llm.GEMINI_3_FLASH_PREVIEW_MINIMAL,
+ _prompt_with_media(media_url),
+ )
+
+ assert engine.tool_runtime.input_images == expected_images
+ assert (
+ factory_arguments["should_extract_assets"]
+ is expected_extraction_enabled
+ )
+ assert session.closed is True
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index af45af18..1cbab73a 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -616,7 +616,11 @@ function App() {
images: inputMode === "image" ? media : [],
videos: inputMode === "video" ? media : [],
},
- isAssetExtractionEnabled,
+ // Asset extraction operates on still screenshots. Video data uses the
+ // same transport shape for Gemini, so explicitly disable extraction
+ // instead of letting the agent try to crop a video payload.
+ isAssetExtractionEnabled:
+ inputMode === "image" && isAssetExtractionEnabled,
variantHistory,
});
}