Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Prevent asset extraction for video-only prompts
Flightdeck Agentf2960cd157f7
diff --git a/backend/agent/engine.py b/backend/agent/engine.pyindex a809639a..9ce6e986 100644--- a/backend/agent/engine.py+++ b/backend/agent/engine.py@@ -81,7 +81,15 @@ class AgentEngine:if not isinstance(image_url, dict):continueurl = cast(object, image_url.get("url"))- if isinstance(url, str) and url:+ # Video parts use the OpenAI-compatible `image_url` shape too,+ # but extract_assets can only crop still-image data URLs. Keep+ # non-image media out of the tool runtime so video-only prompts+ # do not expose a tool that is guaranteed to fail.+ if (+ isinstance(url, str)+ and url.startswith("data:image/")+ and "," in url+ ):images.append(url)return images@@ -266,7 +274,13 @@ class AgentEngine:anthropic_api_key=self.anthropic_api_key,gemini_api_key=self.gemini_api_key,replicate_api_key=self.replicate_api_key,- should_extract_assets=self.should_extract_assets,+ # Only advertise extraction when the request actually contains a+ # still image the runtime can crop. In particular, Gemini videos+ # share the image_url message shape but are not valid extractor+ # inputs.+ should_extract_assets=(+ self.should_extract_assets and bool(self.tool_runtime.input_images)+ ),)try:return await self._run_with_session(session)diff --git a/backend/tests/test_agent_engine.py b/backend/tests/test_agent_engine.pynew file mode 100644index 00000000..7275201f--- /dev/null+++ b/backend/tests/test_agent_engine.py@@ -0,0 +1,109 @@+from typing import Any, cast++import pytest+from openai.types.chat import ChatCompletionMessageParam++from agent.engine import AgentEngine+from agent.providers.base import EventSink, ExecutedToolCall, ProviderTurn+from llm import Llm+++class NoToolProviderSession:+ def __init__(self) -> None:+ self.closed = False++ async def stream_turn(self, on_event: EventSink) -> ProviderTurn:+ return ProviderTurn(assistant_text="", tool_calls=[])++ async def append_tool_results(+ self,+ turn: ProviderTurn,+ executed_tool_calls: list[ExecutedToolCall],+ ) -> None:+ raise AssertionError("A no-tool turn must not append tool results")++ async def close(self) -> None:+ self.closed = True+++def _prompt_with_media(media_url: str) -> list[ChatCompletionMessageParam]:+ return cast(+ list[ChatCompletionMessageParam],+ [+ {"role": "system", "content": "System prompt"},+ {+ "role": "user",+ "content": [+ {+ "type": "image_url",+ "image_url": {"url": media_url, "detail": "high"},+ },+ {"type": "text", "text": "Build this interface."},+ ],+ },+ ],+ )+++ ("media_url", "expected_images", "expected_extraction_enabled"),+ [+ (+ "data:image/png;base64,aW1hZ2U=",+ ["data:image/png;base64,aW1hZ2U="],+ True,+ ),+ ("data:video/mp4;base64,dmlkZW8=", [], False),+ ("data:video/webm;base64,dmlkZW8=", [], False),+ ],+)+async def test_asset_extraction_is_only_offered_for_still_image_inputs(+ monkeypatch: pytest.MonkeyPatch,+ media_url: str,+ expected_images: list[str],+ expected_extraction_enabled: bool,+) -> None:+ factory_arguments: dict[str, Any] = {}+ session = NoToolProviderSession()++ def fake_create_provider_session(**kwargs: Any) -> NoToolProviderSession:+ factory_arguments.update(kwargs)+ return session++ monkeypatch.setattr(+ "agent.engine.create_provider_session", fake_create_provider_session+ )++ async def send_message(+ message_type: str,+ value: str | None,+ variant_index: int,+ data: dict[str, Any] | None,+ event_id: str | None,+ ) -> None:+ return None++ engine = AgentEngine(+ send_message=send_message,+ variant_index=0,+ openai_api_key=None,+ openai_base_url=None,+ anthropic_api_key=None,+ gemini_api_key="gemini-key",+ replicate_api_key=None,+ should_generate_images=True,+ should_extract_assets=True,+ )++ await engine.run(+ Llm.GEMINI_3_FLASH_PREVIEW_MINIMAL,+ _prompt_with_media(media_url),+ )++ assert engine.tool_runtime.input_images == expected_images+ assert (+ factory_arguments["should_extract_assets"]+ is expected_extraction_enabled+ )+ assert session.closed is Truediff --git a/frontend/src/App.tsx b/frontend/src/App.tsxindex af45af18..1cbab73a 100644--- a/frontend/src/App.tsx+++ b/frontend/src/App.tsx@@ -616,7 +616,11 @@ function App() {images: inputMode === "image" ? media : [],videos: inputMode === "video" ? media : [],},- isAssetExtractionEnabled,+ // Asset extraction operates on still screenshots. Video data uses the+ // same transport shape for Gemini, so explicitly disable extraction+ // instead of letting the agent try to crop a video payload.+ isAssetExtractionEnabled:+ inputMode === "image" && isAssetExtractionEnabled,variantHistory,});}