Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Add an asset extraction preference for code generation
Flightdeck Agentb2ca8cb70f32
diff --git a/backend/agent/engine.py b/backend/agent/engine.pyindex e0bd675d..a809639a 100644--- a/backend/agent/engine.py+++ b/backend/agent/engine.py@@ -33,6 +33,7 @@ class AgentEngine:gemini_api_key: Optional[str],replicate_api_key: Optional[str],should_generate_images: bool,+ should_extract_assets: bool = True,asset_base_url: str = "",initial_file_state: Optional[Dict[str, str]] = None,option_codes: Optional[List[str]] = None,@@ -45,6 +46,7 @@ class AgentEngine:self.gemini_api_key = gemini_api_keyself.replicate_api_key = replicate_api_keyself.should_generate_images = should_generate_images+ self.should_extract_assets = should_extract_assetsself.file_state = AgentFileState()if initial_file_state and initial_file_state.get("content"):@@ -264,6 +266,7 @@ class AgentEngine:anthropic_api_key=self.anthropic_api_key,gemini_api_key=self.gemini_api_key,replicate_api_key=self.replicate_api_key,+ should_extract_assets=self.should_extract_assets,)try:return await self._run_with_session(session)diff --git a/backend/agent/providers/factory.py b/backend/agent/providers/factory.pyindex 43f47bff..64ee882c 100644--- a/backend/agent/providers/factory.py+++ b/backend/agent/providers/factory.py@@ -24,13 +24,14 @@ def create_provider_session(anthropic_api_key: Optional[str],gemini_api_key: Optional[str],replicate_api_key: Optional[str],+ should_extract_assets: bool = True,) -> ProviderSession:canonical_tools = canonical_tool_definitions(image_generation_enabled=should_generate_images,# The edit_image tool calls Replicate, so don't offer it without a key.image_editing_enabled=bool(replicate_api_key or REPLICATE_API_KEY),# The extract_assets tool calls Gemini, so don't offer it without a key.- asset_extraction_enabled=bool(gemini_api_key),+ asset_extraction_enabled=should_extract_assets and bool(gemini_api_key),# screenshot_preview needs headless Chromium; skip it if it can't launch.screenshot_enabled=is_screenshot_preview_available(),)diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.pyindex f9401e7b..7ea2feee 100644--- a/backend/routes/generate_code.py+++ b/backend/routes/generate_code.py@@ -262,6 +262,7 @@ class ExtractedParams:history: List[PromptHistoryMessage]file_state: Dict[str, str] | Noneoption_codes: List[str]+ should_extract_assets: bool = Trueasset_base_url: str = ""design_system: str | None = None@@ -320,8 +321,9 @@ class ParameterExtractionStage:if not openai_base_url:print("Using official OpenAI URL")- # Get the image generation flag from the request. Fall back to True if not provided.+ # Feature preferences default to enabled for older clients.should_generate_images = bool(params.get("isImageGenerationEnabled", True))+ should_extract_assets = bool(params.get("isAssetExtractionEnabled", True))# Extract and validate generation typegeneration_type = params.get("generationType", "create")@@ -372,6 +374,7 @@ class ParameterExtractionStage:stack=validated_stack,input_mode=validated_input_mode,should_generate_images=should_generate_images,+ should_extract_assets=should_extract_assets,openai_api_key=openai_api_key,anthropic_api_key=anthropic_api_key,gemini_api_key=gemini_api_key,@@ -555,6 +558,7 @@ class AgenticGenerationStage:file_state: Dict[str, str] | None,asset_base_url: str,option_codes: List[str] | None,+ should_extract_assets: bool = True,):self.send_message = send_messageself.openai_api_key = openai_api_key@@ -563,6 +567,7 @@ class AgenticGenerationStage:self.gemini_api_key = gemini_api_keyself.replicate_api_key = replicate_api_keyself.should_generate_images = should_generate_images+ self.should_extract_assets = should_extract_assetsself.file_state = file_stateself.asset_base_url = asset_base_urlself.option_codes = option_codes or []@@ -622,6 +627,7 @@ class AgenticGenerationStage:gemini_api_key=self.gemini_api_key,replicate_api_key=self.replicate_api_key,should_generate_images=self.should_generate_images,+ should_extract_assets=self.should_extract_assets,asset_base_url=self.asset_base_url,initial_file_state=self.file_state,option_codes=self.option_codes,@@ -804,6 +810,7 @@ class CodeGenerationMiddleware(Middleware):gemini_api_key=context.extracted_params.gemini_api_key,replicate_api_key=context.extracted_params.replicate_api_key,should_generate_images=context.extracted_params.should_generate_images,+ should_extract_assets=context.extracted_params.should_extract_assets,file_state=context.extracted_params.file_state,asset_base_url=context.extracted_params.asset_base_url,option_codes=context.extracted_params.option_codes,diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.pyindex ef5dd01d..2be3f3e7 100644--- a/backend/tests/test_agent_tools.py+++ b/backend/tests/test_agent_tools.py@@ -108,6 +108,24 @@ def test_provider_session_includes_extract_assets_with_gemini_key() -> None:assert "extract_assets" in tool_names+def test_provider_session_excludes_extract_assets_when_user_disables_it() -> None:+ session = create_provider_session(+ model=Llm.GPT_5_5_HIGH,+ prompt_messages=[{"role": "user", "content": "Build a page."}],+ should_generate_images=True,+ openai_api_key="openai-key",+ openai_base_url=None,+ anthropic_api_key=None,+ gemini_api_key="gemini-key",+ replicate_api_key=None,+ should_extract_assets=False,+ )++ tools = cast(list[dict[str, Any]], getattr(session, "_tools"))+ tool_names = [tool["name"] for tool in tools]+ assert "extract_assets" not in tool_names++def test_provider_session_excludes_screenshot_preview_when_chromium_unavailable(monkeypatch: pytest.MonkeyPatch,) -> None:diff --git a/backend/tests/test_parameter_extraction_stage.py b/backend/tests/test_parameter_extraction_stage.pyindex d275783a..291f66cc 100644--- a/backend/tests/test_parameter_extraction_stage.py+++ b/backend/tests/test_parameter_extraction_stage.py@@ -5,6 +5,28 @@ import pytestfrom routes.generate_code import ParameterExtractionStage+ ("request_value", "expected"),+ [(None, True), (False, False), (True, True)],+)+async def test_asset_extraction_preference_defaults_on_and_supports_opt_out(+ request_value: bool | None, expected: bool+) -> None:+ stage = ParameterExtractionStage(AsyncMock())+ params: dict[str, object] = {+ "generatedCodeConfig": "html_tailwind",+ "inputMode": "text",+ "prompt": {"text": "hello"},+ }+ if request_value is not None:+ params["isAssetExtractionEnabled"] = request_value++ extracted = await stage.extract_and_validate(params)++ assert extracted.should_extract_assets is expected++@pytest.mark.asyncioasync def test_extracts_gemini_api_key_from_settings_dialog() -> None:stage = ParameterExtractionStage(AsyncMock())diff --git a/frontend/src/App.tsx b/frontend/src/App.tsxindex 773c2023..778303f0 100644--- a/frontend/src/App.tsx+++ b/frontend/src/App.tsx@@ -188,6 +188,7 @@ function App() {}}, [settings, setSettings]);+useEffect(() => {if (settings.selectedDesignSystemId &&@@ -319,8 +320,8 @@ function App() {// Merge settings with paramsconst updatedParams = {- ...requestParams,...settings,+ ...requestParams,designSystem: selectedDesignSystem?.content ?? null,};@@ -559,7 +560,8 @@ function App() {function doCreate(referenceImages: string[],inputMode: "image" | "video",- textPrompt: string = ""+ textPrompt: string = "",+ isAssetExtractionEnabled = true) {// Reset any existing statereset();@@ -603,6 +605,7 @@ function App() {images: inputMode === "image" ? media : [],videos: inputMode === "video" ? media : [],},+ isAssetExtractionEnabled,variantHistory,});}diff --git a/frontend/src/components/settings/SettingsTab.tsx b/frontend/src/components/settings/SettingsTab.tsxindex 9e5e5afa..6f9a5030 100644--- a/frontend/src/components/settings/SettingsTab.tsx+++ b/frontend/src/components/settings/SettingsTab.tsx@@ -270,10 +270,10 @@ function SettingsTab({ settings, setSettings, appTheme, setAppTheme }: Props) {<Switchid="image-generation"checked={settings.isImageGenerationEnabled}- onCheckedChange={() =>+ onCheckedChange={(checked) =>setSettings((s) => ({...s,- isImageGenerationEnabled: !s.isImageGenerationEnabled,+ isImageGenerationEnabled: checked,}))}/>diff --git a/frontend/src/components/start-pane/StartPane.tsx b/frontend/src/components/start-pane/StartPane.tsxindex 36c831b5..7931ecc3 100644--- a/frontend/src/components/start-pane/StartPane.tsx+++ b/frontend/src/components/start-pane/StartPane.tsx@@ -7,7 +7,8 @@ interface Props {doCreate: (images: string[],inputMode: "image" | "video",- textPrompt?: string+ textPrompt?: string,+ isAssetExtractionEnabled?: boolean) => void;doCreateFromText: (text: string) => void;importFromCode: (code: string, stack: Stack) => void;diff --git a/frontend/src/components/unified-input/UnifiedInputPane.tsx b/frontend/src/components/unified-input/UnifiedInputPane.tsxindex 203ea290..355b3ae2 100644--- a/frontend/src/components/unified-input/UnifiedInputPane.tsx+++ b/frontend/src/components/unified-input/UnifiedInputPane.tsx@@ -12,7 +12,8 @@ interface Props {doCreate: (images: string[],inputMode: "image" | "video",- textPrompt?: string+ textPrompt?: string,+ isAssetExtractionEnabled?: boolean) => void;doCreateFromText: (text: string) => void;importFromCode: (code: string, stack: Stack) => void;diff --git a/frontend/src/components/unified-input/tabs/UploadTab.tsx b/frontend/src/components/unified-input/tabs/UploadTab.tsxindex ed1cb204..0bf12d21 100644--- a/frontend/src/components/unified-input/tabs/UploadTab.tsx+++ b/frontend/src/components/unified-input/tabs/UploadTab.tsx@@ -3,6 +3,7 @@ import { useDropzone } from "react-dropzone";import { toast } from "react-hot-toast";import { Cross2Icon, ImageIcon } from "@radix-ui/react-icons";import { Button } from "../../ui/button";+import { Checkbox } from "../../ui/checkbox";import { ScreenRecorderState } from "../../../types";import ScreenRecorder from "../../recording/ScreenRecorder";import OutputSettingsSection from "../../settings/OutputSettingsSection";@@ -48,7 +49,8 @@ interface Props {doCreate: (referenceImages: string[],inputMode: "image" | "video",- textPrompt?: string+ textPrompt?: string,+ isAssetExtractionEnabled?: boolean) => void;stack: Stack;setStack: (stack: Stack) => void;@@ -63,6 +65,7 @@ function UploadTab({ doCreate, stack, setStack, designSystem }: Props) {>("image");const [textPrompt, setTextPrompt] = useState("");const [showTextPrompt, setShowTextPrompt] = useState(false);+ const [isAssetExtractionEnabled, setIsAssetExtractionEnabled] = useState(true);const [selectedIndex, setSelectedIndex] = useState(0);const textInputRef = useRef<HTMLTextAreaElement>(null);const filesRef = useRef<FileWithPreview[]>([]);@@ -75,9 +78,20 @@ function UploadTab({ doCreate, stack, setStack, designSystem }: Props) {const handleGenerate = useCallback(() => {if (uploadedDataUrls.length > 0) {- doCreate(uploadedDataUrls, uploadedInputMode, textPrompt);+ doCreate(+ uploadedDataUrls,+ uploadedInputMode,+ textPrompt,+ isAssetExtractionEnabled+ );}- }, [uploadedDataUrls, uploadedInputMode, textPrompt, doCreate]);+ }, [+ uploadedDataUrls,+ uploadedInputMode,+ textPrompt,+ isAssetExtractionEnabled,+ doCreate,+ ]);useEffect(() => {if (!hasUploadedFile) return;@@ -467,12 +481,35 @@ function UploadTab({ doCreate, stack, setStack, designSystem }: Props) {</div>)}- <div className="w-full max-w-md">+ <div className="w-full max-w-md space-y-4"><OutputSettingsSectionstack={stack}setStack={setStack}designSystem={designSystem}/>+ {uploadedInputMode === "image" && (+ <label+ htmlFor="asset-extraction"+ className="flex cursor-pointer items-start gap-3 rounded-lg border border-gray-200 bg-gray-50 p-3 dark:border-zinc-700 dark:bg-zinc-800/60"+ >+ <Checkbox+ id="asset-extraction"+ checked={isAssetExtractionEnabled}+ onCheckedChange={(checked) =>+ setIsAssetExtractionEnabled(checked === true)+ }+ className="mt-0.5"+ />+ <span>+ <span className="block text-sm font-medium text-gray-700 dark:text-zinc-200">+ Extract image assets+ </span>+ <span className="mt-0.5 block text-xs text-gray-500 dark:text-zinc-400">+ Reuse visual assets from this screenshot in the generated code.+ </span>+ </span>+ </label>+ )}</div><div className="flex flex-col items-center gap-1 w-full max-w-md">diff --git a/frontend/src/components/unified-input/tabs/UrlTab.tsx b/frontend/src/components/unified-input/tabs/UrlTab.tsxindex dde31171..97cc453b 100644--- a/frontend/src/components/unified-input/tabs/UrlTab.tsx+++ b/frontend/src/components/unified-input/tabs/UrlTab.tsx@@ -1,6 +1,7 @@import { useState } from "react";import { HTTP_BACKEND_URL } from "../../../config";import { Button } from "../../ui/button";+import { Checkbox } from "../../ui/checkbox";import { Input } from "../../ui/input";import { toast } from "react-hot-toast";import OutputSettingsSection from "../../settings/OutputSettingsSection";@@ -13,6 +14,7 @@ interface Props {urls: string[],inputMode: "image" | "video",textPrompt?: string,+ isAssetExtractionEnabled?: boolean,) => void;stack: Stack;setStack: (stack: Stack) => void;@@ -32,6 +34,7 @@ function UrlTab({}: Props) {const [isLoading, setIsLoading] = useState(false);const [referenceUrl, setReferenceUrl] = useState("");+ const [isAssetExtractionEnabled, setIsAssetExtractionEnabled] = useState(true);async function takeScreenshot() {const trimmedReferenceUrl = referenceUrl.trim();@@ -82,7 +85,7 @@ function UrlTab({}const res = await response.json();- doCreate([res.url], "image");+ doCreate([res.url], "image", "", isAssetExtractionEnabled);} catch (error) {console.error(error);toast.error("Failed to capture screenshot. Check console for details.");@@ -142,6 +145,27 @@ function UrlTab({setStack={setStack}designSystem={designSystem}/>+ <label+ htmlFor="url-asset-extraction"+ className="flex cursor-pointer items-start gap-3 rounded-lg border border-gray-200 bg-gray-50 p-3 dark:border-zinc-700 dark:bg-zinc-800/60"+ >+ <Checkbox+ id="url-asset-extraction"+ checked={isAssetExtractionEnabled}+ onCheckedChange={(checked) =>+ setIsAssetExtractionEnabled(checked === true)+ }+ className="mt-0.5"+ />+ <span>+ <span className="block text-sm font-medium text-gray-700 dark:text-zinc-200">+ Extract image assets+ </span>+ <span className="mt-0.5 block text-xs text-gray-500 dark:text-zinc-400">+ Reuse visual assets from this screenshot in the generated code.+ </span>+ </span>+ </label><ButtononClick={takeScreenshot}diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex 57d6420a..70dd12d5 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -84,6 +84,7 @@ export interface CodeGenerationParams {content: string;};optionCodes?: string[];+ isAssetExtractionEnabled?: boolean;}export type FullGenerationSettings = CodeGenerationParams &