Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Add an asset extraction preference for code generation

Flightdeck Agentb2ca8cb70f32
diff --git a/backend/agent/engine.py b/backend/agent/engine.py
index e0bd675d..a809639a 100644
--- a/backend/agent/engine.py
+++ b/backend/agent/engine.py
@@ -33,6 +33,7 @@ class AgentEngine:
gemini_api_key: Optional[str],
replicate_api_key: Optional[str],
should_generate_images: bool,
+ should_extract_assets: bool = True,
asset_base_url: str = "",
initial_file_state: Optional[Dict[str, str]] = None,
option_codes: Optional[List[str]] = None,
@@ -45,6 +46,7 @@ class AgentEngine:
self.gemini_api_key = gemini_api_key
self.replicate_api_key = replicate_api_key
self.should_generate_images = should_generate_images
+ self.should_extract_assets = should_extract_assets
self.file_state = AgentFileState()
if initial_file_state and initial_file_state.get("content"):
@@ -264,6 +266,7 @@ class AgentEngine:
anthropic_api_key=self.anthropic_api_key,
gemini_api_key=self.gemini_api_key,
replicate_api_key=self.replicate_api_key,
+ should_extract_assets=self.should_extract_assets,
)
try:
return await self._run_with_session(session)
diff --git a/backend/agent/providers/factory.py b/backend/agent/providers/factory.py
index 43f47bff..64ee882c 100644
--- a/backend/agent/providers/factory.py
+++ b/backend/agent/providers/factory.py
@@ -24,13 +24,14 @@ def create_provider_session(
anthropic_api_key: Optional[str],
gemini_api_key: Optional[str],
replicate_api_key: Optional[str],
+ should_extract_assets: bool = True,
) -> ProviderSession:
canonical_tools = canonical_tool_definitions(
image_generation_enabled=should_generate_images,
# The edit_image tool calls Replicate, so don't offer it without a key.
image_editing_enabled=bool(replicate_api_key or REPLICATE_API_KEY),
# The extract_assets tool calls Gemini, so don't offer it without a key.
- asset_extraction_enabled=bool(gemini_api_key),
+ asset_extraction_enabled=should_extract_assets and bool(gemini_api_key),
# screenshot_preview needs headless Chromium; skip it if it can't launch.
screenshot_enabled=is_screenshot_preview_available(),
)
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index f9401e7b..7ea2feee 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -262,6 +262,7 @@ class ExtractedParams:
history: List[PromptHistoryMessage]
file_state: Dict[str, str] | None
option_codes: List[str]
+ should_extract_assets: bool = True
asset_base_url: str = ""
design_system: str | None = None
@@ -320,8 +321,9 @@ class ParameterExtractionStage:
if not openai_base_url:
print("Using official OpenAI URL")
- # Get the image generation flag from the request. Fall back to True if not provided.
+ # Feature preferences default to enabled for older clients.
should_generate_images = bool(params.get("isImageGenerationEnabled", True))
+ should_extract_assets = bool(params.get("isAssetExtractionEnabled", True))
# Extract and validate generation type
generation_type = params.get("generationType", "create")
@@ -372,6 +374,7 @@ class ParameterExtractionStage:
stack=validated_stack,
input_mode=validated_input_mode,
should_generate_images=should_generate_images,
+ should_extract_assets=should_extract_assets,
openai_api_key=openai_api_key,
anthropic_api_key=anthropic_api_key,
gemini_api_key=gemini_api_key,
@@ -555,6 +558,7 @@ class AgenticGenerationStage:
file_state: Dict[str, str] | None,
asset_base_url: str,
option_codes: List[str] | None,
+ should_extract_assets: bool = True,
):
self.send_message = send_message
self.openai_api_key = openai_api_key
@@ -563,6 +567,7 @@ class AgenticGenerationStage:
self.gemini_api_key = gemini_api_key
self.replicate_api_key = replicate_api_key
self.should_generate_images = should_generate_images
+ self.should_extract_assets = should_extract_assets
self.file_state = file_state
self.asset_base_url = asset_base_url
self.option_codes = option_codes or []
@@ -622,6 +627,7 @@ class AgenticGenerationStage:
gemini_api_key=self.gemini_api_key,
replicate_api_key=self.replicate_api_key,
should_generate_images=self.should_generate_images,
+ should_extract_assets=self.should_extract_assets,
asset_base_url=self.asset_base_url,
initial_file_state=self.file_state,
option_codes=self.option_codes,
@@ -804,6 +810,7 @@ class CodeGenerationMiddleware(Middleware):
gemini_api_key=context.extracted_params.gemini_api_key,
replicate_api_key=context.extracted_params.replicate_api_key,
should_generate_images=context.extracted_params.should_generate_images,
+ should_extract_assets=context.extracted_params.should_extract_assets,
file_state=context.extracted_params.file_state,
asset_base_url=context.extracted_params.asset_base_url,
option_codes=context.extracted_params.option_codes,
diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.py
index ef5dd01d..2be3f3e7 100644
--- a/backend/tests/test_agent_tools.py
+++ b/backend/tests/test_agent_tools.py
@@ -108,6 +108,24 @@ def test_provider_session_includes_extract_assets_with_gemini_key() -> None:
assert "extract_assets" in tool_names
+def test_provider_session_excludes_extract_assets_when_user_disables_it() -> None:
+ session = create_provider_session(
+ model=Llm.GPT_5_5_HIGH,
+ prompt_messages=[{"role": "user", "content": "Build a page."}],
+ should_generate_images=True,
+ openai_api_key="openai-key",
+ openai_base_url=None,
+ anthropic_api_key=None,
+ gemini_api_key="gemini-key",
+ replicate_api_key=None,
+ should_extract_assets=False,
+ )
+
+ tools = cast(list[dict[str, Any]], getattr(session, "_tools"))
+ tool_names = [tool["name"] for tool in tools]
+ assert "extract_assets" not in tool_names
+
+
def test_provider_session_excludes_screenshot_preview_when_chromium_unavailable(
monkeypatch: pytest.MonkeyPatch,
) -> None:
diff --git a/backend/tests/test_parameter_extraction_stage.py b/backend/tests/test_parameter_extraction_stage.py
index d275783a..291f66cc 100644
--- a/backend/tests/test_parameter_extraction_stage.py
+++ b/backend/tests/test_parameter_extraction_stage.py
@@ -5,6 +5,28 @@ import pytest
from routes.generate_code import ParameterExtractionStage
+ ("request_value", "expected"),
+ [(None, True), (False, False), (True, True)],
+)
+async def test_asset_extraction_preference_defaults_on_and_supports_opt_out(
+ request_value: bool | None, expected: bool
+) -> None:
+ stage = ParameterExtractionStage(AsyncMock())
+ params: dict[str, object] = {
+ "generatedCodeConfig": "html_tailwind",
+ "inputMode": "text",
+ "prompt": {"text": "hello"},
+ }
+ if request_value is not None:
+ params["isAssetExtractionEnabled"] = request_value
+
+ extracted = await stage.extract_and_validate(params)
+
+ assert extracted.should_extract_assets is expected
+
+
@pytest.mark.asyncio
async def test_extracts_gemini_api_key_from_settings_dialog() -> None:
stage = ParameterExtractionStage(AsyncMock())
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 773c2023..778303f0 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -188,6 +188,7 @@ function App() {
}
}, [settings, setSettings]);
+
useEffect(() => {
if (
settings.selectedDesignSystemId &&
@@ -319,8 +320,8 @@ function App() {
// Merge settings with params
const updatedParams = {
- ...requestParams,
...settings,
+ ...requestParams,
designSystem: selectedDesignSystem?.content ?? null,
};
@@ -559,7 +560,8 @@ function App() {
function doCreate(
referenceImages: string[],
inputMode: "image" | "video",
- textPrompt: string = ""
+ textPrompt: string = "",
+ isAssetExtractionEnabled = true
) {
// Reset any existing state
reset();
@@ -603,6 +605,7 @@ function App() {
images: inputMode === "image" ? media : [],
videos: inputMode === "video" ? media : [],
},
+ isAssetExtractionEnabled,
variantHistory,
});
}
diff --git a/frontend/src/components/settings/SettingsTab.tsx b/frontend/src/components/settings/SettingsTab.tsx
index 9e5e5afa..6f9a5030 100644
--- a/frontend/src/components/settings/SettingsTab.tsx
+++ b/frontend/src/components/settings/SettingsTab.tsx
@@ -270,10 +270,10 @@ function SettingsTab({ settings, setSettings, appTheme, setAppTheme }: Props) {
<Switch
id="image-generation"
checked={settings.isImageGenerationEnabled}
- onCheckedChange={() =>
+ onCheckedChange={(checked) =>
setSettings((s) => ({
...s,
- isImageGenerationEnabled: !s.isImageGenerationEnabled,
+ isImageGenerationEnabled: checked,
}))
}
/>
diff --git a/frontend/src/components/start-pane/StartPane.tsx b/frontend/src/components/start-pane/StartPane.tsx
index 36c831b5..7931ecc3 100644
--- a/frontend/src/components/start-pane/StartPane.tsx
+++ b/frontend/src/components/start-pane/StartPane.tsx
@@ -7,7 +7,8 @@ interface Props {
doCreate: (
images: string[],
inputMode: "image" | "video",
- textPrompt?: string
+ textPrompt?: string,
+ isAssetExtractionEnabled?: boolean
) => void;
doCreateFromText: (text: string) => void;
importFromCode: (code: string, stack: Stack) => void;
diff --git a/frontend/src/components/unified-input/UnifiedInputPane.tsx b/frontend/src/components/unified-input/UnifiedInputPane.tsx
index 203ea290..355b3ae2 100644
--- a/frontend/src/components/unified-input/UnifiedInputPane.tsx
+++ b/frontend/src/components/unified-input/UnifiedInputPane.tsx
@@ -12,7 +12,8 @@ interface Props {
doCreate: (
images: string[],
inputMode: "image" | "video",
- textPrompt?: string
+ textPrompt?: string,
+ isAssetExtractionEnabled?: boolean
) => void;
doCreateFromText: (text: string) => void;
importFromCode: (code: string, stack: Stack) => void;
diff --git a/frontend/src/components/unified-input/tabs/UploadTab.tsx b/frontend/src/components/unified-input/tabs/UploadTab.tsx
index ed1cb204..0bf12d21 100644
--- a/frontend/src/components/unified-input/tabs/UploadTab.tsx
+++ b/frontend/src/components/unified-input/tabs/UploadTab.tsx
@@ -3,6 +3,7 @@ import { useDropzone } from "react-dropzone";
import { toast } from "react-hot-toast";
import { Cross2Icon, ImageIcon } from "@radix-ui/react-icons";
import { Button } from "../../ui/button";
+import { Checkbox } from "../../ui/checkbox";
import { ScreenRecorderState } from "../../../types";
import ScreenRecorder from "../../recording/ScreenRecorder";
import OutputSettingsSection from "../../settings/OutputSettingsSection";
@@ -48,7 +49,8 @@ interface Props {
doCreate: (
referenceImages: string[],
inputMode: "image" | "video",
- textPrompt?: string
+ textPrompt?: string,
+ isAssetExtractionEnabled?: boolean
) => void;
stack: Stack;
setStack: (stack: Stack) => void;
@@ -63,6 +65,7 @@ function UploadTab({ doCreate, stack, setStack, designSystem }: Props) {
>("image");
const [textPrompt, setTextPrompt] = useState("");
const [showTextPrompt, setShowTextPrompt] = useState(false);
+ const [isAssetExtractionEnabled, setIsAssetExtractionEnabled] = useState(true);
const [selectedIndex, setSelectedIndex] = useState(0);
const textInputRef = useRef<HTMLTextAreaElement>(null);
const filesRef = useRef<FileWithPreview[]>([]);
@@ -75,9 +78,20 @@ function UploadTab({ doCreate, stack, setStack, designSystem }: Props) {
const handleGenerate = useCallback(() => {
if (uploadedDataUrls.length > 0) {
- doCreate(uploadedDataUrls, uploadedInputMode, textPrompt);
+ doCreate(
+ uploadedDataUrls,
+ uploadedInputMode,
+ textPrompt,
+ isAssetExtractionEnabled
+ );
}
- }, [uploadedDataUrls, uploadedInputMode, textPrompt, doCreate]);
+ }, [
+ uploadedDataUrls,
+ uploadedInputMode,
+ textPrompt,
+ isAssetExtractionEnabled,
+ doCreate,
+ ]);
useEffect(() => {
if (!hasUploadedFile) return;
@@ -467,12 +481,35 @@ function UploadTab({ doCreate, stack, setStack, designSystem }: Props) {
</div>
)}
- <div className="w-full max-w-md">
+ <div className="w-full max-w-md space-y-4">
<OutputSettingsSection
stack={stack}
setStack={setStack}
designSystem={designSystem}
/>
+ {uploadedInputMode === "image" && (
+ <label
+ htmlFor="asset-extraction"
+ className="flex cursor-pointer items-start gap-3 rounded-lg border border-gray-200 bg-gray-50 p-3 dark:border-zinc-700 dark:bg-zinc-800/60"
+ >
+ <Checkbox
+ id="asset-extraction"
+ checked={isAssetExtractionEnabled}
+ onCheckedChange={(checked) =>
+ setIsAssetExtractionEnabled(checked === true)
+ }
+ className="mt-0.5"
+ />
+ <span>
+ <span className="block text-sm font-medium text-gray-700 dark:text-zinc-200">
+ Extract image assets
+ </span>
+ <span className="mt-0.5 block text-xs text-gray-500 dark:text-zinc-400">
+ Reuse visual assets from this screenshot in the generated code.
+ </span>
+ </span>
+ </label>
+ )}
</div>
<div className="flex flex-col items-center gap-1 w-full max-w-md">
diff --git a/frontend/src/components/unified-input/tabs/UrlTab.tsx b/frontend/src/components/unified-input/tabs/UrlTab.tsx
index dde31171..97cc453b 100644
--- a/frontend/src/components/unified-input/tabs/UrlTab.tsx
+++ b/frontend/src/components/unified-input/tabs/UrlTab.tsx
@@ -1,6 +1,7 @@
import { useState } from "react";
import { HTTP_BACKEND_URL } from "../../../config";
import { Button } from "../../ui/button";
+import { Checkbox } from "../../ui/checkbox";
import { Input } from "../../ui/input";
import { toast } from "react-hot-toast";
import OutputSettingsSection from "../../settings/OutputSettingsSection";
@@ -13,6 +14,7 @@ interface Props {
urls: string[],
inputMode: "image" | "video",
textPrompt?: string,
+ isAssetExtractionEnabled?: boolean,
) => void;
stack: Stack;
setStack: (stack: Stack) => void;
@@ -32,6 +34,7 @@ function UrlTab({
}: Props) {
const [isLoading, setIsLoading] = useState(false);
const [referenceUrl, setReferenceUrl] = useState("");
+ const [isAssetExtractionEnabled, setIsAssetExtractionEnabled] = useState(true);
async function takeScreenshot() {
const trimmedReferenceUrl = referenceUrl.trim();
@@ -82,7 +85,7 @@ function UrlTab({
}
const res = await response.json();
- doCreate([res.url], "image");
+ doCreate([res.url], "image", "", isAssetExtractionEnabled);
} catch (error) {
console.error(error);
toast.error("Failed to capture screenshot. Check console for details.");
@@ -142,6 +145,27 @@ function UrlTab({
setStack={setStack}
designSystem={designSystem}
/>
+ <label
+ htmlFor="url-asset-extraction"
+ className="flex cursor-pointer items-start gap-3 rounded-lg border border-gray-200 bg-gray-50 p-3 dark:border-zinc-700 dark:bg-zinc-800/60"
+ >
+ <Checkbox
+ id="url-asset-extraction"
+ checked={isAssetExtractionEnabled}
+ onCheckedChange={(checked) =>
+ setIsAssetExtractionEnabled(checked === true)
+ }
+ className="mt-0.5"
+ />
+ <span>
+ <span className="block text-sm font-medium text-gray-700 dark:text-zinc-200">
+ Extract image assets
+ </span>
+ <span className="mt-0.5 block text-xs text-gray-500 dark:text-zinc-400">
+ Reuse visual assets from this screenshot in the generated code.
+ </span>
+ </span>
+ </label>
<Button
onClick={takeScreenshot}
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 57d6420a..70dd12d5 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -84,6 +84,7 @@ export interface CodeGenerationParams {
content: string;
};
optionCodes?: string[];
+ isAssetExtractionEnabled?: boolean;
}
export type FullGenerationSettings = CodeGenerationParams &