Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Support Replicate key in settings (#613)
Abi Raja02303fa52416
diff --git a/backend/agent/engine.py b/backend/agent/engine.pyindex 18edfabe..e0bd675d 100644--- a/backend/agent/engine.py+++ b/backend/agent/engine.py@@ -31,6 +31,7 @@ class AgentEngine:openai_base_url: Optional[str],anthropic_api_key: Optional[str],gemini_api_key: Optional[str],+ replicate_api_key: Optional[str],should_generate_images: bool,asset_base_url: str = "",initial_file_state: Optional[Dict[str, str]] = None,@@ -42,6 +43,7 @@ class AgentEngine:self.openai_base_url = openai_base_urlself.anthropic_api_key = anthropic_api_keyself.gemini_api_key = gemini_api_key+ self.replicate_api_key = replicate_api_keyself.should_generate_images = should_generate_imagesself.file_state = AgentFileState()@@ -55,6 +57,7 @@ class AgentEngine:openai_api_key=openai_api_key,openai_base_url=openai_base_url,gemini_api_key=gemini_api_key,+ replicate_api_key=replicate_api_key,asset_base_url=asset_base_url,option_codes=option_codes,)@@ -260,6 +263,7 @@ class AgentEngine:openai_base_url=self.openai_base_url,anthropic_api_key=self.anthropic_api_key,gemini_api_key=self.gemini_api_key,+ replicate_api_key=self.replicate_api_key,)try:return await self._run_with_session(session)diff --git a/backend/agent/providers/factory.py b/backend/agent/providers/factory.pyindex 35216bdd..43f47bff 100644--- a/backend/agent/providers/factory.py+++ b/backend/agent/providers/factory.py@@ -23,11 +23,12 @@ def create_provider_session(openai_base_url: Optional[str],anthropic_api_key: Optional[str],gemini_api_key: Optional[str],+ replicate_api_key: Optional[str],) -> ProviderSession:canonical_tools = canonical_tool_definitions(image_generation_enabled=should_generate_images,# The edit_image tool calls Replicate, so don't offer it without a key.- image_editing_enabled=bool(REPLICATE_API_KEY),+ image_editing_enabled=bool(replicate_api_key or REPLICATE_API_KEY),# The extract_assets tool calls Gemini, so don't offer it without a key.asset_extraction_enabled=bool(gemini_api_key),# screenshot_preview needs headless Chromium; skip it if it can't launch.diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.pyindex 91bfd662..59215ba9 100644--- a/backend/agent/tools/runtime.py+++ b/backend/agent/tools/runtime.py@@ -30,6 +30,7 @@ class AgentToolRuntime:openai_api_key: Optional[str],openai_base_url: Optional[str],gemini_api_key: Optional[str] = None,+ replicate_api_key: Optional[str] = None,input_images: Optional[List[str]] = None,asset_base_url: str = "",user_id: Optional[str] = None,@@ -40,11 +41,15 @@ class AgentToolRuntime:self.openai_api_key = openai_api_keyself.openai_base_url = openai_base_urlself.gemini_api_key = gemini_api_key+ self.replicate_api_key = replicate_api_keyself.input_images = input_images or []self.asset_base_url = asset_base_urlself.user_id = user_idself.option_codes = option_codes or []+ def _effective_replicate_api_key(self) -> str | None:+ return self.replicate_api_key or REPLICATE_API_KEY+async def execute(self, tool_call: ToolCall) -> ToolExecutionResult:if "INVALID_JSON" in tool_call.arguments:invalid_json = ensure_str(tool_call.arguments.get("INVALID_JSON"))@@ -272,9 +277,10 @@ class AgentToolRuntime:result={"error": "No valid prompts provided"},summary={"error": "No valid prompts"},)- if REPLICATE_API_KEY:+ replicate_api_key = self._effective_replicate_api_key()+ if replicate_api_key:model = "flux"- api_key = REPLICATE_API_KEY+ api_key = replicate_api_keybase_url = Noneelse:if not self.openai_api_key:@@ -318,7 +324,8 @@ class AgentToolRuntime:)async def _remove_background(self, args: Dict[str, Any]) -> ToolExecutionResult:- if not REPLICATE_API_KEY:+ replicate_api_key = self._effective_replicate_api_key()+ if not replicate_api_key:return ToolExecutionResult(ok=False,result={"error": "Background removal requires REPLICATE_API_KEY."},@@ -350,7 +357,7 @@ class AgentToolRuntime:batch = unique_urls[i : i + batch_size]# Replicate can't fetch localhost; inline local assets as data URLs.tasks = [- remove_background(local_asset_url_to_data_url(url), REPLICATE_API_KEY)+ remove_background(local_asset_url_to_data_url(url), replicate_api_key)for url in batch]raw_results.extend(await asyncio.gather(*tasks, return_exceptions=True))@@ -392,7 +399,8 @@ class AgentToolRuntime:)async def _edit_image(self, args: Dict[str, Any]) -> ToolExecutionResult:- if not REPLICATE_API_KEY:+ replicate_api_key = self._effective_replicate_api_key()+ if not replicate_api_key:return ToolExecutionResult(ok=False,result={"error": "Image editing requires REPLICATE_API_KEY."},@@ -433,7 +441,7 @@ class AgentToolRuntime:result_url = await edit_image(prompt=prompt,image_urls=[local_asset_url_to_data_url(url) for url in unique_urls],- api_token=REPLICATE_API_KEY,+ api_token=replicate_api_key,aspect_ratio=aspect_ratio,)except Exception as exc:diff --git a/backend/evals/core.py b/backend/evals/core.pyindex 1e1495a8..089e2b2d 100644--- a/backend/evals/core.py+++ b/backend/evals/core.py@@ -4,6 +4,7 @@ from config import (LOCAL_ASSET_BASE_URL,OPENAI_API_KEY,OPENAI_BASE_URL,+ REPLICATE_API_KEY,)from llm import Llm, OPENAI_MODELS, ANTHROPIC_MODELS, GEMINI_MODELSfrom agent.runner import Agent@@ -46,6 +47,7 @@ async def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> sopenai_base_url=OPENAI_BASE_URL,anthropic_api_key=ANTHROPIC_API_KEY,gemini_api_key=GEMINI_API_KEY,+ replicate_api_key=REPLICATE_API_KEY,should_generate_images=True,# No websocket to infer the host from, so use the configured base URL;# otherwise extracted/saved assets get hostless /local-assets/ URLs.diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.pyindex 5408cdca..f9401e7b 100644--- a/backend/routes/generate_code.py+++ b/backend/routes/generate_code.py@@ -255,6 +255,7 @@ class ExtractedParams:openai_api_key: str | Noneanthropic_api_key: str | Nonegemini_api_key: str | None+ replicate_api_key: str | Noneopenai_base_url: str | Nonegeneration_type: Literal["create", "update"]prompt: UserTurnInput@@ -305,6 +306,9 @@ class ParameterExtractionStage:gemini_api_key = self._get_from_settings_dialog_or_env(params, "geminiApiKey", GEMINI_API_KEY)+ replicate_api_key = self._get_from_settings_dialog_or_env(+ params, "replicateApiKey", REPLICATE_API_KEY+ )# Base URL for OpenAI APIopenai_base_url: str | None = None@@ -371,6 +375,7 @@ class ParameterExtractionStage:openai_api_key=openai_api_key,anthropic_api_key=anthropic_api_key,gemini_api_key=gemini_api_key,+ replicate_api_key=replicate_api_key,openai_base_url=openai_base_url,generation_type=generation_type,prompt=prompt,@@ -545,6 +550,7 @@ class AgenticGenerationStage:openai_base_url: str | None,anthropic_api_key: str | None,gemini_api_key: str | None,+ replicate_api_key: str | None,should_generate_images: bool,file_state: Dict[str, str] | None,asset_base_url: str,@@ -555,6 +561,7 @@ class AgenticGenerationStage:self.openai_base_url = openai_base_urlself.anthropic_api_key = anthropic_api_keyself.gemini_api_key = gemini_api_key+ self.replicate_api_key = replicate_api_keyself.should_generate_images = should_generate_imagesself.file_state = file_stateself.asset_base_url = asset_base_url@@ -613,6 +620,7 @@ class AgenticGenerationStage:openai_base_url=self.openai_base_url,anthropic_api_key=self.anthropic_api_key,gemini_api_key=self.gemini_api_key,+ replicate_api_key=self.replicate_api_key,should_generate_images=self.should_generate_images,asset_base_url=self.asset_base_url,initial_file_state=self.file_state,@@ -794,6 +802,7 @@ class CodeGenerationMiddleware(Middleware):openai_base_url=context.extracted_params.openai_base_url,anthropic_api_key=context.extracted_params.anthropic_api_key,gemini_api_key=context.extracted_params.gemini_api_key,+ replicate_api_key=context.extracted_params.replicate_api_key,should_generate_images=context.extracted_params.should_generate_images,file_state=context.extracted_params.file_state,asset_base_url=context.extracted_params.asset_base_url,diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.pyindex 2e0306d5..ef5dd01d 100644--- a/backend/tests/test_agent_tools.py+++ b/backend/tests/test_agent_tools.py@@ -83,6 +83,7 @@ def test_provider_session_excludes_extract_assets_without_gemini_key() -> None:openai_base_url=None,anthropic_api_key=None,gemini_api_key=None,+ replicate_api_key=None,)tools = cast(list[dict[str, Any]], getattr(session, "_tools"))@@ -99,6 +100,7 @@ def test_provider_session_includes_extract_assets_with_gemini_key() -> None:openai_base_url=None,anthropic_api_key=None,gemini_api_key="gemini-key",+ replicate_api_key=None,)tools = cast(list[dict[str, Any]], getattr(session, "_tools"))@@ -120,6 +122,7 @@ def test_provider_session_excludes_screenshot_preview_when_chromium_unavailable(openai_base_url=None,anthropic_api_key=None,gemini_api_key=None,+ replicate_api_key=None,)tools = cast(list[dict[str, Any]], getattr(session, "_tools"))@@ -141,6 +144,7 @@ def test_provider_session_includes_screenshot_preview_when_chromium_available(openai_base_url=None,anthropic_api_key=None,gemini_api_key=None,+ replicate_api_key=None,)tools = cast(list[dict[str, Any]], getattr(session, "_tools"))diff --git a/backend/tests/test_parameter_extraction_stage.py b/backend/tests/test_parameter_extraction_stage.pyindex 149dfcc5..d275783a 100644--- a/backend/tests/test_parameter_extraction_stage.py+++ b/backend/tests/test_parameter_extraction_stage.py@@ -39,6 +39,40 @@ async def test_extracts_gemini_api_key_from_env_when_not_in_request(monkeypatch:assert extracted.gemini_api_key == "gemini-from-env"+async def test_extracts_replicate_api_key_from_settings_dialog() -> None:+ stage = ParameterExtractionStage(AsyncMock())++ extracted = await stage.extract_and_validate(+ {+ "generatedCodeConfig": "html_tailwind",+ "inputMode": "text",+ "replicateApiKey": "replicate-from-ui",+ "prompt": {"text": "hello"},+ }+ )++ assert extracted.replicate_api_key == "replicate-from-ui"+++async def test_extracts_replicate_api_key_from_env_when_not_in_request(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr("routes.generate_code.REPLICATE_API_KEY", "replicate-from-env")+ stage = ParameterExtractionStage(AsyncMock())++ extracted = await stage.extract_and_validate(+ {+ "generatedCodeConfig": "html_tailwind",+ "inputMode": "text",+ "prompt": {"text": "hello"},+ }+ )++ assert extracted.replicate_api_key == "replicate-from-env"++@pytest.mark.asyncioasync def test_extracts_design_system_from_request() -> None:stage = ParameterExtractionStage(AsyncMock())diff --git a/backend/tests/test_status_broadcast.py b/backend/tests/test_status_broadcast.pyindex 6036ebdb..9345af74 100644--- a/backend/tests/test_status_broadcast.py+++ b/backend/tests/test_status_broadcast.py@@ -39,6 +39,7 @@ async def test_video_update_broadcasts_two_variants() -> None:openai_api_key=None,anthropic_api_key=None,gemini_api_key="key",+ replicate_api_key=None,openai_base_url=None,generation_type="update",prompt={"text": "Edit this video output", "images": [], "videos": []},@@ -91,6 +92,7 @@ async def test_image_update_broadcasts_two_variants() -> None:openai_api_key="key",anthropic_api_key="key",gemini_api_key=None,+ replicate_api_key=None,openai_base_url=None,generation_type="update",prompt={"text": "Edit this screenshot", "images": ["data:image/png;base64,abc"], "videos": []},diff --git a/backend/tests/test_tool_image_outputs.py b/backend/tests/test_tool_image_outputs.pyindex 12581f20..aa9cf502 100644--- a/backend/tests/test_tool_image_outputs.py+++ b/backend/tests/test_tool_image_outputs.py@@ -223,6 +223,42 @@ async def test_generate_images_emits_url_parts(assert result.multimodal_parts[0].data is None+async def test_generate_images_uses_replicate_key_from_settings(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr("agent.tools.runtime.REPLICATE_API_KEY", None)+ captured: dict[str, Any] = {}++ async def fake_process_tasks(+ prompts: List[str], api_key: str, base_url: Any, model: str+ ) -> List[str]:+ captured["api_key"] = api_key+ captured["base_url"] = base_url+ captured["model"] = model+ return [f"https://replicate.delivery/{p}.png" for p in prompts]++ monkeypatch.setattr("agent.tools.runtime.process_tasks", fake_process_tasks)+ runtime = AgentToolRuntime(+ file_state=AgentFileState(),+ should_generate_images=True,+ openai_api_key=None,+ openai_base_url=None,+ replicate_api_key="replicate-from-ui",+ )++ result = await runtime.execute(+ ToolCall(id="t", name="generate_images", arguments={"prompts": ["a logo"]})+ )++ assert result.ok is True+ assert captured == {+ "api_key": "replicate-from-ui",+ "base_url": None,+ "model": "flux",+ }++@pytest.mark.asyncioasync def test_edit_image_emits_url_part(monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setattr("agent.tools.runtime.REPLICATE_API_KEY", "fake-key")@@ -248,6 +284,38 @@ async def test_edit_image_emits_url_part(monkeypatch: pytest.MonkeyPatch) -> Nonassert result.multimodal_parts[0].image_url == "https://replicate.delivery/edited.png"+async def test_edit_image_uses_replicate_key_from_settings(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ monkeypatch.setattr("agent.tools.runtime.REPLICATE_API_KEY", None)+ captured: dict[str, Any] = {}++ async def fake_edit_image(**kwargs: Any) -> str:+ captured.update(kwargs)+ return "https://replicate.delivery/edited.png"++ monkeypatch.setattr("agent.tools.runtime.edit_image", fake_edit_image)+ runtime = AgentToolRuntime(+ file_state=AgentFileState(),+ should_generate_images=True,+ openai_api_key=None,+ openai_base_url=None,+ replicate_api_key="replicate-from-ui",+ )++ result = await runtime.execute(+ ToolCall(+ id="t",+ name="edit_image",+ arguments={"prompt": "bw", "image_urls": ["https://x/in.png"]},+ )+ )++ assert result.ok is True+ assert captured["api_token"] == "replicate-from-ui"++@pytest.mark.asyncioasync def test_save_assets_emits_byte_parts(monkeypatch: pytest.MonkeyPatch, tmp_path: Pathdiff --git a/frontend/src/App.tsx b/frontend/src/App.tsxindex 85e046e4..773c2023 100644--- a/frontend/src/App.tsx+++ b/frontend/src/App.tsx@@ -90,6 +90,7 @@ function App() {{openAiApiKey: null,openAiBaseURL: null,+ replicateApiKey: null,anthropicApiKey: null,geminiApiKey: null,screenshotOneApiKey: null,diff --git a/frontend/src/components/settings/SettingsTab.tsx b/frontend/src/components/settings/SettingsTab.tsxindex bff7f640..9e5e5afa 100644--- a/frontend/src/components/settings/SettingsTab.tsx+++ b/frontend/src/components/settings/SettingsTab.tsx@@ -223,6 +223,30 @@ function SettingsTab({ settings, setSettings, appTheme, setAppTheme }: Props) {}/></div>++ {!IS_RUNNING_ON_CLOUD && (+ <div>+ <p className="text-sm font-medium text-gray-700 dark:text-zinc-300">+ Replicate API key+ </p>+ <p className="mt-1 text-xs text-gray-500 dark:text-zinc-400">+ Only stored in your browser. Never stored on servers. Overrides+ your .env config for image generation and editing.+ </p>+ <Input+ id="replicate-api-key"+ className="mt-2"+ placeholder="Replicate API key"+ value={settings.replicateApiKey || ""}+ onChange={(e) =>+ setSettings((s) => ({+ ...s,+ replicateApiKey: e.target.value,+ }))+ }+ />+ </div>+ )}</div></div>diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex 8a70ef76..57d6420a 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -15,6 +15,7 @@ export enum AppTheme {export interface Settings {openAiApiKey: string | null;openAiBaseURL: string | null;+ replicateApiKey: string | null;screenshotOneApiKey: string | null;isImageGenerationEnabled: boolean;editorTheme: EditorTheme;