Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Support Replicate key in settings (#613)

Abi Raja02303fa52416
diff --git a/backend/agent/engine.py b/backend/agent/engine.py
index 18edfabe..e0bd675d 100644
--- a/backend/agent/engine.py
+++ b/backend/agent/engine.py
@@ -31,6 +31,7 @@ class AgentEngine:
openai_base_url: Optional[str],
anthropic_api_key: Optional[str],
gemini_api_key: Optional[str],
+ replicate_api_key: Optional[str],
should_generate_images: bool,
asset_base_url: str = "",
initial_file_state: Optional[Dict[str, str]] = None,
@@ -42,6 +43,7 @@ class AgentEngine:
self.openai_base_url = openai_base_url
self.anthropic_api_key = anthropic_api_key
self.gemini_api_key = gemini_api_key
+ self.replicate_api_key = replicate_api_key
self.should_generate_images = should_generate_images
self.file_state = AgentFileState()
@@ -55,6 +57,7 @@ class AgentEngine:
openai_api_key=openai_api_key,
openai_base_url=openai_base_url,
gemini_api_key=gemini_api_key,
+ replicate_api_key=replicate_api_key,
asset_base_url=asset_base_url,
option_codes=option_codes,
)
@@ -260,6 +263,7 @@ class AgentEngine:
openai_base_url=self.openai_base_url,
anthropic_api_key=self.anthropic_api_key,
gemini_api_key=self.gemini_api_key,
+ replicate_api_key=self.replicate_api_key,
)
try:
return await self._run_with_session(session)
diff --git a/backend/agent/providers/factory.py b/backend/agent/providers/factory.py
index 35216bdd..43f47bff 100644
--- a/backend/agent/providers/factory.py
+++ b/backend/agent/providers/factory.py
@@ -23,11 +23,12 @@ def create_provider_session(
openai_base_url: Optional[str],
anthropic_api_key: Optional[str],
gemini_api_key: Optional[str],
+ replicate_api_key: Optional[str],
) -> ProviderSession:
canonical_tools = canonical_tool_definitions(
image_generation_enabled=should_generate_images,
# The edit_image tool calls Replicate, so don't offer it without a key.
- image_editing_enabled=bool(REPLICATE_API_KEY),
+ image_editing_enabled=bool(replicate_api_key or REPLICATE_API_KEY),
# The extract_assets tool calls Gemini, so don't offer it without a key.
asset_extraction_enabled=bool(gemini_api_key),
# screenshot_preview needs headless Chromium; skip it if it can't launch.
diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.py
index 91bfd662..59215ba9 100644
--- a/backend/agent/tools/runtime.py
+++ b/backend/agent/tools/runtime.py
@@ -30,6 +30,7 @@ class AgentToolRuntime:
openai_api_key: Optional[str],
openai_base_url: Optional[str],
gemini_api_key: Optional[str] = None,
+ replicate_api_key: Optional[str] = None,
input_images: Optional[List[str]] = None,
asset_base_url: str = "",
user_id: Optional[str] = None,
@@ -40,11 +41,15 @@ class AgentToolRuntime:
self.openai_api_key = openai_api_key
self.openai_base_url = openai_base_url
self.gemini_api_key = gemini_api_key
+ self.replicate_api_key = replicate_api_key
self.input_images = input_images or []
self.asset_base_url = asset_base_url
self.user_id = user_id
self.option_codes = option_codes or []
+ def _effective_replicate_api_key(self) -> str | None:
+ return self.replicate_api_key or REPLICATE_API_KEY
+
async def execute(self, tool_call: ToolCall) -> ToolExecutionResult:
if "INVALID_JSON" in tool_call.arguments:
invalid_json = ensure_str(tool_call.arguments.get("INVALID_JSON"))
@@ -272,9 +277,10 @@ class AgentToolRuntime:
result={"error": "No valid prompts provided"},
summary={"error": "No valid prompts"},
)
- if REPLICATE_API_KEY:
+ replicate_api_key = self._effective_replicate_api_key()
+ if replicate_api_key:
model = "flux"
- api_key = REPLICATE_API_KEY
+ api_key = replicate_api_key
base_url = None
else:
if not self.openai_api_key:
@@ -318,7 +324,8 @@ class AgentToolRuntime:
)
async def _remove_background(self, args: Dict[str, Any]) -> ToolExecutionResult:
- if not REPLICATE_API_KEY:
+ replicate_api_key = self._effective_replicate_api_key()
+ if not replicate_api_key:
return ToolExecutionResult(
ok=False,
result={"error": "Background removal requires REPLICATE_API_KEY."},
@@ -350,7 +357,7 @@ class AgentToolRuntime:
batch = unique_urls[i : i + batch_size]
# Replicate can't fetch localhost; inline local assets as data URLs.
tasks = [
- remove_background(local_asset_url_to_data_url(url), REPLICATE_API_KEY)
+ remove_background(local_asset_url_to_data_url(url), replicate_api_key)
for url in batch
]
raw_results.extend(await asyncio.gather(*tasks, return_exceptions=True))
@@ -392,7 +399,8 @@ class AgentToolRuntime:
)
async def _edit_image(self, args: Dict[str, Any]) -> ToolExecutionResult:
- if not REPLICATE_API_KEY:
+ replicate_api_key = self._effective_replicate_api_key()
+ if not replicate_api_key:
return ToolExecutionResult(
ok=False,
result={"error": "Image editing requires REPLICATE_API_KEY."},
@@ -433,7 +441,7 @@ class AgentToolRuntime:
result_url = await edit_image(
prompt=prompt,
image_urls=[local_asset_url_to_data_url(url) for url in unique_urls],
- api_token=REPLICATE_API_KEY,
+ api_token=replicate_api_key,
aspect_ratio=aspect_ratio,
)
except Exception as exc:
diff --git a/backend/evals/core.py b/backend/evals/core.py
index 1e1495a8..089e2b2d 100644
--- a/backend/evals/core.py
+++ b/backend/evals/core.py
@@ -4,6 +4,7 @@ from config import (
LOCAL_ASSET_BASE_URL,
OPENAI_API_KEY,
OPENAI_BASE_URL,
+ REPLICATE_API_KEY,
)
from llm import Llm, OPENAI_MODELS, ANTHROPIC_MODELS, GEMINI_MODELS
from agent.runner import Agent
@@ -46,6 +47,7 @@ async def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> s
openai_base_url=OPENAI_BASE_URL,
anthropic_api_key=ANTHROPIC_API_KEY,
gemini_api_key=GEMINI_API_KEY,
+ replicate_api_key=REPLICATE_API_KEY,
should_generate_images=True,
# No websocket to infer the host from, so use the configured base URL;
# otherwise extracted/saved assets get hostless /local-assets/ URLs.
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index 5408cdca..f9401e7b 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -255,6 +255,7 @@ class ExtractedParams:
openai_api_key: str | None
anthropic_api_key: str | None
gemini_api_key: str | None
+ replicate_api_key: str | None
openai_base_url: str | None
generation_type: Literal["create", "update"]
prompt: UserTurnInput
@@ -305,6 +306,9 @@ class ParameterExtractionStage:
gemini_api_key = self._get_from_settings_dialog_or_env(
params, "geminiApiKey", GEMINI_API_KEY
)
+ replicate_api_key = self._get_from_settings_dialog_or_env(
+ params, "replicateApiKey", REPLICATE_API_KEY
+ )
# Base URL for OpenAI API
openai_base_url: str | None = None
@@ -371,6 +375,7 @@ class ParameterExtractionStage:
openai_api_key=openai_api_key,
anthropic_api_key=anthropic_api_key,
gemini_api_key=gemini_api_key,
+ replicate_api_key=replicate_api_key,
openai_base_url=openai_base_url,
generation_type=generation_type,
prompt=prompt,
@@ -545,6 +550,7 @@ class AgenticGenerationStage:
openai_base_url: str | None,
anthropic_api_key: str | None,
gemini_api_key: str | None,
+ replicate_api_key: str | None,
should_generate_images: bool,
file_state: Dict[str, str] | None,
asset_base_url: str,
@@ -555,6 +561,7 @@ class AgenticGenerationStage:
self.openai_base_url = openai_base_url
self.anthropic_api_key = anthropic_api_key
self.gemini_api_key = gemini_api_key
+ self.replicate_api_key = replicate_api_key
self.should_generate_images = should_generate_images
self.file_state = file_state
self.asset_base_url = asset_base_url
@@ -613,6 +620,7 @@ class AgenticGenerationStage:
openai_base_url=self.openai_base_url,
anthropic_api_key=self.anthropic_api_key,
gemini_api_key=self.gemini_api_key,
+ replicate_api_key=self.replicate_api_key,
should_generate_images=self.should_generate_images,
asset_base_url=self.asset_base_url,
initial_file_state=self.file_state,
@@ -794,6 +802,7 @@ class CodeGenerationMiddleware(Middleware):
openai_base_url=context.extracted_params.openai_base_url,
anthropic_api_key=context.extracted_params.anthropic_api_key,
gemini_api_key=context.extracted_params.gemini_api_key,
+ replicate_api_key=context.extracted_params.replicate_api_key,
should_generate_images=context.extracted_params.should_generate_images,
file_state=context.extracted_params.file_state,
asset_base_url=context.extracted_params.asset_base_url,
diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.py
index 2e0306d5..ef5dd01d 100644
--- a/backend/tests/test_agent_tools.py
+++ b/backend/tests/test_agent_tools.py
@@ -83,6 +83,7 @@ def test_provider_session_excludes_extract_assets_without_gemini_key() -> None:
openai_base_url=None,
anthropic_api_key=None,
gemini_api_key=None,
+ replicate_api_key=None,
)
tools = cast(list[dict[str, Any]], getattr(session, "_tools"))
@@ -99,6 +100,7 @@ def test_provider_session_includes_extract_assets_with_gemini_key() -> None:
openai_base_url=None,
anthropic_api_key=None,
gemini_api_key="gemini-key",
+ replicate_api_key=None,
)
tools = cast(list[dict[str, Any]], getattr(session, "_tools"))
@@ -120,6 +122,7 @@ def test_provider_session_excludes_screenshot_preview_when_chromium_unavailable(
openai_base_url=None,
anthropic_api_key=None,
gemini_api_key=None,
+ replicate_api_key=None,
)
tools = cast(list[dict[str, Any]], getattr(session, "_tools"))
@@ -141,6 +144,7 @@ def test_provider_session_includes_screenshot_preview_when_chromium_available(
openai_base_url=None,
anthropic_api_key=None,
gemini_api_key=None,
+ replicate_api_key=None,
)
tools = cast(list[dict[str, Any]], getattr(session, "_tools"))
diff --git a/backend/tests/test_parameter_extraction_stage.py b/backend/tests/test_parameter_extraction_stage.py
index 149dfcc5..d275783a 100644
--- a/backend/tests/test_parameter_extraction_stage.py
+++ b/backend/tests/test_parameter_extraction_stage.py
@@ -39,6 +39,40 @@ async def test_extracts_gemini_api_key_from_env_when_not_in_request(monkeypatch:
assert extracted.gemini_api_key == "gemini-from-env"
+async def test_extracts_replicate_api_key_from_settings_dialog() -> None:
+ stage = ParameterExtractionStage(AsyncMock())
+
+ extracted = await stage.extract_and_validate(
+ {
+ "generatedCodeConfig": "html_tailwind",
+ "inputMode": "text",
+ "replicateApiKey": "replicate-from-ui",
+ "prompt": {"text": "hello"},
+ }
+ )
+
+ assert extracted.replicate_api_key == "replicate-from-ui"
+
+
+async def test_extracts_replicate_api_key_from_env_when_not_in_request(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr("routes.generate_code.REPLICATE_API_KEY", "replicate-from-env")
+ stage = ParameterExtractionStage(AsyncMock())
+
+ extracted = await stage.extract_and_validate(
+ {
+ "generatedCodeConfig": "html_tailwind",
+ "inputMode": "text",
+ "prompt": {"text": "hello"},
+ }
+ )
+
+ assert extracted.replicate_api_key == "replicate-from-env"
+
+
@pytest.mark.asyncio
async def test_extracts_design_system_from_request() -> None:
stage = ParameterExtractionStage(AsyncMock())
diff --git a/backend/tests/test_status_broadcast.py b/backend/tests/test_status_broadcast.py
index 6036ebdb..9345af74 100644
--- a/backend/tests/test_status_broadcast.py
+++ b/backend/tests/test_status_broadcast.py
@@ -39,6 +39,7 @@ async def test_video_update_broadcasts_two_variants() -> None:
openai_api_key=None,
anthropic_api_key=None,
gemini_api_key="key",
+ replicate_api_key=None,
openai_base_url=None,
generation_type="update",
prompt={"text": "Edit this video output", "images": [], "videos": []},
@@ -91,6 +92,7 @@ async def test_image_update_broadcasts_two_variants() -> None:
openai_api_key="key",
anthropic_api_key="key",
gemini_api_key=None,
+ replicate_api_key=None,
openai_base_url=None,
generation_type="update",
prompt={"text": "Edit this screenshot", "images": ["data:image/png;base64,abc"], "videos": []},
diff --git a/backend/tests/test_tool_image_outputs.py b/backend/tests/test_tool_image_outputs.py
index 12581f20..aa9cf502 100644
--- a/backend/tests/test_tool_image_outputs.py
+++ b/backend/tests/test_tool_image_outputs.py
@@ -223,6 +223,42 @@ async def test_generate_images_emits_url_parts(
assert result.multimodal_parts[0].data is None
+async def test_generate_images_uses_replicate_key_from_settings(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr("agent.tools.runtime.REPLICATE_API_KEY", None)
+ captured: dict[str, Any] = {}
+
+ async def fake_process_tasks(
+ prompts: List[str], api_key: str, base_url: Any, model: str
+ ) -> List[str]:
+ captured["api_key"] = api_key
+ captured["base_url"] = base_url
+ captured["model"] = model
+ return [f"https://replicate.delivery/{p}.png" for p in prompts]
+
+ monkeypatch.setattr("agent.tools.runtime.process_tasks", fake_process_tasks)
+ runtime = AgentToolRuntime(
+ file_state=AgentFileState(),
+ should_generate_images=True,
+ openai_api_key=None,
+ openai_base_url=None,
+ replicate_api_key="replicate-from-ui",
+ )
+
+ result = await runtime.execute(
+ ToolCall(id="t", name="generate_images", arguments={"prompts": ["a logo"]})
+ )
+
+ assert result.ok is True
+ assert captured == {
+ "api_key": "replicate-from-ui",
+ "base_url": None,
+ "model": "flux",
+ }
+
+
@pytest.mark.asyncio
async def test_edit_image_emits_url_part(monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setattr("agent.tools.runtime.REPLICATE_API_KEY", "fake-key")
@@ -248,6 +284,38 @@ async def test_edit_image_emits_url_part(monkeypatch: pytest.MonkeyPatch) -> Non
assert result.multimodal_parts[0].image_url == "https://replicate.delivery/edited.png"
+async def test_edit_image_uses_replicate_key_from_settings(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ monkeypatch.setattr("agent.tools.runtime.REPLICATE_API_KEY", None)
+ captured: dict[str, Any] = {}
+
+ async def fake_edit_image(**kwargs: Any) -> str:
+ captured.update(kwargs)
+ return "https://replicate.delivery/edited.png"
+
+ monkeypatch.setattr("agent.tools.runtime.edit_image", fake_edit_image)
+ runtime = AgentToolRuntime(
+ file_state=AgentFileState(),
+ should_generate_images=True,
+ openai_api_key=None,
+ openai_base_url=None,
+ replicate_api_key="replicate-from-ui",
+ )
+
+ result = await runtime.execute(
+ ToolCall(
+ id="t",
+ name="edit_image",
+ arguments={"prompt": "bw", "image_urls": ["https://x/in.png"]},
+ )
+ )
+
+ assert result.ok is True
+ assert captured["api_token"] == "replicate-from-ui"
+
+
@pytest.mark.asyncio
async def test_save_assets_emits_byte_parts(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index 85e046e4..773c2023 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -90,6 +90,7 @@ function App() {
{
openAiApiKey: null,
openAiBaseURL: null,
+ replicateApiKey: null,
anthropicApiKey: null,
geminiApiKey: null,
screenshotOneApiKey: null,
diff --git a/frontend/src/components/settings/SettingsTab.tsx b/frontend/src/components/settings/SettingsTab.tsx
index bff7f640..9e5e5afa 100644
--- a/frontend/src/components/settings/SettingsTab.tsx
+++ b/frontend/src/components/settings/SettingsTab.tsx
@@ -223,6 +223,30 @@ function SettingsTab({ settings, setSettings, appTheme, setAppTheme }: Props) {
}
/>
</div>
+
+ {!IS_RUNNING_ON_CLOUD && (
+ <div>
+ <p className="text-sm font-medium text-gray-700 dark:text-zinc-300">
+ Replicate API key
+ </p>
+ <p className="mt-1 text-xs text-gray-500 dark:text-zinc-400">
+ Only stored in your browser. Never stored on servers. Overrides
+ your .env config for image generation and editing.
+ </p>
+ <Input
+ id="replicate-api-key"
+ className="mt-2"
+ placeholder="Replicate API key"
+ value={settings.replicateApiKey || ""}
+ onChange={(e) =>
+ setSettings((s) => ({
+ ...s,
+ replicateApiKey: e.target.value,
+ }))
+ }
+ />
+ </div>
+ )}
</div>
</div>
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 8a70ef76..57d6420a 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -15,6 +15,7 @@ export enum AppTheme {
export interface Settings {
openAiApiKey: string | null;
openAiBaseURL: string | null;
+ replicateApiKey: string | null;
screenshotOneApiKey: string | null;
isImageGenerationEnabled: boolean;
editorTheme: EditorTheme;