Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Add edit_image agent tool backed by Replicate

Abi Raja33f2d07a4d0b
diff --git a/backend/agent/providers/factory.py b/backend/agent/providers/factory.py
index 91574102..48f326da 100644
--- a/backend/agent/providers/factory.py
+++ b/backend/agent/providers/factory.py
@@ -10,6 +10,7 @@ from agent.providers.base import ProviderSession
from agent.providers.gemini import GeminiProviderSession, serialize_gemini_tools
from agent.providers.openai import OpenAIProviderSession, serialize_openai_tools
from agent.tools import canonical_tool_definitions
+from config import REPLICATE_API_KEY
from llm import ANTHROPIC_MODELS, GEMINI_MODELS, OPENAI_MODELS, Llm
@@ -23,7 +24,9 @@ def create_provider_session(
gemini_api_key: Optional[str],
) -> ProviderSession:
canonical_tools = canonical_tool_definitions(
- image_generation_enabled=should_generate_images
+ image_generation_enabled=should_generate_images,
+ # The edit_image tool calls Replicate, so don't offer it without a key.
+ image_editing_enabled=bool(REPLICATE_API_KEY),
)
if model in OPENAI_MODELS:
diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.py
index 1b796aac..b72aa1cf 100644
--- a/backend/agent/tools/definitions.py
+++ b/backend/agent/tools/definitions.py
@@ -89,6 +89,50 @@ def _remove_background_schema() -> Dict[str, Any]:
}
+def _edit_image_schema() -> Dict[str, Any]:
+ return {
+ "type": "object",
+ "properties": {
+ "prompt": {
+ "type": "string",
+ "description": (
+ "Clear edit instruction. Refer to inputs as image 1, image 2, "
+ "and so on when multiple images are provided."
+ ),
+ },
+ "image_urls": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "description": (
+ "URL of a source/reference image. For editing, put the main "
+ "image first."
+ ),
+ },
+ },
+ "turbo": {
+ "type": "boolean",
+ "description": (
+ "Run faster with additional optimizations. Set false for more "
+ "complicated edits."
+ ),
+ },
+ "aspect_ratio": {
+ "type": "string",
+ "description": (
+ "Aspect ratio for the edited image. Use match_input_image to "
+ "match the first image."
+ ),
+ },
+ "seed": {
+ "type": "integer",
+ "description": "Optional random seed for reproducible edits.",
+ },
+ },
+ "required": ["prompt", "image_urls"],
+ }
+
+
def _retrieve_option_schema() -> Dict[str, Any]:
return {
"type": "object",
@@ -104,6 +148,7 @@ def _retrieve_option_schema() -> Dict[str, Any]:
def canonical_tool_definitions(
image_generation_enabled: bool = True,
+ image_editing_enabled: bool = True,
) -> List[CanonicalToolDefinition]:
tools: List[CanonicalToolDefinition] = [
CanonicalToolDefinition(
@@ -146,6 +191,22 @@ def canonical_tool_definitions(
),
parameters=_remove_background_schema(),
),
+ ]
+ )
+ if image_editing_enabled:
+ tools.append(
+ CanonicalToolDefinition(
+ name="edit_image",
+ description=(
+ "Edit one or more images using a text prompt. Provide the main "
+ "image first, followed by optional reference images. Returns a URL "
+ "to the edited image."
+ ),
+ parameters=_edit_image_schema(),
+ )
+ )
+ tools.extend(
+ [
SAVE_ASSETS_TOOL_DEFINITION,
CanonicalToolDefinition(
name="retrieve_option",
diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.py
index b2c2b0dd..49198c3f 100644
--- a/backend/agent/tools/runtime.py
+++ b/backend/agent/tools/runtime.py
@@ -1,12 +1,16 @@
# pyright: reportUnknownVariableType=false
import asyncio
+import base64
import difflib
+import mimetypes
+import os
+from urllib.parse import unquote, urlparse
from typing import Any, Dict, List, Optional, Tuple, Union
from codegen.utils import extract_html_content
-from config import REPLICATE_API_KEY
+from config import LOCAL_ASSET_DIR, REPLICATE_API_KEY
from image_generation.generation import process_tasks
-from image_generation.replicate import remove_background
+from image_generation.replicate import edit_image, remove_background
from uploaded_assets.tools import run_save_assets
from agent.state import AgentFileState, ensure_str
@@ -14,6 +18,34 @@ from agent.tools.types import ToolCall, ToolExecutionResult
from agent.tools.summaries import summarize_text
+LOCAL_ASSET_HOSTS = {"127.0.0.1", "localhost", "::1"}
+
+
+def _local_asset_url_to_data_url(image_url: str) -> str:
+ """Replicate can't fetch localhost URLs, so inline local assets as data URLs."""
+ parsed = urlparse(image_url)
+ if parsed.scheme not in {"http", "https"}:
+ return image_url
+ if parsed.hostname not in LOCAL_ASSET_HOSTS:
+ return image_url
+ if not parsed.path.startswith("/local-assets/"):
+ return image_url
+
+ relative_path = unquote(parsed.path.removeprefix("/local-assets/"))
+ asset_root = os.path.abspath(LOCAL_ASSET_DIR)
+ asset_path = os.path.abspath(os.path.join(asset_root, relative_path))
+ if not asset_path.startswith(asset_root + os.sep):
+ return image_url
+ if not os.path.isfile(asset_path):
+ return image_url
+
+ content_type = mimetypes.guess_type(asset_path)[0] or "image/png"
+ with open(asset_path, "rb") as file:
+ encoded = base64.b64encode(file.read()).decode("ascii")
+
+ return f"data:{content_type};base64,{encoded}"
+
+
class AgentToolRuntime:
def __init__(
self,
@@ -51,6 +83,8 @@ class AgentToolRuntime:
return await self._generate_images(tool_call.arguments)
if tool_call.name == "remove_background":
return await self._remove_background(tool_call.arguments)
+ if tool_call.name == "edit_image":
+ return await self._edit_image(tool_call.arguments)
if tool_call.name == "save_assets":
return await run_save_assets(tool_call.arguments, user_id=self.user_id)
if tool_call.name == "retrieve_option":
@@ -334,6 +368,102 @@ class AgentToolRuntime:
summary={"images": summary_items},
)
+ async def _edit_image(self, args: Dict[str, Any]) -> ToolExecutionResult:
+ if not REPLICATE_API_KEY:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "Image editing requires REPLICATE_API_KEY."},
+ summary={"error": "Missing Replicate API key"},
+ )
+
+ prompt = ensure_str(args.get("prompt")).strip()
+ if not prompt:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "edit_image requires a non-empty prompt"},
+ summary={"error": "Missing prompt"},
+ )
+
+ image_urls = args.get("image_urls") or args.get("images") or []
+ if not isinstance(image_urls, list) or not image_urls:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "edit_image requires a non-empty image_urls list"},
+ summary={"error": "Missing image_urls"},
+ )
+
+ cleaned = [url.strip() for url in image_urls if isinstance(url, str)]
+ unique_urls = list(dict.fromkeys([u for u in cleaned if u]))
+ if not unique_urls:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "No valid image URLs provided"},
+ summary={"error": "No valid image_urls"},
+ )
+
+ turbo = args.get("turbo")
+ if not isinstance(turbo, bool):
+ turbo = True
+
+ aspect_ratio = ensure_str(args.get("aspect_ratio") or "match_input_image")
+ seed_value = args.get("seed")
+ seed = seed_value if isinstance(seed_value, int) else None
+
+ try:
+ result_url = await edit_image(
+ prompt=prompt,
+ image_urls=[_local_asset_url_to_data_url(url) for url in unique_urls],
+ api_token=REPLICATE_API_KEY,
+ turbo=turbo,
+ aspect_ratio=aspect_ratio,
+ seed=seed,
+ )
+ except Exception as exc:
+ print(f"Image edit failed for {unique_urls}: {exc}")
+ return ToolExecutionResult(
+ ok=True,
+ result={
+ "image": {
+ "prompt": prompt,
+ "image_urls": unique_urls,
+ "result_url": None,
+ "status": "error",
+ }
+ },
+ summary={
+ "image": {
+ "prompt": summarize_text(prompt, 240),
+ "image_urls": [summarize_text(url, 100) for url in unique_urls],
+ "result_url": None,
+ "status": "error",
+ }
+ },
+ )
+
+ result = {
+ "image": {
+ "prompt": prompt,
+ "image_urls": unique_urls,
+ "result_url": result_url,
+ "status": "ok",
+ "turbo": turbo,
+ "aspect_ratio": aspect_ratio,
+ **({"seed": seed} if seed is not None else {}),
+ }
+ }
+ summary = {
+ "image": {
+ "prompt": summarize_text(prompt, 240),
+ "image_urls": [summarize_text(url, 100) for url in unique_urls],
+ "result_url": result_url,
+ "status": "ok",
+ "turbo": turbo,
+ "aspect_ratio": aspect_ratio,
+ **({"seed": seed} if seed is not None else {}),
+ }
+ }
+ return ToolExecutionResult(ok=True, result=result, summary=summary)
+
def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:
raw_option_number = args.get("option_number")
raw_index = args.get("index")
diff --git a/backend/agent/tools/summaries.py b/backend/agent/tools/summaries.py
index bee522e6..2af8decf 100644
--- a/backend/agent/tools/summaries.py
+++ b/backend/agent/tools/summaries.py
@@ -64,6 +64,19 @@ def summarize_tool_input(tool_call: ToolCall, file_state: AgentFileState) -> Dic
}
return {"image_urls": []}
+ if tool_call.name == "edit_image":
+ image_urls = args.get("image_urls") or args.get("images") or []
+ if isinstance(image_urls, list):
+ return {
+ "count": len(image_urls),
+ "prompt": ensure_str(args.get("prompt")),
+ "image_urls": [ensure_str(u) for u in image_urls],
+ "turbo": args.get("turbo"),
+ "aspect_ratio": args.get("aspect_ratio"),
+ "seed": args.get("seed"),
+ }
+ return {"prompt": ensure_str(args.get("prompt")), "image_urls": []}
+
if tool_call.name == "save_assets":
return summarize_save_assets_input(args)
diff --git a/backend/image_generation/replicate.py b/backend/image_generation/replicate.py
index d2116962..e10f021e 100644
--- a/backend/image_generation/replicate.py
+++ b/backend/image_generation/replicate.py
@@ -7,6 +7,7 @@ REPLICATE_API_BASE_URL = "https://api.replicate.com/v1"
ReplicateImageModel = Literal["z_image_turbo", "flux_2_klein"]
Z_IMAGE_TURBO_MODEL_PATH = "prunaai/z-image-turbo"
FLUX_2_KLEIN_MODEL_PATH = "black-forest-labs/flux-2-klein-4b"
+P_IMAGE_EDIT_MODEL_PATH = "prunaai/p-image-edit"
MODEL_PATHS: dict[ReplicateImageModel, str] = {
"z_image_turbo": Z_IMAGE_TURBO_MODEL_PATH,
"flux_2_klein": FLUX_2_KLEIN_MODEL_PATH,
@@ -143,6 +144,28 @@ async def remove_background(image_url: str, api_token: str) -> str:
return _extract_output_url(result, "background remover")
+async def edit_image(
+ prompt: str,
+ image_urls: list[str],
+ api_token: str,
+ *,
+ turbo: bool = True,
+ aspect_ratio: str = "match_input_image",
+ seed: int | None = None,
+) -> str:
+ input: dict[str, Any] = {
+ "prompt": prompt,
+ "images": image_urls,
+ "turbo": turbo,
+ "aspect_ratio": aspect_ratio,
+ }
+ if seed is not None:
+ input["seed"] = seed
+
+ result = await call_replicate_model(P_IMAGE_EDIT_MODEL_PATH, input, api_token)
+ return _extract_output_url(result, "Replicate image edit model prediction")
+
+
async def call_replicate(
input: dict[str, str | int | float | bool],
api_token: str,
diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.py
index c972f57b..43852d82 100644
--- a/backend/tests/test_agent_tools.py
+++ b/backend/tests/test_agent_tools.py
@@ -45,3 +45,46 @@ def test_save_assets_tool_input_summary_uses_asset_ids() -> None:
"count": 2,
"asset_ids": ["asset_one", "asset_two"],
}
+
+
+def test_canonical_tool_definitions_include_edit_image() -> None:
+ tools = canonical_tool_definitions(True)
+ tool_names = [tool.name for tool in tools]
+ assert "edit_image" in tool_names
+ edit_image_tool = next(tool for tool in tools if tool.name == "edit_image")
+ assert edit_image_tool.parameters["required"] == ["prompt", "image_urls"]
+ assert edit_image_tool.parameters["properties"]["image_urls"]["type"] == "array"
+
+
+def test_canonical_tool_definitions_exclude_edit_image_when_disabled() -> None:
+ tool_names = [
+ tool.name
+ for tool in canonical_tool_definitions(True, image_editing_enabled=False)
+ ]
+ assert "edit_image" not in tool_names
+
+
+def test_edit_image_tool_input_summary_uses_prompt_and_image_urls() -> None:
+ summary = summarize_tool_input(
+ ToolCall(
+ id="call-1",
+ name="edit_image",
+ arguments={
+ "prompt": "Make image 1 monochrome",
+ "image_urls": ["https://example.com/input.png"],
+ "turbo": False,
+ "aspect_ratio": "match_input_image",
+ "seed": 123,
+ },
+ ),
+ AgentFileState(),
+ )
+
+ assert summary == {
+ "count": 1,
+ "prompt": "Make image 1 monochrome",
+ "image_urls": ["https://example.com/input.png"],
+ "turbo": False,
+ "aspect_ratio": "match_input_image",
+ "seed": 123,
+ }
diff --git a/backend/tests/test_image_generation_replicate.py b/backend/tests/test_image_generation_replicate.py
index deb165a0..25a6ce13 100644
--- a/backend/tests/test_image_generation_replicate.py
+++ b/backend/tests/test_image_generation_replicate.py
@@ -107,3 +107,40 @@ async def test_remove_background_uses_version_and_normalizes_output(
assert result == "https://example.com/no-bg.png"
assert captured["version"] == replicate.REMOVE_BACKGROUND_VERSION
assert captured["api_token"] == "token"
+
+
+async def test_edit_image_uses_pruna_model_and_normalizes_output(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ captured: dict[str, object] = {}
+
+ async def fake_call_replicate_model(
+ model_path: str, input: dict[str, object], api_token: str
+ ) -> dict[str, str]:
+ captured["model_path"] = model_path
+ captured["input"] = input
+ captured["api_token"] = api_token
+ return {"url": "https://example.com/edited.png"}
+
+ monkeypatch.setattr(replicate, "call_replicate_model", fake_call_replicate_model)
+
+ result = await replicate.edit_image(
+ prompt="Make image 1 monochrome",
+ image_urls=["data:image/png;base64,aW1hZ2U="],
+ api_token="token",
+ turbo=False,
+ aspect_ratio="match_input_image",
+ seed=123,
+ )
+
+ assert result == "https://example.com/edited.png"
+ assert captured["model_path"] == replicate.P_IMAGE_EDIT_MODEL_PATH
+ assert captured["input"] == {
+ "prompt": "Make image 1 monochrome",
+ "images": ["data:image/png;base64,aW1hZ2U="],
+ "turbo": False,
+ "aspect_ratio": "match_input_image",
+ "seed": 123,
+ }
+ assert captured["api_token"] == "token"
diff --git a/frontend/src/components/agent/AgentActivity.tsx b/frontend/src/components/agent/AgentActivity.tsx
index b5a61408..48d23ce0 100644
--- a/frontend/src/components/agent/AgentActivity.tsx
+++ b/frontend/src/components/agent/AgentActivity.tsx
@@ -105,6 +105,9 @@ function getEventIcon(type: AgentEventType, toolName?: string) {
if (toolName === "remove_background") {
return <BsScissors className="text-teal-500" />;
}
+ if (toolName === "edit_image") {
+ return <BsImage className="text-violet-500" />;
+ }
if (toolName === "retrieve_option") {
return <BsFiles className="text-slate-500" />;
}
@@ -148,6 +151,20 @@ function getEventTitle(event: AgentEvent): string {
}
return rbCount > 1 ? `Removed ${rbCount} backgrounds` : "Background removed";
}
+ if (event.toolName === "edit_image") {
+ const editInput = event.input as { image_urls?: unknown[] } | null;
+ const editOutput = event.output as {
+ image?: { image_urls?: unknown[] };
+ } | null;
+ const editCount =
+ editOutput?.image?.image_urls?.length || editInput?.image_urls?.length || 0;
+ if (event.status === "running") {
+ return editCount > 1
+ ? `Editing image with ${editCount} references`
+ : "Editing image";
+ }
+ return "Edited image";
+ }
if (event.toolName === "retrieve_option") {
return event.status === "running"
? "Retrieving option"
@@ -358,6 +375,106 @@ function renderToolDetails(event: AgentEvent, variantCode?: string) {
</div>
)}
+ {event.toolName === "edit_image" && !hasError && (
+ <div>
+ {event.status === "running" && input?.image_urls && Array.isArray(input.image_urls) && (
+ <div className="space-y-3">
+ {input.prompt && (
+ <div className="text-xs text-gray-600 dark:text-gray-300">
+ {input.prompt}
+ </div>
+ )}
+ <div className="divide-y divide-gray-100 dark:divide-gray-800">
+ {input.image_urls.map((url: string, index: number) => (
+ <div key={`${url}-${index}`} className="py-2">
+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1">
+ {index === 0 ? "Main image" : `Reference image ${index + 1}`}
+ </div>
+ <img
+ src={url}
+ alt={index === 0 ? "Main image" : `Reference image ${index + 1}`}
+ className="w-full rounded object-contain bg-gray-50 dark:bg-gray-800"
+ loading="lazy"
+ />
+ </div>
+ ))}
+ </div>
+ </div>
+ )}
+ {event.status !== "running" && output?.image && (
+ <div className="space-y-3">
+ <div className="text-xs text-gray-600 dark:text-gray-300">
+ {output.image.prompt}
+ </div>
+ <div className="grid gap-3 sm:grid-cols-2">
+ <div>
+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1">
+ Main image
+ </div>
+ {Array.isArray(output.image.image_urls) && output.image.image_urls[0] ? (
+ <img
+ src={output.image.image_urls[0]}
+ alt="Main image"
+ className="w-full rounded object-contain bg-gray-50 dark:bg-gray-800"
+ loading="lazy"
+ />
+ ) : (
+ <div className="aspect-square rounded bg-gray-100 dark:bg-gray-800 flex items-center justify-center text-xs text-gray-400">
+ Missing
+ </div>
+ )}
+ </div>
+ <div>
+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1">
+ Edited image
+ </div>
+ {output.image.result_url ? (
+ <img
+ src={output.image.result_url}
+ alt="Edited image"
+ className="w-full rounded object-contain bg-gray-50 dark:bg-gray-800"
+ loading="lazy"
+ />
+ ) : (
+ <div className="aspect-square rounded bg-gray-100 dark:bg-gray-800 flex items-center justify-center text-xs text-gray-400">
+ Failed
+ </div>
+ )}
+ </div>
+ </div>
+ {Array.isArray(output.image.image_urls) && output.image.image_urls.length > 1 && (
+ <div>
+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1">
+ Reference images
+ </div>
+ <div className="grid grid-cols-3 gap-2">
+ {output.image.image_urls.slice(1).map((url: string, index: number) => (
+ <img
+ key={`${url}-${index}`}
+ src={url}
+ alt={`Reference image ${index + 2}`}
+ className="aspect-square w-full rounded object-cover bg-gray-50 dark:bg-gray-800"
+ loading="lazy"
+ />
+ ))}
+ </div>
+ </div>
+ )}
+ {output.image.result_url && (
+ <div>
+ <div className="text-xs text-gray-500 dark:text-gray-400">
+ Result URL
+ </div>
+ <div className="mt-1 break-all rounded bg-gray-50 p-2 font-mono text-xs text-gray-600 dark:bg-gray-800 dark:text-gray-300">
+ {output.image.result_url}
+ </div>
+ </div>
+ )}
+ </div>
+ )}
+ </div>
+ )}
+
{event.toolName === "save_assets" && !hasError && (
<div className="space-y-3">
{event.status === "running" && input?.asset_ids && Array.isArray(input.asset_ids) && (