Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Add edit_image agent tool backed by Replicate
Abi Raja33f2d07a4d0b
diff --git a/backend/agent/providers/factory.py b/backend/agent/providers/factory.pyindex 91574102..48f326da 100644--- a/backend/agent/providers/factory.py+++ b/backend/agent/providers/factory.py@@ -10,6 +10,7 @@ from agent.providers.base import ProviderSessionfrom agent.providers.gemini import GeminiProviderSession, serialize_gemini_toolsfrom agent.providers.openai import OpenAIProviderSession, serialize_openai_toolsfrom agent.tools import canonical_tool_definitions+from config import REPLICATE_API_KEYfrom llm import ANTHROPIC_MODELS, GEMINI_MODELS, OPENAI_MODELS, Llm@@ -23,7 +24,9 @@ def create_provider_session(gemini_api_key: Optional[str],) -> ProviderSession:canonical_tools = canonical_tool_definitions(- image_generation_enabled=should_generate_images+ image_generation_enabled=should_generate_images,+ # The edit_image tool calls Replicate, so don't offer it without a key.+ image_editing_enabled=bool(REPLICATE_API_KEY),)if model in OPENAI_MODELS:diff --git a/backend/agent/tools/definitions.py b/backend/agent/tools/definitions.pyindex 1b796aac..b72aa1cf 100644--- a/backend/agent/tools/definitions.py+++ b/backend/agent/tools/definitions.py@@ -89,6 +89,50 @@ def _remove_background_schema() -> Dict[str, Any]:}+def _edit_image_schema() -> Dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "prompt": {+ "type": "string",+ "description": (+ "Clear edit instruction. Refer to inputs as image 1, image 2, "+ "and so on when multiple images are provided."+ ),+ },+ "image_urls": {+ "type": "array",+ "items": {+ "type": "string",+ "description": (+ "URL of a source/reference image. For editing, put the main "+ "image first."+ ),+ },+ },+ "turbo": {+ "type": "boolean",+ "description": (+ "Run faster with additional optimizations. Set false for more "+ "complicated edits."+ ),+ },+ "aspect_ratio": {+ "type": "string",+ "description": (+ "Aspect ratio for the edited image. Use match_input_image to "+ "match the first image."+ ),+ },+ "seed": {+ "type": "integer",+ "description": "Optional random seed for reproducible edits.",+ },+ },+ "required": ["prompt", "image_urls"],+ }++def _retrieve_option_schema() -> Dict[str, Any]:return {"type": "object",@@ -104,6 +148,7 @@ def _retrieve_option_schema() -> Dict[str, Any]:def canonical_tool_definitions(image_generation_enabled: bool = True,+ image_editing_enabled: bool = True,) -> List[CanonicalToolDefinition]:tools: List[CanonicalToolDefinition] = [CanonicalToolDefinition(@@ -146,6 +191,22 @@ def canonical_tool_definitions(),parameters=_remove_background_schema(),),+ ]+ )+ if image_editing_enabled:+ tools.append(+ CanonicalToolDefinition(+ name="edit_image",+ description=(+ "Edit one or more images using a text prompt. Provide the main "+ "image first, followed by optional reference images. Returns a URL "+ "to the edited image."+ ),+ parameters=_edit_image_schema(),+ )+ )+ tools.extend(+ [SAVE_ASSETS_TOOL_DEFINITION,CanonicalToolDefinition(name="retrieve_option",diff --git a/backend/agent/tools/runtime.py b/backend/agent/tools/runtime.pyindex b2c2b0dd..49198c3f 100644--- a/backend/agent/tools/runtime.py+++ b/backend/agent/tools/runtime.py@@ -1,12 +1,16 @@# pyright: reportUnknownVariableType=falseimport asyncio+import base64import difflib+import mimetypes+import os+from urllib.parse import unquote, urlparsefrom typing import Any, Dict, List, Optional, Tuple, Unionfrom codegen.utils import extract_html_content-from config import REPLICATE_API_KEY+from config import LOCAL_ASSET_DIR, REPLICATE_API_KEYfrom image_generation.generation import process_tasks-from image_generation.replicate import remove_background+from image_generation.replicate import edit_image, remove_backgroundfrom uploaded_assets.tools import run_save_assetsfrom agent.state import AgentFileState, ensure_str@@ -14,6 +18,34 @@ from agent.tools.types import ToolCall, ToolExecutionResultfrom agent.tools.summaries import summarize_text+LOCAL_ASSET_HOSTS = {"127.0.0.1", "localhost", "::1"}+++def _local_asset_url_to_data_url(image_url: str) -> str:+ """Replicate can't fetch localhost URLs, so inline local assets as data URLs."""+ parsed = urlparse(image_url)+ if parsed.scheme not in {"http", "https"}:+ return image_url+ if parsed.hostname not in LOCAL_ASSET_HOSTS:+ return image_url+ if not parsed.path.startswith("/local-assets/"):+ return image_url++ relative_path = unquote(parsed.path.removeprefix("/local-assets/"))+ asset_root = os.path.abspath(LOCAL_ASSET_DIR)+ asset_path = os.path.abspath(os.path.join(asset_root, relative_path))+ if not asset_path.startswith(asset_root + os.sep):+ return image_url+ if not os.path.isfile(asset_path):+ return image_url++ content_type = mimetypes.guess_type(asset_path)[0] or "image/png"+ with open(asset_path, "rb") as file:+ encoded = base64.b64encode(file.read()).decode("ascii")++ return f"data:{content_type};base64,{encoded}"++class AgentToolRuntime:def __init__(self,@@ -51,6 +83,8 @@ class AgentToolRuntime:return await self._generate_images(tool_call.arguments)if tool_call.name == "remove_background":return await self._remove_background(tool_call.arguments)+ if tool_call.name == "edit_image":+ return await self._edit_image(tool_call.arguments)if tool_call.name == "save_assets":return await run_save_assets(tool_call.arguments, user_id=self.user_id)if tool_call.name == "retrieve_option":@@ -334,6 +368,102 @@ class AgentToolRuntime:summary={"images": summary_items},)+ async def _edit_image(self, args: Dict[str, Any]) -> ToolExecutionResult:+ if not REPLICATE_API_KEY:+ return ToolExecutionResult(+ ok=False,+ result={"error": "Image editing requires REPLICATE_API_KEY."},+ summary={"error": "Missing Replicate API key"},+ )++ prompt = ensure_str(args.get("prompt")).strip()+ if not prompt:+ return ToolExecutionResult(+ ok=False,+ result={"error": "edit_image requires a non-empty prompt"},+ summary={"error": "Missing prompt"},+ )++ image_urls = args.get("image_urls") or args.get("images") or []+ if not isinstance(image_urls, list) or not image_urls:+ return ToolExecutionResult(+ ok=False,+ result={"error": "edit_image requires a non-empty image_urls list"},+ summary={"error": "Missing image_urls"},+ )++ cleaned = [url.strip() for url in image_urls if isinstance(url, str)]+ unique_urls = list(dict.fromkeys([u for u in cleaned if u]))+ if not unique_urls:+ return ToolExecutionResult(+ ok=False,+ result={"error": "No valid image URLs provided"},+ summary={"error": "No valid image_urls"},+ )++ turbo = args.get("turbo")+ if not isinstance(turbo, bool):+ turbo = True++ aspect_ratio = ensure_str(args.get("aspect_ratio") or "match_input_image")+ seed_value = args.get("seed")+ seed = seed_value if isinstance(seed_value, int) else None++ try:+ result_url = await edit_image(+ prompt=prompt,+ image_urls=[_local_asset_url_to_data_url(url) for url in unique_urls],+ api_token=REPLICATE_API_KEY,+ turbo=turbo,+ aspect_ratio=aspect_ratio,+ seed=seed,+ )+ except Exception as exc:+ print(f"Image edit failed for {unique_urls}: {exc}")+ return ToolExecutionResult(+ ok=True,+ result={+ "image": {+ "prompt": prompt,+ "image_urls": unique_urls,+ "result_url": None,+ "status": "error",+ }+ },+ summary={+ "image": {+ "prompt": summarize_text(prompt, 240),+ "image_urls": [summarize_text(url, 100) for url in unique_urls],+ "result_url": None,+ "status": "error",+ }+ },+ )++ result = {+ "image": {+ "prompt": prompt,+ "image_urls": unique_urls,+ "result_url": result_url,+ "status": "ok",+ "turbo": turbo,+ "aspect_ratio": aspect_ratio,+ **({"seed": seed} if seed is not None else {}),+ }+ }+ summary = {+ "image": {+ "prompt": summarize_text(prompt, 240),+ "image_urls": [summarize_text(url, 100) for url in unique_urls],+ "result_url": result_url,+ "status": "ok",+ "turbo": turbo,+ "aspect_ratio": aspect_ratio,+ **({"seed": seed} if seed is not None else {}),+ }+ }+ return ToolExecutionResult(ok=True, result=result, summary=summary)+def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:raw_option_number = args.get("option_number")raw_index = args.get("index")diff --git a/backend/agent/tools/summaries.py b/backend/agent/tools/summaries.pyindex bee522e6..2af8decf 100644--- a/backend/agent/tools/summaries.py+++ b/backend/agent/tools/summaries.py@@ -64,6 +64,19 @@ def summarize_tool_input(tool_call: ToolCall, file_state: AgentFileState) -> Dic}return {"image_urls": []}+ if tool_call.name == "edit_image":+ image_urls = args.get("image_urls") or args.get("images") or []+ if isinstance(image_urls, list):+ return {+ "count": len(image_urls),+ "prompt": ensure_str(args.get("prompt")),+ "image_urls": [ensure_str(u) for u in image_urls],+ "turbo": args.get("turbo"),+ "aspect_ratio": args.get("aspect_ratio"),+ "seed": args.get("seed"),+ }+ return {"prompt": ensure_str(args.get("prompt")), "image_urls": []}+if tool_call.name == "save_assets":return summarize_save_assets_input(args)diff --git a/backend/image_generation/replicate.py b/backend/image_generation/replicate.pyindex d2116962..e10f021e 100644--- a/backend/image_generation/replicate.py+++ b/backend/image_generation/replicate.py@@ -7,6 +7,7 @@ REPLICATE_API_BASE_URL = "https://api.replicate.com/v1"ReplicateImageModel = Literal["z_image_turbo", "flux_2_klein"]Z_IMAGE_TURBO_MODEL_PATH = "prunaai/z-image-turbo"FLUX_2_KLEIN_MODEL_PATH = "black-forest-labs/flux-2-klein-4b"+P_IMAGE_EDIT_MODEL_PATH = "prunaai/p-image-edit"MODEL_PATHS: dict[ReplicateImageModel, str] = {"z_image_turbo": Z_IMAGE_TURBO_MODEL_PATH,"flux_2_klein": FLUX_2_KLEIN_MODEL_PATH,@@ -143,6 +144,28 @@ async def remove_background(image_url: str, api_token: str) -> str:return _extract_output_url(result, "background remover")+async def edit_image(+ prompt: str,+ image_urls: list[str],+ api_token: str,+ *,+ turbo: bool = True,+ aspect_ratio: str = "match_input_image",+ seed: int | None = None,+) -> str:+ input: dict[str, Any] = {+ "prompt": prompt,+ "images": image_urls,+ "turbo": turbo,+ "aspect_ratio": aspect_ratio,+ }+ if seed is not None:+ input["seed"] = seed++ result = await call_replicate_model(P_IMAGE_EDIT_MODEL_PATH, input, api_token)+ return _extract_output_url(result, "Replicate image edit model prediction")++async def call_replicate(input: dict[str, str | int | float | bool],api_token: str,diff --git a/backend/tests/test_agent_tools.py b/backend/tests/test_agent_tools.pyindex c972f57b..43852d82 100644--- a/backend/tests/test_agent_tools.py+++ b/backend/tests/test_agent_tools.py@@ -45,3 +45,46 @@ def test_save_assets_tool_input_summary_uses_asset_ids() -> None:"count": 2,"asset_ids": ["asset_one", "asset_two"],}+++def test_canonical_tool_definitions_include_edit_image() -> None:+ tools = canonical_tool_definitions(True)+ tool_names = [tool.name for tool in tools]+ assert "edit_image" in tool_names+ edit_image_tool = next(tool for tool in tools if tool.name == "edit_image")+ assert edit_image_tool.parameters["required"] == ["prompt", "image_urls"]+ assert edit_image_tool.parameters["properties"]["image_urls"]["type"] == "array"+++def test_canonical_tool_definitions_exclude_edit_image_when_disabled() -> None:+ tool_names = [+ tool.name+ for tool in canonical_tool_definitions(True, image_editing_enabled=False)+ ]+ assert "edit_image" not in tool_names+++def test_edit_image_tool_input_summary_uses_prompt_and_image_urls() -> None:+ summary = summarize_tool_input(+ ToolCall(+ id="call-1",+ name="edit_image",+ arguments={+ "prompt": "Make image 1 monochrome",+ "image_urls": ["https://example.com/input.png"],+ "turbo": False,+ "aspect_ratio": "match_input_image",+ "seed": 123,+ },+ ),+ AgentFileState(),+ )++ assert summary == {+ "count": 1,+ "prompt": "Make image 1 monochrome",+ "image_urls": ["https://example.com/input.png"],+ "turbo": False,+ "aspect_ratio": "match_input_image",+ "seed": 123,+ }diff --git a/backend/tests/test_image_generation_replicate.py b/backend/tests/test_image_generation_replicate.pyindex deb165a0..25a6ce13 100644--- a/backend/tests/test_image_generation_replicate.py+++ b/backend/tests/test_image_generation_replicate.py@@ -107,3 +107,40 @@ async def test_remove_background_uses_version_and_normalizes_output(assert result == "https://example.com/no-bg.png"assert captured["version"] == replicate.REMOVE_BACKGROUND_VERSIONassert captured["api_token"] == "token"+++async def test_edit_image_uses_pruna_model_and_normalizes_output(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ captured: dict[str, object] = {}++ async def fake_call_replicate_model(+ model_path: str, input: dict[str, object], api_token: str+ ) -> dict[str, str]:+ captured["model_path"] = model_path+ captured["input"] = input+ captured["api_token"] = api_token+ return {"url": "https://example.com/edited.png"}++ monkeypatch.setattr(replicate, "call_replicate_model", fake_call_replicate_model)++ result = await replicate.edit_image(+ prompt="Make image 1 monochrome",+ image_urls=["data:image/png;base64,aW1hZ2U="],+ api_token="token",+ turbo=False,+ aspect_ratio="match_input_image",+ seed=123,+ )++ assert result == "https://example.com/edited.png"+ assert captured["model_path"] == replicate.P_IMAGE_EDIT_MODEL_PATH+ assert captured["input"] == {+ "prompt": "Make image 1 monochrome",+ "images": ["data:image/png;base64,aW1hZ2U="],+ "turbo": False,+ "aspect_ratio": "match_input_image",+ "seed": 123,+ }+ assert captured["api_token"] == "token"diff --git a/frontend/src/components/agent/AgentActivity.tsx b/frontend/src/components/agent/AgentActivity.tsxindex b5a61408..48d23ce0 100644--- a/frontend/src/components/agent/AgentActivity.tsx+++ b/frontend/src/components/agent/AgentActivity.tsx@@ -105,6 +105,9 @@ function getEventIcon(type: AgentEventType, toolName?: string) {if (toolName === "remove_background") {return <BsScissors className="text-teal-500" />;}+ if (toolName === "edit_image") {+ return <BsImage className="text-violet-500" />;+ }if (toolName === "retrieve_option") {return <BsFiles className="text-slate-500" />;}@@ -148,6 +151,20 @@ function getEventTitle(event: AgentEvent): string {}return rbCount > 1 ? `Removed ${rbCount} backgrounds` : "Background removed";}+ if (event.toolName === "edit_image") {+ const editInput = event.input as { image_urls?: unknown[] } | null;+ const editOutput = event.output as {+ image?: { image_urls?: unknown[] };+ } | null;+ const editCount =+ editOutput?.image?.image_urls?.length || editInput?.image_urls?.length || 0;+ if (event.status === "running") {+ return editCount > 1+ ? `Editing image with ${editCount} references`+ : "Editing image";+ }+ return "Edited image";+ }if (event.toolName === "retrieve_option") {return event.status === "running"? "Retrieving option"@@ -358,6 +375,106 @@ function renderToolDetails(event: AgentEvent, variantCode?: string) {</div>)}+ {event.toolName === "edit_image" && !hasError && (+ <div>+ {event.status === "running" && input?.image_urls && Array.isArray(input.image_urls) && (+ <div className="space-y-3">+ {input.prompt && (+ <div className="text-xs text-gray-600 dark:text-gray-300">+ {input.prompt}+ </div>+ )}+ <div className="divide-y divide-gray-100 dark:divide-gray-800">+ {input.image_urls.map((url: string, index: number) => (+ <div key={`${url}-${index}`} className="py-2">+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1">+ {index === 0 ? "Main image" : `Reference image ${index + 1}`}+ </div>+ <img+ src={url}+ alt={index === 0 ? "Main image" : `Reference image ${index + 1}`}+ className="w-full rounded object-contain bg-gray-50 dark:bg-gray-800"+ loading="lazy"+ />+ </div>+ ))}+ </div>+ </div>+ )}+ {event.status !== "running" && output?.image && (+ <div className="space-y-3">+ <div className="text-xs text-gray-600 dark:text-gray-300">+ {output.image.prompt}+ </div>+ <div className="grid gap-3 sm:grid-cols-2">+ <div>+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1">+ Main image+ </div>+ {Array.isArray(output.image.image_urls) && output.image.image_urls[0] ? (+ <img+ src={output.image.image_urls[0]}+ alt="Main image"+ className="w-full rounded object-contain bg-gray-50 dark:bg-gray-800"+ loading="lazy"+ />+ ) : (+ <div className="aspect-square rounded bg-gray-100 dark:bg-gray-800 flex items-center justify-center text-xs text-gray-400">+ Missing+ </div>+ )}+ </div>+ <div>+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1">+ Edited image+ </div>+ {output.image.result_url ? (+ <img+ src={output.image.result_url}+ alt="Edited image"+ className="w-full rounded object-contain bg-gray-50 dark:bg-gray-800"+ loading="lazy"+ />+ ) : (+ <div className="aspect-square rounded bg-gray-100 dark:bg-gray-800 flex items-center justify-center text-xs text-gray-400">+ Failed+ </div>+ )}+ </div>+ </div>+ {Array.isArray(output.image.image_urls) && output.image.image_urls.length > 1 && (+ <div>+ <div className="text-xs text-gray-500 dark:text-gray-400 mb-1">+ Reference images+ </div>+ <div className="grid grid-cols-3 gap-2">+ {output.image.image_urls.slice(1).map((url: string, index: number) => (+ <img+ key={`${url}-${index}`}+ src={url}+ alt={`Reference image ${index + 2}`}+ className="aspect-square w-full rounded object-cover bg-gray-50 dark:bg-gray-800"+ loading="lazy"+ />+ ))}+ </div>+ </div>+ )}+ {output.image.result_url && (+ <div>+ <div className="text-xs text-gray-500 dark:text-gray-400">+ Result URL+ </div>+ <div className="mt-1 break-all rounded bg-gray-50 p-2 font-mono text-xs text-gray-600 dark:bg-gray-800 dark:text-gray-300">+ {output.image.result_url}+ </div>+ </div>+ )}+ </div>+ )}+ </div>+ )}+{event.toolName === "save_assets" && !hasError && (<div className="space-y-3">{event.status === "running" && input?.asset_ids && Array.isArray(input.asset_ids) && (