Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Add background removal tool for agentic workflow

Abi Raja5f5b02e60f61
diff --git a/backend/agentic/runner.py b/backend/agentic/runner.py
index 0d30f634..7bbfa65b 100644
--- a/backend/agentic/runner.py
+++ b/backend/agentic/runner.py
@@ -13,6 +13,7 @@ from google.genai import types
from codegen.utils import extract_html_content
from config import REPLICATE_API_KEY
from image_generation.core import process_tasks
+from image_generation.replicate import remove_background
from llm import (
Llm,
OPENAI_MODELS,
@@ -209,6 +210,8 @@ class AgentToolbox:
return self._edit_file(tool_call.arguments)
if tool_call.name == "generate_images":
return await self._generate_images(tool_call.arguments)
+ if tool_call.name == "remove_background":
+ return await self._remove_background(tool_call.arguments)
return ToolExecutionResult(
ok=False,
result={"error": f"Unknown tool: {tool_call.name}"},
@@ -408,6 +411,38 @@ class AgentToolbox:
summary = {"images": summary_items}
return ToolExecutionResult(ok=True, result=result, summary=summary)
+ async def _remove_background(self, args: Dict[str, Any]) -> ToolExecutionResult:
+ if not REPLICATE_API_KEY:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "Background removal requires REPLICATE_API_KEY."},
+ summary={"error": "Missing Replicate API key"},
+ )
+
+ image_url = _ensure_str(args.get("image_url"))
+ if not image_url:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "remove_background requires an image_url"},
+ summary={"error": "Missing image_url"},
+ )
+
+ try:
+ result_url = await remove_background(image_url, REPLICATE_API_KEY)
+ result = {"image_url": image_url, "result_url": result_url}
+ summary = {
+ "image_url": _summarize_text(image_url, 100),
+ "result_url": result_url,
+ "status": "ok",
+ }
+ return ToolExecutionResult(ok=True, result=result, summary=summary)
+ except Exception as e:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": str(e), "image_url": image_url},
+ summary={"error": _summarize_text(str(e), 100)},
+ )
+
class AgenticRunner:
def __init__(
@@ -569,6 +604,9 @@ class AgenticRunner:
"count": len(prompts),
"prompts": [_summarize_text(_ensure_str(p), 140) for p in prompts],
}
+ if tool_call.name == "remove_background":
+ image_url = _ensure_str(args.get("image_url"))
+ return {"image_url": _summarize_text(image_url, 100)}
return args
def _tool_schemas(self) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[types.Tool]]:
@@ -632,6 +670,16 @@ class AgenticRunner:
},
"required": ["prompts"],
}
+ remove_bg_schema = {
+ "type": "object",
+ "properties": {
+ "image_url": {
+ "type": "string",
+ "description": "URL of the image to remove the background from.",
+ },
+ },
+ "required": ["image_url"],
+ }
openai_tools = [
{
@@ -658,6 +706,14 @@ class AgenticRunner:
"parameters": image_schema,
},
},
+ {
+ "type": "function",
+ "function": {
+ "name": "remove_background",
+ "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",
+ "parameters": remove_bg_schema,
+ },
+ },
]
anthropic_tools = [
@@ -676,6 +732,11 @@ class AgenticRunner:
"description": "Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",
"input_schema": image_schema,
},
+ {
+ "name": "remove_background",
+ "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",
+ "input_schema": remove_bg_schema,
+ },
]
gemini_tools = [
@@ -696,6 +757,11 @@ class AgenticRunner:
description="Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",
parameters_json_schema=image_schema,
),
+ types.FunctionDeclaration(
+ name="remove_background",
+ description="Remove the background from an image. Returns a URL to the processed image with transparent background.",
+ parameters_json_schema=remove_bg_schema,
+ ),
]
)
]
@@ -768,6 +834,17 @@ class AgenticRunner:
},
"required": ["prompts"],
}
+ remove_bg_schema = {
+ "type": "object",
+ "additionalProperties": False,
+ "properties": {
+ "image_url": {
+ "type": "string",
+ "description": "URL of the image to remove the background from.",
+ },
+ },
+ "required": ["image_url"],
+ }
return [
{
"type": "function",
@@ -790,6 +867,13 @@ class AgenticRunner:
"parameters": image_schema,
"strict": True,
},
+ {
+ "type": "function",
+ "name": "remove_background",
+ "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",
+ "parameters": remove_bg_schema,
+ "strict": True,
+ },
]
async def run(self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]) -> str:
@@ -814,7 +898,7 @@ class AgenticRunner:
openai_tools, _, _ = self._tool_schemas()
messages: List[Dict[str, Any]] = [dict(m) for m in prompt_messages]
- max_steps = 8
+ max_steps = 20
try:
for _ in range(max_steps):
assistant_event_id = self._next_event_id("assistant")
@@ -982,7 +1066,7 @@ class AgenticRunner:
_convert_message_to_responses_input(m) for m in prompt_messages
]
- max_steps = 8
+ max_steps = 20
try:
for _ in range(max_steps):
assistant_event_id = self._next_event_id("assistant")
@@ -1295,7 +1379,7 @@ class AgenticRunner:
client = AsyncAnthropic(api_key=self.anthropic_api_key)
_, anthropic_tools, _ = self._tool_schemas()
- max_steps = 8
+ max_steps = 20
try:
for _ in range(max_steps):
assistant_event_id = self._next_event_id("assistant")
@@ -1517,7 +1601,7 @@ class AgenticRunner:
_, _, gemini_tools = self._tool_schemas()
client = genai.Client(api_key=self.gemini_api_key)
- max_steps = 8
+ max_steps = 20
for _ in range(max_steps):
assistant_event_id = self._next_event_id("assistant")
thinking_event_id = self._next_event_id("thinking")
diff --git a/backend/image_generation/replicate.py b/backend/image_generation/replicate.py
index 5905e42c..474d199f 100644
--- a/backend/image_generation/replicate.py
+++ b/backend/image_generation/replicate.py
@@ -1,5 +1,148 @@
import asyncio
import httpx
+from typing import Any
+
+
+async def call_replicate_model(
+ model_path: str, input: dict[str, Any], api_token: str
+) -> Any:
+ headers = {
+ "Authorization": f"Bearer {api_token}",
+ "Content-Type": "application/json",
+ }
+
+ data = {"input": input}
+
+ async with httpx.AsyncClient() as client:
+ try:
+ response = await client.post(
+ f"https://api.replicate.com/v1/models/{model_path}/predictions",
+ headers=headers,
+ json=data,
+ )
+ response.raise_for_status()
+ response_json = response.json()
+
+ prediction_id = response_json.get("id")
+ if not prediction_id:
+ raise ValueError("Prediction ID not found in initial response.")
+
+ num_polls = 0
+ max_polls = 100
+ while num_polls < max_polls:
+ num_polls += 1
+
+ await asyncio.sleep(0.1)
+
+ status_check_url = (
+ f"https://api.replicate.com/v1/predictions/{prediction_id}"
+ )
+ status_response = await client.get(status_check_url, headers=headers)
+ status_response.raise_for_status()
+ status_response_json = status_response.json()
+ status = status_response_json.get("status")
+
+ if status == "succeeded":
+ return status_response_json["output"]
+ elif status == "error":
+ raise ValueError(
+ f"Inference errored out: {status_response_json.get('error', 'Unknown error')}"
+ )
+ elif status == "failed":
+ raise ValueError("Inference failed")
+
+ raise TimeoutError("Inference timed out")
+
+ except httpx.HTTPStatusError as e:
+ raise ValueError(f"HTTP error occurred: {e}")
+ except httpx.RequestError as e:
+ raise ValueError(f"An error occurred while requesting: {e}")
+ except asyncio.TimeoutError:
+ raise TimeoutError("Request timed out")
+ except Exception as e:
+ raise ValueError(f"An unexpected error occurred: {e}")
+
+
+async def call_replicate_version(
+ version: str, input: dict[str, Any], api_token: str
+) -> Any:
+ headers = {
+ "Authorization": f"Bearer {api_token}",
+ "Content-Type": "application/json",
+ }
+
+ data = {"version": version, "input": input}
+
+ async with httpx.AsyncClient() as client:
+ try:
+ response = await client.post(
+ "https://api.replicate.com/v1/predictions",
+ headers=headers,
+ json=data,
+ )
+ response.raise_for_status()
+ response_json = response.json()
+
+ prediction_id = response_json.get("id")
+ if not prediction_id:
+ raise ValueError("Prediction ID not found in initial response.")
+
+ num_polls = 0
+ max_polls = 100
+ while num_polls < max_polls:
+ num_polls += 1
+
+ await asyncio.sleep(0.1)
+
+ status_check_url = (
+ f"https://api.replicate.com/v1/predictions/{prediction_id}"
+ )
+ status_response = await client.get(status_check_url, headers=headers)
+ status_response.raise_for_status()
+ status_response_json = status_response.json()
+ status = status_response_json.get("status")
+
+ if status == "succeeded":
+ return status_response_json["output"]
+ elif status == "error":
+ raise ValueError(
+ f"Inference errored out: {status_response_json.get('error', 'Unknown error')}"
+ )
+ elif status == "failed":
+ raise ValueError("Inference failed")
+
+ raise TimeoutError("Inference timed out")
+
+ except httpx.HTTPStatusError as e:
+ raise ValueError(f"HTTP error occurred: {e}")
+ except httpx.RequestError as e:
+ raise ValueError(f"An error occurred while requesting: {e}")
+ except asyncio.TimeoutError:
+ raise TimeoutError("Request timed out")
+ except Exception as e:
+ raise ValueError(f"An unexpected error occurred: {e}")
+
+
+async def remove_background(image_url: str, api_token: str) -> str:
+ result = await call_replicate_version(
+ "a029dff38972b5fda4ec5d75d7d1cd25aeff621d2cf4946a41055d7db66b80bc",
+ {
+ "image": image_url,
+ "format": "png",
+ "reverse": False,
+ "threshold": 0,
+ "background_type": "rgba",
+ },
+ api_token,
+ )
+ # Handle different response formats
+ if isinstance(result, str):
+ return result
+ if isinstance(result, dict) and "url" in result:
+ return result["url"]
+ if isinstance(result, list) and len(result) > 0:
+ return result[0] if isinstance(result[0], str) else result[0].get("url", "")
+ raise ValueError(f"Unexpected response from background remover: {result}")
async def call_replicate(input: dict[str, str | int], api_token: str) -> str:
diff --git a/frontend/src/components/agent/AgentActivity.tsx b/frontend/src/components/agent/AgentActivity.tsx
index 3a3dc896..ea3e68a4 100644
--- a/frontend/src/components/agent/AgentActivity.tsx
+++ b/frontend/src/components/agent/AgentActivity.tsx
@@ -15,6 +15,7 @@ import {
BsFileEarmarkPlus,
BsPencilSquare,
BsImage,
+ BsScissors,
} from "react-icons/bs";
import ReactMarkdown from "react-markdown";
@@ -100,6 +101,9 @@ function getEventIcon(type: AgentEventType, toolName?: string) {
if (toolName === "generate_images") {
return <BsImage className="text-pink-500" />;
}
+ if (toolName === "remove_background") {
+ return <BsScissors className="text-teal-500" />;
+ }
return <BsFileEarmarkPlus className="text-gray-500" />;
}
@@ -124,6 +128,11 @@ function getEventTitle(event: AgentEvent): string {
? "Generating images"
: "Generated images";
}
+ if (event.toolName === "remove_background") {
+ return event.status === "running"
+ ? "Removing background"
+ : "Background removed";
+ }
return event.status === "running" ? "Running tool" : "Tool completed";
}
return "Activity";
@@ -150,6 +159,9 @@ function getToolSummary(event: AgentEvent): string | null {
? `${images} image${images > 1 ? "s" : ""} generated`
: "Images generated";
}
+ if (event.toolName === "remove_background") {
+ return output?.result_url ? "Background removed successfully" : "Processing image";
+ }
return "Tool completed";
}
@@ -294,6 +306,60 @@ function renderToolDetails(event: AgentEvent) {
</div>
)}
+ {event.toolName === "remove_background" && !hasError && (
+ <div className="rounded-md border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-900/60 p-3">
+ <div className="flex items-center gap-4">
+ <div className="flex-1">
+ <div className="text-xs text-gray-500 mb-2">Before</div>
+ {output?.image_url ? (
+ <img
+ src={output.image_url}
+ alt="Original image"
+ className="w-full max-w-[120px] rounded-md border border-gray-200 dark:border-gray-700"
+ loading="lazy"
+ />
+ ) : (
+ <div className="h-20 w-20 rounded-md bg-gray-100 dark:bg-gray-800 flex items-center justify-center text-xs text-gray-400">
+ N/A
+ </div>
+ )}
+ </div>
+ <div className="text-gray-400 text-xl">→</div>
+ <div className="flex-1">
+ <div className="text-xs text-gray-500 mb-2">After</div>
+ {output?.result_url ? (
+ <div className="relative">
+ <div
+ className="absolute inset-0 rounded-md max-w-[120px]"
+ style={{
+ backgroundImage:
+ "linear-gradient(45deg, #e5e7eb 25%, transparent 25%), linear-gradient(-45deg, #e5e7eb 25%, transparent 25%), linear-gradient(45deg, transparent 75%, #e5e7eb 75%), linear-gradient(-45deg, transparent 75%, #e5e7eb 75%)",
+ backgroundSize: "10px 10px",
+ backgroundPosition: "0 0, 0 5px, 5px -5px, -5px 0px",
+ }}
+ />
+ <img
+ src={output.result_url}
+ alt="Background removed"
+ className="relative w-full max-w-[120px] rounded-md border border-gray-200 dark:border-gray-700"
+ loading="lazy"
+ />
+ </div>
+ ) : (
+ <div className="h-20 w-20 rounded-md bg-gray-100 dark:bg-gray-800 flex items-center justify-center text-xs text-gray-400">
+ N/A
+ </div>
+ )}
+ </div>
+ </div>
+ {output?.result_url && (
+ <div className="text-xs text-gray-500 mt-3 truncate">
+ {output.result_url}
+ </div>
+ )}
+ </div>
+ )}
+
{!event.toolName && !hasError && (
<>
{event.input && (