Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Add background removal tool for agentic workflow
Abi Raja5f5b02e60f61
diff --git a/backend/agentic/runner.py b/backend/agentic/runner.pyindex 0d30f634..7bbfa65b 100644--- a/backend/agentic/runner.py+++ b/backend/agentic/runner.py@@ -13,6 +13,7 @@ from google.genai import typesfrom codegen.utils import extract_html_contentfrom config import REPLICATE_API_KEYfrom image_generation.core import process_tasks+from image_generation.replicate import remove_backgroundfrom llm import (Llm,OPENAI_MODELS,@@ -209,6 +210,8 @@ class AgentToolbox:return self._edit_file(tool_call.arguments)if tool_call.name == "generate_images":return await self._generate_images(tool_call.arguments)+ if tool_call.name == "remove_background":+ return await self._remove_background(tool_call.arguments)return ToolExecutionResult(ok=False,result={"error": f"Unknown tool: {tool_call.name}"},@@ -408,6 +411,38 @@ class AgentToolbox:summary = {"images": summary_items}return ToolExecutionResult(ok=True, result=result, summary=summary)+ async def _remove_background(self, args: Dict[str, Any]) -> ToolExecutionResult:+ if not REPLICATE_API_KEY:+ return ToolExecutionResult(+ ok=False,+ result={"error": "Background removal requires REPLICATE_API_KEY."},+ summary={"error": "Missing Replicate API key"},+ )++ image_url = _ensure_str(args.get("image_url"))+ if not image_url:+ return ToolExecutionResult(+ ok=False,+ result={"error": "remove_background requires an image_url"},+ summary={"error": "Missing image_url"},+ )++ try:+ result_url = await remove_background(image_url, REPLICATE_API_KEY)+ result = {"image_url": image_url, "result_url": result_url}+ summary = {+ "image_url": _summarize_text(image_url, 100),+ "result_url": result_url,+ "status": "ok",+ }+ return ToolExecutionResult(ok=True, result=result, summary=summary)+ except Exception as e:+ return ToolExecutionResult(+ ok=False,+ result={"error": str(e), "image_url": image_url},+ summary={"error": _summarize_text(str(e), 100)},+ )+class AgenticRunner:def __init__(@@ -569,6 +604,9 @@ class AgenticRunner:"count": len(prompts),"prompts": [_summarize_text(_ensure_str(p), 140) for p in prompts],}+ if tool_call.name == "remove_background":+ image_url = _ensure_str(args.get("image_url"))+ return {"image_url": _summarize_text(image_url, 100)}return argsdef _tool_schemas(self) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[types.Tool]]:@@ -632,6 +670,16 @@ class AgenticRunner:},"required": ["prompts"],}+ remove_bg_schema = {+ "type": "object",+ "properties": {+ "image_url": {+ "type": "string",+ "description": "URL of the image to remove the background from.",+ },+ },+ "required": ["image_url"],+ }openai_tools = [{@@ -658,6 +706,14 @@ class AgenticRunner:"parameters": image_schema,},},+ {+ "type": "function",+ "function": {+ "name": "remove_background",+ "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",+ "parameters": remove_bg_schema,+ },+ },]anthropic_tools = [@@ -676,6 +732,11 @@ class AgenticRunner:"description": "Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.","input_schema": image_schema,},+ {+ "name": "remove_background",+ "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",+ "input_schema": remove_bg_schema,+ },]gemini_tools = [@@ -696,6 +757,11 @@ class AgenticRunner:description="Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",parameters_json_schema=image_schema,),+ types.FunctionDeclaration(+ name="remove_background",+ description="Remove the background from an image. Returns a URL to the processed image with transparent background.",+ parameters_json_schema=remove_bg_schema,+ ),])]@@ -768,6 +834,17 @@ class AgenticRunner:},"required": ["prompts"],}+ remove_bg_schema = {+ "type": "object",+ "additionalProperties": False,+ "properties": {+ "image_url": {+ "type": "string",+ "description": "URL of the image to remove the background from.",+ },+ },+ "required": ["image_url"],+ }return [{"type": "function",@@ -790,6 +867,13 @@ class AgenticRunner:"parameters": image_schema,"strict": True,},+ {+ "type": "function",+ "name": "remove_background",+ "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",+ "parameters": remove_bg_schema,+ "strict": True,+ },]async def run(self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]) -> str:@@ -814,7 +898,7 @@ class AgenticRunner:openai_tools, _, _ = self._tool_schemas()messages: List[Dict[str, Any]] = [dict(m) for m in prompt_messages]- max_steps = 8+ max_steps = 20try:for _ in range(max_steps):assistant_event_id = self._next_event_id("assistant")@@ -982,7 +1066,7 @@ class AgenticRunner:_convert_message_to_responses_input(m) for m in prompt_messages]- max_steps = 8+ max_steps = 20try:for _ in range(max_steps):assistant_event_id = self._next_event_id("assistant")@@ -1295,7 +1379,7 @@ class AgenticRunner:client = AsyncAnthropic(api_key=self.anthropic_api_key)_, anthropic_tools, _ = self._tool_schemas()- max_steps = 8+ max_steps = 20try:for _ in range(max_steps):assistant_event_id = self._next_event_id("assistant")@@ -1517,7 +1601,7 @@ class AgenticRunner:_, _, gemini_tools = self._tool_schemas()client = genai.Client(api_key=self.gemini_api_key)- max_steps = 8+ max_steps = 20for _ in range(max_steps):assistant_event_id = self._next_event_id("assistant")thinking_event_id = self._next_event_id("thinking")diff --git a/backend/image_generation/replicate.py b/backend/image_generation/replicate.pyindex 5905e42c..474d199f 100644--- a/backend/image_generation/replicate.py+++ b/backend/image_generation/replicate.py@@ -1,5 +1,148 @@import asyncioimport httpx+from typing import Any+++async def call_replicate_model(+ model_path: str, input: dict[str, Any], api_token: str+) -> Any:+ headers = {+ "Authorization": f"Bearer {api_token}",+ "Content-Type": "application/json",+ }++ data = {"input": input}++ async with httpx.AsyncClient() as client:+ try:+ response = await client.post(+ f"https://api.replicate.com/v1/models/{model_path}/predictions",+ headers=headers,+ json=data,+ )+ response.raise_for_status()+ response_json = response.json()++ prediction_id = response_json.get("id")+ if not prediction_id:+ raise ValueError("Prediction ID not found in initial response.")++ num_polls = 0+ max_polls = 100+ while num_polls < max_polls:+ num_polls += 1++ await asyncio.sleep(0.1)++ status_check_url = (+ f"https://api.replicate.com/v1/predictions/{prediction_id}"+ )+ status_response = await client.get(status_check_url, headers=headers)+ status_response.raise_for_status()+ status_response_json = status_response.json()+ status = status_response_json.get("status")++ if status == "succeeded":+ return status_response_json["output"]+ elif status == "error":+ raise ValueError(+ f"Inference errored out: {status_response_json.get('error', 'Unknown error')}"+ )+ elif status == "failed":+ raise ValueError("Inference failed")++ raise TimeoutError("Inference timed out")++ except httpx.HTTPStatusError as e:+ raise ValueError(f"HTTP error occurred: {e}")+ except httpx.RequestError as e:+ raise ValueError(f"An error occurred while requesting: {e}")+ except asyncio.TimeoutError:+ raise TimeoutError("Request timed out")+ except Exception as e:+ raise ValueError(f"An unexpected error occurred: {e}")+++async def call_replicate_version(+ version: str, input: dict[str, Any], api_token: str+) -> Any:+ headers = {+ "Authorization": f"Bearer {api_token}",+ "Content-Type": "application/json",+ }++ data = {"version": version, "input": input}++ async with httpx.AsyncClient() as client:+ try:+ response = await client.post(+ "https://api.replicate.com/v1/predictions",+ headers=headers,+ json=data,+ )+ response.raise_for_status()+ response_json = response.json()++ prediction_id = response_json.get("id")+ if not prediction_id:+ raise ValueError("Prediction ID not found in initial response.")++ num_polls = 0+ max_polls = 100+ while num_polls < max_polls:+ num_polls += 1++ await asyncio.sleep(0.1)++ status_check_url = (+ f"https://api.replicate.com/v1/predictions/{prediction_id}"+ )+ status_response = await client.get(status_check_url, headers=headers)+ status_response.raise_for_status()+ status_response_json = status_response.json()+ status = status_response_json.get("status")++ if status == "succeeded":+ return status_response_json["output"]+ elif status == "error":+ raise ValueError(+ f"Inference errored out: {status_response_json.get('error', 'Unknown error')}"+ )+ elif status == "failed":+ raise ValueError("Inference failed")++ raise TimeoutError("Inference timed out")++ except httpx.HTTPStatusError as e:+ raise ValueError(f"HTTP error occurred: {e}")+ except httpx.RequestError as e:+ raise ValueError(f"An error occurred while requesting: {e}")+ except asyncio.TimeoutError:+ raise TimeoutError("Request timed out")+ except Exception as e:+ raise ValueError(f"An unexpected error occurred: {e}")+++async def remove_background(image_url: str, api_token: str) -> str:+ result = await call_replicate_version(+ "a029dff38972b5fda4ec5d75d7d1cd25aeff621d2cf4946a41055d7db66b80bc",+ {+ "image": image_url,+ "format": "png",+ "reverse": False,+ "threshold": 0,+ "background_type": "rgba",+ },+ api_token,+ )+ # Handle different response formats+ if isinstance(result, str):+ return result+ if isinstance(result, dict) and "url" in result:+ return result["url"]+ if isinstance(result, list) and len(result) > 0:+ return result[0] if isinstance(result[0], str) else result[0].get("url", "")+ raise ValueError(f"Unexpected response from background remover: {result}")async def call_replicate(input: dict[str, str | int], api_token: str) -> str:diff --git a/frontend/src/components/agent/AgentActivity.tsx b/frontend/src/components/agent/AgentActivity.tsxindex 3a3dc896..ea3e68a4 100644--- a/frontend/src/components/agent/AgentActivity.tsx+++ b/frontend/src/components/agent/AgentActivity.tsx@@ -15,6 +15,7 @@ import {BsFileEarmarkPlus,BsPencilSquare,BsImage,+ BsScissors,} from "react-icons/bs";import ReactMarkdown from "react-markdown";@@ -100,6 +101,9 @@ function getEventIcon(type: AgentEventType, toolName?: string) {if (toolName === "generate_images") {return <BsImage className="text-pink-500" />;}+ if (toolName === "remove_background") {+ return <BsScissors className="text-teal-500" />;+ }return <BsFileEarmarkPlus className="text-gray-500" />;}@@ -124,6 +128,11 @@ function getEventTitle(event: AgentEvent): string {? "Generating images": "Generated images";}+ if (event.toolName === "remove_background") {+ return event.status === "running"+ ? "Removing background"+ : "Background removed";+ }return event.status === "running" ? "Running tool" : "Tool completed";}return "Activity";@@ -150,6 +159,9 @@ function getToolSummary(event: AgentEvent): string | null {? `${images} image${images > 1 ? "s" : ""} generated`: "Images generated";}+ if (event.toolName === "remove_background") {+ return output?.result_url ? "Background removed successfully" : "Processing image";+ }return "Tool completed";}@@ -294,6 +306,60 @@ function renderToolDetails(event: AgentEvent) {</div>)}+ {event.toolName === "remove_background" && !hasError && (+ <div className="rounded-md border border-gray-200 dark:border-gray-700 bg-white dark:bg-gray-900/60 p-3">+ <div className="flex items-center gap-4">+ <div className="flex-1">+ <div className="text-xs text-gray-500 mb-2">Before</div>+ {output?.image_url ? (+ <img+ src={output.image_url}+ alt="Original image"+ className="w-full max-w-[120px] rounded-md border border-gray-200 dark:border-gray-700"+ loading="lazy"+ />+ ) : (+ <div className="h-20 w-20 rounded-md bg-gray-100 dark:bg-gray-800 flex items-center justify-center text-xs text-gray-400">+ N/A+ </div>+ )}+ </div>+ <div className="text-gray-400 text-xl">→</div>+ <div className="flex-1">+ <div className="text-xs text-gray-500 mb-2">After</div>+ {output?.result_url ? (+ <div className="relative">+ <div+ className="absolute inset-0 rounded-md max-w-[120px]"+ style={{+ backgroundImage:+ "linear-gradient(45deg, #e5e7eb 25%, transparent 25%), linear-gradient(-45deg, #e5e7eb 25%, transparent 25%), linear-gradient(45deg, transparent 75%, #e5e7eb 75%), linear-gradient(-45deg, transparent 75%, #e5e7eb 75%)",+ backgroundSize: "10px 10px",+ backgroundPosition: "0 0, 0 5px, 5px -5px, -5px 0px",+ }}+ />+ <img+ src={output.result_url}+ alt="Background removed"+ className="relative w-full max-w-[120px] rounded-md border border-gray-200 dark:border-gray-700"+ loading="lazy"+ />+ </div>+ ) : (+ <div className="h-20 w-20 rounded-md bg-gray-100 dark:bg-gray-800 flex items-center justify-center text-xs text-gray-400">+ N/A+ </div>+ )}+ </div>+ </div>+ {output?.result_url && (+ <div className="text-xs text-gray-500 mt-3 truncate">+ {output.result_url}+ </div>+ )}+ </div>+ )}+{!event.toolName && !hasError && (<>{event.input && (