Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Add retrieve option tool for variants

Abi Rajacf3c3de9f99d
diff --git a/backend/agentic/runner.py b/backend/agentic/runner.py
index 7bbfa65b..ab240ed2 100644
--- a/backend/agentic/runner.py
+++ b/backend/agentic/runner.py
@@ -196,12 +196,14 @@ class AgentToolbox:
should_generate_images: bool,
openai_api_key: Optional[str],
openai_base_url: Optional[str],
+ option_codes: Optional[List[str]] = None,
):
self.file_state = file_state
self.image_cache = image_cache
self.should_generate_images = should_generate_images
self.openai_api_key = openai_api_key
self.openai_base_url = openai_base_url
+ self.option_codes = option_codes or []
async def execute(self, tool_call: ToolCall) -> ToolExecutionResult:
if tool_call.name == "create_file":
@@ -212,6 +214,8 @@ class AgentToolbox:
return await self._generate_images(tool_call.arguments)
if tool_call.name == "remove_background":
return await self._remove_background(tool_call.arguments)
+ if tool_call.name == "retrieve_option":
+ return self._retrieve_option(tool_call.arguments)
return ToolExecutionResult(
ok=False,
result={"error": f"Unknown tool: {tool_call.name}"},
@@ -443,6 +447,69 @@ class AgentToolbox:
summary={"error": _summarize_text(str(e), 100)},
)
+ def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:
+ raw_option_number = args.get("option_number")
+ raw_index = args.get("index")
+
+ def _coerce_int(value: Any) -> Optional[int]:
+ if value is None:
+ return None
+ try:
+ return int(value)
+ except (TypeError, ValueError):
+ return None
+
+ option_number = _coerce_int(raw_option_number)
+ index = _coerce_int(raw_index)
+
+ if option_number is None and index is None:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "retrieve_option requires option_number"},
+ summary={"error": "Missing option_number"},
+ )
+
+ resolved_index = index if option_number is None else option_number - 1
+ if resolved_index is None:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "Invalid option_number"},
+ summary={"error": "Invalid option_number"},
+ )
+
+ if resolved_index < 0 or resolved_index >= len(self.option_codes):
+ return ToolExecutionResult(
+ ok=False,
+ result={
+ "error": "Option index out of range",
+ "option_number": resolved_index + 1,
+ "available": len(self.option_codes),
+ },
+ summary={
+ "error": "Option index out of range",
+ "available": len(self.option_codes),
+ },
+ )
+
+ code = _ensure_str(self.option_codes[resolved_index])
+ if not code.strip():
+ return ToolExecutionResult(
+ ok=False,
+ result={
+ "error": "Option code is empty or unavailable",
+ "option_number": resolved_index + 1,
+ },
+ summary={"error": "Option code unavailable"},
+ )
+
+ summary = {
+ "option_number": resolved_index + 1,
+ "contentLength": len(code),
+ "preview": _summarize_text(code, 200),
+ }
+ result = {"option_number": resolved_index + 1, "code": code}
+ return ToolExecutionResult(ok=True, result=result, summary=summary)
+
class AgenticRunner:
def __init__(
@@ -456,6 +523,7 @@ class AgenticRunner:
should_generate_images: bool,
image_cache: Dict[str, str],
initial_file_state: Optional[Dict[str, str]] = None,
+ option_codes: Optional[List[str]] = None,
):
self.send_message = send_message
self.variant_index = variant_index
@@ -476,6 +544,7 @@ class AgenticRunner:
should_generate_images,
openai_api_key,
openai_base_url,
+ option_codes,
)
self.assistant_step = 0
self.thinking_step = 0
@@ -607,6 +676,11 @@ class AgenticRunner:
if tool_call.name == "remove_background":
image_url = _ensure_str(args.get("image_url"))
return {"image_url": _summarize_text(image_url, 100)}
+ if tool_call.name == "retrieve_option":
+ return {
+ "option_number": args.get("option_number"),
+ "index": args.get("index"),
+ }
return args
def _tool_schemas(self) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[types.Tool]]:
@@ -680,6 +754,16 @@ class AgenticRunner:
},
"required": ["image_url"],
}
+ retrieve_option_schema = {
+ "type": "object",
+ "properties": {
+ "option_number": {
+ "type": "integer",
+ "description": "1-based option number to retrieve (Option 1, Option 2, etc.).",
+ }
+ },
+ "required": ["option_number"],
+ }
openai_tools = [
{
@@ -714,6 +798,14 @@ class AgenticRunner:
"parameters": remove_bg_schema,
},
},
+ {
+ "type": "function",
+ "function": {
+ "name": "retrieve_option",
+ "description": "Retrieve the full HTML for a specific option (variant) so you can reference it.",
+ "parameters": retrieve_option_schema,
+ },
+ },
]
anthropic_tools = [
@@ -737,6 +829,11 @@ class AgenticRunner:
"description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",
"input_schema": remove_bg_schema,
},
+ {
+ "name": "retrieve_option",
+ "description": "Retrieve the full HTML for a specific option (variant) so you can reference it.",
+ "input_schema": retrieve_option_schema,
+ },
]
gemini_tools = [
@@ -762,6 +859,11 @@ class AgenticRunner:
description="Remove the background from an image. Returns a URL to the processed image with transparent background.",
parameters_json_schema=remove_bg_schema,
),
+ types.FunctionDeclaration(
+ name="retrieve_option",
+ description="Retrieve the full HTML for a specific option (variant) so you can reference it.",
+ parameters_json_schema=retrieve_option_schema,
+ ),
]
)
]
@@ -845,6 +947,17 @@ class AgenticRunner:
},
"required": ["image_url"],
}
+ retrieve_option_schema = {
+ "type": "object",
+ "additionalProperties": False,
+ "properties": {
+ "option_number": {
+ "type": "integer",
+ "description": "1-based option number to retrieve (Option 1, Option 2, etc.).",
+ },
+ },
+ "required": ["option_number"],
+ }
return [
{
"type": "function",
@@ -874,6 +987,13 @@ class AgenticRunner:
"parameters": remove_bg_schema,
"strict": True,
},
+ {
+ "type": "function",
+ "name": "retrieve_option",
+ "description": "Retrieve the full HTML for a specific option (variant) so you can reference it.",
+ "parameters": retrieve_option_schema,
+ "strict": True,
+ },
]
async def run(self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]) -> str:
diff --git a/backend/prompts/agentic_instructions.py b/backend/prompts/agentic_instructions.py
index ba276fdc..aecfa9e6 100644
--- a/backend/prompts/agentic_instructions.py
+++ b/backend/prompts/agentic_instructions.py
@@ -1,11 +1,12 @@
TOOL_USE_INSTRUCTIONS = """
Tooling instructions:
-- You have access to tools: create_file, edit_file, generate_images.
+- You have access to tools: create_file, edit_file, generate_images, retrieve_option.
- The main file is a single HTML file. Use path "index.html" unless told otherwise.
- For a brand new app, call create_file exactly once with the full HTML.
- For updates, call edit_file using exact string replacements. Do NOT regenerate the entire file.
- Do not output raw HTML in chat. Any code changes must go through tools.
- Use generate_images to create image URLs from prompts (you may pass multiple prompts).
+- Use retrieve_option to fetch the full HTML for a specific option (1-based option_number) when a user references another option.
- Any non-tool output should be short, user-facing chat updates.
"""
diff --git a/backend/routes/generate_code.py b/backend/routes/generate_code.py
index 06e3ab0b..5d098e91 100644
--- a/backend/routes/generate_code.py
+++ b/backend/routes/generate_code.py
@@ -248,6 +248,7 @@ class ExtractedParams:
history: List[Dict[str, Any]]
is_imported_from_code: bool
file_state: Dict[str, str] | None
+ option_codes: List[str]
class ParameterExtractionStage:
@@ -321,6 +322,17 @@ class ParameterExtractionStage:
path = raw_file_state.get("path") or "index.html"
file_state = {"path": path, "content": content}
+ raw_option_codes = params.get("optionCodes")
+ option_codes: List[str] = []
+ if isinstance(raw_option_codes, list):
+ for entry in raw_option_codes:
+ if isinstance(entry, str):
+ option_codes.append(entry)
+ elif entry is None:
+ option_codes.append("")
+ else:
+ option_codes.append(str(entry))
+
return ExtractedParams(
stack=validated_stack,
input_mode=validated_input_mode,
@@ -333,6 +345,7 @@ class ParameterExtractionStage:
history=history,
is_imported_from_code=is_imported_from_code,
file_state=file_state,
+ option_codes=option_codes,
)
def _get_from_settings_dialog_or_env(
@@ -848,6 +861,7 @@ class AgenticGenerationStage:
should_generate_images: bool,
image_cache: Dict[str, str],
file_state: Dict[str, str] | None,
+ option_codes: List[str] | None,
):
self.send_message = send_message
self.openai_api_key = openai_api_key
@@ -857,6 +871,7 @@ class AgenticGenerationStage:
self.should_generate_images = should_generate_images
self.image_cache = image_cache
self.file_state = file_state
+ self.option_codes = option_codes or []
async def process_variants(
self,
@@ -899,6 +914,7 @@ class AgenticGenerationStage:
should_generate_images=self.should_generate_images,
image_cache=self.image_cache,
initial_file_state=self.file_state,
+ option_codes=self.option_codes,
)
completion = await runner.run(model, prompt_messages)
if completion:
@@ -1117,6 +1133,7 @@ class CodeGenerationMiddleware(Middleware):
should_generate_images=context.extracted_params.should_generate_images,
image_cache=context.image_cache,
file_state=context.extracted_params.file_state,
+ option_codes=context.extracted_params.option_codes,
)
context.variant_completions = (
diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx
index a872e294..7177ea3b 100644
--- a/frontend/src/App.tsx
+++ b/frontend/src/App.tsx
@@ -435,6 +435,9 @@ function App() {
const currentCommit = commits[head];
const currentCode =
currentCommit?.variants[currentCommit.selectedVariantIndex]?.code || "";
+ const optionCodes = currentCommit?.variants.map(
+ (variant) => variant.code || ""
+ );
let historyTree;
try {
@@ -470,6 +473,7 @@ function App() {
: { text: "", images: [referenceImages[0]] },
history: updatedHistory,
isImportedFromCode,
+ optionCodes,
fileState: currentCode
? {
path: "index.html",
diff --git a/frontend/src/components/agent/AgentActivity.tsx b/frontend/src/components/agent/AgentActivity.tsx
index ea3e68a4..a16477b6 100644
--- a/frontend/src/components/agent/AgentActivity.tsx
+++ b/frontend/src/components/agent/AgentActivity.tsx
@@ -16,6 +16,7 @@ import {
BsPencilSquare,
BsImage,
BsScissors,
+ BsFiles,
} from "react-icons/bs";
import ReactMarkdown from "react-markdown";
@@ -104,6 +105,9 @@ function getEventIcon(type: AgentEventType, toolName?: string) {
if (toolName === "remove_background") {
return <BsScissors className="text-teal-500" />;
}
+ if (toolName === "retrieve_option") {
+ return <BsFiles className="text-slate-500" />;
+ }
return <BsFileEarmarkPlus className="text-gray-500" />;
}
@@ -133,6 +137,11 @@ function getEventTitle(event: AgentEvent): string {
? "Removing background"
: "Background removed";
}
+ if (event.toolName === "retrieve_option") {
+ return event.status === "running"
+ ? "Retrieving option"
+ : "Retrieved option";
+ }
return event.status === "running" ? "Running tool" : "Tool completed";
}
return "Activity";
@@ -162,6 +171,17 @@ function getToolSummary(event: AgentEvent): string | null {
if (event.toolName === "remove_background") {
return output?.result_url ? "Background removed successfully" : "Processing image";
}
+ if (event.toolName === "retrieve_option") {
+ const optionNumber = output?.option_number;
+ const length = output?.contentLength;
+ if (optionNumber && length) {
+ return `Retrieved option ${optionNumber} · ${length} chars`;
+ }
+ if (optionNumber) {
+ return `Retrieved option ${optionNumber}`;
+ }
+ return "Retrieved option";
+ }
return "Tool completed";
}
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 1fe01eff..260e657e 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -46,6 +46,7 @@ export interface CodeGenerationParams {
path: string;
content: string;
};
+ optionCodes?: string[];
}
export type FullGenerationSettings = CodeGenerationParams & Settings;