Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Refactor agentic runner modules and clean targeted typing

Abi Raja4b6d4ebf7726
diff --git a/backend/agentic/runner.py b/backend/agentic/runner.py
index ab240ed2..5ac5f45c 100644
--- a/backend/agentic/runner.py
+++ b/backend/agentic/runner.py
@@ -1,520 +1,52 @@
import asyncio
-import json
import uuid
-from dataclasses import dataclass
-from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple
+from typing import Any, Awaitable, Callable, Dict, List, Optional
-from openai import AsyncOpenAI
-from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
from anthropic import AsyncAnthropic
from google import genai
-from google.genai import types
+from openai import AsyncOpenAI
+from openai.types.chat import ChatCompletionMessageParam
from codegen.utils import extract_html_content
-from config import REPLICATE_API_KEY
-from image_generation.core import process_tasks
-from image_generation.replicate import remove_background
from llm import (
- Llm,
- OPENAI_MODELS,
ANTHROPIC_MODELS,
GEMINI_MODELS,
OPENAI_CODEX_MODELS,
- get_openai_api_name,
- get_openai_reasoning_effort,
+ OPENAI_MODELS,
+ Llm,
)
-
-@dataclass
-class ToolCall:
- id: str
- name: str
- arguments: Dict[str, Any]
-
-
-@dataclass
-class ToolExecutionResult:
- ok: bool
- result: Dict[str, Any]
- summary: Dict[str, Any]
- updated_content: Optional[str] = None
-
-
-@dataclass
-class AgentFileState:
- path: str = "index.html"
- content: str = ""
-
-
-def _summarize_text(value: str, limit: int = 240) -> str:
- if len(value) <= limit:
- return value
- return value[:limit] + "..."
-
-
-def _ensure_str(value: Any) -> str:
- if value is None:
- return ""
- return str(value)
-
-
-def _parse_json_arguments(raw_args: Any) -> Tuple[Dict[str, Any], Optional[str]]:
- if isinstance(raw_args, dict):
- return raw_args, None
- if raw_args is None:
- return {}, None
- raw_text = _ensure_str(raw_args).strip()
- if not raw_text:
- return {}, None
- try:
- return json.loads(raw_text), None
- except json.JSONDecodeError as exc:
- return {}, f"Invalid JSON arguments: {exc}"
-
-
-def _convert_message_to_responses_input(
- message: ChatCompletionMessageParam,
-) -> Dict[str, Any]:
- role = message.get("role", "user")
- content = message.get("content", "")
-
- if isinstance(content, str):
- return {"role": role, "content": content}
-
- parts: List[Dict[str, Any]] = []
- if isinstance(content, list):
- for part in content:
- if not isinstance(part, dict):
- continue
- if part.get("type") == "text":
- parts.append({"type": "input_text", "text": part.get("text", "")})
- elif part.get("type") == "image_url":
- image_url = part.get("image_url", {})
- parts.append(
- {
- "type": "input_image",
- "image_url": image_url.get("url", ""),
- "detail": image_url.get("detail", "auto"),
- }
- )
-
- return {"role": role, "content": parts}
-
-
-def _get_event_attr(event: Any, key: str, default: Any = None) -> Any:
- if hasattr(event, key):
- return getattr(event, key)
- if isinstance(event, dict):
- return event.get(key, default)
- return default
-
-
-def _strip_incomplete_escape(value: str) -> str:
- if not value:
- return value
- trailing = 0
- for ch in reversed(value):
- if ch == "\\":
- trailing += 1
- else:
- break
- if trailing % 2 == 1:
- return value[:-1]
- return value
-
-
-def _extract_partial_json_string(raw_text: str, key: str) -> Optional[str]:
- if not raw_text:
- return None
- token = f"\"{key}\""
- idx = raw_text.find(token)
- if idx == -1:
- return None
- colon = raw_text.find(":", idx + len(token))
- if colon == -1:
- return None
- cursor = colon + 1
- while cursor < len(raw_text) and raw_text[cursor].isspace():
- cursor += 1
- if cursor >= len(raw_text) or raw_text[cursor] != "\"":
- return None
- start = cursor + 1
- last_quote: Optional[int] = None
- cursor = start
- while cursor < len(raw_text):
- if raw_text[cursor] == "\"":
- backslashes = 0
- back = cursor - 1
- while back >= start and raw_text[back] == "\\":
- backslashes += 1
- back -= 1
- if backslashes % 2 == 0:
- last_quote = cursor
- cursor += 1
- if last_quote is None:
- partial = raw_text[start:]
- else:
- partial = raw_text[start:last_quote]
- partial = _strip_incomplete_escape(partial)
- if not partial:
- return ""
- try:
- return json.loads(f"\"{partial}\"")
- except Exception:
- return (
- partial.replace("\\n", "\n")
- .replace("\\t", "\t")
- .replace("\\r", "\r")
- .replace("\\\"", "\"")
- .replace("\\\\", "\\")
- )
-
-
-def _extract_content_from_args(raw_args: Any) -> Optional[str]:
- if isinstance(raw_args, dict):
- content = raw_args.get("content")
- if content is None:
- return None
- return _ensure_str(content)
- raw_text = _ensure_str(raw_args)
- return _extract_partial_json_string(raw_text, "content")
-
-
-def _extract_path_from_args(raw_args: Any) -> Optional[str]:
- if isinstance(raw_args, dict):
- path = raw_args.get("path")
- return _ensure_str(path) if path is not None else None
- raw_text = _ensure_str(raw_args)
- return _extract_partial_json_string(raw_text, "path")
-
-
-class AgentToolbox:
- def __init__(
- self,
- file_state: AgentFileState,
- image_cache: Dict[str, str],
- should_generate_images: bool,
- openai_api_key: Optional[str],
- openai_base_url: Optional[str],
- option_codes: Optional[List[str]] = None,
- ):
- self.file_state = file_state
- self.image_cache = image_cache
- self.should_generate_images = should_generate_images
- self.openai_api_key = openai_api_key
- self.openai_base_url = openai_base_url
- self.option_codes = option_codes or []
-
- async def execute(self, tool_call: ToolCall) -> ToolExecutionResult:
- if tool_call.name == "create_file":
- return self._create_file(tool_call.arguments)
- if tool_call.name == "edit_file":
- return self._edit_file(tool_call.arguments)
- if tool_call.name == "generate_images":
- return await self._generate_images(tool_call.arguments)
- if tool_call.name == "remove_background":
- return await self._remove_background(tool_call.arguments)
- if tool_call.name == "retrieve_option":
- return self._retrieve_option(tool_call.arguments)
- return ToolExecutionResult(
- ok=False,
- result={"error": f"Unknown tool: {tool_call.name}"},
- summary={"error": f"Unknown tool: {tool_call.name}"},
- )
-
- def _create_file(self, args: Dict[str, Any]) -> ToolExecutionResult:
- path = _ensure_str(args.get("path") or self.file_state.path or "index.html")
- content = _ensure_str(args.get("content"))
- if not content:
- return ToolExecutionResult(
- ok=False,
- result={"error": "create_file requires non-empty content"},
- summary={"error": "Missing content"},
- )
-
- extracted = extract_html_content(content)
- self.file_state.path = path
- self.file_state.content = extracted or content
-
- summary = {
- "path": self.file_state.path,
- "contentLength": len(self.file_state.content),
- "preview": _summarize_text(self.file_state.content, 320),
- }
- result = {
- "path": self.file_state.path,
- "content": self.file_state.content,
- }
- return ToolExecutionResult(
- ok=True,
- result=result,
- summary=summary,
- updated_content=self.file_state.content,
- )
-
- def _apply_single_edit(
- self, content: str, old_text: str, new_text: str, count: Optional[int]
- ) -> Tuple[str, int]:
- if old_text not in content:
- return content, 0
-
- if count is None:
- replace_count = 1
- elif count < 0:
- replace_count = content.count(old_text)
- else:
- replace_count = count
-
- updated = content.replace(old_text, new_text, replace_count)
- return updated, min(replace_count, content.count(old_text))
-
- def _edit_file(self, args: Dict[str, Any]) -> ToolExecutionResult:
- if not self.file_state.content:
- return ToolExecutionResult(
- ok=False,
- result={"error": "No file exists yet. Call create_file first."},
- summary={"error": "No file to edit"},
- )
-
- edits = args.get("edits")
- if not edits:
- old_text = _ensure_str(args.get("old_text"))
- new_text = _ensure_str(args.get("new_text"))
- count = args.get("count")
- edits = [
- {"old_text": old_text, "new_text": new_text, "count": count}
- ]
-
- if not isinstance(edits, list):
- return ToolExecutionResult(
- ok=False,
- result={"error": "edits must be a list"},
- summary={"error": "Invalid edits payload"},
- )
-
- content = self.file_state.content
- summary_edits: List[Dict[str, Any]] = []
- for edit in edits:
- old_text = _ensure_str(edit.get("old_text"))
- new_text = _ensure_str(edit.get("new_text"))
- count = edit.get("count")
- if not old_text:
- return ToolExecutionResult(
- ok=False,
- result={"error": "edit_file requires old_text"},
- summary={"error": "Missing old_text"},
- )
-
- content, replaced = self._apply_single_edit(
- content, old_text, new_text, count
- )
- if replaced == 0:
- return ToolExecutionResult(
- ok=False,
- result={"error": "old_text not found", "old_text": old_text},
- summary={
- "error": "old_text not found",
- "old_text": _summarize_text(old_text, 160),
- },
- )
-
- summary_edits.append(
- {
- "old_text": _summarize_text(old_text, 140),
- "new_text": _summarize_text(new_text, 140),
- "replaced": replaced,
- }
- )
-
- self.file_state.content = content
- summary = {
- "path": self.file_state.path,
- "edits": summary_edits,
- "contentLength": len(self.file_state.content),
- }
- result = {
- "path": self.file_state.path,
- "content": self.file_state.content,
- }
- return ToolExecutionResult(
- ok=True,
- result=result,
- summary=summary,
- updated_content=self.file_state.content,
- )
-
- async def _generate_images(self, args: Dict[str, Any]) -> ToolExecutionResult:
- if not self.should_generate_images:
- return ToolExecutionResult(
- ok=False,
- result={"error": "Image generation is disabled."},
- summary={"error": "Image generation disabled"},
- )
-
- prompts = args.get("prompts") or []
- if not isinstance(prompts, list) or not prompts:
- return ToolExecutionResult(
- ok=False,
- result={"error": "generate_images requires a non-empty prompts list"},
- summary={"error": "Missing prompts"},
- )
-
- cleaned = [prompt.strip() for prompt in prompts if isinstance(prompt, str)]
- unique_prompts = list(dict.fromkeys([p for p in cleaned if p]))
- if not unique_prompts:
- return ToolExecutionResult(
- ok=False,
- result={"error": "No valid prompts provided"},
- summary={"error": "No valid prompts"},
- )
-
- cached_results: Dict[str, Optional[str]] = {}
- to_generate: List[str] = []
- for prompt in unique_prompts:
- if prompt in self.image_cache:
- cached_results[prompt] = self.image_cache[prompt]
- else:
- to_generate.append(prompt)
-
- generated_results: Dict[str, Optional[str]] = {}
-
- if to_generate:
- if REPLICATE_API_KEY:
- model = "flux"
- api_key = REPLICATE_API_KEY
- base_url = None
- else:
- if not self.openai_api_key:
- return ToolExecutionResult(
- ok=False,
- result={
- "error": "No API key available for image generation."
- },
- summary={"error": "Missing image generation API key"},
- )
- model = "dalle3"
- api_key = self.openai_api_key
- base_url = self.openai_base_url
-
- generated = await process_tasks(to_generate, api_key, base_url, model) # type: ignore
- for prompt, url in zip(to_generate, generated):
- generated_results[prompt] = url
- if url:
- self.image_cache[prompt] = url
-
- merged_results = {**cached_results, **generated_results}
- summary_items = [
- {
- "prompt": _summarize_text(prompt, 160),
- "url": url,
- "status": "ok" if url else "error",
- }
- for prompt, url in merged_results.items()
- ]
- result = {"images": merged_results}
- summary = {"images": summary_items}
- return ToolExecutionResult(ok=True, result=result, summary=summary)
-
- async def _remove_background(self, args: Dict[str, Any]) -> ToolExecutionResult:
- if not REPLICATE_API_KEY:
- return ToolExecutionResult(
- ok=False,
- result={"error": "Background removal requires REPLICATE_API_KEY."},
- summary={"error": "Missing Replicate API key"},
- )
-
- image_url = _ensure_str(args.get("image_url"))
- if not image_url:
- return ToolExecutionResult(
- ok=False,
- result={"error": "remove_background requires an image_url"},
- summary={"error": "Missing image_url"},
- )
-
- try:
- result_url = await remove_background(image_url, REPLICATE_API_KEY)
- result = {"image_url": image_url, "result_url": result_url}
- summary = {
- "image_url": _summarize_text(image_url, 100),
- "result_url": result_url,
- "status": "ok",
- }
- return ToolExecutionResult(ok=True, result=result, summary=summary)
- except Exception as e:
- return ToolExecutionResult(
- ok=False,
- result={"error": str(e), "image_url": image_url},
- summary={"error": _summarize_text(str(e), 100)},
- )
-
- def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:
- raw_option_number = args.get("option_number")
- raw_index = args.get("index")
-
- def _coerce_int(value: Any) -> Optional[int]:
- if value is None:
- return None
- try:
- return int(value)
- except (TypeError, ValueError):
- return None
-
- option_number = _coerce_int(raw_option_number)
- index = _coerce_int(raw_index)
-
- if option_number is None and index is None:
- return ToolExecutionResult(
- ok=False,
- result={"error": "retrieve_option requires option_number"},
- summary={"error": "Missing option_number"},
- )
-
- resolved_index = index if option_number is None else option_number - 1
- if resolved_index is None:
- return ToolExecutionResult(
- ok=False,
- result={"error": "Invalid option_number"},
- summary={"error": "Invalid option_number"},
- )
-
- if resolved_index < 0 or resolved_index >= len(self.option_codes):
- return ToolExecutionResult(
- ok=False,
- result={
- "error": "Option index out of range",
- "option_number": resolved_index + 1,
- "available": len(self.option_codes),
- },
- summary={
- "error": "Option index out of range",
- "available": len(self.option_codes),
- },
- )
-
- code = _ensure_str(self.option_codes[resolved_index])
- if not code.strip():
- return ToolExecutionResult(
- ok=False,
- result={
- "error": "Option code is empty or unavailable",
- "option_number": resolved_index + 1,
- },
- summary={"error": "Option code unavailable"},
- )
-
- summary = {
- "option_number": resolved_index + 1,
- "contentLength": len(code),
- "preview": _summarize_text(code, 200),
- }
- result = {"option_number": resolved_index + 1, "code": code}
- return ToolExecutionResult(ok=True, result=result, summary=summary)
+from agentic.state import AgentFileState, seed_file_state_from_messages
+from agentic.streaming import (
+ AnthropicAdapter,
+ ExecutedToolCall,
+ GeminiAdapter,
+ OpenAIChatAdapter,
+ OpenAIResponsesAdapter,
+ ProviderAdapter,
+ StreamEvent,
+)
+from agentic.tools import (
+ AgentToolbox,
+ canonical_tool_definitions,
+ extract_content_from_args,
+ extract_path_from_args,
+ serialize_anthropic_tools,
+ serialize_gemini_tools,
+ serialize_openai_chat_tools,
+ serialize_openai_responses_tools,
+ summarize_text,
+ summarize_tool_input,
+)
class AgenticRunner:
def __init__(
self,
- send_message: Callable[[str, Optional[str], int, Optional[Dict[str, Any]], Optional[str]], Awaitable[None]],
+ send_message: Callable[
+ [str, Optional[str], int, Optional[Dict[str, Any]], Optional[str]],
+ Awaitable[None],
+ ],
variant_index: int,
openai_api_key: Optional[str],
openai_base_url: Optional[str],
@@ -531,25 +63,34 @@ class AgenticRunner:
self.openai_base_url = openai_base_url
self.anthropic_api_key = anthropic_api_key
self.gemini_api_key = gemini_api_key
- self.should_generate_images = should_generate_images
- self.image_cache = image_cache
self.file_state = AgentFileState()
if initial_file_state and initial_file_state.get("content"):
self.file_state.path = initial_file_state.get("path") or "index.html"
self.file_state.content = initial_file_state["content"]
+
self.toolbox = AgentToolbox(
self.file_state,
- self.image_cache,
+ image_cache,
should_generate_images,
openai_api_key,
openai_base_url,
option_codes,
)
- self.assistant_step = 0
- self.thinking_step = 0
self._tool_preview_lengths: Dict[str, int] = {}
+ def _next_event_id(self, prefix: str) -> str:
+ return f"{prefix}-{self.variant_index}-{uuid.uuid4().hex[:8]}"
+
+ async def _send(
+ self,
+ msg_type: str,
+ value: Optional[str] = None,
+ data: Optional[Dict[str, Any]] = None,
+ event_id: Optional[str] = None,
+ ) -> None:
+ await self.send_message(msg_type, value, self.variant_index, data, event_id)
+
def _mark_preview_length(self, tool_event_id: Optional[str], length: int) -> None:
if not tool_event_id:
return
@@ -560,1308 +101,236 @@ class AgenticRunner:
async def _stream_code_preview(self, tool_event_id: Optional[str], content: str) -> None:
if not tool_event_id or not content:
return
+
already_sent = self._tool_preview_lengths.get(tool_event_id, 0)
total_len = len(content)
if already_sent >= total_len:
return
+
max_chunks = 18
min_step = 200
step = max(min_step, total_len // max_chunks)
start = already_sent if already_sent > 0 else 0
+
for end in range(start + step, total_len, step):
await self._send("setCode", content[:end])
self._mark_preview_length(tool_event_id, end)
await asyncio.sleep(0.01)
+
await self._send("setCode", content)
self._mark_preview_length(tool_event_id, total_len)
- def _next_event_id(self, prefix: str) -> str:
- return f"{prefix}-{self.variant_index}-{uuid.uuid4().hex[:8]}"
-
- def _extract_text_content(self, message: ChatCompletionMessageParam) -> str:
- content = message.get("content", "")
- if isinstance(content, str):
- return content
- if isinstance(content, list):
- for part in content:
- if isinstance(part, dict) and part.get("type") == "text":
- return _ensure_str(part.get("text"))
- return ""
-
- def _seed_file_state_from_messages(
- self, prompt_messages: List[ChatCompletionMessageParam]
+ async def _handle_streamed_tool_delta(
+ self,
+ event: StreamEvent,
+ started_tool_ids: set[str],
+ streamed_lengths: Dict[str, int],
) -> None:
- if self.file_state.content:
+ if event.type != "tool_call_delta":
return
- for message in reversed(prompt_messages):
- if message.get("role") != "assistant":
- continue
- raw_text = self._extract_text_content(message)
- if not raw_text:
- continue
- extracted = extract_html_content(raw_text)
- self.file_state.content = extracted or raw_text
- if not self.file_state.path:
- self.file_state.path = "index.html"
+ if event.tool_name != "create_file":
+ return
+ if not event.tool_call_id:
return
- if prompt_messages:
- system_message = prompt_messages[0]
- if system_message.get("role") == "system":
- system_text = self._extract_text_content(system_message)
- markers = [
- "Here is the code of the app:",
- "Here is the code of the SVG:",
- ]
- for marker in markers:
- if marker in system_text:
- raw_text = system_text.split(marker, 1)[1].strip()
- extracted = extract_html_content(raw_text)
- self.file_state.content = extracted or raw_text
- if not self.file_state.path:
- self.file_state.path = "index.html"
- return
-
- async def _send(
- self,
- msg_type: str,
- value: Optional[str] = None,
- data: Optional[Dict[str, Any]] = None,
- event_id: Optional[str] = None,
- ) -> None:
- await self.send_message(msg_type, value, self.variant_index, data, event_id)
-
- def _summarize_tool_input(self, tool_call: ToolCall) -> Dict[str, Any]:
- args = tool_call.arguments or {}
- if tool_call.name == "create_file":
- content = _ensure_str(args.get("content"))
- return {
- "path": args.get("path") or self.file_state.path,
- "contentLength": len(content),
- "preview": _summarize_text(content, 200),
- }
- if tool_call.name == "edit_file":
- edits = args.get("edits")
- if not edits:
- edits = [
- {
- "old_text": args.get("old_text"),
- "new_text": args.get("new_text"),
- "count": args.get("count"),
- }
- ]
- summary_edits = []
- for edit in edits if isinstance(edits, list) else []:
- summary_edits.append(
- {
- "old_text": _summarize_text(
- _ensure_str(edit.get("old_text")), 160
- ),
- "new_text": _summarize_text(
- _ensure_str(edit.get("new_text")), 160
- ),
- "count": edit.get("count"),
- }
- )
- return {
- "path": args.get("path") or self.file_state.path,
- "edits": summary_edits,
- }
- if tool_call.name == "generate_images":
- prompts = args.get("prompts") or []
- if isinstance(prompts, list):
- return {
- "count": len(prompts),
- "prompts": [_summarize_text(_ensure_str(p), 140) for p in prompts],
- }
- if tool_call.name == "remove_background":
- image_url = _ensure_str(args.get("image_url"))
- return {"image_url": _summarize_text(image_url, 100)}
- if tool_call.name == "retrieve_option":
- return {
- "option_number": args.get("option_number"),
- "index": args.get("index"),
- }
- return args
-
- def _tool_schemas(self) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[types.Tool]]:
- create_schema = {
- "type": "object",
- "properties": {
- "path": {
- "type": "string",
- "description": "Path for the main HTML file. Use index.html if unsure.",
- },
- "content": {
- "type": "string",
- "description": "Full HTML for the single-file app.",
- },
- },
- "required": ["content"],
- }
- edit_schema = {
- "type": "object",
- "properties": {
- "path": {
- "type": "string",
- "description": "Path for the main HTML file.",
- },
- "old_text": {
- "type": "string",
- "description": "Exact text to replace. Must match the file contents.",
- },
- "new_text": {
- "type": "string",
- "description": "Replacement text.",
- },
- "count": {
- "type": "integer",
- "description": "How many occurrences to replace. Use -1 for all.",
- },
- "edits": {
- "type": "array",
- "items": {
- "type": "object",
- "properties": {
- "old_text": {"type": "string"},
- "new_text": {"type": "string"},
- "count": {"type": "integer"},
- },
- "required": ["old_text", "new_text"],
- },
- },
- },
- }
- image_schema = {
- "type": "object",
- "properties": {
- "prompts": {
- "type": "array",
- "items": {
- "type": "string",
- "description": "Prompt describing a single image to generate.",
- },
- }
- },
- "required": ["prompts"],
- }
- remove_bg_schema = {
- "type": "object",
- "properties": {
- "image_url": {
- "type": "string",
- "description": "URL of the image to remove the background from.",
- },
- },
- "required": ["image_url"],
- }
- retrieve_option_schema = {
- "type": "object",
- "properties": {
- "option_number": {
- "type": "integer",
- "description": "1-based option number to retrieve (Option 1, Option 2, etc.).",
- }
- },
- "required": ["option_number"],
- }
-
- openai_tools = [
- {
- "type": "function",
- "function": {
- "name": "create_file",
- "description": "Create the main HTML file for the app. Use exactly once to write the full HTML. Returns the updated file content.",
- "parameters": create_schema,
- },
- },
- {
- "type": "function",
- "function": {
- "name": "edit_file",
- "description": "Edit the main HTML file using exact string replacements. Do not regenerate the entire file. Returns the updated file content.",
- "parameters": edit_schema,
- },
- },
- {
- "type": "function",
- "function": {
- "name": "generate_images",
- "description": "Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",
- "parameters": image_schema,
- },
- },
- {
- "type": "function",
- "function": {
- "name": "remove_background",
- "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",
- "parameters": remove_bg_schema,
- },
- },
- {
- "type": "function",
- "function": {
- "name": "retrieve_option",
- "description": "Retrieve the full HTML for a specific option (variant) so you can reference it.",
- "parameters": retrieve_option_schema,
- },
- },
- ]
- anthropic_tools = [
- {
- "name": "create_file",
- "description": "Create the main HTML file for the app. Use exactly once to write the full HTML. Returns the updated file content.",
- "input_schema": create_schema,
- },
- {
- "name": "edit_file",
- "description": "Edit the main HTML file using exact string replacements. Do not regenerate the entire file. Returns the updated file content.",
- "input_schema": edit_schema,
- },
- {
- "name": "generate_images",
- "description": "Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",
- "input_schema": image_schema,
- },
- {
- "name": "remove_background",
- "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",
- "input_schema": remove_bg_schema,
- },
- {
- "name": "retrieve_option",
- "description": "Retrieve the full HTML for a specific option (variant) so you can reference it.",
- "input_schema": retrieve_option_schema,
- },
- ]
+ content = extract_content_from_args(event.tool_arguments)
+ if content is None:
+ return
- gemini_tools = [
- types.Tool(
- function_declarations=[
- types.FunctionDeclaration(
- name="create_file",
- description="Create the main HTML file for the app. Use exactly once to write the full HTML. Returns the updated file content.",
- parameters_json_schema=create_schema,
- ),
- types.FunctionDeclaration(
- name="edit_file",
- description="Edit the main HTML file using exact string replacements. Do not regenerate the entire file. Returns the updated file content.",
- parameters_json_schema=edit_schema,
- ),
- types.FunctionDeclaration(
- name="generate_images",
- description="Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",
- parameters_json_schema=image_schema,
- ),
- types.FunctionDeclaration(
- name="remove_background",
- description="Remove the background from an image. Returns a URL to the processed image with transparent background.",
- parameters_json_schema=remove_bg_schema,
- ),
- types.FunctionDeclaration(
- name="retrieve_option",
- description="Retrieve the full HTML for a specific option (variant) so you can reference it.",
- parameters_json_schema=retrieve_option_schema,
- ),
- ]
+ tool_event_id = event.tool_call_id
+ if tool_event_id not in started_tool_ids:
+ path = (
+ extract_path_from_args(event.tool_arguments)
+ or self.file_state.path
+ or "index.html"
)
- ]
-
- return openai_tools, anthropic_tools, gemini_tools
-
- def _openai_responses_tools(self) -> List[Dict[str, Any]]:
- create_schema = {
- "type": "object",
- "additionalProperties": False,
- "properties": {
- "path": {
- "type": ["string", "null"],
- "description": "Path for the main HTML file. Use index.html if unsure.",
- },
- "content": {
- "type": "string",
- "description": "The full HTML content for the app (single file).",
- },
- },
- "required": ["path", "content"],
- }
- edit_schema = {
- "type": "object",
- "additionalProperties": False,
- "properties": {
- "path": {
- "type": ["string", "null"],
- "description": "Path for the main HTML file.",
- },
- "old_text": {
- "type": ["string", "null"],
- "description": "Exact text to replace.",
- },
- "new_text": {
- "type": ["string", "null"],
- "description": "Replacement text.",
- },
- "count": {
- "type": ["integer", "null"],
- "description": "How many occurrences to replace. Use -1 for all.",
- },
- "edits": {
- "type": ["array", "null"],
- "items": {
- "type": "object",
- "additionalProperties": False,
- "properties": {
- "old_text": {"type": ["string", "null"]},
- "new_text": {"type": ["string", "null"]},
- "count": {"type": ["integer", "null"]},
- },
- "required": ["old_text", "new_text", "count"],
- },
- },
- },
- "required": ["path", "old_text", "new_text", "count", "edits"],
- }
- image_schema = {
- "type": "object",
- "additionalProperties": False,
- "properties": {
- "prompts": {
- "type": ["array", "null"],
- "items": {
- "type": "string",
- "description": "Prompt describing a single image to generate.",
+ await self._send(
+ "toolStart",
+ data={
+ "name": "create_file",
+ "input": {
+ "path": path,
+ "contentLength": len(content),
+ "preview": summarize_text(content, 200),
},
- }
- },
- "required": ["prompts"],
- }
- remove_bg_schema = {
- "type": "object",
- "additionalProperties": False,
- "properties": {
- "image_url": {
- "type": "string",
- "description": "URL of the image to remove the background from.",
- },
- },
- "required": ["image_url"],
- }
- retrieve_option_schema = {
- "type": "object",
- "additionalProperties": False,
- "properties": {
- "option_number": {
- "type": "integer",
- "description": "1-based option number to retrieve (Option 1, Option 2, etc.).",
},
- },
- "required": ["option_number"],
- }
- return [
- {
- "type": "function",
- "name": "create_file",
- "description": "Create the main HTML file for the app. Use exactly once to write the full HTML. Returns the updated file content.",
- "parameters": create_schema,
- "strict": True,
- },
- {
- "type": "function",
- "name": "edit_file",
- "description": "Edit the main HTML file using exact string replacements. Do not regenerate the entire file. Returns the updated file content.",
- "parameters": edit_schema,
- "strict": True,
- },
- {
- "type": "function",
- "name": "generate_images",
- "description": "Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",
- "parameters": image_schema,
- "strict": True,
- },
- {
- "type": "function",
- "name": "remove_background",
- "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",
- "parameters": remove_bg_schema,
- "strict": True,
- },
- {
- "type": "function",
- "name": "retrieve_option",
- "description": "Retrieve the full HTML for a specific option (variant) so you can reference it.",
- "parameters": retrieve_option_schema,
- "strict": True,
- },
- ]
-
- async def run(self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]) -> str:
- self._seed_file_state_from_messages(prompt_messages)
- if model in OPENAI_MODELS:
- return await self._run_openai(model, prompt_messages)
- if model in ANTHROPIC_MODELS:
- return await self._run_anthropic(model, prompt_messages)
- if model in GEMINI_MODELS:
- return await self._run_gemini(model, prompt_messages)
- raise ValueError(f"Unsupported model: {model.value}")
-
- async def _run_openai(
- self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]
- ) -> str:
- if model in OPENAI_CODEX_MODELS:
- return await self._run_openai_responses(model, prompt_messages)
- if not self.openai_api_key:
- raise Exception("OpenAI API key is missing.")
-
- client = AsyncOpenAI(api_key=self.openai_api_key, base_url=self.openai_base_url)
- openai_tools, _, _ = self._tool_schemas()
- messages: List[Dict[str, Any]] = [dict(m) for m in prompt_messages]
-
+ event_id=tool_event_id,
+ )
+ started_tool_ids.add(tool_event_id)
+
+ last_len = streamed_lengths.get(tool_event_id, 0)
+ if last_len == 0 and content:
+ streamed_lengths[tool_event_id] = len(content)
+ await self._send("setCode", content)
+ self._mark_preview_length(tool_event_id, len(content))
+ elif len(content) - last_len >= 40:
+ streamed_lengths[tool_event_id] = len(content)
+ await self._send("setCode", content)
+ self._mark_preview_length(tool_event_id, len(content))
+
+ async def _run_with_adapter(self, adapter: ProviderAdapter) -> str:
max_steps = 20
- try:
- for _ in range(max_steps):
- assistant_event_id = self._next_event_id("assistant")
- assistant_text = ""
- tool_calls: Dict[int, Dict[str, Any]] = {}
- started_tool_ids: set[str] = set()
-
- params: Dict[str, Any] = {
- "model": get_openai_api_name(model),
- "messages": messages,
- "tools": openai_tools,
- "tool_choice": "auto",
- "temperature": 0,
- "stream": True,
- }
- params["max_tokens"] = 30000
- stream = await client.chat.completions.create(**params)
+ for _ in range(max_steps):
+ assistant_event_id = self._next_event_id("assistant")
+ thinking_event_id = self._next_event_id("thinking")
+ started_tool_ids: set[str] = set()
+ streamed_lengths: Dict[str, int] = {}
- async for chunk in stream: # type: ignore
- assert isinstance(chunk, ChatCompletionChunk)
- if not chunk.choices:
- continue
- delta = chunk.choices[0].delta
- if delta.content:
- assistant_text += delta.content
+ async def on_event(event: StreamEvent) -> None:
+ if event.type == "assistant_delta":
+ if event.text:
await self._send(
- "assistant", delta.content, event_id=assistant_event_id
+ "assistant",
+ event.text,
+ event_id=assistant_event_id,
)
- if delta.tool_calls:
- for tc in delta.tool_calls:
- idx = tc.index or 0
- entry = tool_calls.setdefault(
- idx,
- {
- "id": tc.id or f"tool-{idx}-{uuid.uuid4().hex[:6]}",
- "name": None,
- "arguments": "",
- "started": False,
- "last_len": 0,
- },
- )
- if tc.id:
- entry["id"] = tc.id
- if tc.function and tc.function.name:
- entry["name"] = tc.function.name
- if tc.function and tc.function.arguments:
- entry["arguments"] += tc.function.arguments
- if entry.get("name") == "create_file":
- content = _extract_content_from_args(entry.get("arguments"))
- if content is not None:
- if not entry["started"]:
- path = (
- _extract_path_from_args(entry.get("arguments"))
- or self.file_state.path
- or "index.html"
- )
- await self._send(
- "toolStart",
- data={
- "name": "create_file",
- "input": {
- "path": path,
- "contentLength": len(content),
- "preview": _summarize_text(content, 200),
- },
- },
- event_id=entry["id"],
- )
- entry["started"] = True
- started_tool_ids.add(entry["id"])
- if entry["last_len"] == 0 and content:
- entry["last_len"] = len(content)
- await self._send("setCode", content)
- self._mark_preview_length(entry["id"], len(content))
- elif len(content) - entry["last_len"] >= 40:
- entry["last_len"] = len(content)
- await self._send("setCode", content)
- self._mark_preview_length(entry["id"], len(content))
+ return
- tool_call_list: List[ToolCall] = []
- for entry in tool_calls.values():
- args, error = _parse_json_arguments(entry.get("arguments"))
- if error:
- args = {"_raw": entry.get("arguments")}
- tool_call_id = entry.get("id") or f"tool-{uuid.uuid4().hex[:6]}"
- tool_call_list.append(
- ToolCall(
- id=tool_call_id,
- name=entry.get("name") or "unknown_tool",
- arguments=args,
+ if event.type == "thinking_delta":
+ if event.text:
+ await self._send(
+ "thinking",
+ event.text,
+ event_id=thinking_event_id,
)
- )
+ return
- if not tool_call_list:
- return await self._finalize_response(assistant_text)
+ if event.type == "tool_call_delta":
+ await self._handle_streamed_tool_delta(
+ event,
+ started_tool_ids,
+ streamed_lengths,
+ )
- assistant_message = {
- "role": "assistant",
- "content": assistant_text or None,
- "tool_calls": [
- {
- "id": tc.id,
- "type": "function",
- "function": {
- "name": tc.name,
- "arguments": json.dumps(tc.arguments),
- },
- }
- for tc in tool_call_list
- ],
- }
- messages.append(assistant_message)
+ step_result = await adapter.run_step(on_event)
- for tool_call in tool_call_list:
- tool_event_id = tool_call.id or self._next_event_id("tool")
- if tool_event_id not in started_tool_ids:
- await self._send(
- "toolStart",
- data={
- "name": tool_call.name,
- "input": self._summarize_tool_input(tool_call),
- },
- event_id=tool_event_id,
- )
- if tool_call.name == "create_file":
- content = _extract_content_from_args(tool_call.arguments)
- if content:
- await self._stream_code_preview(tool_event_id, content)
- tool_result = await self.toolbox.execute(tool_call)
- if tool_result.updated_content:
- await self._send("setCode", tool_result.updated_content)
+ if not step_result.tool_calls:
+ return await self._finalize_response(step_result.assistant_text)
+ executed_tool_calls: List[ExecutedToolCall] = []
+ for tool_call in step_result.tool_calls:
+ tool_event_id = tool_call.id or self._next_event_id("tool")
+ if tool_event_id not in started_tool_ids:
await self._send(
- "toolResult",
+ "toolStart",
data={
"name": tool_call.name,
- "output": tool_result.summary,
- "ok": tool_result.ok,
+ "input": summarize_tool_input(tool_call, self.file_state),
},
event_id=tool_event_id,
)
- messages.append(
- {
- "role": "tool",
- "tool_call_id": tool_call.id,
- "content": json.dumps(tool_result.result),
- }
- )
-
- raise Exception("Agent exceeded max tool turns")
- finally:
- await client.close()
-
- async def _run_openai_responses(
- self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]
- ) -> str:
- if not self.openai_api_key:
- raise Exception("OpenAI API key is missing.")
-
- client = AsyncOpenAI(api_key=self.openai_api_key, base_url=self.openai_base_url)
- openai_tools = self._openai_responses_tools()
- input_items: List[Dict[str, Any]] = [
- _convert_message_to_responses_input(m) for m in prompt_messages
- ]
-
- max_steps = 20
- try:
- for _ in range(max_steps):
- assistant_event_id = self._next_event_id("assistant")
- thinking_event_id = self._next_event_id("thinking")
- assistant_text = ""
- tool_calls: Dict[str, Dict[str, Any]] = {}
- item_to_call_id: Dict[str, str] = {}
- output_items_by_index: Dict[int, Dict[str, Any]] = {}
- started_tool_ids: set[str] = set()
-
- params: Dict[str, Any] = {
- "model": get_openai_api_name(model),
- "input": input_items,
- "tools": openai_tools,
- "tool_choice": "auto",
- "stream": True,
- "max_output_tokens": 30000,
- }
- reasoning_effort = get_openai_reasoning_effort(model)
- if reasoning_effort:
- params["reasoning"] = {"effort": reasoning_effort}
-
- responses_client = getattr(client, "responses", None)
- if responses_client is None:
- raise Exception(
- "OpenAI SDK is too old for GPT-5.2 Codex. Please upgrade the 'openai' package to a version that supports the Responses API."
- )
- stream = await responses_client.create(**params) # type: ignore
+ if tool_call.name == "create_file":
+ content = extract_content_from_args(tool_call.arguments)
+ if content:
+ await self._stream_code_preview(tool_event_id, content)
- async for event in stream: # type: ignore
- event_type = _get_event_attr(event, "type")
- if event_type in (
- "response.created",
- "response.completed",
- "response.done",
- "response.output_item.done",
- ):
- if event_type == "response.output_item.done":
- output_index = _get_event_attr(event, "output_index")
- item = _get_event_attr(event, "item")
- if isinstance(output_index, int) and item:
- output_items_by_index[output_index] = item
- continue
- if event_type == "response.output_text.delta":
- delta = _get_event_attr(event, "delta", "")
- if delta:
- assistant_text += delta
- await self._send(
- "assistant", delta, event_id=assistant_event_id
- )
- continue
- if event_type in (
- "response.reasoning_text.delta",
- "response.reasoning_summary_text.delta",
- ):
- delta = _get_event_attr(event, "delta", "")
- if delta:
- await self._send("thinking", delta, event_id=thinking_event_id)
- continue
- if event_type in (
- "response.reasoning_summary_part.added",
- "response.reasoning_summary_part.done",
- ):
- part = _get_event_attr(event, "part") or {}
- text = _get_event_attr(part, "text", "")
- if text:
- await self._send("thinking", text, event_id=thinking_event_id)
- continue
- if event_type == "response.output_item.added":
- item = _get_event_attr(event, "item")
- item_type = _get_event_attr(item, "type") if item else None
- output_index = _get_event_attr(event, "output_index")
- if isinstance(output_index, int) and item:
- output_items_by_index.setdefault(output_index, item)
- if item and item_type in ("function_call", "custom_tool_call"):
- item_id = _get_event_attr(item, "id")
- call_id = _get_event_attr(item, "call_id") or item_id
- if item_id and call_id:
- item_to_call_id[item_id] = call_id
- if call_id:
- if item_id and item_id in tool_calls and item_id != call_id:
- existing = tool_calls.pop(item_id)
- tool_calls[call_id] = {
- **existing,
- "id": call_id,
- }
- args_value = _get_event_attr(item, "arguments")
- if args_value is None and item_type == "custom_tool_call":
- args_value = _get_event_attr(item, "input")
- tool_calls.setdefault(
- call_id,
- {
- "id": call_id,
- "name": _get_event_attr(item, "name"),
- "arguments": args_value or "",
- "started": False,
- "last_len": 0,
- },
- )
- continue
- if event_type in (
- "response.function_call_arguments.delta",
- "response.mcp_call_arguments.delta",
- "response.custom_tool_call_input.delta",
- ):
- item_id = _get_event_attr(event, "item_id")
- call_id = _get_event_attr(event, "call_id")
- if call_id and item_id:
- item_to_call_id[item_id] = call_id
- if not call_id:
- call_id = item_to_call_id.get(item_id) if item_id else None
- if not call_id and item_id:
- call_id = item_id
- if not call_id:
- continue
- entry = tool_calls.setdefault(
- call_id,
- {
- "id": call_id,
- "name": _get_event_attr(event, "name"),
- "arguments": "",
- "started": False,
- "last_len": 0,
- },
- )
- delta_value = _get_event_attr(event, "delta")
- if delta_value is None:
- delta_value = _get_event_attr(event, "input")
- entry["arguments"] += delta_value or ""
- if entry.get("name") == "create_file":
- content = _extract_content_from_args(entry.get("arguments"))
- if content is not None:
- if not entry["started"]:
- path = (
- _extract_path_from_args(entry.get("arguments"))
- or self.file_state.path
- or "index.html"
- )
- await self._send(
- "toolStart",
- data={
- "name": "create_file",
- "input": {
- "path": path,
- "contentLength": len(content),
- "preview": _summarize_text(content, 200),
- },
- },
- event_id=entry["id"],
- )
- entry["started"] = True
- started_tool_ids.add(entry["id"])
- if entry["last_len"] == 0 and content:
- entry["last_len"] = len(content)
- await self._send("setCode", content)
- self._mark_preview_length(entry["id"], len(content))
- elif len(content) - entry["last_len"] >= 40:
- entry["last_len"] = len(content)
- await self._send("setCode", content)
- self._mark_preview_length(entry["id"], len(content))
- continue
- if event_type in (
- "response.function_call_arguments.done",
- "response.mcp_call_arguments.done",
- "response.custom_tool_call_input.done",
- ):
- item_id = _get_event_attr(event, "item_id")
- call_id = _get_event_attr(event, "call_id")
- if call_id and item_id:
- item_to_call_id[item_id] = call_id
- if not call_id:
- call_id = item_to_call_id.get(item_id) if item_id else None
- if not call_id and item_id:
- call_id = item_id
- if not call_id:
- continue
- entry = tool_calls.setdefault(
- call_id,
- {
- "id": call_id,
- "name": _get_event_attr(event, "name"),
- "arguments": "",
- "started": False,
- "last_len": 0,
- },
- )
- final_value = _get_event_attr(event, "arguments")
- if final_value is None:
- final_value = _get_event_attr(event, "input")
- if final_value is None:
- final_value = entry["arguments"]
- entry["arguments"] = final_value
- if _get_event_attr(event, "name"):
- entry["name"] = _get_event_attr(event, "name")
- output_index = _get_event_attr(event, "output_index")
- if (
- item_id
- and isinstance(output_index, int)
- and isinstance(output_items_by_index.get(output_index), dict)
- ):
- output_items_by_index[output_index] = {
- **output_items_by_index[output_index],
- "arguments": entry["arguments"],
- "call_id": call_id,
- "name": entry.get("name"),
- }
+ tool_result = await self.toolbox.execute(tool_call)
+ if tool_result.updated_content:
+ await self._send("setCode", tool_result.updated_content)
- output_items = [
- output_items_by_index[idx]
- for idx in sorted(output_items_by_index.keys())
- if output_items_by_index.get(idx)
- ]
- tool_items = [
- item
- for item in output_items
- if isinstance(item, dict)
- and item.get("type") in ("function_call", "custom_tool_call")
- ]
- if not tool_items and not tool_calls:
- return await self._finalize_response(assistant_text)
+ await self._send(
+ "toolResult",
+ data={
+ "name": tool_call.name,
+ "output": tool_result.summary,
+ "ok": tool_result.ok,
+ },
+ event_id=tool_event_id,
+ )
+ executed_tool_calls.append(
+ ExecutedToolCall(tool_call=tool_call, result=tool_result)
+ )
- tool_call_list: List[ToolCall] = []
- if tool_items:
- for item in tool_items:
- raw_args = item.get("arguments")
- if raw_args is None and item.get("type") == "custom_tool_call":
- raw_args = item.get("input")
- args, error = _parse_json_arguments(raw_args)
- if error:
- args = {"_raw": raw_args}
- call_id = item.get("call_id") or item.get("id")
- tool_call_list.append(
- ToolCall(
- id=call_id or f"call-{uuid.uuid4().hex[:6]}",
- name=item.get("name") or "unknown_tool",
- arguments=args,
- )
- )
- else:
- for entry in tool_calls.values():
- args, error = _parse_json_arguments(entry.get("arguments"))
- if error:
- args = {"_raw": entry.get("arguments")}
- call_id = entry.get("id") or entry.get("call_id")
- tool_call_list.append(
- ToolCall(
- id=call_id or f"call-{uuid.uuid4().hex[:6]}",
- name=entry.get("name") or "unknown_tool",
- arguments=args,
- )
- )
+ adapter.apply_tool_results(step_result, executed_tool_calls)
- tool_output_items: List[Dict[str, Any]] = []
- for tool_call in tool_call_list:
- tool_event_id = tool_call.id or self._next_event_id("tool")
- if tool_event_id not in started_tool_ids:
- await self._send(
- "toolStart",
- data={
- "name": tool_call.name,
- "input": self._summarize_tool_input(tool_call),
- },
- event_id=tool_event_id,
- )
- if tool_call.name == "create_file":
- content = _extract_content_from_args(tool_call.arguments)
- if content:
- await self._stream_code_preview(tool_event_id, content)
+ raise Exception("Agent exceeded max tool turns")
- tool_result = await self.toolbox.execute(tool_call)
- if tool_result.updated_content:
- await self._send("setCode", tool_result.updated_content)
+ async def run(self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]) -> str:
+ seed_file_state_from_messages(self.file_state, prompt_messages)
- await self._send(
- "toolResult",
- data={
- "name": tool_call.name,
- "output": tool_result.summary,
- "ok": tool_result.ok,
- },
- event_id=tool_event_id,
- )
+ if model in OPENAI_MODELS:
+ return await self._run_openai(model, prompt_messages)
+ if model in ANTHROPIC_MODELS:
+ return await self._run_anthropic(model, prompt_messages)
+ if model in GEMINI_MODELS:
+ return await self._run_gemini(model, prompt_messages)
+ raise ValueError(f"Unsupported model: {model.value}")
- tool_output_items.append(
- {
- "type": "function_call_output",
- "call_id": tool_call.id,
- "output": json.dumps(tool_result.result),
- }
- )
+ async def _run_openai(
+ self,
+ model: Llm,
+ prompt_messages: List[ChatCompletionMessageParam],
+ ) -> str:
+ if not self.openai_api_key:
+ raise Exception("OpenAI API key is missing.")
- input_items.extend(output_items)
- input_items.extend(tool_output_items)
+ client = AsyncOpenAI(api_key=self.openai_api_key, base_url=self.openai_base_url)
+ canonical_tools = canonical_tool_definitions()
- raise Exception("Agent exceeded max tool turns")
+ try:
+ if model in OPENAI_CODEX_MODELS:
+ tools = serialize_openai_responses_tools(canonical_tools)
+ adapter: ProviderAdapter = OpenAIResponsesAdapter(
+ client=client,
+ model=model,
+ prompt_messages=prompt_messages,
+ tools=tools,
+ )
+ else:
+ tools = serialize_openai_chat_tools(canonical_tools)
+ adapter = OpenAIChatAdapter(
+ client=client,
+ model=model,
+ prompt_messages=prompt_messages,
+ tools=tools,
+ )
+ return await self._run_with_adapter(adapter)
finally:
await client.close()
async def _run_anthropic(
- self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]
+ self,
+ model: Llm,
+ prompt_messages: List[ChatCompletionMessageParam],
) -> str:
if not self.anthropic_api_key:
raise Exception("Anthropic API key is missing.")
- from models.claude import convert_openai_messages_to_claude
-
- system_prompt, claude_messages = convert_openai_messages_to_claude(
- prompt_messages
- )
client = AsyncAnthropic(api_key=self.anthropic_api_key)
- _, anthropic_tools, _ = self._tool_schemas()
+ canonical_tools = canonical_tool_definitions()
+ tools = serialize_anthropic_tools(canonical_tools)
- max_steps = 20
try:
- for _ in range(max_steps):
- assistant_event_id = self._next_event_id("assistant")
- thinking_event_id = self._next_event_id("thinking")
- assistant_text = ""
- tool_blocks: Dict[int, Dict[str, Any]] = {}
- tool_json_buffers: Dict[int, str] = {}
- started_tool_ids: set[str] = set()
-
- thinking_models = [
- Llm.CLAUDE_4_5_SONNET_2025_09_29.value,
- Llm.CLAUDE_4_5_OPUS_2025_11_01.value,
- ]
-
- stream_kwargs: Dict[str, Any] = {
- "model": model.value,
- "max_tokens": 30000,
- "system": system_prompt,
- "messages": claude_messages,
- "tools": anthropic_tools,
- }
-
- if model.value in thinking_models:
- stream_kwargs["thinking"] = {
- "type": "enabled",
- "budget_tokens": 10000,
- }
- else:
- stream_kwargs["temperature"] = 0.0
-
- stream_ctx = None
- stream_client = None
- if hasattr(client, "beta") and hasattr(client.beta, "messages"):
- stream_client = client.beta.messages
- else:
- stream_client = client.messages
- try:
- stream_ctx = stream_client.stream(
- **stream_kwargs,
- betas=["fine-grained-tool-streaming-2025-05-14"],
- )
- except TypeError:
- stream_ctx = stream_client.stream(**stream_kwargs)
-
- async with stream_ctx as stream:
- async for event in stream:
- if event.type == "content_block_start":
- block = event.content_block
- if getattr(block, "type", None) == "tool_use":
- tool_id = getattr(block, "id", None) or f"tool-{uuid.uuid4().hex[:6]}"
- tool_name = getattr(block, "name", None) or "unknown_tool"
- tool_blocks[event.index] = {
- "id": tool_id,
- "name": tool_name,
- }
- tool_json_buffers[event.index] = ""
- if tool_name == "create_file":
- args = getattr(block, "input", None)
- content = _extract_content_from_args(args)
- if tool_id not in started_tool_ids:
- path = (
- _extract_path_from_args(args)
- or self.file_state.path
- or "index.html"
- )
- await self._send(
- "toolStart",
- data={
- "name": "create_file",
- "input": {
- "path": path,
- "contentLength": len(content or ""),
- "preview": _summarize_text(content or "", 200),
- },
- },
- event_id=tool_id,
- )
- started_tool_ids.add(tool_id)
- if content:
- await self._stream_code_preview(tool_id, content)
- if event.type == "content_block_delta":
- if event.delta.type == "thinking_delta":
- await self._send(
- "thinking",
- event.delta.thinking,
- event_id=thinking_event_id,
- )
- elif event.delta.type == "text_delta":
- assistant_text += event.delta.text
- await self._send(
- "assistant",
- event.delta.text,
- event_id=assistant_event_id,
- )
- elif event.delta.type == "input_json_delta":
- partial_json = getattr(event.delta, "partial_json", None) or ""
- if partial_json:
- buffer = tool_json_buffers.get(event.index, "") + partial_json
- tool_json_buffers[event.index] = buffer
- meta = tool_blocks.get(event.index)
- if meta and meta.get("name") == "create_file":
- content = _extract_content_from_args(buffer)
- if meta["id"] not in started_tool_ids:
- path = (
- _extract_path_from_args(buffer)
- or self.file_state.path
- or "index.html"
- )
- await self._send(
- "toolStart",
- data={
- "name": "create_file",
- "input": {
- "path": path,
- "contentLength": len(content or ""),
- "preview": _summarize_text(content or "", 200),
- },
- },
- event_id=meta["id"],
- )
- started_tool_ids.add(meta["id"])
- if content:
- await self._stream_code_preview(meta["id"], content)
-
- final_message = await stream.get_final_message()
-
- tool_calls: List[ToolCall] = []
- if final_message and final_message.content:
- for block in final_message.content:
- if block.type == "tool_use":
- tool_calls.append(
- ToolCall(
- id=block.id,
- name=block.name,
- arguments=block.input,
- )
- )
-
- if not tool_calls:
- return await self._finalize_response(assistant_text)
-
- assistant_blocks: List[Dict[str, Any]] = []
- if assistant_text:
- assistant_blocks.append({"type": "text", "text": assistant_text})
- for call in tool_calls:
- assistant_blocks.append(
- {
- "type": "tool_use",
- "id": call.id,
- "name": call.name,
- "input": call.arguments,
- }
- )
-
- claude_messages.append({"role": "assistant", "content": assistant_blocks})
-
- tool_result_blocks: List[Dict[str, Any]] = []
- for tool_call in tool_calls:
- tool_event_id = tool_call.id or self._next_event_id("tool")
- if tool_event_id not in started_tool_ids:
- await self._send(
- "toolStart",
- data={
- "name": tool_call.name,
- "input": self._summarize_tool_input(tool_call),
- },
- event_id=tool_event_id,
- )
- if tool_call.name == "create_file":
- content = _extract_content_from_args(tool_call.arguments)
- if content:
- await self._stream_code_preview(tool_event_id, content)
- tool_result = await self.toolbox.execute(tool_call)
- if tool_result.updated_content:
- await self._send("setCode", tool_result.updated_content)
-
- await self._send(
- "toolResult",
- data={
- "name": tool_call.name,
- "output": tool_result.summary,
- "ok": tool_result.ok,
- },
- event_id=tool_event_id,
- )
-
- tool_result_blocks.append(
- {
- "type": "tool_result",
- "tool_use_id": tool_call.id,
- "content": json.dumps(tool_result.result),
- "is_error": not tool_result.ok,
- }
- )
-
- claude_messages.append({"role": "user", "content": tool_result_blocks})
-
- raise Exception("Agent exceeded max tool turns")
+ adapter: ProviderAdapter = AnthropicAdapter(
+ client=client,
+ model=model,
+ prompt_messages=prompt_messages,
+ tools=tools,
+ )
+ return await self._run_with_adapter(adapter)
finally:
await client.close()
async def _run_gemini(
- self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]
+ self,
+ model: Llm,
+ prompt_messages: List[ChatCompletionMessageParam],
) -> str:
if not self.gemini_api_key:
raise Exception("Gemini API key is missing.")
- from models.gemini import (
- convert_message_to_gemini_content,
- get_gemini_api_model_name,
- get_thinking_level_for_model,
- )
-
- system_prompt = str(prompt_messages[0].get("content", ""))
- gemini_contents: List[types.Content] = [
- convert_message_to_gemini_content(msg) for msg in prompt_messages[1:]
- ]
-
- _, _, gemini_tools = self._tool_schemas()
client = genai.Client(api_key=self.gemini_api_key)
-
- max_steps = 20
- for _ in range(max_steps):
- assistant_event_id = self._next_event_id("assistant")
- thinking_event_id = self._next_event_id("thinking")
- assistant_text = ""
- tool_calls: List[ToolCall] = []
- function_call_parts: List[types.Part] = []
- tool_stream_state: Dict[str, Dict[str, Any]] = {}
- started_tool_ids: set[str] = set()
-
- thinking_level = get_thinking_level_for_model(model)
- config = types.GenerateContentConfig(
- temperature=0,
- max_output_tokens=30000,
- system_instruction=system_prompt,
- thinking_config=types.ThinkingConfig(
- thinking_level=thinking_level, include_thoughts=True
- ),
- tools=gemini_tools,
- )
-
- api_model_name = get_gemini_api_model_name(model)
-
- async for chunk in await client.aio.models.generate_content_stream(
- model=api_model_name,
- contents=gemini_contents,
- config=config,
- ):
- if not chunk.candidates:
- continue
- for part in chunk.candidates[0].content.parts:
- if getattr(part, "thought", False) and part.text:
- await self._send(
- "thinking", part.text, event_id=thinking_event_id
- )
- continue
- if part.function_call:
- function_call_parts.append(part)
- args = part.function_call.args or {}
- tool_id = part.function_call.id or f"tool-{uuid.uuid4().hex[:6]}"
- tool_name = part.function_call.name or "unknown_tool"
- if tool_name == "create_file":
- content = _extract_content_from_args(args)
- if content is not None:
- state = tool_stream_state.setdefault(
- tool_id, {"started": False, "last_len": 0}
- )
- if not state["started"]:
- path = (
- _extract_path_from_args(args)
- or self.file_state.path
- or "index.html"
- )
- await self._send(
- "toolStart",
- data={
- "name": "create_file",
- "input": {
- "path": path,
- "contentLength": len(content),
- "preview": _summarize_text(
- content, 200
- ),
- },
- },
- event_id=tool_id,
- )
- state["started"] = True
- started_tool_ids.add(tool_id)
- if state["last_len"] == 0 and content:
- state["last_len"] = len(content)
- await self._send("setCode", content)
- self._mark_preview_length(tool_id, len(content))
- elif len(content) - state["last_len"] >= 40:
- state["last_len"] = len(content)
- await self._send("setCode", content)
- self._mark_preview_length(tool_id, len(content))
- tool_calls.append(
- ToolCall(
- id=tool_id,
- name=tool_name,
- arguments=args,
- )
- )
- continue
- if part.text:
- assistant_text += part.text
- await self._send(
- "assistant", part.text, event_id=assistant_event_id
- )
-
- if not tool_calls:
- return await self._finalize_response(assistant_text)
-
- model_parts: List[Any] = []
- if assistant_text:
- model_parts.append({"text": assistant_text}) # type: ignore
- model_parts.extend(function_call_parts)
-
- model_content = types.Content(role="model", parts=model_parts)
- gemini_contents.append(model_content)
-
- tool_result_parts: List[types.Part] = []
- for tool_call in tool_calls:
- tool_event_id = tool_call.id or self._next_event_id("tool")
- if tool_event_id not in started_tool_ids:
- await self._send(
- "toolStart",
- data={
- "name": tool_call.name,
- "input": self._summarize_tool_input(tool_call),
- },
- event_id=tool_event_id,
- )
- if tool_call.name == "create_file":
- content = _extract_content_from_args(tool_call.arguments)
- if content:
- await self._stream_code_preview(tool_event_id, content)
- tool_result = await self.toolbox.execute(tool_call)
- if tool_result.updated_content:
- await self._send("setCode", tool_result.updated_content)
-
- await self._send(
- "toolResult",
- data={
- "name": tool_call.name,
- "output": tool_result.summary,
- "ok": tool_result.ok,
- },
- event_id=tool_event_id,
- )
-
- tool_result_parts.append(
- types.Part.from_function_response(
- name=tool_call.name, response=tool_result.result
- )
- )
-
- gemini_contents.append(types.Content(role="tool", parts=tool_result_parts))
-
- raise Exception("Agent exceeded max tool turns")
+ canonical_tools = canonical_tool_definitions()
+ tools = serialize_gemini_tools(canonical_tools)
+ adapter: ProviderAdapter = GeminiAdapter(
+ client=client,
+ model=model,
+ prompt_messages=prompt_messages,
+ tools=tools,
+ )
+ return await self._run_with_adapter(adapter)
async def _finalize_response(self, assistant_text: str) -> str:
if self.file_state.content:
@@ -1871,4 +340,5 @@ class AgenticRunner:
if html:
self.file_state.content = html
await self._send("setCode", html)
+
return self.file_state.content
diff --git a/backend/agentic/state.py b/backend/agentic/state.py
new file mode 100644
index 00000000..b8d8bb2f
--- /dev/null
+++ b/backend/agentic/state.py
@@ -0,0 +1,71 @@
+from dataclasses import dataclass
+from typing import Any, List
+
+from openai.types.chat import ChatCompletionMessageParam
+
+from codegen.utils import extract_html_content
+
+
+@dataclass
+class AgentFileState:
+ path: str = "index.html"
+ content: str = ""
+
+
+def ensure_str(value: Any) -> str:
+ if value is None:
+ return ""
+ return str(value)
+
+
+def extract_text_content(message: ChatCompletionMessageParam) -> str:
+ content = message.get("content", "")
+ if isinstance(content, str):
+ return content
+ if isinstance(content, list):
+ for part in content:
+ if isinstance(part, dict) and part.get("type") == "text":
+ return ensure_str(part.get("text"))
+ return ""
+
+
+def seed_file_state_from_messages(
+ file_state: AgentFileState,
+ prompt_messages: List[ChatCompletionMessageParam],
+) -> None:
+ if file_state.content:
+ return
+
+ for message in reversed(prompt_messages):
+ if message.get("role") != "assistant":
+ continue
+ raw_text = extract_text_content(message)
+ if not raw_text:
+ continue
+ extracted = extract_html_content(raw_text)
+ file_state.content = extracted or raw_text
+ if not file_state.path:
+ file_state.path = "index.html"
+ return
+
+ if not prompt_messages:
+ return
+
+ system_message = prompt_messages[0]
+ if system_message.get("role") != "system":
+ return
+
+ system_text = extract_text_content(system_message)
+ markers = [
+ "Here is the code of the app:",
+ "Here is the code of the SVG:",
+ ]
+ for marker in markers:
+ if marker not in system_text:
+ continue
+ raw_text = system_text.split(marker, 1)[1].strip()
+ extracted = extract_html_content(raw_text)
+ file_state.content = extracted or raw_text
+ if not file_state.path:
+ file_state.path = "index.html"
+ return
diff --git a/backend/agentic/streaming/__init__.py b/backend/agentic/streaming/__init__.py
new file mode 100644
index 00000000..42dfac00
--- /dev/null
+++ b/backend/agentic/streaming/__init__.py
@@ -0,0 +1,18 @@
+from agentic.streaming.anthropic import AnthropicAdapter
+from agentic.streaming.base import ProviderAdapter
+from agentic.streaming.gemini import GeminiAdapter
+from agentic.streaming.openai_chat import OpenAIChatAdapter
+from agentic.streaming.openai_responses import OpenAIResponsesAdapter
+from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult, StreamEvent
+
+__all__ = [
+ "AnthropicAdapter",
+ "ProviderAdapter",
+ "GeminiAdapter",
+ "OpenAIChatAdapter",
+ "OpenAIResponsesAdapter",
+ "EventSink",
+ "ExecutedToolCall",
+ "StepResult",
+ "StreamEvent",
+]
diff --git a/backend/agentic/streaming/anthropic.py b/backend/agentic/streaming/anthropic.py
new file mode 100644
index 00000000..600e389c
--- /dev/null
+++ b/backend/agentic/streaming/anthropic.py
@@ -0,0 +1,171 @@
+import json
+import uuid
+from typing import Any, Dict, List
+
+from anthropic import AsyncAnthropic
+from openai.types.chat import ChatCompletionMessageParam
+
+from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult, StreamEvent
+from agentic.tools import ToolCall
+from llm import Llm
+from models.claude import convert_openai_messages_to_claude
+
+
+THINKING_MODELS = {
+ Llm.CLAUDE_4_5_SONNET_2025_09_29.value,
+ Llm.CLAUDE_4_5_OPUS_2025_11_01.value,
+}
+
+
+class AnthropicAdapter:
+ def __init__(
+ self,
+ client: AsyncAnthropic,
+ model: Llm,
+ prompt_messages: List[ChatCompletionMessageParam],
+ tools: List[Dict[str, Any]],
+ ):
+ self._client = client
+ self._model = model
+ self._tools = tools
+ system_prompt, claude_messages = convert_openai_messages_to_claude(prompt_messages)
+ self._system_prompt = system_prompt
+ self._messages = claude_messages
+
+ async def run_step(self, on_event: EventSink) -> StepResult:
+ assistant_text = ""
+ tool_blocks: Dict[int, Dict[str, Any]] = {}
+ tool_json_buffers: Dict[int, str] = {}
+
+ stream_kwargs: Dict[str, Any] = {
+ "model": self._model.value,
+ "max_tokens": 30000,
+ "system": self._system_prompt,
+ "messages": self._messages,
+ "tools": self._tools,
+ }
+
+ if self._model.value in THINKING_MODELS:
+ stream_kwargs["thinking"] = {
+ "type": "enabled",
+ "budget_tokens": 10000,
+ }
+ else:
+ stream_kwargs["temperature"] = 0.0
+
+ if hasattr(self._client, "beta") and hasattr(self._client.beta, "messages"):
+ stream_client = self._client.beta.messages
+ else:
+ stream_client = self._client.messages
+
+ try:
+ stream_kwargs_with_betas = dict(stream_kwargs)
+ stream_kwargs_with_betas["betas"] = ["fine-grained-tool-streaming-2025-05-14"]
+ stream_ctx = stream_client.stream(**stream_kwargs_with_betas)
+ except TypeError:
+ stream_ctx = stream_client.stream(**stream_kwargs)
+
+ async with stream_ctx as stream:
+ async for event in stream:
+ if event.type == "content_block_start":
+ block = event.content_block
+ if getattr(block, "type", None) == "tool_use":
+ tool_id = getattr(block, "id", None) or f"tool-{uuid.uuid4().hex[:6]}"
+ tool_name = getattr(block, "name", None) or "unknown_tool"
+ args = getattr(block, "input", None)
+ tool_blocks[event.index] = {
+ "id": tool_id,
+ "name": tool_name,
+ }
+ tool_json_buffers[event.index] = ""
+ if args:
+ await on_event(
+ StreamEvent(
+ type="tool_call_delta",
+ tool_call_id=tool_id,
+ tool_name=tool_name,
+ tool_arguments=args,
+ )
+ )
+ continue
+
+ if event.type != "content_block_delta":
+ continue
+
+ if event.delta.type == "thinking_delta":
+ await on_event(
+ StreamEvent(type="thinking_delta", text=event.delta.thinking)
+ )
+ elif event.delta.type == "text_delta":
+ assistant_text += event.delta.text
+ await on_event(
+ StreamEvent(type="assistant_delta", text=event.delta.text)
+ )
+ elif event.delta.type == "input_json_delta":
+ partial_json = getattr(event.delta, "partial_json", None) or ""
+ if not partial_json:
+ continue
+ buffer = tool_json_buffers.get(event.index, "") + partial_json
+ tool_json_buffers[event.index] = buffer
+ meta = tool_blocks.get(event.index)
+ if not meta:
+ continue
+ await on_event(
+ StreamEvent(
+ type="tool_call_delta",
+ tool_call_id=meta.get("id"),
+ tool_name=meta.get("name"),
+ tool_arguments=buffer,
+ )
+ )
+
+ final_message = await stream.get_final_message()
+
+ tool_calls: List[ToolCall] = []
+ if final_message and final_message.content:
+ for block in final_message.content:
+ if block.type != "tool_use":
+ continue
+ tool_calls.append(
+ ToolCall(
+ id=block.id,
+ name=block.name,
+ arguments=block.input,
+ )
+ )
+
+ return StepResult(assistant_text=assistant_text, tool_calls=tool_calls)
+
+ def apply_tool_results(
+ self,
+ step_result: StepResult,
+ executed_tool_calls: list[ExecutedToolCall],
+ ) -> None:
+ assistant_blocks: List[Dict[str, Any]] = []
+ if step_result.assistant_text:
+ assistant_blocks.append({"type": "text", "text": step_result.assistant_text})
+
+ for call in step_result.tool_calls:
+ assistant_blocks.append(
+ {
+ "type": "tool_use",
+ "id": call.id,
+ "name": call.name,
+ "input": call.arguments,
+ }
+ )
+
+ self._messages.append({"role": "assistant", "content": assistant_blocks})
+
+ tool_result_blocks: List[Dict[str, Any]] = []
+ for executed in executed_tool_calls:
+ tool_result_blocks.append(
+ {
+ "type": "tool_result",
+ "tool_use_id": executed.tool_call.id,
+ "content": json.dumps(executed.result.result),
+ "is_error": not executed.result.ok,
+ }
+ )
+
+ self._messages.append({"role": "user", "content": tool_result_blocks})
diff --git a/backend/agentic/streaming/base.py b/backend/agentic/streaming/base.py
new file mode 100644
index 00000000..4ec373ab
--- /dev/null
+++ b/backend/agentic/streaming/base.py
@@ -0,0 +1,15 @@
+from typing import Protocol
+
+from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult
+
+
+class ProviderAdapter(Protocol):
+ async def run_step(self, on_event: EventSink) -> StepResult:
+ ...
+
+ def apply_tool_results(
+ self,
+ step_result: StepResult,
+ executed_tool_calls: list[ExecutedToolCall],
+ ) -> None:
+ ...
diff --git a/backend/agentic/streaming/gemini.py b/backend/agentic/streaming/gemini.py
new file mode 100644
index 00000000..8aafdeca
--- /dev/null
+++ b/backend/agentic/streaming/gemini.py
@@ -0,0 +1,132 @@
+# pyright: reportUnknownVariableType=false
+import uuid
+from typing import Any, Dict, List, cast
+
+from google import genai
+from google.genai import types
+from openai.types.chat import ChatCompletionMessageParam
+
+from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult, StreamEvent
+from agentic.tools import ToolCall
+from llm import Llm
+from models.gemini import (
+ convert_message_to_gemini_content,
+ get_gemini_api_model_name,
+ get_thinking_level_for_model,
+)
+
+
+class GeminiAdapter:
+ def __init__(
+ self,
+ client: genai.Client,
+ model: Llm,
+ prompt_messages: List[ChatCompletionMessageParam],
+ tools: List[types.Tool],
+ ):
+ self._client = client
+ self._model = model
+ self._tools = tools
+
+ self._system_prompt = str(prompt_messages[0].get("content", ""))
+ self._contents: List[types.Content] = [
+ convert_message_to_gemini_content(msg) for msg in prompt_messages[1:]
+ ]
+
+ async def run_step(self, on_event: EventSink) -> StepResult:
+ assistant_text = ""
+ tool_calls: List[ToolCall] = []
+ function_call_parts: List[types.Part] = []
+
+ thinking_level = get_thinking_level_for_model(self._model)
+ config = types.GenerateContentConfig(
+ temperature=0,
+ max_output_tokens=30000,
+ system_instruction=self._system_prompt,
+ thinking_config=types.ThinkingConfig(
+ thinking_level=cast(Any, thinking_level),
+ include_thoughts=True,
+ ),
+ tools=self._tools,
+ )
+
+ api_model_name = get_gemini_api_model_name(self._model)
+
+ async for chunk in await self._client.aio.models.generate_content_stream(
+ model=api_model_name,
+ contents=cast(Any, self._contents),
+ config=config,
+ ):
+ if not chunk.candidates:
+ continue
+
+ candidate_content = chunk.candidates[0].content
+ if not candidate_content or not candidate_content.parts:
+ continue
+
+ for part in candidate_content.parts:
+ if getattr(part, "thought", False) and part.text:
+ await on_event(StreamEvent(type="thinking_delta", text=part.text))
+ continue
+
+ if part.function_call:
+ function_call_parts.append(part)
+ args = part.function_call.args or {}
+ tool_id = part.function_call.id or f"tool-{uuid.uuid4().hex[:6]}"
+ tool_name = part.function_call.name or "unknown_tool"
+
+ await on_event(
+ StreamEvent(
+ type="tool_call_delta",
+ tool_call_id=tool_id,
+ tool_name=tool_name,
+ tool_arguments=args,
+ )
+ )
+
+ tool_calls.append(
+ ToolCall(
+ id=tool_id,
+ name=tool_name,
+ arguments=args,
+ )
+ )
+ continue
+
+ if part.text:
+ assistant_text += part.text
+ await on_event(StreamEvent(type="assistant_delta", text=part.text))
+
+ return StepResult(
+ assistant_text=assistant_text,
+ tool_calls=tool_calls,
+ provider_state={"function_call_parts": function_call_parts},
+ )
+
+ def apply_tool_results(
+ self,
+ step_result: StepResult,
+ executed_tool_calls: list[ExecutedToolCall],
+ ) -> None:
+ function_call_parts = cast(
+ List[types.Part],
+ ((step_result.provider_state or {}).get("function_call_parts") or []),
+ )
+
+ model_parts: List[Any] = []
+ if step_result.assistant_text:
+ model_parts.append({"text": step_result.assistant_text}) # type: ignore
+ model_parts.extend(function_call_parts)
+
+ self._contents.append(types.Content(role="model", parts=model_parts))
+
+ tool_result_parts: List[types.Part] = []
+ for executed in executed_tool_calls:
+ tool_result_parts.append(
+ types.Part.from_function_response(
+ name=executed.tool_call.name,
+ response=executed.result.result,
+ )
+ )
+
+ self._contents.append(types.Content(role="tool", parts=tool_result_parts))
diff --git a/backend/agentic/streaming/openai_chat.py b/backend/agentic/streaming/openai_chat.py
new file mode 100644
index 00000000..41c01001
--- /dev/null
+++ b/backend/agentic/streaming/openai_chat.py
@@ -0,0 +1,138 @@
+# pyright: reportUnknownVariableType=false
+import json
+import uuid
+from typing import Any, Dict, List
+
+from openai import AsyncOpenAI
+from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam
+
+from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult, StreamEvent
+from agentic.tools import ToolCall, parse_json_arguments
+from llm import Llm, get_openai_api_name
+
+
+class OpenAIChatAdapter:
+ def __init__(
+ self,
+ client: AsyncOpenAI,
+ model: Llm,
+ prompt_messages: List[ChatCompletionMessageParam],
+ tools: List[Dict[str, Any]],
+ ):
+ self._client = client
+ self._model = model
+ self._tools = tools
+ self._messages: List[Dict[str, Any]] = [dict(m) for m in prompt_messages]
+
+ async def run_step(self, on_event: EventSink) -> StepResult:
+ assistant_text = ""
+ tool_calls: Dict[int, Dict[str, Any]] = {}
+
+ params: Dict[str, Any] = {
+ "model": get_openai_api_name(self._model),
+ "messages": self._messages,
+ "tools": self._tools,
+ "tool_choice": "auto",
+ "temperature": 0,
+ "stream": True,
+ "max_tokens": 30000,
+ }
+
+ stream = await self._client.chat.completions.create(**params)
+
+ async for chunk in stream: # type: ignore
+ assert isinstance(chunk, ChatCompletionChunk)
+ if not chunk.choices:
+ continue
+
+ delta = chunk.choices[0].delta
+ if delta.content:
+ assistant_text += delta.content
+ await on_event(StreamEvent(type="assistant_delta", text=delta.content))
+
+ if not delta.tool_calls:
+ continue
+
+ for tc in delta.tool_calls:
+ idx = tc.index or 0
+ entry = tool_calls.setdefault(
+ idx,
+ {
+ "id": tc.id or f"tool-{idx}-{uuid.uuid4().hex[:6]}",
+ "name": None,
+ "arguments": "",
+ },
+ )
+ if tc.id:
+ entry["id"] = tc.id
+ if tc.function and tc.function.name:
+ entry["name"] = tc.function.name
+ if tc.function and tc.function.arguments:
+ entry["arguments"] += tc.function.arguments
+ await on_event(
+ StreamEvent(
+ type="tool_call_delta",
+ tool_call_id=entry["id"],
+ tool_name=entry.get("name"),
+ tool_arguments=entry.get("arguments"),
+ )
+ )
+
+ tool_call_list: List[ToolCall] = []
+ for entry in tool_calls.values():
+ args, error = parse_json_arguments(entry.get("arguments"))
+ if error:
+ args = {"_raw": entry.get("arguments")}
+ tool_call_id = entry.get("id") or f"tool-{uuid.uuid4().hex[:6]}"
+ tool_call_list.append(
+ ToolCall(
+ id=tool_call_id,
+ name=entry.get("name") or "unknown_tool",
+ arguments=args,
+ )
+ )
+
+ provider_state = None
+ if tool_call_list:
+ provider_state = {
+ "assistant_message": {
+ "role": "assistant",
+ "content": assistant_text or None,
+ "tool_calls": [
+ {
+ "id": tc.id,
+ "type": "function",
+ "function": {
+ "name": tc.name,
+ "arguments": json.dumps(tc.arguments),
+ },
+ }
+ for tc in tool_call_list
+ ],
+ }
+ }
+
+ return StepResult(
+ assistant_text=assistant_text,
+ tool_calls=tool_call_list,
+ provider_state=provider_state,
+ )
+
+ def apply_tool_results(
+ self,
+ step_result: StepResult,
+ executed_tool_calls: list[ExecutedToolCall],
+ ) -> None:
+ provider_state = step_result.provider_state or {}
+ assistant_message = provider_state.get("assistant_message")
+ if assistant_message:
+ self._messages.append(assistant_message)
+
+ for executed in executed_tool_calls:
+ self._messages.append(
+ {
+ "role": "tool",
+ "tool_call_id": executed.tool_call.id,
+ "content": json.dumps(executed.result.result),
+ }
+ )
diff --git a/backend/agentic/streaming/openai_responses.py b/backend/agentic/streaming/openai_responses.py
new file mode 100644
index 00000000..53f4e9e0
--- /dev/null
+++ b/backend/agentic/streaming/openai_responses.py
@@ -0,0 +1,345 @@
+# pyright: reportUnknownVariableType=false
+import json
+import uuid
+from typing import Any, Dict, List
+
+from openai import AsyncOpenAI
+from openai.types.chat import ChatCompletionMessageParam
+
+from agentic.state import ensure_str
+from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult, StreamEvent
+from agentic.tools import ToolCall, parse_json_arguments
+from llm import Llm, get_openai_api_name, get_openai_reasoning_effort
+
+
+def _convert_message_to_responses_input(
+ message: ChatCompletionMessageParam,
+) -> Dict[str, Any]:
+ role = message.get("role", "user")
+ content = message.get("content", "")
+
+ if isinstance(content, str):
+ return {"role": role, "content": content}
+
+ parts: List[Dict[str, Any]] = []
+ if isinstance(content, list):
+ for part in content:
+ if not isinstance(part, dict):
+ continue
+ if part.get("type") == "text":
+ parts.append({"type": "input_text", "text": part.get("text", "")})
+ elif part.get("type") == "image_url":
+ image_url = part.get("image_url", {})
+ parts.append(
+ {
+ "type": "input_image",
+ "image_url": image_url.get("url", ""),
+ "detail": image_url.get("detail", "auto"),
+ }
+ )
+
+ return {"role": role, "content": parts}
+
+
+def _get_event_attr(event: Any, key: str, default: Any = None) -> Any:
+ if hasattr(event, key):
+ return getattr(event, key)
+ if isinstance(event, dict):
+ return event.get(key, default)
+ return default
+
+
+class OpenAIResponsesAdapter:
+ def __init__(
+ self,
+ client: AsyncOpenAI,
+ model: Llm,
+ prompt_messages: List[ChatCompletionMessageParam],
+ tools: List[Dict[str, Any]],
+ ):
+ self._client = client
+ self._model = model
+ self._tools = tools
+ self._input_items: List[Dict[str, Any]] = [
+ _convert_message_to_responses_input(m) for m in prompt_messages
+ ]
+
+ async def run_step(self, on_event: EventSink) -> StepResult:
+ assistant_text = ""
+ tool_calls: Dict[str, Dict[str, Any]] = {}
+ item_to_call_id: Dict[str, str] = {}
+ output_items_by_index: Dict[int, Dict[str, Any]] = {}
+
+ params: Dict[str, Any] = {
+ "model": get_openai_api_name(self._model),
+ "input": self._input_items,
+ "tools": self._tools,
+ "tool_choice": "auto",
+ "stream": True,
+ "max_output_tokens": 30000,
+ }
+
+ reasoning_effort = get_openai_reasoning_effort(self._model)
+ if reasoning_effort:
+ params["reasoning"] = {"effort": reasoning_effort}
+
+ responses_client = getattr(self._client, "responses", None)
+ if responses_client is None:
+ raise Exception(
+ "OpenAI SDK is too old for GPT-5.2 Codex. Please upgrade the 'openai' package to a version that supports the Responses API."
+ )
+
+ stream = await responses_client.create(**params) # type: ignore
+
+ async for event in stream: # type: ignore
+ event_type = _get_event_attr(event, "type")
+ if event_type in (
+ "response.created",
+ "response.completed",
+ "response.done",
+ "response.output_item.done",
+ ):
+ if event_type == "response.output_item.done":
+ output_index = _get_event_attr(event, "output_index")
+ item = _get_event_attr(event, "item")
+ if isinstance(output_index, int) and item:
+ output_items_by_index[output_index] = item
+ continue
+
+ if event_type == "response.output_text.delta":
+ delta = _get_event_attr(event, "delta", "")
+ if delta:
+ assistant_text += delta
+ await on_event(StreamEvent(type="assistant_delta", text=delta))
+ continue
+
+ if event_type in (
+ "response.reasoning_text.delta",
+ "response.reasoning_summary_text.delta",
+ ):
+ delta = _get_event_attr(event, "delta", "")
+ if delta:
+ await on_event(StreamEvent(type="thinking_delta", text=delta))
+ continue
+
+ if event_type in (
+ "response.reasoning_summary_part.added",
+ "response.reasoning_summary_part.done",
+ ):
+ part = _get_event_attr(event, "part") or {}
+ text = _get_event_attr(part, "text", "")
+ if text:
+ await on_event(StreamEvent(type="thinking_delta", text=text))
+ continue
+
+ if event_type == "response.output_item.added":
+ item = _get_event_attr(event, "item")
+ item_type = _get_event_attr(item, "type") if item else None
+ output_index = _get_event_attr(event, "output_index")
+ if isinstance(output_index, int) and item:
+ output_items_by_index.setdefault(output_index, item)
+
+ if item and item_type in ("function_call", "custom_tool_call"):
+ item_id = _get_event_attr(item, "id")
+ call_id = _get_event_attr(item, "call_id") or item_id
+ if item_id and call_id:
+ item_to_call_id[item_id] = call_id
+ if call_id:
+ if item_id and item_id in tool_calls and item_id != call_id:
+ existing = tool_calls.pop(item_id)
+ tool_calls[call_id] = {
+ **existing,
+ "id": call_id,
+ }
+ args_value = _get_event_attr(item, "arguments")
+ if args_value is None and item_type == "custom_tool_call":
+ args_value = _get_event_attr(item, "input")
+ tool_calls.setdefault(
+ call_id,
+ {
+ "id": call_id,
+ "name": _get_event_attr(item, "name"),
+ "arguments": args_value or "",
+ },
+ )
+ if args_value:
+ await on_event(
+ StreamEvent(
+ type="tool_call_delta",
+ tool_call_id=call_id,
+ tool_name=_get_event_attr(item, "name"),
+ tool_arguments=args_value,
+ )
+ )
+ continue
+
+ if event_type in (
+ "response.function_call_arguments.delta",
+ "response.mcp_call_arguments.delta",
+ "response.custom_tool_call_input.delta",
+ ):
+ item_id = _get_event_attr(event, "item_id")
+ call_id = _get_event_attr(event, "call_id")
+ if call_id and item_id:
+ item_to_call_id[item_id] = call_id
+ if not call_id:
+ call_id = item_to_call_id.get(item_id) if item_id else None
+ if not call_id and item_id:
+ call_id = item_id
+ if not call_id:
+ continue
+
+ entry = tool_calls.setdefault(
+ call_id,
+ {
+ "id": call_id,
+ "name": _get_event_attr(event, "name"),
+ "arguments": "",
+ },
+ )
+ delta_value = _get_event_attr(event, "delta")
+ if delta_value is None:
+ delta_value = _get_event_attr(event, "input")
+ entry["arguments"] += ensure_str(delta_value)
+
+ await on_event(
+ StreamEvent(
+ type="tool_call_delta",
+ tool_call_id=call_id,
+ tool_name=entry.get("name"),
+ tool_arguments=entry.get("arguments"),
+ )
+ )
+ continue
+
+ if event_type in (
+ "response.function_call_arguments.done",
+ "response.mcp_call_arguments.done",
+ "response.custom_tool_call_input.done",
+ ):
+ item_id = _get_event_attr(event, "item_id")
+ call_id = _get_event_attr(event, "call_id")
+ if call_id and item_id:
+ item_to_call_id[item_id] = call_id
+ if not call_id:
+ call_id = item_to_call_id.get(item_id) if item_id else None
+ if not call_id and item_id:
+ call_id = item_id
+ if not call_id:
+ continue
+
+ entry = tool_calls.setdefault(
+ call_id,
+ {
+ "id": call_id,
+ "name": _get_event_attr(event, "name"),
+ "arguments": "",
+ },
+ )
+ final_value = _get_event_attr(event, "arguments")
+ if final_value is None:
+ final_value = _get_event_attr(event, "input")
+ if final_value is None:
+ final_value = entry["arguments"]
+ entry["arguments"] = final_value
+ if _get_event_attr(event, "name"):
+ entry["name"] = _get_event_attr(event, "name")
+
+ await on_event(
+ StreamEvent(
+ type="tool_call_delta",
+ tool_call_id=call_id,
+ tool_name=entry.get("name"),
+ tool_arguments=entry.get("arguments"),
+ )
+ )
+
+ output_index = _get_event_attr(event, "output_index")
+ if (
+ item_id
+ and isinstance(output_index, int)
+ and isinstance(output_items_by_index.get(output_index), dict)
+ ):
+ output_items_by_index[output_index] = {
+ **output_items_by_index[output_index],
+ "arguments": entry["arguments"],
+ "call_id": call_id,
+ "name": entry.get("name"),
+ }
+
+ output_items = [
+ output_items_by_index[idx]
+ for idx in sorted(output_items_by_index.keys())
+ if output_items_by_index.get(idx)
+ ]
+
+ tool_items = [
+ item
+ for item in output_items
+ if isinstance(item, dict)
+ and item.get("type") in ("function_call", "custom_tool_call")
+ ]
+
+ tool_call_list: List[ToolCall] = []
+ if tool_items:
+ for item in tool_items:
+ raw_args = item.get("arguments")
+ if raw_args is None and item.get("type") == "custom_tool_call":
+ raw_args = item.get("input")
+ args, error = parse_json_arguments(raw_args)
+ if error:
+ args = {"_raw": raw_args}
+ call_id = item.get("call_id") or item.get("id")
+ tool_call_list.append(
+ ToolCall(
+ id=call_id or f"call-{uuid.uuid4().hex[:6]}",
+ name=item.get("name") or "unknown_tool",
+ arguments=args,
+ )
+ )
+ else:
+ for entry in tool_calls.values():
+ args, error = parse_json_arguments(entry.get("arguments"))
+ if error:
+ args = {"_raw": entry.get("arguments")}
+ call_id = entry.get("id") or entry.get("call_id")
+ tool_call_list.append(
+ ToolCall(
+ id=call_id or f"call-{uuid.uuid4().hex[:6]}",
+ name=entry.get("name") or "unknown_tool",
+ arguments=args,
+ )
+ )
+
+ provider_state = None
+ if tool_call_list:
+ provider_state = {
+ "output_items": output_items,
+ }
+
+ return StepResult(
+ assistant_text=assistant_text,
+ tool_calls=tool_call_list,
+ provider_state=provider_state,
+ )
+
+ def apply_tool_results(
+ self,
+ step_result: StepResult,
+ executed_tool_calls: list[ExecutedToolCall],
+ ) -> None:
+ provider_state = step_result.provider_state or {}
+ output_items = provider_state.get("output_items") or []
+
+ tool_output_items: List[Dict[str, Any]] = []
+ for executed in executed_tool_calls:
+ tool_output_items.append(
+ {
+ "type": "function_call_output",
+ "call_id": executed.tool_call.id,
+ "output": json.dumps(executed.result.result),
+ }
+ )
+
+ self._input_items.extend(output_items)
+ self._input_items.extend(tool_output_items)
diff --git a/backend/agentic/streaming/types.py b/backend/agentic/streaming/types.py
new file mode 100644
index 00000000..0dacf133
--- /dev/null
+++ b/backend/agentic/streaming/types.py
@@ -0,0 +1,36 @@
+from dataclasses import dataclass
+from typing import Any, Awaitable, Callable, List, Literal, Optional
+
+from agentic.tools import ToolCall, ToolExecutionResult
+
+
+StreamEventType = Literal[
+ "assistant_delta",
+ "thinking_delta",
+ "tool_call_delta",
+]
+
+
+@dataclass
+class StreamEvent:
+ type: StreamEventType
+ text: str = ""
+ tool_call_id: Optional[str] = None
+ tool_name: Optional[str] = None
+ tool_arguments: Any = None
+
+
+@dataclass
+class StepResult:
+ assistant_text: str
+ tool_calls: List[ToolCall]
+ provider_state: Any = None
+
+
+@dataclass
+class ExecutedToolCall:
+ tool_call: ToolCall
+ result: ToolExecutionResult
+
+
+EventSink = Callable[[StreamEvent], Awaitable[None]]
diff --git a/backend/agentic/tools.py b/backend/agentic/tools.py
new file mode 100644
index 00000000..cadccc92
--- /dev/null
+++ b/backend/agentic/tools.py
@@ -0,0 +1,758 @@
+# pyright: reportUnknownVariableType=false
+import copy
+import json
+from dataclasses import dataclass
+from typing import Any, Dict, List, Optional, Tuple
+
+from google.genai import types
+
+from codegen.utils import extract_html_content
+from config import REPLICATE_API_KEY
+from image_generation.core import process_tasks
+from image_generation.replicate import remove_background
+
+from agentic.state import AgentFileState, ensure_str
+
+
+@dataclass(frozen=True)
+class ToolCall:
+ id: str
+ name: str
+ arguments: Dict[str, Any]
+
+
+@dataclass
+class ToolExecutionResult:
+ ok: bool
+ result: Dict[str, Any]
+ summary: Dict[str, Any]
+ updated_content: Optional[str] = None
+
+
+@dataclass(frozen=True)
+class CanonicalToolDefinition:
+ name: str
+ description: str
+ parameters: Dict[str, Any]
+
+
+def summarize_text(value: str, limit: int = 240) -> str:
+ if len(value) <= limit:
+ return value
+ return value[:limit] + "..."
+
+
+def parse_json_arguments(raw_args: Any) -> Tuple[Dict[str, Any], Optional[str]]:
+ if isinstance(raw_args, dict):
+ return raw_args, None
+ if raw_args is None:
+ return {}, None
+ raw_text = ensure_str(raw_args).strip()
+ if not raw_text:
+ return {}, None
+ try:
+ return json.loads(raw_text), None
+ except json.JSONDecodeError as exc:
+ return {}, f"Invalid JSON arguments: {exc}"
+
+
+def _strip_incomplete_escape(value: str) -> str:
+ if not value:
+ return value
+ trailing = 0
+ for ch in reversed(value):
+ if ch == "\\":
+ trailing += 1
+ else:
+ break
+ if trailing % 2 == 1:
+ return value[:-1]
+ return value
+
+
+def _extract_partial_json_string(raw_text: str, key: str) -> Optional[str]:
+ if not raw_text:
+ return None
+ token = f'"{key}"'
+ idx = raw_text.find(token)
+ if idx == -1:
+ return None
+ colon = raw_text.find(":", idx + len(token))
+ if colon == -1:
+ return None
+ cursor = colon + 1
+ while cursor < len(raw_text) and raw_text[cursor].isspace():
+ cursor += 1
+ if cursor >= len(raw_text) or raw_text[cursor] != '"':
+ return None
+
+ start = cursor + 1
+ last_quote: Optional[int] = None
+ cursor = start
+ while cursor < len(raw_text):
+ if raw_text[cursor] == '"':
+ backslashes = 0
+ back = cursor - 1
+ while back >= start and raw_text[back] == "\\":
+ backslashes += 1
+ back -= 1
+ if backslashes % 2 == 0:
+ last_quote = cursor
+ cursor += 1
+
+ partial = raw_text[start:] if last_quote is None else raw_text[start:last_quote]
+ partial = _strip_incomplete_escape(partial)
+ if not partial:
+ return ""
+
+ try:
+ return json.loads(f'"{partial}"')
+ except Exception:
+ return (
+ partial.replace("\\n", "\n")
+ .replace("\\t", "\t")
+ .replace("\\r", "\r")
+ .replace('\\"', '"')
+ .replace("\\\\", "\\")
+ )
+
+
+def extract_content_from_args(raw_args: Any) -> Optional[str]:
+ if isinstance(raw_args, dict):
+ content = raw_args.get("content")
+ if content is None:
+ return None
+ return ensure_str(content)
+ raw_text = ensure_str(raw_args)
+ return _extract_partial_json_string(raw_text, "content")
+
+
+def extract_path_from_args(raw_args: Any) -> Optional[str]:
+ if isinstance(raw_args, dict):
+ path = raw_args.get("path")
+ return ensure_str(path) if path is not None else None
+ raw_text = ensure_str(raw_args)
+ return _extract_partial_json_string(raw_text, "path")
+
+
+def _create_schema() -> Dict[str, Any]:
+ return {
+ "type": "object",
+ "properties": {
+ "path": {
+ "type": "string",
+ "description": "Path for the main HTML file. Use index.html if unsure.",
+ },
+ "content": {
+ "type": "string",
+ "description": "Full HTML for the single-file app.",
+ },
+ },
+ "required": ["content"],
+ }
+
+
+def _edit_schema() -> Dict[str, Any]:
+ return {
+ "type": "object",
+ "properties": {
+ "path": {
+ "type": "string",
+ "description": "Path for the main HTML file.",
+ },
+ "old_text": {
+ "type": "string",
+ "description": "Exact text to replace. Must match the file contents.",
+ },
+ "new_text": {
+ "type": "string",
+ "description": "Replacement text.",
+ },
+ "count": {
+ "type": "integer",
+ "description": "How many occurrences to replace. Use -1 for all.",
+ },
+ "edits": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "old_text": {"type": "string"},
+ "new_text": {"type": "string"},
+ "count": {"type": "integer"},
+ },
+ "required": ["old_text", "new_text"],
+ },
+ },
+ },
+ }
+
+
+def _image_schema() -> Dict[str, Any]:
+ return {
+ "type": "object",
+ "properties": {
+ "prompts": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "description": "Prompt describing a single image to generate.",
+ },
+ }
+ },
+ "required": ["prompts"],
+ }
+
+
+def _remove_background_schema() -> Dict[str, Any]:
+ return {
+ "type": "object",
+ "properties": {
+ "image_url": {
+ "type": "string",
+ "description": "URL of the image to remove the background from.",
+ },
+ },
+ "required": ["image_url"],
+ }
+
+
+def _retrieve_option_schema() -> Dict[str, Any]:
+ return {
+ "type": "object",
+ "properties": {
+ "option_number": {
+ "type": "integer",
+ "description": "1-based option number to retrieve (Option 1, Option 2, etc.).",
+ }
+ },
+ "required": ["option_number"],
+ }
+
+
+def canonical_tool_definitions() -> List[CanonicalToolDefinition]:
+ return [
+ CanonicalToolDefinition(
+ name="create_file",
+ description=(
+ "Create the main HTML file for the app. Use exactly once to write the "
+ "full HTML. Returns the updated file content."
+ ),
+ parameters=_create_schema(),
+ ),
+ CanonicalToolDefinition(
+ name="edit_file",
+ description=(
+ "Edit the main HTML file using exact string replacements. Do not "
+ "regenerate the entire file. Returns the updated file content."
+ ),
+ parameters=_edit_schema(),
+ ),
+ CanonicalToolDefinition(
+ name="generate_images",
+ description=(
+ "Generate image URLs from prompts. Use to replace placeholder images. "
+ "You can pass multiple prompts at once."
+ ),
+ parameters=_image_schema(),
+ ),
+ CanonicalToolDefinition(
+ name="remove_background",
+ description=(
+ "Remove the background from an image. Returns a URL to the processed "
+ "image with transparent background."
+ ),
+ parameters=_remove_background_schema(),
+ ),
+ CanonicalToolDefinition(
+ name="retrieve_option",
+ description=(
+ "Retrieve the full HTML for a specific option (variant) so you can "
+ "reference it."
+ ),
+ parameters=_retrieve_option_schema(),
+ ),
+ ]
+
+
+def _copy_schema(schema: Dict[str, Any]) -> Dict[str, Any]:
+ return copy.deepcopy(schema)
+
+
+def serialize_openai_chat_tools(
+ tools: List[CanonicalToolDefinition],
+) -> List[Dict[str, Any]]:
+ return [
+ {
+ "type": "function",
+ "function": {
+ "name": tool.name,
+ "description": tool.description,
+ "parameters": _copy_schema(tool.parameters),
+ },
+ }
+ for tool in tools
+ ]
+
+
+def _nullable_type(type_value: Any) -> Any:
+ if isinstance(type_value, list):
+ if "null" not in type_value:
+ return [*type_value, "null"]
+ return type_value
+ if isinstance(type_value, str):
+ return [type_value, "null"]
+ return type_value
+
+
+def _make_responses_schema_strict(schema: Dict[str, Any]) -> Dict[str, Any]:
+ schema_copy: Dict[str, Any] = _copy_schema(schema)
+
+ def transform(node: Dict[str, Any], in_object_property: bool = False) -> None:
+ node_type = node.get("type")
+
+ if node_type == "object":
+ node["additionalProperties"] = False
+ properties = node.get("properties") or {}
+ if isinstance(properties, dict):
+ node["required"] = list(properties.keys())
+ for prop in properties.values():
+ if isinstance(prop, dict):
+ transform(prop, in_object_property=True)
+ return
+
+ if node_type == "array":
+ if in_object_property:
+ node["type"] = _nullable_type(node_type)
+ items = node.get("items")
+ if isinstance(items, dict):
+ transform(items, in_object_property=False)
+ return
+
+ if in_object_property and node_type is not None:
+ node["type"] = _nullable_type(node_type)
+
+ transform(schema_copy, in_object_property=False)
+ return schema_copy
+
+
+def serialize_openai_responses_tools(
+ tools: List[CanonicalToolDefinition],
+) -> List[Dict[str, Any]]:
+ serialized: List[Dict[str, Any]] = []
+ for tool in tools:
+ schema = _make_responses_schema_strict(tool.parameters)
+ serialized.append(
+ {
+ "type": "function",
+ "name": tool.name,
+ "description": tool.description,
+ "parameters": schema,
+ "strict": True,
+ }
+ )
+ return serialized
+
+
+def serialize_anthropic_tools(
+ tools: List[CanonicalToolDefinition],
+) -> List[Dict[str, Any]]:
+ return [
+ {
+ "name": tool.name,
+ "description": tool.description,
+ "input_schema": _copy_schema(tool.parameters),
+ }
+ for tool in tools
+ ]
+
+
+def serialize_gemini_tools(
+ tools: List[CanonicalToolDefinition],
+) -> List[types.Tool]:
+ declarations = [
+ types.FunctionDeclaration(
+ name=tool.name,
+ description=tool.description,
+ parameters_json_schema=_copy_schema(tool.parameters),
+ )
+ for tool in tools
+ ]
+ return [types.Tool(function_declarations=declarations)]
+
+
+def summarize_tool_input(tool_call: ToolCall, file_state: AgentFileState) -> Dict[str, Any]:
+ args = tool_call.arguments or {}
+
+ if tool_call.name == "create_file":
+ content = ensure_str(args.get("content"))
+ return {
+ "path": args.get("path") or file_state.path,
+ "contentLength": len(content),
+ "preview": summarize_text(content, 200),
+ }
+
+ if tool_call.name == "edit_file":
+ edits = args.get("edits")
+ if not edits:
+ edits = [
+ {
+ "old_text": args.get("old_text"),
+ "new_text": args.get("new_text"),
+ "count": args.get("count"),
+ }
+ ]
+ summary_edits = []
+ for edit in edits if isinstance(edits, list) else []:
+ summary_edits.append(
+ {
+ "old_text": summarize_text(ensure_str(edit.get("old_text")), 160),
+ "new_text": summarize_text(ensure_str(edit.get("new_text")), 160),
+ "count": edit.get("count"),
+ }
+ )
+ return {
+ "path": args.get("path") or file_state.path,
+ "edits": summary_edits,
+ }
+
+ if tool_call.name == "generate_images":
+ prompts = args.get("prompts") or []
+ if isinstance(prompts, list):
+ return {
+ "count": len(prompts),
+ "prompts": [summarize_text(ensure_str(p), 140) for p in prompts],
+ }
+
+ if tool_call.name == "remove_background":
+ image_url = ensure_str(args.get("image_url"))
+ return {"image_url": summarize_text(image_url, 100)}
+
+ if tool_call.name == "retrieve_option":
+ return {
+ "option_number": args.get("option_number"),
+ "index": args.get("index"),
+ }
+
+ return args
+
+
+class AgentToolbox:
+ def __init__(
+ self,
+ file_state: AgentFileState,
+ image_cache: Dict[str, str],
+ should_generate_images: bool,
+ openai_api_key: Optional[str],
+ openai_base_url: Optional[str],
+ option_codes: Optional[List[str]] = None,
+ ):
+ self.file_state = file_state
+ self.image_cache = image_cache
+ self.should_generate_images = should_generate_images
+ self.openai_api_key = openai_api_key
+ self.openai_base_url = openai_base_url
+ self.option_codes = option_codes or []
+
+ async def execute(self, tool_call: ToolCall) -> ToolExecutionResult:
+ if tool_call.name == "create_file":
+ return self._create_file(tool_call.arguments)
+ if tool_call.name == "edit_file":
+ return self._edit_file(tool_call.arguments)
+ if tool_call.name == "generate_images":
+ return await self._generate_images(tool_call.arguments)
+ if tool_call.name == "remove_background":
+ return await self._remove_background(tool_call.arguments)
+ if tool_call.name == "retrieve_option":
+ return self._retrieve_option(tool_call.arguments)
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": f"Unknown tool: {tool_call.name}"},
+ summary={"error": f"Unknown tool: {tool_call.name}"},
+ )
+
+ def _create_file(self, args: Dict[str, Any]) -> ToolExecutionResult:
+ path = ensure_str(args.get("path") or self.file_state.path or "index.html")
+ content = ensure_str(args.get("content"))
+ if not content:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "create_file requires non-empty content"},
+ summary={"error": "Missing content"},
+ )
+
+ extracted = extract_html_content(content)
+ self.file_state.path = path
+ self.file_state.content = extracted or content
+
+ summary = {
+ "path": self.file_state.path,
+ "contentLength": len(self.file_state.content),
+ "preview": summarize_text(self.file_state.content, 320),
+ }
+ result = {
+ "path": self.file_state.path,
+ "content": self.file_state.content,
+ }
+ return ToolExecutionResult(
+ ok=True,
+ result=result,
+ summary=summary,
+ updated_content=self.file_state.content,
+ )
+
+ def _apply_single_edit(
+ self,
+ content: str,
+ old_text: str,
+ new_text: str,
+ count: Optional[int],
+ ) -> Tuple[str, int]:
+ if old_text not in content:
+ return content, 0
+
+ if count is None:
+ replace_count = 1
+ elif count < 0:
+ replace_count = content.count(old_text)
+ else:
+ replace_count = count
+
+ updated = content.replace(old_text, new_text, replace_count)
+ return updated, min(replace_count, content.count(old_text))
+
+ def _edit_file(self, args: Dict[str, Any]) -> ToolExecutionResult:
+ if not self.file_state.content:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "No file exists yet. Call create_file first."},
+ summary={"error": "No file to edit"},
+ )
+
+ edits = args.get("edits")
+ if not edits:
+ old_text = ensure_str(args.get("old_text"))
+ new_text = ensure_str(args.get("new_text"))
+ count = args.get("count")
+ edits = [{"old_text": old_text, "new_text": new_text, "count": count}]
+
+ if not isinstance(edits, list):
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "edits must be a list"},
+ summary={"error": "Invalid edits payload"},
+ )
+
+ content = self.file_state.content
+ summary_edits: List[Dict[str, Any]] = []
+ for edit in edits:
+ old_text = ensure_str(edit.get("old_text"))
+ new_text = ensure_str(edit.get("new_text"))
+ count = edit.get("count")
+ if not old_text:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "edit_file requires old_text"},
+ summary={"error": "Missing old_text"},
+ )
+
+ content, replaced = self._apply_single_edit(content, old_text, new_text, count)
+ if replaced == 0:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "old_text not found", "old_text": old_text},
+ summary={
+ "error": "old_text not found",
+ "old_text": summarize_text(old_text, 160),
+ },
+ )
+
+ summary_edits.append(
+ {
+ "old_text": summarize_text(old_text, 140),
+ "new_text": summarize_text(new_text, 140),
+ "replaced": replaced,
+ }
+ )
+
+ self.file_state.content = content
+ summary = {
+ "path": self.file_state.path,
+ "edits": summary_edits,
+ "contentLength": len(self.file_state.content),
+ }
+ result = {
+ "path": self.file_state.path,
+ "content": self.file_state.content,
+ }
+ return ToolExecutionResult(
+ ok=True,
+ result=result,
+ summary=summary,
+ updated_content=self.file_state.content,
+ )
+
+ async def _generate_images(self, args: Dict[str, Any]) -> ToolExecutionResult:
+ if not self.should_generate_images:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "Image generation is disabled."},
+ summary={"error": "Image generation disabled"},
+ )
+
+ prompts = args.get("prompts") or []
+ if not isinstance(prompts, list) or not prompts:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "generate_images requires a non-empty prompts list"},
+ summary={"error": "Missing prompts"},
+ )
+
+ cleaned = [prompt.strip() for prompt in prompts if isinstance(prompt, str)]
+ unique_prompts = list(dict.fromkeys([p for p in cleaned if p]))
+ if not unique_prompts:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "No valid prompts provided"},
+ summary={"error": "No valid prompts"},
+ )
+
+ cached_results: Dict[str, Optional[str]] = {}
+ to_generate: List[str] = []
+ for prompt in unique_prompts:
+ if prompt in self.image_cache:
+ cached_results[prompt] = self.image_cache[prompt]
+ else:
+ to_generate.append(prompt)
+
+ generated_results: Dict[str, Optional[str]] = {}
+
+ if to_generate:
+ if REPLICATE_API_KEY:
+ model = "flux"
+ api_key = REPLICATE_API_KEY
+ base_url = None
+ else:
+ if not self.openai_api_key:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "No API key available for image generation."},
+ summary={"error": "Missing image generation API key"},
+ )
+ model = "dalle3"
+ api_key = self.openai_api_key
+ base_url = self.openai_base_url
+
+ generated = await process_tasks(to_generate, api_key, base_url, model) # type: ignore
+ for prompt, url in zip(to_generate, generated):
+ generated_results[prompt] = url
+ if url:
+ self.image_cache[prompt] = url
+
+ merged_results = {**cached_results, **generated_results}
+ summary_items = [
+ {
+ "prompt": summarize_text(prompt, 160),
+ "url": url,
+ "status": "ok" if url else "error",
+ }
+ for prompt, url in merged_results.items()
+ ]
+ result = {"images": merged_results}
+ summary = {"images": summary_items}
+ return ToolExecutionResult(ok=True, result=result, summary=summary)
+
+ async def _remove_background(self, args: Dict[str, Any]) -> ToolExecutionResult:
+ if not REPLICATE_API_KEY:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "Background removal requires REPLICATE_API_KEY."},
+ summary={"error": "Missing Replicate API key"},
+ )
+
+ image_url = ensure_str(args.get("image_url"))
+ if not image_url:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "remove_background requires an image_url"},
+ summary={"error": "Missing image_url"},
+ )
+
+ try:
+ result_url = await remove_background(image_url, REPLICATE_API_KEY)
+ result = {"image_url": image_url, "result_url": result_url}
+ summary = {
+ "image_url": summarize_text(image_url, 100),
+ "result_url": result_url,
+ "status": "ok",
+ }
+ return ToolExecutionResult(ok=True, result=result, summary=summary)
+ except Exception as exc:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": str(exc), "image_url": image_url},
+ summary={"error": summarize_text(str(exc), 100)},
+ )
+
+ def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:
+ raw_option_number = args.get("option_number")
+ raw_index = args.get("index")
+
+ def coerce_int(value: Any) -> Optional[int]:
+ if value is None:
+ return None
+ try:
+ return int(value)
+ except (TypeError, ValueError):
+ return None
+
+ option_number = coerce_int(raw_option_number)
+ index = coerce_int(raw_index)
+
+ if option_number is None and index is None:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "retrieve_option requires option_number"},
+ summary={"error": "Missing option_number"},
+ )
+
+ resolved_index = index if option_number is None else option_number - 1
+ if resolved_index is None:
+ return ToolExecutionResult(
+ ok=False,
+ result={"error": "Invalid option_number"},
+ summary={"error": "Invalid option_number"},
+ )
+
+ if resolved_index < 0 or resolved_index >= len(self.option_codes):
+ return ToolExecutionResult(
+ ok=False,
+ result={
+ "error": "Option index out of range",
+ "option_number": resolved_index + 1,
+ "available": len(self.option_codes),
+ },
+ summary={
+ "error": "Option index out of range",
+ "available": len(self.option_codes),
+ },
+ )
+
+ code = ensure_str(self.option_codes[resolved_index])
+ if not code.strip():
+ return ToolExecutionResult(
+ ok=False,
+ result={
+ "error": "Option code is empty or unavailable",
+ "option_number": resolved_index + 1,
+ },
+ summary={"error": "Option code unavailable"},
+ )
+
+ summary = {
+ "option_number": resolved_index + 1,
+ "contentLength": len(code),
+ "preview": summarize_text(code, 200),
+ }
+ result = {"option_number": resolved_index + 1, "code": code}
+ return ToolExecutionResult(ok=True, result=result, summary=summary)
diff --git a/backend/evals/core.py b/backend/evals/core.py
index 394295c5..0c7a397d 100644
--- a/backend/evals/core.py
+++ b/backend/evals/core.py
@@ -1,57 +1,63 @@
-from config import ANTHROPIC_API_KEY, GEMINI_API_KEY, OPENAI_API_KEY
-from llm import Llm, ANTHROPIC_MODELS, GEMINI_MODELS
-from models import (
- stream_claude_response,
- stream_gemini_response,
- stream_openai_response,
+from config import (
+ ANTHROPIC_API_KEY,
+ GEMINI_API_KEY,
+ OPENAI_API_KEY,
+ OPENAI_BASE_URL,
)
+from llm import Llm, OPENAI_MODELS, ANTHROPIC_MODELS, GEMINI_MODELS
+from agentic.runner import AgenticRunner
from prompts import assemble_prompt
+from prompts.agentic_instructions import apply_tool_instructions
from prompts.types import Stack
from openai.types.chat import ChatCompletionMessageParam
+from typing import Any, cast
async def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> str:
prompt_messages = assemble_prompt([image_url], stack)
+ if prompt_messages:
+ first_message = cast(dict[str, Any], prompt_messages[0])
+ first_content = first_message.get("content")
+ if first_content:
+ first_message["content"] = apply_tool_instructions(
+ str(first_content),
+ should_generate_images=True,
+ )
return await generate_code_core(prompt_messages, model)
async def generate_code_core(
prompt_messages: list[ChatCompletionMessageParam], model: Llm
) -> str:
-
- async def process_chunk(_: str):
- pass
-
- if model in ANTHROPIC_MODELS:
- if not ANTHROPIC_API_KEY:
- raise Exception("Anthropic API key not found")
-
- completion = await stream_claude_response(
- prompt_messages,
- api_key=ANTHROPIC_API_KEY,
- callback=lambda x: process_chunk(x),
- model_name=model.value,
- )
- elif model in GEMINI_MODELS:
- if not GEMINI_API_KEY:
- raise Exception("Gemini API key not found")
-
- completion = await stream_gemini_response(
- prompt_messages,
- api_key=GEMINI_API_KEY,
- callback=lambda x: process_chunk(x),
- model=model,
- )
- else:
- if not OPENAI_API_KEY:
- raise Exception("OpenAI API key not found")
-
- completion = await stream_openai_response(
- prompt_messages,
- api_key=OPENAI_API_KEY,
- base_url=None,
- callback=lambda x: process_chunk(x),
- model=model,
- )
-
- return completion["code"]
+ async def send_message(
+ _: str,
+ __: str | None,
+ ___: int,
+ ____: dict[str, Any] | None = None,
+ _____: str | None = None,
+ ) -> None:
+ # Evals do not stream tool/assistant messages to a frontend.
+ return None
+
+ if model in ANTHROPIC_MODELS and not ANTHROPIC_API_KEY:
+ raise Exception("Anthropic API key not found")
+ if model in GEMINI_MODELS and not GEMINI_API_KEY:
+ raise Exception("Gemini API key not found")
+ if model in OPENAI_MODELS and not OPENAI_API_KEY:
+ raise Exception("OpenAI API key not found")
+
+ print(f"[EVALS] Using agentic runner for model: {model.value}")
+
+ runner = AgenticRunner(
+ send_message=send_message,
+ variant_index=0,
+ openai_api_key=OPENAI_API_KEY,
+ openai_base_url=OPENAI_BASE_URL,
+ anthropic_api_key=ANTHROPIC_API_KEY,
+ gemini_api_key=GEMINI_API_KEY,
+ should_generate_images=True,
+ image_cache={},
+ initial_file_state=None,
+ option_codes=None,
+ )
+ return await runner.run(model, prompt_messages)