Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Refactor agentic runner modules and clean targeted typing
Abi Raja4b6d4ebf7726
diff --git a/backend/agentic/runner.py b/backend/agentic/runner.pyindex ab240ed2..5ac5f45c 100644--- a/backend/agentic/runner.py+++ b/backend/agentic/runner.py@@ -1,520 +1,52 @@import asyncio-import jsonimport uuid-from dataclasses import dataclass-from typing import Any, Awaitable, Callable, Dict, List, Optional, Tuple+from typing import Any, Awaitable, Callable, Dict, List, Optional-from openai import AsyncOpenAI-from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParamfrom anthropic import AsyncAnthropicfrom google import genai-from google.genai import types+from openai import AsyncOpenAI+from openai.types.chat import ChatCompletionMessageParamfrom codegen.utils import extract_html_content-from config import REPLICATE_API_KEY-from image_generation.core import process_tasks-from image_generation.replicate import remove_backgroundfrom llm import (- Llm,- OPENAI_MODELS,ANTHROPIC_MODELS,GEMINI_MODELS,OPENAI_CODEX_MODELS,- get_openai_api_name,- get_openai_reasoning_effort,+ OPENAI_MODELS,+ Llm,)--@dataclass-class ToolCall:- id: str- name: str- arguments: Dict[str, Any]---@dataclass-class ToolExecutionResult:- ok: bool- result: Dict[str, Any]- summary: Dict[str, Any]- updated_content: Optional[str] = None---@dataclass-class AgentFileState:- path: str = "index.html"- content: str = ""---def _summarize_text(value: str, limit: int = 240) -> str:- if len(value) <= limit:- return value- return value[:limit] + "..."---def _ensure_str(value: Any) -> str:- if value is None:- return ""- return str(value)---def _parse_json_arguments(raw_args: Any) -> Tuple[Dict[str, Any], Optional[str]]:- if isinstance(raw_args, dict):- return raw_args, None- if raw_args is None:- return {}, None- raw_text = _ensure_str(raw_args).strip()- if not raw_text:- return {}, None- try:- return json.loads(raw_text), None- except json.JSONDecodeError as exc:- return {}, f"Invalid JSON arguments: {exc}"---def _convert_message_to_responses_input(- message: ChatCompletionMessageParam,-) -> Dict[str, Any]:- role = message.get("role", "user")- content = message.get("content", "")-- if isinstance(content, str):- return {"role": role, "content": content}-- parts: List[Dict[str, Any]] = []- if isinstance(content, list):- for part in content:- if not isinstance(part, dict):- continue- if part.get("type") == "text":- parts.append({"type": "input_text", "text": part.get("text", "")})- elif part.get("type") == "image_url":- image_url = part.get("image_url", {})- parts.append(- {- "type": "input_image",- "image_url": image_url.get("url", ""),- "detail": image_url.get("detail", "auto"),- }- )-- return {"role": role, "content": parts}---def _get_event_attr(event: Any, key: str, default: Any = None) -> Any:- if hasattr(event, key):- return getattr(event, key)- if isinstance(event, dict):- return event.get(key, default)- return default---def _strip_incomplete_escape(value: str) -> str:- if not value:- return value- trailing = 0- for ch in reversed(value):- if ch == "\\":- trailing += 1- else:- break- if trailing % 2 == 1:- return value[:-1]- return value---def _extract_partial_json_string(raw_text: str, key: str) -> Optional[str]:- if not raw_text:- return None- token = f"\"{key}\""- idx = raw_text.find(token)- if idx == -1:- return None- colon = raw_text.find(":", idx + len(token))- if colon == -1:- return None- cursor = colon + 1- while cursor < len(raw_text) and raw_text[cursor].isspace():- cursor += 1- if cursor >= len(raw_text) or raw_text[cursor] != "\"":- return None- start = cursor + 1- last_quote: Optional[int] = None- cursor = start- while cursor < len(raw_text):- if raw_text[cursor] == "\"":- backslashes = 0- back = cursor - 1- while back >= start and raw_text[back] == "\\":- backslashes += 1- back -= 1- if backslashes % 2 == 0:- last_quote = cursor- cursor += 1- if last_quote is None:- partial = raw_text[start:]- else:- partial = raw_text[start:last_quote]- partial = _strip_incomplete_escape(partial)- if not partial:- return ""- try:- return json.loads(f"\"{partial}\"")- except Exception:- return (- partial.replace("\\n", "\n")- .replace("\\t", "\t")- .replace("\\r", "\r")- .replace("\\\"", "\"")- .replace("\\\\", "\\")- )---def _extract_content_from_args(raw_args: Any) -> Optional[str]:- if isinstance(raw_args, dict):- content = raw_args.get("content")- if content is None:- return None- return _ensure_str(content)- raw_text = _ensure_str(raw_args)- return _extract_partial_json_string(raw_text, "content")---def _extract_path_from_args(raw_args: Any) -> Optional[str]:- if isinstance(raw_args, dict):- path = raw_args.get("path")- return _ensure_str(path) if path is not None else None- raw_text = _ensure_str(raw_args)- return _extract_partial_json_string(raw_text, "path")---class AgentToolbox:- def __init__(- self,- file_state: AgentFileState,- image_cache: Dict[str, str],- should_generate_images: bool,- openai_api_key: Optional[str],- openai_base_url: Optional[str],- option_codes: Optional[List[str]] = None,- ):- self.file_state = file_state- self.image_cache = image_cache- self.should_generate_images = should_generate_images- self.openai_api_key = openai_api_key- self.openai_base_url = openai_base_url- self.option_codes = option_codes or []-- async def execute(self, tool_call: ToolCall) -> ToolExecutionResult:- if tool_call.name == "create_file":- return self._create_file(tool_call.arguments)- if tool_call.name == "edit_file":- return self._edit_file(tool_call.arguments)- if tool_call.name == "generate_images":- return await self._generate_images(tool_call.arguments)- if tool_call.name == "remove_background":- return await self._remove_background(tool_call.arguments)- if tool_call.name == "retrieve_option":- return self._retrieve_option(tool_call.arguments)- return ToolExecutionResult(- ok=False,- result={"error": f"Unknown tool: {tool_call.name}"},- summary={"error": f"Unknown tool: {tool_call.name}"},- )-- def _create_file(self, args: Dict[str, Any]) -> ToolExecutionResult:- path = _ensure_str(args.get("path") or self.file_state.path or "index.html")- content = _ensure_str(args.get("content"))- if not content:- return ToolExecutionResult(- ok=False,- result={"error": "create_file requires non-empty content"},- summary={"error": "Missing content"},- )-- extracted = extract_html_content(content)- self.file_state.path = path- self.file_state.content = extracted or content-- summary = {- "path": self.file_state.path,- "contentLength": len(self.file_state.content),- "preview": _summarize_text(self.file_state.content, 320),- }- result = {- "path": self.file_state.path,- "content": self.file_state.content,- }- return ToolExecutionResult(- ok=True,- result=result,- summary=summary,- updated_content=self.file_state.content,- )-- def _apply_single_edit(- self, content: str, old_text: str, new_text: str, count: Optional[int]- ) -> Tuple[str, int]:- if old_text not in content:- return content, 0-- if count is None:- replace_count = 1- elif count < 0:- replace_count = content.count(old_text)- else:- replace_count = count-- updated = content.replace(old_text, new_text, replace_count)- return updated, min(replace_count, content.count(old_text))-- def _edit_file(self, args: Dict[str, Any]) -> ToolExecutionResult:- if not self.file_state.content:- return ToolExecutionResult(- ok=False,- result={"error": "No file exists yet. Call create_file first."},- summary={"error": "No file to edit"},- )-- edits = args.get("edits")- if not edits:- old_text = _ensure_str(args.get("old_text"))- new_text = _ensure_str(args.get("new_text"))- count = args.get("count")- edits = [- {"old_text": old_text, "new_text": new_text, "count": count}- ]-- if not isinstance(edits, list):- return ToolExecutionResult(- ok=False,- result={"error": "edits must be a list"},- summary={"error": "Invalid edits payload"},- )-- content = self.file_state.content- summary_edits: List[Dict[str, Any]] = []- for edit in edits:- old_text = _ensure_str(edit.get("old_text"))- new_text = _ensure_str(edit.get("new_text"))- count = edit.get("count")- if not old_text:- return ToolExecutionResult(- ok=False,- result={"error": "edit_file requires old_text"},- summary={"error": "Missing old_text"},- )-- content, replaced = self._apply_single_edit(- content, old_text, new_text, count- )- if replaced == 0:- return ToolExecutionResult(- ok=False,- result={"error": "old_text not found", "old_text": old_text},- summary={- "error": "old_text not found",- "old_text": _summarize_text(old_text, 160),- },- )-- summary_edits.append(- {- "old_text": _summarize_text(old_text, 140),- "new_text": _summarize_text(new_text, 140),- "replaced": replaced,- }- )-- self.file_state.content = content- summary = {- "path": self.file_state.path,- "edits": summary_edits,- "contentLength": len(self.file_state.content),- }- result = {- "path": self.file_state.path,- "content": self.file_state.content,- }- return ToolExecutionResult(- ok=True,- result=result,- summary=summary,- updated_content=self.file_state.content,- )-- async def _generate_images(self, args: Dict[str, Any]) -> ToolExecutionResult:- if not self.should_generate_images:- return ToolExecutionResult(- ok=False,- result={"error": "Image generation is disabled."},- summary={"error": "Image generation disabled"},- )-- prompts = args.get("prompts") or []- if not isinstance(prompts, list) or not prompts:- return ToolExecutionResult(- ok=False,- result={"error": "generate_images requires a non-empty prompts list"},- summary={"error": "Missing prompts"},- )-- cleaned = [prompt.strip() for prompt in prompts if isinstance(prompt, str)]- unique_prompts = list(dict.fromkeys([p for p in cleaned if p]))- if not unique_prompts:- return ToolExecutionResult(- ok=False,- result={"error": "No valid prompts provided"},- summary={"error": "No valid prompts"},- )-- cached_results: Dict[str, Optional[str]] = {}- to_generate: List[str] = []- for prompt in unique_prompts:- if prompt in self.image_cache:- cached_results[prompt] = self.image_cache[prompt]- else:- to_generate.append(prompt)-- generated_results: Dict[str, Optional[str]] = {}-- if to_generate:- if REPLICATE_API_KEY:- model = "flux"- api_key = REPLICATE_API_KEY- base_url = None- else:- if not self.openai_api_key:- return ToolExecutionResult(- ok=False,- result={- "error": "No API key available for image generation."- },- summary={"error": "Missing image generation API key"},- )- model = "dalle3"- api_key = self.openai_api_key- base_url = self.openai_base_url-- generated = await process_tasks(to_generate, api_key, base_url, model) # type: ignore- for prompt, url in zip(to_generate, generated):- generated_results[prompt] = url- if url:- self.image_cache[prompt] = url-- merged_results = {**cached_results, **generated_results}- summary_items = [- {- "prompt": _summarize_text(prompt, 160),- "url": url,- "status": "ok" if url else "error",- }- for prompt, url in merged_results.items()- ]- result = {"images": merged_results}- summary = {"images": summary_items}- return ToolExecutionResult(ok=True, result=result, summary=summary)-- async def _remove_background(self, args: Dict[str, Any]) -> ToolExecutionResult:- if not REPLICATE_API_KEY:- return ToolExecutionResult(- ok=False,- result={"error": "Background removal requires REPLICATE_API_KEY."},- summary={"error": "Missing Replicate API key"},- )-- image_url = _ensure_str(args.get("image_url"))- if not image_url:- return ToolExecutionResult(- ok=False,- result={"error": "remove_background requires an image_url"},- summary={"error": "Missing image_url"},- )-- try:- result_url = await remove_background(image_url, REPLICATE_API_KEY)- result = {"image_url": image_url, "result_url": result_url}- summary = {- "image_url": _summarize_text(image_url, 100),- "result_url": result_url,- "status": "ok",- }- return ToolExecutionResult(ok=True, result=result, summary=summary)- except Exception as e:- return ToolExecutionResult(- ok=False,- result={"error": str(e), "image_url": image_url},- summary={"error": _summarize_text(str(e), 100)},- )-- def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:- raw_option_number = args.get("option_number")- raw_index = args.get("index")-- def _coerce_int(value: Any) -> Optional[int]:- if value is None:- return None- try:- return int(value)- except (TypeError, ValueError):- return None-- option_number = _coerce_int(raw_option_number)- index = _coerce_int(raw_index)-- if option_number is None and index is None:- return ToolExecutionResult(- ok=False,- result={"error": "retrieve_option requires option_number"},- summary={"error": "Missing option_number"},- )-- resolved_index = index if option_number is None else option_number - 1- if resolved_index is None:- return ToolExecutionResult(- ok=False,- result={"error": "Invalid option_number"},- summary={"error": "Invalid option_number"},- )-- if resolved_index < 0 or resolved_index >= len(self.option_codes):- return ToolExecutionResult(- ok=False,- result={- "error": "Option index out of range",- "option_number": resolved_index + 1,- "available": len(self.option_codes),- },- summary={- "error": "Option index out of range",- "available": len(self.option_codes),- },- )-- code = _ensure_str(self.option_codes[resolved_index])- if not code.strip():- return ToolExecutionResult(- ok=False,- result={- "error": "Option code is empty or unavailable",- "option_number": resolved_index + 1,- },- summary={"error": "Option code unavailable"},- )-- summary = {- "option_number": resolved_index + 1,- "contentLength": len(code),- "preview": _summarize_text(code, 200),- }- result = {"option_number": resolved_index + 1, "code": code}- return ToolExecutionResult(ok=True, result=result, summary=summary)+from agentic.state import AgentFileState, seed_file_state_from_messages+from agentic.streaming import (+ AnthropicAdapter,+ ExecutedToolCall,+ GeminiAdapter,+ OpenAIChatAdapter,+ OpenAIResponsesAdapter,+ ProviderAdapter,+ StreamEvent,+)+from agentic.tools import (+ AgentToolbox,+ canonical_tool_definitions,+ extract_content_from_args,+ extract_path_from_args,+ serialize_anthropic_tools,+ serialize_gemini_tools,+ serialize_openai_chat_tools,+ serialize_openai_responses_tools,+ summarize_text,+ summarize_tool_input,+)class AgenticRunner:def __init__(self,- send_message: Callable[[str, Optional[str], int, Optional[Dict[str, Any]], Optional[str]], Awaitable[None]],+ send_message: Callable[+ [str, Optional[str], int, Optional[Dict[str, Any]], Optional[str]],+ Awaitable[None],+ ],variant_index: int,openai_api_key: Optional[str],openai_base_url: Optional[str],@@ -531,25 +63,34 @@ class AgenticRunner:self.openai_base_url = openai_base_urlself.anthropic_api_key = anthropic_api_keyself.gemini_api_key = gemini_api_key- self.should_generate_images = should_generate_images- self.image_cache = image_cacheself.file_state = AgentFileState()if initial_file_state and initial_file_state.get("content"):self.file_state.path = initial_file_state.get("path") or "index.html"self.file_state.content = initial_file_state["content"]+self.toolbox = AgentToolbox(self.file_state,- self.image_cache,+ image_cache,should_generate_images,openai_api_key,openai_base_url,option_codes,)- self.assistant_step = 0- self.thinking_step = 0self._tool_preview_lengths: Dict[str, int] = {}+ def _next_event_id(self, prefix: str) -> str:+ return f"{prefix}-{self.variant_index}-{uuid.uuid4().hex[:8]}"++ async def _send(+ self,+ msg_type: str,+ value: Optional[str] = None,+ data: Optional[Dict[str, Any]] = None,+ event_id: Optional[str] = None,+ ) -> None:+ await self.send_message(msg_type, value, self.variant_index, data, event_id)+def _mark_preview_length(self, tool_event_id: Optional[str], length: int) -> None:if not tool_event_id:return@@ -560,1308 +101,236 @@ class AgenticRunner:async def _stream_code_preview(self, tool_event_id: Optional[str], content: str) -> None:if not tool_event_id or not content:return+already_sent = self._tool_preview_lengths.get(tool_event_id, 0)total_len = len(content)if already_sent >= total_len:return+max_chunks = 18min_step = 200step = max(min_step, total_len // max_chunks)start = already_sent if already_sent > 0 else 0+for end in range(start + step, total_len, step):await self._send("setCode", content[:end])self._mark_preview_length(tool_event_id, end)await asyncio.sleep(0.01)+await self._send("setCode", content)self._mark_preview_length(tool_event_id, total_len)- def _next_event_id(self, prefix: str) -> str:- return f"{prefix}-{self.variant_index}-{uuid.uuid4().hex[:8]}"-- def _extract_text_content(self, message: ChatCompletionMessageParam) -> str:- content = message.get("content", "")- if isinstance(content, str):- return content- if isinstance(content, list):- for part in content:- if isinstance(part, dict) and part.get("type") == "text":- return _ensure_str(part.get("text"))- return ""-- def _seed_file_state_from_messages(- self, prompt_messages: List[ChatCompletionMessageParam]+ async def _handle_streamed_tool_delta(+ self,+ event: StreamEvent,+ started_tool_ids: set[str],+ streamed_lengths: Dict[str, int],) -> None:- if self.file_state.content:+ if event.type != "tool_call_delta":return- for message in reversed(prompt_messages):- if message.get("role") != "assistant":- continue- raw_text = self._extract_text_content(message)- if not raw_text:- continue- extracted = extract_html_content(raw_text)- self.file_state.content = extracted or raw_text- if not self.file_state.path:- self.file_state.path = "index.html"+ if event.tool_name != "create_file":+ return+ if not event.tool_call_id:return- if prompt_messages:- system_message = prompt_messages[0]- if system_message.get("role") == "system":- system_text = self._extract_text_content(system_message)- markers = [- "Here is the code of the app:",- "Here is the code of the SVG:",- ]- for marker in markers:- if marker in system_text:- raw_text = system_text.split(marker, 1)[1].strip()- extracted = extract_html_content(raw_text)- self.file_state.content = extracted or raw_text- if not self.file_state.path:- self.file_state.path = "index.html"- return-- async def _send(- self,- msg_type: str,- value: Optional[str] = None,- data: Optional[Dict[str, Any]] = None,- event_id: Optional[str] = None,- ) -> None:- await self.send_message(msg_type, value, self.variant_index, data, event_id)-- def _summarize_tool_input(self, tool_call: ToolCall) -> Dict[str, Any]:- args = tool_call.arguments or {}- if tool_call.name == "create_file":- content = _ensure_str(args.get("content"))- return {- "path": args.get("path") or self.file_state.path,- "contentLength": len(content),- "preview": _summarize_text(content, 200),- }- if tool_call.name == "edit_file":- edits = args.get("edits")- if not edits:- edits = [- {- "old_text": args.get("old_text"),- "new_text": args.get("new_text"),- "count": args.get("count"),- }- ]- summary_edits = []- for edit in edits if isinstance(edits, list) else []:- summary_edits.append(- {- "old_text": _summarize_text(- _ensure_str(edit.get("old_text")), 160- ),- "new_text": _summarize_text(- _ensure_str(edit.get("new_text")), 160- ),- "count": edit.get("count"),- }- )- return {- "path": args.get("path") or self.file_state.path,- "edits": summary_edits,- }- if tool_call.name == "generate_images":- prompts = args.get("prompts") or []- if isinstance(prompts, list):- return {- "count": len(prompts),- "prompts": [_summarize_text(_ensure_str(p), 140) for p in prompts],- }- if tool_call.name == "remove_background":- image_url = _ensure_str(args.get("image_url"))- return {"image_url": _summarize_text(image_url, 100)}- if tool_call.name == "retrieve_option":- return {- "option_number": args.get("option_number"),- "index": args.get("index"),- }- return args-- def _tool_schemas(self) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[types.Tool]]:- create_schema = {- "type": "object",- "properties": {- "path": {- "type": "string",- "description": "Path for the main HTML file. Use index.html if unsure.",- },- "content": {- "type": "string",- "description": "Full HTML for the single-file app.",- },- },- "required": ["content"],- }- edit_schema = {- "type": "object",- "properties": {- "path": {- "type": "string",- "description": "Path for the main HTML file.",- },- "old_text": {- "type": "string",- "description": "Exact text to replace. Must match the file contents.",- },- "new_text": {- "type": "string",- "description": "Replacement text.",- },- "count": {- "type": "integer",- "description": "How many occurrences to replace. Use -1 for all.",- },- "edits": {- "type": "array",- "items": {- "type": "object",- "properties": {- "old_text": {"type": "string"},- "new_text": {"type": "string"},- "count": {"type": "integer"},- },- "required": ["old_text", "new_text"],- },- },- },- }- image_schema = {- "type": "object",- "properties": {- "prompts": {- "type": "array",- "items": {- "type": "string",- "description": "Prompt describing a single image to generate.",- },- }- },- "required": ["prompts"],- }- remove_bg_schema = {- "type": "object",- "properties": {- "image_url": {- "type": "string",- "description": "URL of the image to remove the background from.",- },- },- "required": ["image_url"],- }- retrieve_option_schema = {- "type": "object",- "properties": {- "option_number": {- "type": "integer",- "description": "1-based option number to retrieve (Option 1, Option 2, etc.).",- }- },- "required": ["option_number"],- }-- openai_tools = [- {- "type": "function",- "function": {- "name": "create_file",- "description": "Create the main HTML file for the app. Use exactly once to write the full HTML. Returns the updated file content.",- "parameters": create_schema,- },- },- {- "type": "function",- "function": {- "name": "edit_file",- "description": "Edit the main HTML file using exact string replacements. Do not regenerate the entire file. Returns the updated file content.",- "parameters": edit_schema,- },- },- {- "type": "function",- "function": {- "name": "generate_images",- "description": "Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",- "parameters": image_schema,- },- },- {- "type": "function",- "function": {- "name": "remove_background",- "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",- "parameters": remove_bg_schema,- },- },- {- "type": "function",- "function": {- "name": "retrieve_option",- "description": "Retrieve the full HTML for a specific option (variant) so you can reference it.",- "parameters": retrieve_option_schema,- },- },- ]- anthropic_tools = [- {- "name": "create_file",- "description": "Create the main HTML file for the app. Use exactly once to write the full HTML. Returns the updated file content.",- "input_schema": create_schema,- },- {- "name": "edit_file",- "description": "Edit the main HTML file using exact string replacements. Do not regenerate the entire file. Returns the updated file content.",- "input_schema": edit_schema,- },- {- "name": "generate_images",- "description": "Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",- "input_schema": image_schema,- },- {- "name": "remove_background",- "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",- "input_schema": remove_bg_schema,- },- {- "name": "retrieve_option",- "description": "Retrieve the full HTML for a specific option (variant) so you can reference it.",- "input_schema": retrieve_option_schema,- },- ]+ content = extract_content_from_args(event.tool_arguments)+ if content is None:+ return- gemini_tools = [- types.Tool(- function_declarations=[- types.FunctionDeclaration(- name="create_file",- description="Create the main HTML file for the app. Use exactly once to write the full HTML. Returns the updated file content.",- parameters_json_schema=create_schema,- ),- types.FunctionDeclaration(- name="edit_file",- description="Edit the main HTML file using exact string replacements. Do not regenerate the entire file. Returns the updated file content.",- parameters_json_schema=edit_schema,- ),- types.FunctionDeclaration(- name="generate_images",- description="Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",- parameters_json_schema=image_schema,- ),- types.FunctionDeclaration(- name="remove_background",- description="Remove the background from an image. Returns a URL to the processed image with transparent background.",- parameters_json_schema=remove_bg_schema,- ),- types.FunctionDeclaration(- name="retrieve_option",- description="Retrieve the full HTML for a specific option (variant) so you can reference it.",- parameters_json_schema=retrieve_option_schema,- ),- ]+ tool_event_id = event.tool_call_id+ if tool_event_id not in started_tool_ids:+ path = (+ extract_path_from_args(event.tool_arguments)+ or self.file_state.path+ or "index.html")- ]-- return openai_tools, anthropic_tools, gemini_tools-- def _openai_responses_tools(self) -> List[Dict[str, Any]]:- create_schema = {- "type": "object",- "additionalProperties": False,- "properties": {- "path": {- "type": ["string", "null"],- "description": "Path for the main HTML file. Use index.html if unsure.",- },- "content": {- "type": "string",- "description": "The full HTML content for the app (single file).",- },- },- "required": ["path", "content"],- }- edit_schema = {- "type": "object",- "additionalProperties": False,- "properties": {- "path": {- "type": ["string", "null"],- "description": "Path for the main HTML file.",- },- "old_text": {- "type": ["string", "null"],- "description": "Exact text to replace.",- },- "new_text": {- "type": ["string", "null"],- "description": "Replacement text.",- },- "count": {- "type": ["integer", "null"],- "description": "How many occurrences to replace. Use -1 for all.",- },- "edits": {- "type": ["array", "null"],- "items": {- "type": "object",- "additionalProperties": False,- "properties": {- "old_text": {"type": ["string", "null"]},- "new_text": {"type": ["string", "null"]},- "count": {"type": ["integer", "null"]},- },- "required": ["old_text", "new_text", "count"],- },- },- },- "required": ["path", "old_text", "new_text", "count", "edits"],- }- image_schema = {- "type": "object",- "additionalProperties": False,- "properties": {- "prompts": {- "type": ["array", "null"],- "items": {- "type": "string",- "description": "Prompt describing a single image to generate.",+ await self._send(+ "toolStart",+ data={+ "name": "create_file",+ "input": {+ "path": path,+ "contentLength": len(content),+ "preview": summarize_text(content, 200),},- }- },- "required": ["prompts"],- }- remove_bg_schema = {- "type": "object",- "additionalProperties": False,- "properties": {- "image_url": {- "type": "string",- "description": "URL of the image to remove the background from.",- },- },- "required": ["image_url"],- }- retrieve_option_schema = {- "type": "object",- "additionalProperties": False,- "properties": {- "option_number": {- "type": "integer",- "description": "1-based option number to retrieve (Option 1, Option 2, etc.).",},- },- "required": ["option_number"],- }- return [- {- "type": "function",- "name": "create_file",- "description": "Create the main HTML file for the app. Use exactly once to write the full HTML. Returns the updated file content.",- "parameters": create_schema,- "strict": True,- },- {- "type": "function",- "name": "edit_file",- "description": "Edit the main HTML file using exact string replacements. Do not regenerate the entire file. Returns the updated file content.",- "parameters": edit_schema,- "strict": True,- },- {- "type": "function",- "name": "generate_images",- "description": "Generate image URLs from prompts. Use to replace placeholder images. You can pass multiple prompts at once.",- "parameters": image_schema,- "strict": True,- },- {- "type": "function",- "name": "remove_background",- "description": "Remove the background from an image. Returns a URL to the processed image with transparent background.",- "parameters": remove_bg_schema,- "strict": True,- },- {- "type": "function",- "name": "retrieve_option",- "description": "Retrieve the full HTML for a specific option (variant) so you can reference it.",- "parameters": retrieve_option_schema,- "strict": True,- },- ]-- async def run(self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]) -> str:- self._seed_file_state_from_messages(prompt_messages)- if model in OPENAI_MODELS:- return await self._run_openai(model, prompt_messages)- if model in ANTHROPIC_MODELS:- return await self._run_anthropic(model, prompt_messages)- if model in GEMINI_MODELS:- return await self._run_gemini(model, prompt_messages)- raise ValueError(f"Unsupported model: {model.value}")-- async def _run_openai(- self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]- ) -> str:- if model in OPENAI_CODEX_MODELS:- return await self._run_openai_responses(model, prompt_messages)- if not self.openai_api_key:- raise Exception("OpenAI API key is missing.")-- client = AsyncOpenAI(api_key=self.openai_api_key, base_url=self.openai_base_url)- openai_tools, _, _ = self._tool_schemas()- messages: List[Dict[str, Any]] = [dict(m) for m in prompt_messages]-+ event_id=tool_event_id,+ )+ started_tool_ids.add(tool_event_id)++ last_len = streamed_lengths.get(tool_event_id, 0)+ if last_len == 0 and content:+ streamed_lengths[tool_event_id] = len(content)+ await self._send("setCode", content)+ self._mark_preview_length(tool_event_id, len(content))+ elif len(content) - last_len >= 40:+ streamed_lengths[tool_event_id] = len(content)+ await self._send("setCode", content)+ self._mark_preview_length(tool_event_id, len(content))++ async def _run_with_adapter(self, adapter: ProviderAdapter) -> str:max_steps = 20- try:- for _ in range(max_steps):- assistant_event_id = self._next_event_id("assistant")- assistant_text = ""- tool_calls: Dict[int, Dict[str, Any]] = {}- started_tool_ids: set[str] = set()-- params: Dict[str, Any] = {- "model": get_openai_api_name(model),- "messages": messages,- "tools": openai_tools,- "tool_choice": "auto",- "temperature": 0,- "stream": True,- }- params["max_tokens"] = 30000- stream = await client.chat.completions.create(**params)+ for _ in range(max_steps):+ assistant_event_id = self._next_event_id("assistant")+ thinking_event_id = self._next_event_id("thinking")+ started_tool_ids: set[str] = set()+ streamed_lengths: Dict[str, int] = {}- async for chunk in stream: # type: ignore- assert isinstance(chunk, ChatCompletionChunk)- if not chunk.choices:- continue- delta = chunk.choices[0].delta- if delta.content:- assistant_text += delta.content+ async def on_event(event: StreamEvent) -> None:+ if event.type == "assistant_delta":+ if event.text:await self._send(- "assistant", delta.content, event_id=assistant_event_id+ "assistant",+ event.text,+ event_id=assistant_event_id,)- if delta.tool_calls:- for tc in delta.tool_calls:- idx = tc.index or 0- entry = tool_calls.setdefault(- idx,- {- "id": tc.id or f"tool-{idx}-{uuid.uuid4().hex[:6]}",- "name": None,- "arguments": "",- "started": False,- "last_len": 0,- },- )- if tc.id:- entry["id"] = tc.id- if tc.function and tc.function.name:- entry["name"] = tc.function.name- if tc.function and tc.function.arguments:- entry["arguments"] += tc.function.arguments- if entry.get("name") == "create_file":- content = _extract_content_from_args(entry.get("arguments"))- if content is not None:- if not entry["started"]:- path = (- _extract_path_from_args(entry.get("arguments"))- or self.file_state.path- or "index.html"- )- await self._send(- "toolStart",- data={- "name": "create_file",- "input": {- "path": path,- "contentLength": len(content),- "preview": _summarize_text(content, 200),- },- },- event_id=entry["id"],- )- entry["started"] = True- started_tool_ids.add(entry["id"])- if entry["last_len"] == 0 and content:- entry["last_len"] = len(content)- await self._send("setCode", content)- self._mark_preview_length(entry["id"], len(content))- elif len(content) - entry["last_len"] >= 40:- entry["last_len"] = len(content)- await self._send("setCode", content)- self._mark_preview_length(entry["id"], len(content))+ return- tool_call_list: List[ToolCall] = []- for entry in tool_calls.values():- args, error = _parse_json_arguments(entry.get("arguments"))- if error:- args = {"_raw": entry.get("arguments")}- tool_call_id = entry.get("id") or f"tool-{uuid.uuid4().hex[:6]}"- tool_call_list.append(- ToolCall(- id=tool_call_id,- name=entry.get("name") or "unknown_tool",- arguments=args,+ if event.type == "thinking_delta":+ if event.text:+ await self._send(+ "thinking",+ event.text,+ event_id=thinking_event_id,)- )+ return- if not tool_call_list:- return await self._finalize_response(assistant_text)+ if event.type == "tool_call_delta":+ await self._handle_streamed_tool_delta(+ event,+ started_tool_ids,+ streamed_lengths,+ )- assistant_message = {- "role": "assistant",- "content": assistant_text or None,- "tool_calls": [- {- "id": tc.id,- "type": "function",- "function": {- "name": tc.name,- "arguments": json.dumps(tc.arguments),- },- }- for tc in tool_call_list- ],- }- messages.append(assistant_message)+ step_result = await adapter.run_step(on_event)- for tool_call in tool_call_list:- tool_event_id = tool_call.id or self._next_event_id("tool")- if tool_event_id not in started_tool_ids:- await self._send(- "toolStart",- data={- "name": tool_call.name,- "input": self._summarize_tool_input(tool_call),- },- event_id=tool_event_id,- )- if tool_call.name == "create_file":- content = _extract_content_from_args(tool_call.arguments)- if content:- await self._stream_code_preview(tool_event_id, content)- tool_result = await self.toolbox.execute(tool_call)- if tool_result.updated_content:- await self._send("setCode", tool_result.updated_content)+ if not step_result.tool_calls:+ return await self._finalize_response(step_result.assistant_text)+ executed_tool_calls: List[ExecutedToolCall] = []+ for tool_call in step_result.tool_calls:+ tool_event_id = tool_call.id or self._next_event_id("tool")+ if tool_event_id not in started_tool_ids:await self._send(- "toolResult",+ "toolStart",data={"name": tool_call.name,- "output": tool_result.summary,- "ok": tool_result.ok,+ "input": summarize_tool_input(tool_call, self.file_state),},event_id=tool_event_id,)- messages.append(- {- "role": "tool",- "tool_call_id": tool_call.id,- "content": json.dumps(tool_result.result),- }- )-- raise Exception("Agent exceeded max tool turns")- finally:- await client.close()-- async def _run_openai_responses(- self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]- ) -> str:- if not self.openai_api_key:- raise Exception("OpenAI API key is missing.")-- client = AsyncOpenAI(api_key=self.openai_api_key, base_url=self.openai_base_url)- openai_tools = self._openai_responses_tools()- input_items: List[Dict[str, Any]] = [- _convert_message_to_responses_input(m) for m in prompt_messages- ]-- max_steps = 20- try:- for _ in range(max_steps):- assistant_event_id = self._next_event_id("assistant")- thinking_event_id = self._next_event_id("thinking")- assistant_text = ""- tool_calls: Dict[str, Dict[str, Any]] = {}- item_to_call_id: Dict[str, str] = {}- output_items_by_index: Dict[int, Dict[str, Any]] = {}- started_tool_ids: set[str] = set()-- params: Dict[str, Any] = {- "model": get_openai_api_name(model),- "input": input_items,- "tools": openai_tools,- "tool_choice": "auto",- "stream": True,- "max_output_tokens": 30000,- }- reasoning_effort = get_openai_reasoning_effort(model)- if reasoning_effort:- params["reasoning"] = {"effort": reasoning_effort}-- responses_client = getattr(client, "responses", None)- if responses_client is None:- raise Exception(- "OpenAI SDK is too old for GPT-5.2 Codex. Please upgrade the 'openai' package to a version that supports the Responses API."- )- stream = await responses_client.create(**params) # type: ignore+ if tool_call.name == "create_file":+ content = extract_content_from_args(tool_call.arguments)+ if content:+ await self._stream_code_preview(tool_event_id, content)- async for event in stream: # type: ignore- event_type = _get_event_attr(event, "type")- if event_type in (- "response.created",- "response.completed",- "response.done",- "response.output_item.done",- ):- if event_type == "response.output_item.done":- output_index = _get_event_attr(event, "output_index")- item = _get_event_attr(event, "item")- if isinstance(output_index, int) and item:- output_items_by_index[output_index] = item- continue- if event_type == "response.output_text.delta":- delta = _get_event_attr(event, "delta", "")- if delta:- assistant_text += delta- await self._send(- "assistant", delta, event_id=assistant_event_id- )- continue- if event_type in (- "response.reasoning_text.delta",- "response.reasoning_summary_text.delta",- ):- delta = _get_event_attr(event, "delta", "")- if delta:- await self._send("thinking", delta, event_id=thinking_event_id)- continue- if event_type in (- "response.reasoning_summary_part.added",- "response.reasoning_summary_part.done",- ):- part = _get_event_attr(event, "part") or {}- text = _get_event_attr(part, "text", "")- if text:- await self._send("thinking", text, event_id=thinking_event_id)- continue- if event_type == "response.output_item.added":- item = _get_event_attr(event, "item")- item_type = _get_event_attr(item, "type") if item else None- output_index = _get_event_attr(event, "output_index")- if isinstance(output_index, int) and item:- output_items_by_index.setdefault(output_index, item)- if item and item_type in ("function_call", "custom_tool_call"):- item_id = _get_event_attr(item, "id")- call_id = _get_event_attr(item, "call_id") or item_id- if item_id and call_id:- item_to_call_id[item_id] = call_id- if call_id:- if item_id and item_id in tool_calls and item_id != call_id:- existing = tool_calls.pop(item_id)- tool_calls[call_id] = {- **existing,- "id": call_id,- }- args_value = _get_event_attr(item, "arguments")- if args_value is None and item_type == "custom_tool_call":- args_value = _get_event_attr(item, "input")- tool_calls.setdefault(- call_id,- {- "id": call_id,- "name": _get_event_attr(item, "name"),- "arguments": args_value or "",- "started": False,- "last_len": 0,- },- )- continue- if event_type in (- "response.function_call_arguments.delta",- "response.mcp_call_arguments.delta",- "response.custom_tool_call_input.delta",- ):- item_id = _get_event_attr(event, "item_id")- call_id = _get_event_attr(event, "call_id")- if call_id and item_id:- item_to_call_id[item_id] = call_id- if not call_id:- call_id = item_to_call_id.get(item_id) if item_id else None- if not call_id and item_id:- call_id = item_id- if not call_id:- continue- entry = tool_calls.setdefault(- call_id,- {- "id": call_id,- "name": _get_event_attr(event, "name"),- "arguments": "",- "started": False,- "last_len": 0,- },- )- delta_value = _get_event_attr(event, "delta")- if delta_value is None:- delta_value = _get_event_attr(event, "input")- entry["arguments"] += delta_value or ""- if entry.get("name") == "create_file":- content = _extract_content_from_args(entry.get("arguments"))- if content is not None:- if not entry["started"]:- path = (- _extract_path_from_args(entry.get("arguments"))- or self.file_state.path- or "index.html"- )- await self._send(- "toolStart",- data={- "name": "create_file",- "input": {- "path": path,- "contentLength": len(content),- "preview": _summarize_text(content, 200),- },- },- event_id=entry["id"],- )- entry["started"] = True- started_tool_ids.add(entry["id"])- if entry["last_len"] == 0 and content:- entry["last_len"] = len(content)- await self._send("setCode", content)- self._mark_preview_length(entry["id"], len(content))- elif len(content) - entry["last_len"] >= 40:- entry["last_len"] = len(content)- await self._send("setCode", content)- self._mark_preview_length(entry["id"], len(content))- continue- if event_type in (- "response.function_call_arguments.done",- "response.mcp_call_arguments.done",- "response.custom_tool_call_input.done",- ):- item_id = _get_event_attr(event, "item_id")- call_id = _get_event_attr(event, "call_id")- if call_id and item_id:- item_to_call_id[item_id] = call_id- if not call_id:- call_id = item_to_call_id.get(item_id) if item_id else None- if not call_id and item_id:- call_id = item_id- if not call_id:- continue- entry = tool_calls.setdefault(- call_id,- {- "id": call_id,- "name": _get_event_attr(event, "name"),- "arguments": "",- "started": False,- "last_len": 0,- },- )- final_value = _get_event_attr(event, "arguments")- if final_value is None:- final_value = _get_event_attr(event, "input")- if final_value is None:- final_value = entry["arguments"]- entry["arguments"] = final_value- if _get_event_attr(event, "name"):- entry["name"] = _get_event_attr(event, "name")- output_index = _get_event_attr(event, "output_index")- if (- item_id- and isinstance(output_index, int)- and isinstance(output_items_by_index.get(output_index), dict)- ):- output_items_by_index[output_index] = {- **output_items_by_index[output_index],- "arguments": entry["arguments"],- "call_id": call_id,- "name": entry.get("name"),- }+ tool_result = await self.toolbox.execute(tool_call)+ if tool_result.updated_content:+ await self._send("setCode", tool_result.updated_content)- output_items = [- output_items_by_index[idx]- for idx in sorted(output_items_by_index.keys())- if output_items_by_index.get(idx)- ]- tool_items = [- item- for item in output_items- if isinstance(item, dict)- and item.get("type") in ("function_call", "custom_tool_call")- ]- if not tool_items and not tool_calls:- return await self._finalize_response(assistant_text)+ await self._send(+ "toolResult",+ data={+ "name": tool_call.name,+ "output": tool_result.summary,+ "ok": tool_result.ok,+ },+ event_id=tool_event_id,+ )+ executed_tool_calls.append(+ ExecutedToolCall(tool_call=tool_call, result=tool_result)+ )- tool_call_list: List[ToolCall] = []- if tool_items:- for item in tool_items:- raw_args = item.get("arguments")- if raw_args is None and item.get("type") == "custom_tool_call":- raw_args = item.get("input")- args, error = _parse_json_arguments(raw_args)- if error:- args = {"_raw": raw_args}- call_id = item.get("call_id") or item.get("id")- tool_call_list.append(- ToolCall(- id=call_id or f"call-{uuid.uuid4().hex[:6]}",- name=item.get("name") or "unknown_tool",- arguments=args,- )- )- else:- for entry in tool_calls.values():- args, error = _parse_json_arguments(entry.get("arguments"))- if error:- args = {"_raw": entry.get("arguments")}- call_id = entry.get("id") or entry.get("call_id")- tool_call_list.append(- ToolCall(- id=call_id or f"call-{uuid.uuid4().hex[:6]}",- name=entry.get("name") or "unknown_tool",- arguments=args,- )- )+ adapter.apply_tool_results(step_result, executed_tool_calls)- tool_output_items: List[Dict[str, Any]] = []- for tool_call in tool_call_list:- tool_event_id = tool_call.id or self._next_event_id("tool")- if tool_event_id not in started_tool_ids:- await self._send(- "toolStart",- data={- "name": tool_call.name,- "input": self._summarize_tool_input(tool_call),- },- event_id=tool_event_id,- )- if tool_call.name == "create_file":- content = _extract_content_from_args(tool_call.arguments)- if content:- await self._stream_code_preview(tool_event_id, content)+ raise Exception("Agent exceeded max tool turns")- tool_result = await self.toolbox.execute(tool_call)- if tool_result.updated_content:- await self._send("setCode", tool_result.updated_content)+ async def run(self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]) -> str:+ seed_file_state_from_messages(self.file_state, prompt_messages)- await self._send(- "toolResult",- data={- "name": tool_call.name,- "output": tool_result.summary,- "ok": tool_result.ok,- },- event_id=tool_event_id,- )+ if model in OPENAI_MODELS:+ return await self._run_openai(model, prompt_messages)+ if model in ANTHROPIC_MODELS:+ return await self._run_anthropic(model, prompt_messages)+ if model in GEMINI_MODELS:+ return await self._run_gemini(model, prompt_messages)+ raise ValueError(f"Unsupported model: {model.value}")- tool_output_items.append(- {- "type": "function_call_output",- "call_id": tool_call.id,- "output": json.dumps(tool_result.result),- }- )+ async def _run_openai(+ self,+ model: Llm,+ prompt_messages: List[ChatCompletionMessageParam],+ ) -> str:+ if not self.openai_api_key:+ raise Exception("OpenAI API key is missing.")- input_items.extend(output_items)- input_items.extend(tool_output_items)+ client = AsyncOpenAI(api_key=self.openai_api_key, base_url=self.openai_base_url)+ canonical_tools = canonical_tool_definitions()- raise Exception("Agent exceeded max tool turns")+ try:+ if model in OPENAI_CODEX_MODELS:+ tools = serialize_openai_responses_tools(canonical_tools)+ adapter: ProviderAdapter = OpenAIResponsesAdapter(+ client=client,+ model=model,+ prompt_messages=prompt_messages,+ tools=tools,+ )+ else:+ tools = serialize_openai_chat_tools(canonical_tools)+ adapter = OpenAIChatAdapter(+ client=client,+ model=model,+ prompt_messages=prompt_messages,+ tools=tools,+ )+ return await self._run_with_adapter(adapter)finally:await client.close()async def _run_anthropic(- self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]+ self,+ model: Llm,+ prompt_messages: List[ChatCompletionMessageParam],) -> str:if not self.anthropic_api_key:raise Exception("Anthropic API key is missing.")- from models.claude import convert_openai_messages_to_claude-- system_prompt, claude_messages = convert_openai_messages_to_claude(- prompt_messages- )client = AsyncAnthropic(api_key=self.anthropic_api_key)- _, anthropic_tools, _ = self._tool_schemas()+ canonical_tools = canonical_tool_definitions()+ tools = serialize_anthropic_tools(canonical_tools)- max_steps = 20try:- for _ in range(max_steps):- assistant_event_id = self._next_event_id("assistant")- thinking_event_id = self._next_event_id("thinking")- assistant_text = ""- tool_blocks: Dict[int, Dict[str, Any]] = {}- tool_json_buffers: Dict[int, str] = {}- started_tool_ids: set[str] = set()-- thinking_models = [- Llm.CLAUDE_4_5_SONNET_2025_09_29.value,- Llm.CLAUDE_4_5_OPUS_2025_11_01.value,- ]-- stream_kwargs: Dict[str, Any] = {- "model": model.value,- "max_tokens": 30000,- "system": system_prompt,- "messages": claude_messages,- "tools": anthropic_tools,- }-- if model.value in thinking_models:- stream_kwargs["thinking"] = {- "type": "enabled",- "budget_tokens": 10000,- }- else:- stream_kwargs["temperature"] = 0.0-- stream_ctx = None- stream_client = None- if hasattr(client, "beta") and hasattr(client.beta, "messages"):- stream_client = client.beta.messages- else:- stream_client = client.messages- try:- stream_ctx = stream_client.stream(- **stream_kwargs,- betas=["fine-grained-tool-streaming-2025-05-14"],- )- except TypeError:- stream_ctx = stream_client.stream(**stream_kwargs)-- async with stream_ctx as stream:- async for event in stream:- if event.type == "content_block_start":- block = event.content_block- if getattr(block, "type", None) == "tool_use":- tool_id = getattr(block, "id", None) or f"tool-{uuid.uuid4().hex[:6]}"- tool_name = getattr(block, "name", None) or "unknown_tool"- tool_blocks[event.index] = {- "id": tool_id,- "name": tool_name,- }- tool_json_buffers[event.index] = ""- if tool_name == "create_file":- args = getattr(block, "input", None)- content = _extract_content_from_args(args)- if tool_id not in started_tool_ids:- path = (- _extract_path_from_args(args)- or self.file_state.path- or "index.html"- )- await self._send(- "toolStart",- data={- "name": "create_file",- "input": {- "path": path,- "contentLength": len(content or ""),- "preview": _summarize_text(content or "", 200),- },- },- event_id=tool_id,- )- started_tool_ids.add(tool_id)- if content:- await self._stream_code_preview(tool_id, content)- if event.type == "content_block_delta":- if event.delta.type == "thinking_delta":- await self._send(- "thinking",- event.delta.thinking,- event_id=thinking_event_id,- )- elif event.delta.type == "text_delta":- assistant_text += event.delta.text- await self._send(- "assistant",- event.delta.text,- event_id=assistant_event_id,- )- elif event.delta.type == "input_json_delta":- partial_json = getattr(event.delta, "partial_json", None) or ""- if partial_json:- buffer = tool_json_buffers.get(event.index, "") + partial_json- tool_json_buffers[event.index] = buffer- meta = tool_blocks.get(event.index)- if meta and meta.get("name") == "create_file":- content = _extract_content_from_args(buffer)- if meta["id"] not in started_tool_ids:- path = (- _extract_path_from_args(buffer)- or self.file_state.path- or "index.html"- )- await self._send(- "toolStart",- data={- "name": "create_file",- "input": {- "path": path,- "contentLength": len(content or ""),- "preview": _summarize_text(content or "", 200),- },- },- event_id=meta["id"],- )- started_tool_ids.add(meta["id"])- if content:- await self._stream_code_preview(meta["id"], content)-- final_message = await stream.get_final_message()-- tool_calls: List[ToolCall] = []- if final_message and final_message.content:- for block in final_message.content:- if block.type == "tool_use":- tool_calls.append(- ToolCall(- id=block.id,- name=block.name,- arguments=block.input,- )- )-- if not tool_calls:- return await self._finalize_response(assistant_text)-- assistant_blocks: List[Dict[str, Any]] = []- if assistant_text:- assistant_blocks.append({"type": "text", "text": assistant_text})- for call in tool_calls:- assistant_blocks.append(- {- "type": "tool_use",- "id": call.id,- "name": call.name,- "input": call.arguments,- }- )-- claude_messages.append({"role": "assistant", "content": assistant_blocks})-- tool_result_blocks: List[Dict[str, Any]] = []- for tool_call in tool_calls:- tool_event_id = tool_call.id or self._next_event_id("tool")- if tool_event_id not in started_tool_ids:- await self._send(- "toolStart",- data={- "name": tool_call.name,- "input": self._summarize_tool_input(tool_call),- },- event_id=tool_event_id,- )- if tool_call.name == "create_file":- content = _extract_content_from_args(tool_call.arguments)- if content:- await self._stream_code_preview(tool_event_id, content)- tool_result = await self.toolbox.execute(tool_call)- if tool_result.updated_content:- await self._send("setCode", tool_result.updated_content)-- await self._send(- "toolResult",- data={- "name": tool_call.name,- "output": tool_result.summary,- "ok": tool_result.ok,- },- event_id=tool_event_id,- )-- tool_result_blocks.append(- {- "type": "tool_result",- "tool_use_id": tool_call.id,- "content": json.dumps(tool_result.result),- "is_error": not tool_result.ok,- }- )-- claude_messages.append({"role": "user", "content": tool_result_blocks})-- raise Exception("Agent exceeded max tool turns")+ adapter: ProviderAdapter = AnthropicAdapter(+ client=client,+ model=model,+ prompt_messages=prompt_messages,+ tools=tools,+ )+ return await self._run_with_adapter(adapter)finally:await client.close()async def _run_gemini(- self, model: Llm, prompt_messages: List[ChatCompletionMessageParam]+ self,+ model: Llm,+ prompt_messages: List[ChatCompletionMessageParam],) -> str:if not self.gemini_api_key:raise Exception("Gemini API key is missing.")- from models.gemini import (- convert_message_to_gemini_content,- get_gemini_api_model_name,- get_thinking_level_for_model,- )-- system_prompt = str(prompt_messages[0].get("content", ""))- gemini_contents: List[types.Content] = [- convert_message_to_gemini_content(msg) for msg in prompt_messages[1:]- ]-- _, _, gemini_tools = self._tool_schemas()client = genai.Client(api_key=self.gemini_api_key)-- max_steps = 20- for _ in range(max_steps):- assistant_event_id = self._next_event_id("assistant")- thinking_event_id = self._next_event_id("thinking")- assistant_text = ""- tool_calls: List[ToolCall] = []- function_call_parts: List[types.Part] = []- tool_stream_state: Dict[str, Dict[str, Any]] = {}- started_tool_ids: set[str] = set()-- thinking_level = get_thinking_level_for_model(model)- config = types.GenerateContentConfig(- temperature=0,- max_output_tokens=30000,- system_instruction=system_prompt,- thinking_config=types.ThinkingConfig(- thinking_level=thinking_level, include_thoughts=True- ),- tools=gemini_tools,- )-- api_model_name = get_gemini_api_model_name(model)-- async for chunk in await client.aio.models.generate_content_stream(- model=api_model_name,- contents=gemini_contents,- config=config,- ):- if not chunk.candidates:- continue- for part in chunk.candidates[0].content.parts:- if getattr(part, "thought", False) and part.text:- await self._send(- "thinking", part.text, event_id=thinking_event_id- )- continue- if part.function_call:- function_call_parts.append(part)- args = part.function_call.args or {}- tool_id = part.function_call.id or f"tool-{uuid.uuid4().hex[:6]}"- tool_name = part.function_call.name or "unknown_tool"- if tool_name == "create_file":- content = _extract_content_from_args(args)- if content is not None:- state = tool_stream_state.setdefault(- tool_id, {"started": False, "last_len": 0}- )- if not state["started"]:- path = (- _extract_path_from_args(args)- or self.file_state.path- or "index.html"- )- await self._send(- "toolStart",- data={- "name": "create_file",- "input": {- "path": path,- "contentLength": len(content),- "preview": _summarize_text(- content, 200- ),- },- },- event_id=tool_id,- )- state["started"] = True- started_tool_ids.add(tool_id)- if state["last_len"] == 0 and content:- state["last_len"] = len(content)- await self._send("setCode", content)- self._mark_preview_length(tool_id, len(content))- elif len(content) - state["last_len"] >= 40:- state["last_len"] = len(content)- await self._send("setCode", content)- self._mark_preview_length(tool_id, len(content))- tool_calls.append(- ToolCall(- id=tool_id,- name=tool_name,- arguments=args,- )- )- continue- if part.text:- assistant_text += part.text- await self._send(- "assistant", part.text, event_id=assistant_event_id- )-- if not tool_calls:- return await self._finalize_response(assistant_text)-- model_parts: List[Any] = []- if assistant_text:- model_parts.append({"text": assistant_text}) # type: ignore- model_parts.extend(function_call_parts)-- model_content = types.Content(role="model", parts=model_parts)- gemini_contents.append(model_content)-- tool_result_parts: List[types.Part] = []- for tool_call in tool_calls:- tool_event_id = tool_call.id or self._next_event_id("tool")- if tool_event_id not in started_tool_ids:- await self._send(- "toolStart",- data={- "name": tool_call.name,- "input": self._summarize_tool_input(tool_call),- },- event_id=tool_event_id,- )- if tool_call.name == "create_file":- content = _extract_content_from_args(tool_call.arguments)- if content:- await self._stream_code_preview(tool_event_id, content)- tool_result = await self.toolbox.execute(tool_call)- if tool_result.updated_content:- await self._send("setCode", tool_result.updated_content)-- await self._send(- "toolResult",- data={- "name": tool_call.name,- "output": tool_result.summary,- "ok": tool_result.ok,- },- event_id=tool_event_id,- )-- tool_result_parts.append(- types.Part.from_function_response(- name=tool_call.name, response=tool_result.result- )- )-- gemini_contents.append(types.Content(role="tool", parts=tool_result_parts))-- raise Exception("Agent exceeded max tool turns")+ canonical_tools = canonical_tool_definitions()+ tools = serialize_gemini_tools(canonical_tools)+ adapter: ProviderAdapter = GeminiAdapter(+ client=client,+ model=model,+ prompt_messages=prompt_messages,+ tools=tools,+ )+ return await self._run_with_adapter(adapter)async def _finalize_response(self, assistant_text: str) -> str:if self.file_state.content:@@ -1871,4 +340,5 @@ class AgenticRunner:if html:self.file_state.content = htmlawait self._send("setCode", html)+return self.file_state.contentdiff --git a/backend/agentic/state.py b/backend/agentic/state.pynew file mode 100644index 00000000..b8d8bb2f--- /dev/null+++ b/backend/agentic/state.py@@ -0,0 +1,71 @@+from dataclasses import dataclass+from typing import Any, List++from openai.types.chat import ChatCompletionMessageParam++from codegen.utils import extract_html_content+++@dataclass+class AgentFileState:+ path: str = "index.html"+ content: str = ""+++def ensure_str(value: Any) -> str:+ if value is None:+ return ""+ return str(value)+++def extract_text_content(message: ChatCompletionMessageParam) -> str:+ content = message.get("content", "")+ if isinstance(content, str):+ return content+ if isinstance(content, list):+ for part in content:+ if isinstance(part, dict) and part.get("type") == "text":+ return ensure_str(part.get("text"))+ return ""+++def seed_file_state_from_messages(+ file_state: AgentFileState,+ prompt_messages: List[ChatCompletionMessageParam],+) -> None:+ if file_state.content:+ return++ for message in reversed(prompt_messages):+ if message.get("role") != "assistant":+ continue+ raw_text = extract_text_content(message)+ if not raw_text:+ continue+ extracted = extract_html_content(raw_text)+ file_state.content = extracted or raw_text+ if not file_state.path:+ file_state.path = "index.html"+ return++ if not prompt_messages:+ return++ system_message = prompt_messages[0]+ if system_message.get("role") != "system":+ return++ system_text = extract_text_content(system_message)+ markers = [+ "Here is the code of the app:",+ "Here is the code of the SVG:",+ ]+ for marker in markers:+ if marker not in system_text:+ continue+ raw_text = system_text.split(marker, 1)[1].strip()+ extracted = extract_html_content(raw_text)+ file_state.content = extracted or raw_text+ if not file_state.path:+ file_state.path = "index.html"+ returndiff --git a/backend/agentic/streaming/__init__.py b/backend/agentic/streaming/__init__.pynew file mode 100644index 00000000..42dfac00--- /dev/null+++ b/backend/agentic/streaming/__init__.py@@ -0,0 +1,18 @@+from agentic.streaming.anthropic import AnthropicAdapter+from agentic.streaming.base import ProviderAdapter+from agentic.streaming.gemini import GeminiAdapter+from agentic.streaming.openai_chat import OpenAIChatAdapter+from agentic.streaming.openai_responses import OpenAIResponsesAdapter+from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult, StreamEvent++__all__ = [+ "AnthropicAdapter",+ "ProviderAdapter",+ "GeminiAdapter",+ "OpenAIChatAdapter",+ "OpenAIResponsesAdapter",+ "EventSink",+ "ExecutedToolCall",+ "StepResult",+ "StreamEvent",+]diff --git a/backend/agentic/streaming/anthropic.py b/backend/agentic/streaming/anthropic.pynew file mode 100644index 00000000..600e389c--- /dev/null+++ b/backend/agentic/streaming/anthropic.py@@ -0,0 +1,171 @@+import json+import uuid+from typing import Any, Dict, List++from anthropic import AsyncAnthropic+from openai.types.chat import ChatCompletionMessageParam++from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult, StreamEvent+from agentic.tools import ToolCall+from llm import Llm+from models.claude import convert_openai_messages_to_claude+++THINKING_MODELS = {+ Llm.CLAUDE_4_5_SONNET_2025_09_29.value,+ Llm.CLAUDE_4_5_OPUS_2025_11_01.value,+}+++class AnthropicAdapter:+ def __init__(+ self,+ client: AsyncAnthropic,+ model: Llm,+ prompt_messages: List[ChatCompletionMessageParam],+ tools: List[Dict[str, Any]],+ ):+ self._client = client+ self._model = model+ self._tools = tools+ system_prompt, claude_messages = convert_openai_messages_to_claude(prompt_messages)+ self._system_prompt = system_prompt+ self._messages = claude_messages++ async def run_step(self, on_event: EventSink) -> StepResult:+ assistant_text = ""+ tool_blocks: Dict[int, Dict[str, Any]] = {}+ tool_json_buffers: Dict[int, str] = {}++ stream_kwargs: Dict[str, Any] = {+ "model": self._model.value,+ "max_tokens": 30000,+ "system": self._system_prompt,+ "messages": self._messages,+ "tools": self._tools,+ }++ if self._model.value in THINKING_MODELS:+ stream_kwargs["thinking"] = {+ "type": "enabled",+ "budget_tokens": 10000,+ }+ else:+ stream_kwargs["temperature"] = 0.0++ if hasattr(self._client, "beta") and hasattr(self._client.beta, "messages"):+ stream_client = self._client.beta.messages+ else:+ stream_client = self._client.messages++ try:+ stream_kwargs_with_betas = dict(stream_kwargs)+ stream_kwargs_with_betas["betas"] = ["fine-grained-tool-streaming-2025-05-14"]+ stream_ctx = stream_client.stream(**stream_kwargs_with_betas)+ except TypeError:+ stream_ctx = stream_client.stream(**stream_kwargs)++ async with stream_ctx as stream:+ async for event in stream:+ if event.type == "content_block_start":+ block = event.content_block+ if getattr(block, "type", None) == "tool_use":+ tool_id = getattr(block, "id", None) or f"tool-{uuid.uuid4().hex[:6]}"+ tool_name = getattr(block, "name", None) or "unknown_tool"+ args = getattr(block, "input", None)+ tool_blocks[event.index] = {+ "id": tool_id,+ "name": tool_name,+ }+ tool_json_buffers[event.index] = ""+ if args:+ await on_event(+ StreamEvent(+ type="tool_call_delta",+ tool_call_id=tool_id,+ tool_name=tool_name,+ tool_arguments=args,+ )+ )+ continue++ if event.type != "content_block_delta":+ continue++ if event.delta.type == "thinking_delta":+ await on_event(+ StreamEvent(type="thinking_delta", text=event.delta.thinking)+ )+ elif event.delta.type == "text_delta":+ assistant_text += event.delta.text+ await on_event(+ StreamEvent(type="assistant_delta", text=event.delta.text)+ )+ elif event.delta.type == "input_json_delta":+ partial_json = getattr(event.delta, "partial_json", None) or ""+ if not partial_json:+ continue+ buffer = tool_json_buffers.get(event.index, "") + partial_json+ tool_json_buffers[event.index] = buffer+ meta = tool_blocks.get(event.index)+ if not meta:+ continue+ await on_event(+ StreamEvent(+ type="tool_call_delta",+ tool_call_id=meta.get("id"),+ tool_name=meta.get("name"),+ tool_arguments=buffer,+ )+ )++ final_message = await stream.get_final_message()++ tool_calls: List[ToolCall] = []+ if final_message and final_message.content:+ for block in final_message.content:+ if block.type != "tool_use":+ continue+ tool_calls.append(+ ToolCall(+ id=block.id,+ name=block.name,+ arguments=block.input,+ )+ )++ return StepResult(assistant_text=assistant_text, tool_calls=tool_calls)++ def apply_tool_results(+ self,+ step_result: StepResult,+ executed_tool_calls: list[ExecutedToolCall],+ ) -> None:+ assistant_blocks: List[Dict[str, Any]] = []+ if step_result.assistant_text:+ assistant_blocks.append({"type": "text", "text": step_result.assistant_text})++ for call in step_result.tool_calls:+ assistant_blocks.append(+ {+ "type": "tool_use",+ "id": call.id,+ "name": call.name,+ "input": call.arguments,+ }+ )++ self._messages.append({"role": "assistant", "content": assistant_blocks})++ tool_result_blocks: List[Dict[str, Any]] = []+ for executed in executed_tool_calls:+ tool_result_blocks.append(+ {+ "type": "tool_result",+ "tool_use_id": executed.tool_call.id,+ "content": json.dumps(executed.result.result),+ "is_error": not executed.result.ok,+ }+ )++ self._messages.append({"role": "user", "content": tool_result_blocks})diff --git a/backend/agentic/streaming/base.py b/backend/agentic/streaming/base.pynew file mode 100644index 00000000..4ec373ab--- /dev/null+++ b/backend/agentic/streaming/base.py@@ -0,0 +1,15 @@+from typing import Protocol++from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult+++class ProviderAdapter(Protocol):+ async def run_step(self, on_event: EventSink) -> StepResult:+ ...++ def apply_tool_results(+ self,+ step_result: StepResult,+ executed_tool_calls: list[ExecutedToolCall],+ ) -> None:+ ...diff --git a/backend/agentic/streaming/gemini.py b/backend/agentic/streaming/gemini.pynew file mode 100644index 00000000..8aafdeca--- /dev/null+++ b/backend/agentic/streaming/gemini.py@@ -0,0 +1,132 @@+# pyright: reportUnknownVariableType=false+import uuid+from typing import Any, Dict, List, cast++from google import genai+from google.genai import types+from openai.types.chat import ChatCompletionMessageParam++from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult, StreamEvent+from agentic.tools import ToolCall+from llm import Llm+from models.gemini import (+ convert_message_to_gemini_content,+ get_gemini_api_model_name,+ get_thinking_level_for_model,+)+++class GeminiAdapter:+ def __init__(+ self,+ client: genai.Client,+ model: Llm,+ prompt_messages: List[ChatCompletionMessageParam],+ tools: List[types.Tool],+ ):+ self._client = client+ self._model = model+ self._tools = tools++ self._system_prompt = str(prompt_messages[0].get("content", ""))+ self._contents: List[types.Content] = [+ convert_message_to_gemini_content(msg) for msg in prompt_messages[1:]+ ]++ async def run_step(self, on_event: EventSink) -> StepResult:+ assistant_text = ""+ tool_calls: List[ToolCall] = []+ function_call_parts: List[types.Part] = []++ thinking_level = get_thinking_level_for_model(self._model)+ config = types.GenerateContentConfig(+ temperature=0,+ max_output_tokens=30000,+ system_instruction=self._system_prompt,+ thinking_config=types.ThinkingConfig(+ thinking_level=cast(Any, thinking_level),+ include_thoughts=True,+ ),+ tools=self._tools,+ )++ api_model_name = get_gemini_api_model_name(self._model)++ async for chunk in await self._client.aio.models.generate_content_stream(+ model=api_model_name,+ contents=cast(Any, self._contents),+ config=config,+ ):+ if not chunk.candidates:+ continue++ candidate_content = chunk.candidates[0].content+ if not candidate_content or not candidate_content.parts:+ continue++ for part in candidate_content.parts:+ if getattr(part, "thought", False) and part.text:+ await on_event(StreamEvent(type="thinking_delta", text=part.text))+ continue++ if part.function_call:+ function_call_parts.append(part)+ args = part.function_call.args or {}+ tool_id = part.function_call.id or f"tool-{uuid.uuid4().hex[:6]}"+ tool_name = part.function_call.name or "unknown_tool"++ await on_event(+ StreamEvent(+ type="tool_call_delta",+ tool_call_id=tool_id,+ tool_name=tool_name,+ tool_arguments=args,+ )+ )++ tool_calls.append(+ ToolCall(+ id=tool_id,+ name=tool_name,+ arguments=args,+ )+ )+ continue++ if part.text:+ assistant_text += part.text+ await on_event(StreamEvent(type="assistant_delta", text=part.text))++ return StepResult(+ assistant_text=assistant_text,+ tool_calls=tool_calls,+ provider_state={"function_call_parts": function_call_parts},+ )++ def apply_tool_results(+ self,+ step_result: StepResult,+ executed_tool_calls: list[ExecutedToolCall],+ ) -> None:+ function_call_parts = cast(+ List[types.Part],+ ((step_result.provider_state or {}).get("function_call_parts") or []),+ )++ model_parts: List[Any] = []+ if step_result.assistant_text:+ model_parts.append({"text": step_result.assistant_text}) # type: ignore+ model_parts.extend(function_call_parts)++ self._contents.append(types.Content(role="model", parts=model_parts))++ tool_result_parts: List[types.Part] = []+ for executed in executed_tool_calls:+ tool_result_parts.append(+ types.Part.from_function_response(+ name=executed.tool_call.name,+ response=executed.result.result,+ )+ )++ self._contents.append(types.Content(role="tool", parts=tool_result_parts))diff --git a/backend/agentic/streaming/openai_chat.py b/backend/agentic/streaming/openai_chat.pynew file mode 100644index 00000000..41c01001--- /dev/null+++ b/backend/agentic/streaming/openai_chat.py@@ -0,0 +1,138 @@+# pyright: reportUnknownVariableType=false+import json+import uuid+from typing import Any, Dict, List++from openai import AsyncOpenAI+from openai.types.chat import ChatCompletionChunk, ChatCompletionMessageParam++from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult, StreamEvent+from agentic.tools import ToolCall, parse_json_arguments+from llm import Llm, get_openai_api_name+++class OpenAIChatAdapter:+ def __init__(+ self,+ client: AsyncOpenAI,+ model: Llm,+ prompt_messages: List[ChatCompletionMessageParam],+ tools: List[Dict[str, Any]],+ ):+ self._client = client+ self._model = model+ self._tools = tools+ self._messages: List[Dict[str, Any]] = [dict(m) for m in prompt_messages]++ async def run_step(self, on_event: EventSink) -> StepResult:+ assistant_text = ""+ tool_calls: Dict[int, Dict[str, Any]] = {}++ params: Dict[str, Any] = {+ "model": get_openai_api_name(self._model),+ "messages": self._messages,+ "tools": self._tools,+ "tool_choice": "auto",+ "temperature": 0,+ "stream": True,+ "max_tokens": 30000,+ }++ stream = await self._client.chat.completions.create(**params)++ async for chunk in stream: # type: ignore+ assert isinstance(chunk, ChatCompletionChunk)+ if not chunk.choices:+ continue++ delta = chunk.choices[0].delta+ if delta.content:+ assistant_text += delta.content+ await on_event(StreamEvent(type="assistant_delta", text=delta.content))++ if not delta.tool_calls:+ continue++ for tc in delta.tool_calls:+ idx = tc.index or 0+ entry = tool_calls.setdefault(+ idx,+ {+ "id": tc.id or f"tool-{idx}-{uuid.uuid4().hex[:6]}",+ "name": None,+ "arguments": "",+ },+ )+ if tc.id:+ entry["id"] = tc.id+ if tc.function and tc.function.name:+ entry["name"] = tc.function.name+ if tc.function and tc.function.arguments:+ entry["arguments"] += tc.function.arguments+ await on_event(+ StreamEvent(+ type="tool_call_delta",+ tool_call_id=entry["id"],+ tool_name=entry.get("name"),+ tool_arguments=entry.get("arguments"),+ )+ )++ tool_call_list: List[ToolCall] = []+ for entry in tool_calls.values():+ args, error = parse_json_arguments(entry.get("arguments"))+ if error:+ args = {"_raw": entry.get("arguments")}+ tool_call_id = entry.get("id") or f"tool-{uuid.uuid4().hex[:6]}"+ tool_call_list.append(+ ToolCall(+ id=tool_call_id,+ name=entry.get("name") or "unknown_tool",+ arguments=args,+ )+ )++ provider_state = None+ if tool_call_list:+ provider_state = {+ "assistant_message": {+ "role": "assistant",+ "content": assistant_text or None,+ "tool_calls": [+ {+ "id": tc.id,+ "type": "function",+ "function": {+ "name": tc.name,+ "arguments": json.dumps(tc.arguments),+ },+ }+ for tc in tool_call_list+ ],+ }+ }++ return StepResult(+ assistant_text=assistant_text,+ tool_calls=tool_call_list,+ provider_state=provider_state,+ )++ def apply_tool_results(+ self,+ step_result: StepResult,+ executed_tool_calls: list[ExecutedToolCall],+ ) -> None:+ provider_state = step_result.provider_state or {}+ assistant_message = provider_state.get("assistant_message")+ if assistant_message:+ self._messages.append(assistant_message)++ for executed in executed_tool_calls:+ self._messages.append(+ {+ "role": "tool",+ "tool_call_id": executed.tool_call.id,+ "content": json.dumps(executed.result.result),+ }+ )diff --git a/backend/agentic/streaming/openai_responses.py b/backend/agentic/streaming/openai_responses.pynew file mode 100644index 00000000..53f4e9e0--- /dev/null+++ b/backend/agentic/streaming/openai_responses.py@@ -0,0 +1,345 @@+# pyright: reportUnknownVariableType=false+import json+import uuid+from typing import Any, Dict, List++from openai import AsyncOpenAI+from openai.types.chat import ChatCompletionMessageParam++from agentic.state import ensure_str+from agentic.streaming.types import EventSink, ExecutedToolCall, StepResult, StreamEvent+from agentic.tools import ToolCall, parse_json_arguments+from llm import Llm, get_openai_api_name, get_openai_reasoning_effort+++def _convert_message_to_responses_input(+ message: ChatCompletionMessageParam,+) -> Dict[str, Any]:+ role = message.get("role", "user")+ content = message.get("content", "")++ if isinstance(content, str):+ return {"role": role, "content": content}++ parts: List[Dict[str, Any]] = []+ if isinstance(content, list):+ for part in content:+ if not isinstance(part, dict):+ continue+ if part.get("type") == "text":+ parts.append({"type": "input_text", "text": part.get("text", "")})+ elif part.get("type") == "image_url":+ image_url = part.get("image_url", {})+ parts.append(+ {+ "type": "input_image",+ "image_url": image_url.get("url", ""),+ "detail": image_url.get("detail", "auto"),+ }+ )++ return {"role": role, "content": parts}+++def _get_event_attr(event: Any, key: str, default: Any = None) -> Any:+ if hasattr(event, key):+ return getattr(event, key)+ if isinstance(event, dict):+ return event.get(key, default)+ return default+++class OpenAIResponsesAdapter:+ def __init__(+ self,+ client: AsyncOpenAI,+ model: Llm,+ prompt_messages: List[ChatCompletionMessageParam],+ tools: List[Dict[str, Any]],+ ):+ self._client = client+ self._model = model+ self._tools = tools+ self._input_items: List[Dict[str, Any]] = [+ _convert_message_to_responses_input(m) for m in prompt_messages+ ]++ async def run_step(self, on_event: EventSink) -> StepResult:+ assistant_text = ""+ tool_calls: Dict[str, Dict[str, Any]] = {}+ item_to_call_id: Dict[str, str] = {}+ output_items_by_index: Dict[int, Dict[str, Any]] = {}++ params: Dict[str, Any] = {+ "model": get_openai_api_name(self._model),+ "input": self._input_items,+ "tools": self._tools,+ "tool_choice": "auto",+ "stream": True,+ "max_output_tokens": 30000,+ }++ reasoning_effort = get_openai_reasoning_effort(self._model)+ if reasoning_effort:+ params["reasoning"] = {"effort": reasoning_effort}++ responses_client = getattr(self._client, "responses", None)+ if responses_client is None:+ raise Exception(+ "OpenAI SDK is too old for GPT-5.2 Codex. Please upgrade the 'openai' package to a version that supports the Responses API."+ )++ stream = await responses_client.create(**params) # type: ignore++ async for event in stream: # type: ignore+ event_type = _get_event_attr(event, "type")+ if event_type in (+ "response.created",+ "response.completed",+ "response.done",+ "response.output_item.done",+ ):+ if event_type == "response.output_item.done":+ output_index = _get_event_attr(event, "output_index")+ item = _get_event_attr(event, "item")+ if isinstance(output_index, int) and item:+ output_items_by_index[output_index] = item+ continue++ if event_type == "response.output_text.delta":+ delta = _get_event_attr(event, "delta", "")+ if delta:+ assistant_text += delta+ await on_event(StreamEvent(type="assistant_delta", text=delta))+ continue++ if event_type in (+ "response.reasoning_text.delta",+ "response.reasoning_summary_text.delta",+ ):+ delta = _get_event_attr(event, "delta", "")+ if delta:+ await on_event(StreamEvent(type="thinking_delta", text=delta))+ continue++ if event_type in (+ "response.reasoning_summary_part.added",+ "response.reasoning_summary_part.done",+ ):+ part = _get_event_attr(event, "part") or {}+ text = _get_event_attr(part, "text", "")+ if text:+ await on_event(StreamEvent(type="thinking_delta", text=text))+ continue++ if event_type == "response.output_item.added":+ item = _get_event_attr(event, "item")+ item_type = _get_event_attr(item, "type") if item else None+ output_index = _get_event_attr(event, "output_index")+ if isinstance(output_index, int) and item:+ output_items_by_index.setdefault(output_index, item)++ if item and item_type in ("function_call", "custom_tool_call"):+ item_id = _get_event_attr(item, "id")+ call_id = _get_event_attr(item, "call_id") or item_id+ if item_id and call_id:+ item_to_call_id[item_id] = call_id+ if call_id:+ if item_id and item_id in tool_calls and item_id != call_id:+ existing = tool_calls.pop(item_id)+ tool_calls[call_id] = {+ **existing,+ "id": call_id,+ }+ args_value = _get_event_attr(item, "arguments")+ if args_value is None and item_type == "custom_tool_call":+ args_value = _get_event_attr(item, "input")+ tool_calls.setdefault(+ call_id,+ {+ "id": call_id,+ "name": _get_event_attr(item, "name"),+ "arguments": args_value or "",+ },+ )+ if args_value:+ await on_event(+ StreamEvent(+ type="tool_call_delta",+ tool_call_id=call_id,+ tool_name=_get_event_attr(item, "name"),+ tool_arguments=args_value,+ )+ )+ continue++ if event_type in (+ "response.function_call_arguments.delta",+ "response.mcp_call_arguments.delta",+ "response.custom_tool_call_input.delta",+ ):+ item_id = _get_event_attr(event, "item_id")+ call_id = _get_event_attr(event, "call_id")+ if call_id and item_id:+ item_to_call_id[item_id] = call_id+ if not call_id:+ call_id = item_to_call_id.get(item_id) if item_id else None+ if not call_id and item_id:+ call_id = item_id+ if not call_id:+ continue++ entry = tool_calls.setdefault(+ call_id,+ {+ "id": call_id,+ "name": _get_event_attr(event, "name"),+ "arguments": "",+ },+ )+ delta_value = _get_event_attr(event, "delta")+ if delta_value is None:+ delta_value = _get_event_attr(event, "input")+ entry["arguments"] += ensure_str(delta_value)++ await on_event(+ StreamEvent(+ type="tool_call_delta",+ tool_call_id=call_id,+ tool_name=entry.get("name"),+ tool_arguments=entry.get("arguments"),+ )+ )+ continue++ if event_type in (+ "response.function_call_arguments.done",+ "response.mcp_call_arguments.done",+ "response.custom_tool_call_input.done",+ ):+ item_id = _get_event_attr(event, "item_id")+ call_id = _get_event_attr(event, "call_id")+ if call_id and item_id:+ item_to_call_id[item_id] = call_id+ if not call_id:+ call_id = item_to_call_id.get(item_id) if item_id else None+ if not call_id and item_id:+ call_id = item_id+ if not call_id:+ continue++ entry = tool_calls.setdefault(+ call_id,+ {+ "id": call_id,+ "name": _get_event_attr(event, "name"),+ "arguments": "",+ },+ )+ final_value = _get_event_attr(event, "arguments")+ if final_value is None:+ final_value = _get_event_attr(event, "input")+ if final_value is None:+ final_value = entry["arguments"]+ entry["arguments"] = final_value+ if _get_event_attr(event, "name"):+ entry["name"] = _get_event_attr(event, "name")++ await on_event(+ StreamEvent(+ type="tool_call_delta",+ tool_call_id=call_id,+ tool_name=entry.get("name"),+ tool_arguments=entry.get("arguments"),+ )+ )++ output_index = _get_event_attr(event, "output_index")+ if (+ item_id+ and isinstance(output_index, int)+ and isinstance(output_items_by_index.get(output_index), dict)+ ):+ output_items_by_index[output_index] = {+ **output_items_by_index[output_index],+ "arguments": entry["arguments"],+ "call_id": call_id,+ "name": entry.get("name"),+ }++ output_items = [+ output_items_by_index[idx]+ for idx in sorted(output_items_by_index.keys())+ if output_items_by_index.get(idx)+ ]++ tool_items = [+ item+ for item in output_items+ if isinstance(item, dict)+ and item.get("type") in ("function_call", "custom_tool_call")+ ]++ tool_call_list: List[ToolCall] = []+ if tool_items:+ for item in tool_items:+ raw_args = item.get("arguments")+ if raw_args is None and item.get("type") == "custom_tool_call":+ raw_args = item.get("input")+ args, error = parse_json_arguments(raw_args)+ if error:+ args = {"_raw": raw_args}+ call_id = item.get("call_id") or item.get("id")+ tool_call_list.append(+ ToolCall(+ id=call_id or f"call-{uuid.uuid4().hex[:6]}",+ name=item.get("name") or "unknown_tool",+ arguments=args,+ )+ )+ else:+ for entry in tool_calls.values():+ args, error = parse_json_arguments(entry.get("arguments"))+ if error:+ args = {"_raw": entry.get("arguments")}+ call_id = entry.get("id") or entry.get("call_id")+ tool_call_list.append(+ ToolCall(+ id=call_id or f"call-{uuid.uuid4().hex[:6]}",+ name=entry.get("name") or "unknown_tool",+ arguments=args,+ )+ )++ provider_state = None+ if tool_call_list:+ provider_state = {+ "output_items": output_items,+ }++ return StepResult(+ assistant_text=assistant_text,+ tool_calls=tool_call_list,+ provider_state=provider_state,+ )++ def apply_tool_results(+ self,+ step_result: StepResult,+ executed_tool_calls: list[ExecutedToolCall],+ ) -> None:+ provider_state = step_result.provider_state or {}+ output_items = provider_state.get("output_items") or []++ tool_output_items: List[Dict[str, Any]] = []+ for executed in executed_tool_calls:+ tool_output_items.append(+ {+ "type": "function_call_output",+ "call_id": executed.tool_call.id,+ "output": json.dumps(executed.result.result),+ }+ )++ self._input_items.extend(output_items)+ self._input_items.extend(tool_output_items)diff --git a/backend/agentic/streaming/types.py b/backend/agentic/streaming/types.pynew file mode 100644index 00000000..0dacf133--- /dev/null+++ b/backend/agentic/streaming/types.py@@ -0,0 +1,36 @@+from dataclasses import dataclass+from typing import Any, Awaitable, Callable, List, Literal, Optional++from agentic.tools import ToolCall, ToolExecutionResult+++StreamEventType = Literal[+ "assistant_delta",+ "thinking_delta",+ "tool_call_delta",+]+++@dataclass+class StreamEvent:+ type: StreamEventType+ text: str = ""+ tool_call_id: Optional[str] = None+ tool_name: Optional[str] = None+ tool_arguments: Any = None+++@dataclass+class StepResult:+ assistant_text: str+ tool_calls: List[ToolCall]+ provider_state: Any = None+++@dataclass+class ExecutedToolCall:+ tool_call: ToolCall+ result: ToolExecutionResult+++EventSink = Callable[[StreamEvent], Awaitable[None]]diff --git a/backend/agentic/tools.py b/backend/agentic/tools.pynew file mode 100644index 00000000..cadccc92--- /dev/null+++ b/backend/agentic/tools.py@@ -0,0 +1,758 @@+# pyright: reportUnknownVariableType=false+import copy+import json+from dataclasses import dataclass+from typing import Any, Dict, List, Optional, Tuple++from google.genai import types++from codegen.utils import extract_html_content+from config import REPLICATE_API_KEY+from image_generation.core import process_tasks+from image_generation.replicate import remove_background++from agentic.state import AgentFileState, ensure_str+++@dataclass(frozen=True)+class ToolCall:+ id: str+ name: str+ arguments: Dict[str, Any]+++@dataclass+class ToolExecutionResult:+ ok: bool+ result: Dict[str, Any]+ summary: Dict[str, Any]+ updated_content: Optional[str] = None+++@dataclass(frozen=True)+class CanonicalToolDefinition:+ name: str+ description: str+ parameters: Dict[str, Any]+++def summarize_text(value: str, limit: int = 240) -> str:+ if len(value) <= limit:+ return value+ return value[:limit] + "..."+++def parse_json_arguments(raw_args: Any) -> Tuple[Dict[str, Any], Optional[str]]:+ if isinstance(raw_args, dict):+ return raw_args, None+ if raw_args is None:+ return {}, None+ raw_text = ensure_str(raw_args).strip()+ if not raw_text:+ return {}, None+ try:+ return json.loads(raw_text), None+ except json.JSONDecodeError as exc:+ return {}, f"Invalid JSON arguments: {exc}"+++def _strip_incomplete_escape(value: str) -> str:+ if not value:+ return value+ trailing = 0+ for ch in reversed(value):+ if ch == "\\":+ trailing += 1+ else:+ break+ if trailing % 2 == 1:+ return value[:-1]+ return value+++def _extract_partial_json_string(raw_text: str, key: str) -> Optional[str]:+ if not raw_text:+ return None+ token = f'"{key}"'+ idx = raw_text.find(token)+ if idx == -1:+ return None+ colon = raw_text.find(":", idx + len(token))+ if colon == -1:+ return None+ cursor = colon + 1+ while cursor < len(raw_text) and raw_text[cursor].isspace():+ cursor += 1+ if cursor >= len(raw_text) or raw_text[cursor] != '"':+ return None++ start = cursor + 1+ last_quote: Optional[int] = None+ cursor = start+ while cursor < len(raw_text):+ if raw_text[cursor] == '"':+ backslashes = 0+ back = cursor - 1+ while back >= start and raw_text[back] == "\\":+ backslashes += 1+ back -= 1+ if backslashes % 2 == 0:+ last_quote = cursor+ cursor += 1++ partial = raw_text[start:] if last_quote is None else raw_text[start:last_quote]+ partial = _strip_incomplete_escape(partial)+ if not partial:+ return ""++ try:+ return json.loads(f'"{partial}"')+ except Exception:+ return (+ partial.replace("\\n", "\n")+ .replace("\\t", "\t")+ .replace("\\r", "\r")+ .replace('\\"', '"')+ .replace("\\\\", "\\")+ )+++def extract_content_from_args(raw_args: Any) -> Optional[str]:+ if isinstance(raw_args, dict):+ content = raw_args.get("content")+ if content is None:+ return None+ return ensure_str(content)+ raw_text = ensure_str(raw_args)+ return _extract_partial_json_string(raw_text, "content")+++def extract_path_from_args(raw_args: Any) -> Optional[str]:+ if isinstance(raw_args, dict):+ path = raw_args.get("path")+ return ensure_str(path) if path is not None else None+ raw_text = ensure_str(raw_args)+ return _extract_partial_json_string(raw_text, "path")+++def _create_schema() -> Dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "path": {+ "type": "string",+ "description": "Path for the main HTML file. Use index.html if unsure.",+ },+ "content": {+ "type": "string",+ "description": "Full HTML for the single-file app.",+ },+ },+ "required": ["content"],+ }+++def _edit_schema() -> Dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "path": {+ "type": "string",+ "description": "Path for the main HTML file.",+ },+ "old_text": {+ "type": "string",+ "description": "Exact text to replace. Must match the file contents.",+ },+ "new_text": {+ "type": "string",+ "description": "Replacement text.",+ },+ "count": {+ "type": "integer",+ "description": "How many occurrences to replace. Use -1 for all.",+ },+ "edits": {+ "type": "array",+ "items": {+ "type": "object",+ "properties": {+ "old_text": {"type": "string"},+ "new_text": {"type": "string"},+ "count": {"type": "integer"},+ },+ "required": ["old_text", "new_text"],+ },+ },+ },+ }+++def _image_schema() -> Dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "prompts": {+ "type": "array",+ "items": {+ "type": "string",+ "description": "Prompt describing a single image to generate.",+ },+ }+ },+ "required": ["prompts"],+ }+++def _remove_background_schema() -> Dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "image_url": {+ "type": "string",+ "description": "URL of the image to remove the background from.",+ },+ },+ "required": ["image_url"],+ }+++def _retrieve_option_schema() -> Dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "option_number": {+ "type": "integer",+ "description": "1-based option number to retrieve (Option 1, Option 2, etc.).",+ }+ },+ "required": ["option_number"],+ }+++def canonical_tool_definitions() -> List[CanonicalToolDefinition]:+ return [+ CanonicalToolDefinition(+ name="create_file",+ description=(+ "Create the main HTML file for the app. Use exactly once to write the "+ "full HTML. Returns the updated file content."+ ),+ parameters=_create_schema(),+ ),+ CanonicalToolDefinition(+ name="edit_file",+ description=(+ "Edit the main HTML file using exact string replacements. Do not "+ "regenerate the entire file. Returns the updated file content."+ ),+ parameters=_edit_schema(),+ ),+ CanonicalToolDefinition(+ name="generate_images",+ description=(+ "Generate image URLs from prompts. Use to replace placeholder images. "+ "You can pass multiple prompts at once."+ ),+ parameters=_image_schema(),+ ),+ CanonicalToolDefinition(+ name="remove_background",+ description=(+ "Remove the background from an image. Returns a URL to the processed "+ "image with transparent background."+ ),+ parameters=_remove_background_schema(),+ ),+ CanonicalToolDefinition(+ name="retrieve_option",+ description=(+ "Retrieve the full HTML for a specific option (variant) so you can "+ "reference it."+ ),+ parameters=_retrieve_option_schema(),+ ),+ ]+++def _copy_schema(schema: Dict[str, Any]) -> Dict[str, Any]:+ return copy.deepcopy(schema)+++def serialize_openai_chat_tools(+ tools: List[CanonicalToolDefinition],+) -> List[Dict[str, Any]]:+ return [+ {+ "type": "function",+ "function": {+ "name": tool.name,+ "description": tool.description,+ "parameters": _copy_schema(tool.parameters),+ },+ }+ for tool in tools+ ]+++def _nullable_type(type_value: Any) -> Any:+ if isinstance(type_value, list):+ if "null" not in type_value:+ return [*type_value, "null"]+ return type_value+ if isinstance(type_value, str):+ return [type_value, "null"]+ return type_value+++def _make_responses_schema_strict(schema: Dict[str, Any]) -> Dict[str, Any]:+ schema_copy: Dict[str, Any] = _copy_schema(schema)++ def transform(node: Dict[str, Any], in_object_property: bool = False) -> None:+ node_type = node.get("type")++ if node_type == "object":+ node["additionalProperties"] = False+ properties = node.get("properties") or {}+ if isinstance(properties, dict):+ node["required"] = list(properties.keys())+ for prop in properties.values():+ if isinstance(prop, dict):+ transform(prop, in_object_property=True)+ return++ if node_type == "array":+ if in_object_property:+ node["type"] = _nullable_type(node_type)+ items = node.get("items")+ if isinstance(items, dict):+ transform(items, in_object_property=False)+ return++ if in_object_property and node_type is not None:+ node["type"] = _nullable_type(node_type)++ transform(schema_copy, in_object_property=False)+ return schema_copy+++def serialize_openai_responses_tools(+ tools: List[CanonicalToolDefinition],+) -> List[Dict[str, Any]]:+ serialized: List[Dict[str, Any]] = []+ for tool in tools:+ schema = _make_responses_schema_strict(tool.parameters)+ serialized.append(+ {+ "type": "function",+ "name": tool.name,+ "description": tool.description,+ "parameters": schema,+ "strict": True,+ }+ )+ return serialized+++def serialize_anthropic_tools(+ tools: List[CanonicalToolDefinition],+) -> List[Dict[str, Any]]:+ return [+ {+ "name": tool.name,+ "description": tool.description,+ "input_schema": _copy_schema(tool.parameters),+ }+ for tool in tools+ ]+++def serialize_gemini_tools(+ tools: List[CanonicalToolDefinition],+) -> List[types.Tool]:+ declarations = [+ types.FunctionDeclaration(+ name=tool.name,+ description=tool.description,+ parameters_json_schema=_copy_schema(tool.parameters),+ )+ for tool in tools+ ]+ return [types.Tool(function_declarations=declarations)]+++def summarize_tool_input(tool_call: ToolCall, file_state: AgentFileState) -> Dict[str, Any]:+ args = tool_call.arguments or {}++ if tool_call.name == "create_file":+ content = ensure_str(args.get("content"))+ return {+ "path": args.get("path") or file_state.path,+ "contentLength": len(content),+ "preview": summarize_text(content, 200),+ }++ if tool_call.name == "edit_file":+ edits = args.get("edits")+ if not edits:+ edits = [+ {+ "old_text": args.get("old_text"),+ "new_text": args.get("new_text"),+ "count": args.get("count"),+ }+ ]+ summary_edits = []+ for edit in edits if isinstance(edits, list) else []:+ summary_edits.append(+ {+ "old_text": summarize_text(ensure_str(edit.get("old_text")), 160),+ "new_text": summarize_text(ensure_str(edit.get("new_text")), 160),+ "count": edit.get("count"),+ }+ )+ return {+ "path": args.get("path") or file_state.path,+ "edits": summary_edits,+ }++ if tool_call.name == "generate_images":+ prompts = args.get("prompts") or []+ if isinstance(prompts, list):+ return {+ "count": len(prompts),+ "prompts": [summarize_text(ensure_str(p), 140) for p in prompts],+ }++ if tool_call.name == "remove_background":+ image_url = ensure_str(args.get("image_url"))+ return {"image_url": summarize_text(image_url, 100)}++ if tool_call.name == "retrieve_option":+ return {+ "option_number": args.get("option_number"),+ "index": args.get("index"),+ }++ return args+++class AgentToolbox:+ def __init__(+ self,+ file_state: AgentFileState,+ image_cache: Dict[str, str],+ should_generate_images: bool,+ openai_api_key: Optional[str],+ openai_base_url: Optional[str],+ option_codes: Optional[List[str]] = None,+ ):+ self.file_state = file_state+ self.image_cache = image_cache+ self.should_generate_images = should_generate_images+ self.openai_api_key = openai_api_key+ self.openai_base_url = openai_base_url+ self.option_codes = option_codes or []++ async def execute(self, tool_call: ToolCall) -> ToolExecutionResult:+ if tool_call.name == "create_file":+ return self._create_file(tool_call.arguments)+ if tool_call.name == "edit_file":+ return self._edit_file(tool_call.arguments)+ if tool_call.name == "generate_images":+ return await self._generate_images(tool_call.arguments)+ if tool_call.name == "remove_background":+ return await self._remove_background(tool_call.arguments)+ if tool_call.name == "retrieve_option":+ return self._retrieve_option(tool_call.arguments)+ return ToolExecutionResult(+ ok=False,+ result={"error": f"Unknown tool: {tool_call.name}"},+ summary={"error": f"Unknown tool: {tool_call.name}"},+ )++ def _create_file(self, args: Dict[str, Any]) -> ToolExecutionResult:+ path = ensure_str(args.get("path") or self.file_state.path or "index.html")+ content = ensure_str(args.get("content"))+ if not content:+ return ToolExecutionResult(+ ok=False,+ result={"error": "create_file requires non-empty content"},+ summary={"error": "Missing content"},+ )++ extracted = extract_html_content(content)+ self.file_state.path = path+ self.file_state.content = extracted or content++ summary = {+ "path": self.file_state.path,+ "contentLength": len(self.file_state.content),+ "preview": summarize_text(self.file_state.content, 320),+ }+ result = {+ "path": self.file_state.path,+ "content": self.file_state.content,+ }+ return ToolExecutionResult(+ ok=True,+ result=result,+ summary=summary,+ updated_content=self.file_state.content,+ )++ def _apply_single_edit(+ self,+ content: str,+ old_text: str,+ new_text: str,+ count: Optional[int],+ ) -> Tuple[str, int]:+ if old_text not in content:+ return content, 0++ if count is None:+ replace_count = 1+ elif count < 0:+ replace_count = content.count(old_text)+ else:+ replace_count = count++ updated = content.replace(old_text, new_text, replace_count)+ return updated, min(replace_count, content.count(old_text))++ def _edit_file(self, args: Dict[str, Any]) -> ToolExecutionResult:+ if not self.file_state.content:+ return ToolExecutionResult(+ ok=False,+ result={"error": "No file exists yet. Call create_file first."},+ summary={"error": "No file to edit"},+ )++ edits = args.get("edits")+ if not edits:+ old_text = ensure_str(args.get("old_text"))+ new_text = ensure_str(args.get("new_text"))+ count = args.get("count")+ edits = [{"old_text": old_text, "new_text": new_text, "count": count}]++ if not isinstance(edits, list):+ return ToolExecutionResult(+ ok=False,+ result={"error": "edits must be a list"},+ summary={"error": "Invalid edits payload"},+ )++ content = self.file_state.content+ summary_edits: List[Dict[str, Any]] = []+ for edit in edits:+ old_text = ensure_str(edit.get("old_text"))+ new_text = ensure_str(edit.get("new_text"))+ count = edit.get("count")+ if not old_text:+ return ToolExecutionResult(+ ok=False,+ result={"error": "edit_file requires old_text"},+ summary={"error": "Missing old_text"},+ )++ content, replaced = self._apply_single_edit(content, old_text, new_text, count)+ if replaced == 0:+ return ToolExecutionResult(+ ok=False,+ result={"error": "old_text not found", "old_text": old_text},+ summary={+ "error": "old_text not found",+ "old_text": summarize_text(old_text, 160),+ },+ )++ summary_edits.append(+ {+ "old_text": summarize_text(old_text, 140),+ "new_text": summarize_text(new_text, 140),+ "replaced": replaced,+ }+ )++ self.file_state.content = content+ summary = {+ "path": self.file_state.path,+ "edits": summary_edits,+ "contentLength": len(self.file_state.content),+ }+ result = {+ "path": self.file_state.path,+ "content": self.file_state.content,+ }+ return ToolExecutionResult(+ ok=True,+ result=result,+ summary=summary,+ updated_content=self.file_state.content,+ )++ async def _generate_images(self, args: Dict[str, Any]) -> ToolExecutionResult:+ if not self.should_generate_images:+ return ToolExecutionResult(+ ok=False,+ result={"error": "Image generation is disabled."},+ summary={"error": "Image generation disabled"},+ )++ prompts = args.get("prompts") or []+ if not isinstance(prompts, list) or not prompts:+ return ToolExecutionResult(+ ok=False,+ result={"error": "generate_images requires a non-empty prompts list"},+ summary={"error": "Missing prompts"},+ )++ cleaned = [prompt.strip() for prompt in prompts if isinstance(prompt, str)]+ unique_prompts = list(dict.fromkeys([p for p in cleaned if p]))+ if not unique_prompts:+ return ToolExecutionResult(+ ok=False,+ result={"error": "No valid prompts provided"},+ summary={"error": "No valid prompts"},+ )++ cached_results: Dict[str, Optional[str]] = {}+ to_generate: List[str] = []+ for prompt in unique_prompts:+ if prompt in self.image_cache:+ cached_results[prompt] = self.image_cache[prompt]+ else:+ to_generate.append(prompt)++ generated_results: Dict[str, Optional[str]] = {}++ if to_generate:+ if REPLICATE_API_KEY:+ model = "flux"+ api_key = REPLICATE_API_KEY+ base_url = None+ else:+ if not self.openai_api_key:+ return ToolExecutionResult(+ ok=False,+ result={"error": "No API key available for image generation."},+ summary={"error": "Missing image generation API key"},+ )+ model = "dalle3"+ api_key = self.openai_api_key+ base_url = self.openai_base_url++ generated = await process_tasks(to_generate, api_key, base_url, model) # type: ignore+ for prompt, url in zip(to_generate, generated):+ generated_results[prompt] = url+ if url:+ self.image_cache[prompt] = url++ merged_results = {**cached_results, **generated_results}+ summary_items = [+ {+ "prompt": summarize_text(prompt, 160),+ "url": url,+ "status": "ok" if url else "error",+ }+ for prompt, url in merged_results.items()+ ]+ result = {"images": merged_results}+ summary = {"images": summary_items}+ return ToolExecutionResult(ok=True, result=result, summary=summary)++ async def _remove_background(self, args: Dict[str, Any]) -> ToolExecutionResult:+ if not REPLICATE_API_KEY:+ return ToolExecutionResult(+ ok=False,+ result={"error": "Background removal requires REPLICATE_API_KEY."},+ summary={"error": "Missing Replicate API key"},+ )++ image_url = ensure_str(args.get("image_url"))+ if not image_url:+ return ToolExecutionResult(+ ok=False,+ result={"error": "remove_background requires an image_url"},+ summary={"error": "Missing image_url"},+ )++ try:+ result_url = await remove_background(image_url, REPLICATE_API_KEY)+ result = {"image_url": image_url, "result_url": result_url}+ summary = {+ "image_url": summarize_text(image_url, 100),+ "result_url": result_url,+ "status": "ok",+ }+ return ToolExecutionResult(ok=True, result=result, summary=summary)+ except Exception as exc:+ return ToolExecutionResult(+ ok=False,+ result={"error": str(exc), "image_url": image_url},+ summary={"error": summarize_text(str(exc), 100)},+ )++ def _retrieve_option(self, args: Dict[str, Any]) -> ToolExecutionResult:+ raw_option_number = args.get("option_number")+ raw_index = args.get("index")++ def coerce_int(value: Any) -> Optional[int]:+ if value is None:+ return None+ try:+ return int(value)+ except (TypeError, ValueError):+ return None++ option_number = coerce_int(raw_option_number)+ index = coerce_int(raw_index)++ if option_number is None and index is None:+ return ToolExecutionResult(+ ok=False,+ result={"error": "retrieve_option requires option_number"},+ summary={"error": "Missing option_number"},+ )++ resolved_index = index if option_number is None else option_number - 1+ if resolved_index is None:+ return ToolExecutionResult(+ ok=False,+ result={"error": "Invalid option_number"},+ summary={"error": "Invalid option_number"},+ )++ if resolved_index < 0 or resolved_index >= len(self.option_codes):+ return ToolExecutionResult(+ ok=False,+ result={+ "error": "Option index out of range",+ "option_number": resolved_index + 1,+ "available": len(self.option_codes),+ },+ summary={+ "error": "Option index out of range",+ "available": len(self.option_codes),+ },+ )++ code = ensure_str(self.option_codes[resolved_index])+ if not code.strip():+ return ToolExecutionResult(+ ok=False,+ result={+ "error": "Option code is empty or unavailable",+ "option_number": resolved_index + 1,+ },+ summary={"error": "Option code unavailable"},+ )++ summary = {+ "option_number": resolved_index + 1,+ "contentLength": len(code),+ "preview": summarize_text(code, 200),+ }+ result = {"option_number": resolved_index + 1, "code": code}+ return ToolExecutionResult(ok=True, result=result, summary=summary)diff --git a/backend/evals/core.py b/backend/evals/core.pyindex 394295c5..0c7a397d 100644--- a/backend/evals/core.py+++ b/backend/evals/core.py@@ -1,57 +1,63 @@-from config import ANTHROPIC_API_KEY, GEMINI_API_KEY, OPENAI_API_KEY-from llm import Llm, ANTHROPIC_MODELS, GEMINI_MODELS-from models import (- stream_claude_response,- stream_gemini_response,- stream_openai_response,+from config import (+ ANTHROPIC_API_KEY,+ GEMINI_API_KEY,+ OPENAI_API_KEY,+ OPENAI_BASE_URL,)+from llm import Llm, OPENAI_MODELS, ANTHROPIC_MODELS, GEMINI_MODELS+from agentic.runner import AgenticRunnerfrom prompts import assemble_prompt+from prompts.agentic_instructions import apply_tool_instructionsfrom prompts.types import Stackfrom openai.types.chat import ChatCompletionMessageParam+from typing import Any, castasync def generate_code_for_image(image_url: str, stack: Stack, model: Llm) -> str:prompt_messages = assemble_prompt([image_url], stack)+ if prompt_messages:+ first_message = cast(dict[str, Any], prompt_messages[0])+ first_content = first_message.get("content")+ if first_content:+ first_message["content"] = apply_tool_instructions(+ str(first_content),+ should_generate_images=True,+ )return await generate_code_core(prompt_messages, model)async def generate_code_core(prompt_messages: list[ChatCompletionMessageParam], model: Llm) -> str:-- async def process_chunk(_: str):- pass-- if model in ANTHROPIC_MODELS:- if not ANTHROPIC_API_KEY:- raise Exception("Anthropic API key not found")-- completion = await stream_claude_response(- prompt_messages,- api_key=ANTHROPIC_API_KEY,- callback=lambda x: process_chunk(x),- model_name=model.value,- )- elif model in GEMINI_MODELS:- if not GEMINI_API_KEY:- raise Exception("Gemini API key not found")-- completion = await stream_gemini_response(- prompt_messages,- api_key=GEMINI_API_KEY,- callback=lambda x: process_chunk(x),- model=model,- )- else:- if not OPENAI_API_KEY:- raise Exception("OpenAI API key not found")-- completion = await stream_openai_response(- prompt_messages,- api_key=OPENAI_API_KEY,- base_url=None,- callback=lambda x: process_chunk(x),- model=model,- )-- return completion["code"]+ async def send_message(+ _: str,+ __: str | None,+ ___: int,+ ____: dict[str, Any] | None = None,+ _____: str | None = None,+ ) -> None:+ # Evals do not stream tool/assistant messages to a frontend.+ return None++ if model in ANTHROPIC_MODELS and not ANTHROPIC_API_KEY:+ raise Exception("Anthropic API key not found")+ if model in GEMINI_MODELS and not GEMINI_API_KEY:+ raise Exception("Gemini API key not found")+ if model in OPENAI_MODELS and not OPENAI_API_KEY:+ raise Exception("OpenAI API key not found")++ print(f"[EVALS] Using agentic runner for model: {model.value}")++ runner = AgenticRunner(+ send_message=send_message,+ variant_index=0,+ openai_api_key=OPENAI_API_KEY,+ openai_base_url=OPENAI_BASE_URL,+ anthropic_api_key=ANTHROPIC_API_KEY,+ gemini_api_key=GEMINI_API_KEY,+ should_generate_images=True,+ image_cache={},+ initial_file_state=None,+ option_codes=None,+ )+ return await runner.run(model, prompt_messages)