Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
# pyright: reportUnknownVariableType=false
import asyncio
import base64
import copy
import io
import json
import math
import uuid
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional
from PIL import Image
from openai import AsyncOpenAI
from openai.types.chat import ChatCompletionMessageParam
from agent.providers.base import (
EventSink,
ExecutedToolCall,
ProviderSession,
ProviderTurn,
StreamEvent,
)
from costs.pricing import MODEL_PRICING
from costs.token_usage import TokenUsage
from agent.state import ensure_str
from agent.tools import CanonicalToolDefinition, ToolCall, parse_json_arguments
from fs_logging.agent_runs import AgentRunRecorder
from fs_logging.prompt_reports import PromptReportLogger
from llm import Llm, get_openai_api_name, get_openai_reasoning_effort
# gpt-6.1-sol rejects "original"-detail images that need more than 30,000
# 32x32 patches (a 2724x11746 landing-page screenshot needs 31,648).
IMAGE_PATCH_SIZE = 32
PATCH_LIMITED_MODEL_MAX_PATCHES: Dict[str, int] = {"gpt-6.1-sol": 30000}
def _patch_count(width: int, height: int) -> int:
return math.ceil(width / IMAGE_PATCH_SIZE) * math.ceil(height / IMAGE_PATCH_SIZE)
def _fit_image_to_patch_limit(image_url: str, max_patches: Optional[int]) -> str:
"""Downscale a base64 data-URL image that exceeds the model's patch limit.
Remote URLs and undecodable data are returned unchanged.
"""
if max_patches is None or not image_url.startswith("data:"):
return image_url
try:
header, encoded = image_url.split(",", 1)
image = Image.open(io.BytesIO(base64.b64decode(encoded)))
width, height = image.size
except Exception:
return image_url
if _patch_count(width, height) <= max_patches:
return image_url
scale = math.sqrt(max_patches / _patch_count(width, height))
new_width, new_height = width, height
while _patch_count(new_width, new_height) > max_patches:
new_width = max(1, math.floor(width * scale))
new_height = max(1, math.floor(height * scale))
scale *= 0.99
resized = image.resize((new_width, new_height), Image.Resampling.LANCZOS)
output_format = image.format if image.format in ("PNG", "JPEG", "WEBP") else "PNG"
if output_format == "JPEG" and resized.mode not in ("RGB", "L"):
resized = resized.convert("RGB")
buffer = io.BytesIO()
resized.save(buffer, format=output_format)
mime_type = f"image/{output_format.lower()}"
print(
f"[OPENAI IMAGE] Downscaled {width}x{height} -> {new_width}x{new_height} "
f"to fit {max_patches} patches ({header})"
)
return f"data:{mime_type};base64,{base64.b64encode(buffer.getvalue()).decode('ascii')}"
def _get_max_patches_for_model(model: Llm) -> Optional[int]:
return PATCH_LIMITED_MODEL_MAX_PATCHES.get(get_openai_api_name(model))
async def _fit_image_off_loop(image_url: str, max_patches: int) -> str:
# Resizing a tall screenshot takes ~2s, so keep it off the event loop.
return await asyncio.to_thread(_fit_image_to_patch_limit, image_url, max_patches)
def _convert_message_to_responses_input(
message: ChatCompletionMessageParam,
image_detail: str = "high",
) -> Dict[str, Any]:
role = message.get("role", "user")
content = message.get("content", "")
if isinstance(content, str):
return {"role": role, "content": content}
parts: List[Dict[str, Any]] = []
if isinstance(content, list):
for part in content:
if not isinstance(part, dict):
continue
if part.get("type") == "text":
parts.append({"type": "input_text", "text": part.get("text", "")})
elif part.get("type") == "image_url":
image_url = part.get("image_url", {})
parts.append(
{
"type": "input_image",
"image_url": image_url.get("url", ""),
"detail": image_detail,
}
)
return {"role": role, "content": parts}
def _get_image_detail_for_model(model: Llm) -> str:
if get_openai_api_name(model) in {
"gpt-5.5",
"gpt-5.6-sol",
"gpt-5.6-terra",
"gpt-6.1-sol",
}:
return "original"
return "high"
def _get_event_attr(event: Any, key: str, default: Any = None) -> Any:
if hasattr(event, key):
return getattr(event, key)
if isinstance(event, dict):
return event.get(key, default)
return default
def _copy_schema(schema: Dict[str, Any]) -> Dict[str, Any]:
return copy.deepcopy(schema)
def _nullable_type(type_value: Any) -> Any:
if isinstance(type_value, list):
if "null" not in type_value:
return [*type_value, "null"]
return type_value
if isinstance(type_value, str):
return [type_value, "null"]
return type_value
def _make_responses_schema_strict(schema: Dict[str, Any]) -> Dict[str, Any]:
schema_copy: Dict[str, Any] = _copy_schema(schema)
def transform(node: Dict[str, Any], in_object_property: bool = False) -> None:
node_type = node.get("type")
if node_type == "object":
node["additionalProperties"] = False
properties = node.get("properties") or {}
if isinstance(properties, dict):
node["required"] = list(properties.keys())
for prop in properties.values():
if isinstance(prop, dict):
transform(prop, in_object_property=True)
return
if node_type == "array":
if in_object_property:
node["type"] = _nullable_type(node_type)
items = node.get("items")
if isinstance(items, dict):
transform(items, in_object_property=False)
return
if in_object_property and node_type is not None:
node["type"] = _nullable_type(node_type)
transform(schema_copy, in_object_property=False)
return schema_copy
def serialize_openai_tools(
tools: List[CanonicalToolDefinition],
) -> List[Dict[str, Any]]:
serialized: List[Dict[str, Any]] = []
for tool in tools:
schema = _make_responses_schema_strict(tool.parameters)
serialized.append(
{
"type": "function",
"name": tool.name,
"description": tool.description,
"parameters": schema,
"strict": True,
}
)
return serialized
@dataclass
class OpenAIResponsesParseState:
assistant_text: str = ""
tool_calls: Dict[str, Dict[str, Any]] = field(default_factory=dict)
item_to_call_id: Dict[str, str] = field(default_factory=dict)
output_items_by_index: Dict[int, Dict[str, Any]] = field(default_factory=dict)
saw_reasoning_summary_text_delta: bool = False
last_emitted_reasoning_summary_part: str = ""
turn_usage: TokenUsage | None = None
def _extract_openai_usage(response: Any) -> TokenUsage:
"""Extract unified token usage from an OpenAI Responses ``response.completed`` event.
OpenAI includes cached tokens inside ``input_tokens``, so they are subtracted
to get the non-cached input count.
"""
usage = _get_event_attr(response, "usage")
if usage is None:
return TokenUsage()
input_tokens = _get_event_attr(usage, "input_tokens", 0) or 0
output_tokens = _get_event_attr(usage, "output_tokens", 0) or 0
total_tokens = _get_event_attr(usage, "total_tokens", 0) or 0
details = _get_event_attr(usage, "input_tokens_details") or {}
cached_tokens = _get_event_attr(details, "cached_tokens", 0) or 0
return TokenUsage(
input=input_tokens - cached_tokens,
output=output_tokens,
cache_read=cached_tokens,
cache_write=0,
total=total_tokens,
)
async def parse_event(
event: Any,
state: OpenAIResponsesParseState,
on_event: EventSink,
) -> None:
event_type = _get_event_attr(event, "type")
if event_type in (
"response.created",
"response.completed",
"response.done",
"response.output_item.done",
):
if event_type == "response.completed":
response = _get_event_attr(event, "response")
if response:
state.turn_usage = _extract_openai_usage(response)
if event_type == "response.output_item.done":
output_index = _get_event_attr(event, "output_index")
item = _get_event_attr(event, "item")
if isinstance(output_index, int) and item:
state.output_items_by_index[output_index] = item
return
if event_type == "response.output_text.delta":
delta = _get_event_attr(event, "delta", "")
if delta:
state.assistant_text += delta
await on_event(StreamEvent(type="assistant_delta", text=delta))
return
if event_type in (
"response.reasoning_text.delta",
"response.reasoning_summary_text.delta",
):
delta = _get_event_attr(event, "delta", "")
if delta:
if event_type == "response.reasoning_summary_text.delta":
state.saw_reasoning_summary_text_delta = True
await on_event(StreamEvent(type="thinking_delta", text=delta))
return
if event_type in (
"response.reasoning_summary_part.added",
"response.reasoning_summary_part.done",
):
if state.saw_reasoning_summary_text_delta:
return
part = _get_event_attr(event, "part") or {}
text = _get_event_attr(part, "text", "")
if text and text != state.last_emitted_reasoning_summary_part:
state.last_emitted_reasoning_summary_part = text
await on_event(StreamEvent(type="thinking_delta", text=text))
return
if event_type == "response.output_item.added":
item = _get_event_attr(event, "item")
item_type = _get_event_attr(item, "type") if item else None
output_index = _get_event_attr(event, "output_index")
if isinstance(output_index, int) and item:
state.output_items_by_index.setdefault(output_index, item)
if item and item_type in ("function_call", "custom_tool_call"):
item_id = _get_event_attr(item, "id")
call_id = _get_event_attr(item, "call_id") or item_id
if item_id and call_id:
state.item_to_call_id[item_id] = call_id
if call_id:
if item_id and item_id in state.tool_calls and item_id != call_id:
existing = state.tool_calls.pop(item_id)
state.tool_calls[call_id] = {
**existing,
"id": call_id,
}
args_value = _get_event_attr(item, "arguments")
if args_value is None and item_type == "custom_tool_call":
args_value = _get_event_attr(item, "input")
state.tool_calls.setdefault(
call_id,
{
"id": call_id,
"name": _get_event_attr(item, "name"),
"arguments": args_value or "",
},
)
if args_value:
await on_event(
StreamEvent(
type="tool_call_delta",
tool_call_id=call_id,
tool_name=_get_event_attr(item, "name"),
tool_arguments=args_value,
)
)
return
if event_type in (
"response.function_call_arguments.delta",
"response.mcp_call_arguments.delta",
"response.custom_tool_call_input.delta",
):
item_id = _get_event_attr(event, "item_id")
call_id = _get_event_attr(event, "call_id")
if call_id and item_id:
state.item_to_call_id[item_id] = call_id
if not call_id:
call_id = state.item_to_call_id.get(item_id) if item_id else None
if not call_id and item_id:
call_id = item_id
if not call_id:
return
entry = state.tool_calls.setdefault(
call_id,
{
"id": call_id,
"name": _get_event_attr(event, "name"),
"arguments": "",
},
)
delta_value = _get_event_attr(event, "delta")
if delta_value is None:
delta_value = _get_event_attr(event, "input")
entry["arguments"] += ensure_str(delta_value)
await on_event(
StreamEvent(
type="tool_call_delta",
tool_call_id=call_id,
tool_name=entry.get("name"),
tool_arguments=entry.get("arguments"),
)
)
return
if event_type not in (
"response.function_call_arguments.done",
"response.mcp_call_arguments.done",
"response.custom_tool_call_input.done",
):
return
item_id = _get_event_attr(event, "item_id")
call_id = _get_event_attr(event, "call_id")
if call_id and item_id:
state.item_to_call_id[item_id] = call_id
if not call_id:
call_id = state.item_to_call_id.get(item_id) if item_id else None
if not call_id and item_id:
call_id = item_id
if not call_id:
return
entry = state.tool_calls.setdefault(
call_id,
{
"id": call_id,
"name": _get_event_attr(event, "name"),
"arguments": "",
},
)
final_value = _get_event_attr(event, "arguments")
if final_value is None:
final_value = _get_event_attr(event, "input")
if final_value is None:
final_value = entry["arguments"]
entry["arguments"] = final_value
if _get_event_attr(event, "name"):
entry["name"] = _get_event_attr(event, "name")
await on_event(
StreamEvent(
type="tool_call_delta",
tool_call_id=call_id,
tool_name=entry.get("name"),
tool_arguments=entry.get("arguments"),
)
)
output_index = _get_event_attr(event, "output_index")
if (
item_id
and isinstance(output_index, int)
and isinstance(state.output_items_by_index.get(output_index), dict)
):
state.output_items_by_index[output_index] = {
**state.output_items_by_index[output_index],
"arguments": entry["arguments"],
"call_id": call_id,
"name": entry.get("name"),
}
def _build_provider_turn(state: OpenAIResponsesParseState) -> ProviderTurn:
output_items = [
state.output_items_by_index[idx]
for idx in sorted(state.output_items_by_index.keys())
if state.output_items_by_index.get(idx)
]
tool_items = [
item
for item in output_items
if isinstance(item, dict)
and item.get("type") in ("function_call", "custom_tool_call")
]
tool_calls: List[ToolCall] = []
if tool_items:
for item in tool_items:
raw_args = item.get("arguments")
if raw_args is None and item.get("type") == "custom_tool_call":
raw_args = item.get("input")
args, error = parse_json_arguments(raw_args)
if error:
args = {"INVALID_JSON": ensure_str(raw_args)}
call_id = item.get("call_id") or item.get("id")
tool_calls.append(
ToolCall(
id=call_id or f"call-{uuid.uuid4().hex[:6]}",
name=item.get("name") or "unknown_tool",
arguments=args,
)
)
else:
for entry in state.tool_calls.values():
args, error = parse_json_arguments(entry.get("arguments"))
if error:
args = {"INVALID_JSON": ensure_str(entry.get("arguments"))}
call_id = entry.get("id") or entry.get("call_id")
tool_calls.append(
ToolCall(
id=call_id or f"call-{uuid.uuid4().hex[:6]}",
name=entry.get("name") or "unknown_tool",
arguments=args,
)
)
assistant_turn: List[Dict[str, Any]] = output_items if tool_calls else []
return ProviderTurn(
assistant_text=state.assistant_text,
tool_calls=tool_calls,
assistant_turn=assistant_turn,
)
class OpenAIProviderSession(ProviderSession):
def __init__(
self,
client: AsyncOpenAI,
model: Llm,
prompt_messages: List[ChatCompletionMessageParam],
tools: List[Dict[str, Any]],
recorder: Optional[AgentRunRecorder] = None,
):
self._client = client
self._model = model
self._tools = tools
self._total_usage = TokenUsage()
self._recorder = recorder
self._prompt_report_logger = PromptReportLogger(
provider="openai",
model=model,
api_model_name=get_openai_api_name(model),
)
image_detail = _get_image_detail_for_model(model)
self._input_items: List[Dict[str, Any]] = [
_convert_message_to_responses_input(message, image_detail=image_detail)
for message in prompt_messages
]
self._max_patches = _get_max_patches_for_model(model)
# Prompt images are fitted on the first turn, where we can await.
self._prompt_images_fitted = self._max_patches is None
async def _fit_prompt_images(self) -> None:
if self._prompt_images_fitted or self._max_patches is None:
return
for item in self._input_items:
content = item.get("content")
if not isinstance(content, list):
continue
for part in content:
if part.get("type") == "input_image":
part["image_url"] = await _fit_image_off_loop(
part["image_url"], self._max_patches
)
self._prompt_images_fitted = True
async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
await self._fit_prompt_images()
model_name = get_openai_api_name(self._model)
params: Dict[str, Any] = {
"model": model_name,
"input": self._input_items,
"tools": self._tools,
"tool_choice": "auto",
"stream": True,
"max_output_tokens": 50000,
}
if model_name == "gpt-5.4-2026-03-05":
params["prompt_cache_retention"] = "24h"
reasoning_effort = get_openai_reasoning_effort(self._model)
if reasoning_effort:
params["reasoning"] = {"effort": reasoning_effort, "summary": "auto"}
self._prompt_report_logger.record_request(params)
if self._recorder is not None:
self._recorder.record_llm_request("openai", model_name, params)
state = OpenAIResponsesParseState()
stream = await self._client.responses.create(**params) # type: ignore
async for event in stream: # type: ignore
await parse_event(event, state, on_event)
if state.turn_usage is not None:
self._prompt_report_logger.record_usage(state.turn_usage)
self._total_usage.accumulate(state.turn_usage)
turn = _build_provider_turn(state)
if self._recorder is not None:
self._recorder.record_llm_response(
turn.assistant_text, turn.tool_calls, state.turn_usage
)
return turn
def total_cost_usd(self) -> float | None:
pricing = MODEL_PRICING.get(get_openai_api_name(self._model))
if pricing is None:
return None
return self._total_usage.cost(pricing)
@staticmethod
def _image_ref(part: Any) -> str | None:
"""A public URL is sent as-is; local bytes become a base64 data URL."""
if part.image_url:
return part.image_url
if part.data is not None:
encoded = base64.b64encode(part.data).decode("ascii")
return f"data:{part.mime_type};base64,{encoded}"
return None
async def append_tool_results(
self,
turn: ProviderTurn,
executed_tool_calls: list[ExecutedToolCall],
) -> None:
assistant_output_items = turn.assistant_turn or []
if assistant_output_items:
self._input_items.extend(assistant_output_items)
image_detail = _get_image_detail_for_model(self._model)
tool_output_items: List[Dict[str, Any]] = []
for executed in executed_tool_calls:
result_json = json.dumps(executed.result.result)
parts = executed.result.multimodal_parts or []
output: Any = result_json
if parts and executed.result.ok:
# Responses lets function_call_output carry image content; keep
# the text result first so the model still gets the structured
# data (URLs, status) alongside the rendered images.
output = [{"type": "input_text", "text": result_json}]
for part in parts:
image_url = self._image_ref(part)
if image_url is None:
continue
if self._max_patches is not None:
image_url = await _fit_image_off_loop(
image_url, self._max_patches
)
output.append(
{
"type": "input_image",
"detail": image_detail,
"image_url": image_url,
}
)
tool_output_items.append(
{
"type": "function_call_output",
"call_id": executed.tool_call.id,
"output": output,
}
)
self._input_items.extend(tool_output_items)
async def close(self) -> None:
u = self._total_usage
model_name = get_openai_api_name(self._model)
pricing = MODEL_PRICING.get(model_name)
cost_str = f" cost=${u.cost(pricing):.4f}" if pricing else ""
cache_hit_rate_str = f" cache_hit_rate={u.cache_hit_rate_percent():.2f}%"
print(
f"[TOKEN USAGE] provider=openai model={model_name} | "
f"input={u.input} output={u.output} "
f"cache_read={u.cache_read} cache_write={u.cache_write} "
f"total={u.total}{cache_hit_rate_str}{cost_str}"
)
await self._client.close()