Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
# pyright: reportUnknownVariableType=false
import base64
import copy
import uuid
import httpx
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, cast
from google import genai
from google.genai import types
from openai.types.chat import ChatCompletionMessageParam
from agent.providers.base import (
EventSink,
ExecutedToolCall,
ProviderSession,
ProviderTurn,
StreamEvent,
)
from costs.pricing import MODEL_PRICING
from costs.token_usage import TokenUsage
from agent.tools import CanonicalToolDefinition, ToolCall
from fs_logging.agent_runs import AgentRunRecorder
from fs_logging.prompt_reports import PromptReportLogger
from llm import Llm
DEFAULT_VIDEO_FPS = 10
def serialize_gemini_tools(tools: List[CanonicalToolDefinition]) -> List[types.Tool]:
declarations = [
types.FunctionDeclaration(
name=tool.name,
description=tool.description,
parameters_json_schema=copy.deepcopy(tool.parameters),
)
for tool in tools
]
return [types.Tool(function_declarations=declarations)]
def _get_gemini_api_model_name(model: Llm) -> str:
if model in [
Llm.GEMINI_3_FLASH_PREVIEW_HIGH,
Llm.GEMINI_3_FLASH_PREVIEW_MINIMAL,
]:
return "gemini-3-flash-preview"
if model in [
Llm.GEMINI_3_5_FLASH_HIGH,
Llm.GEMINI_3_5_FLASH_MEDIUM,
Llm.GEMINI_3_5_FLASH_LOW,
Llm.GEMINI_3_5_FLASH_MINIMAL,
]:
return "gemini-3.5-flash"
if model in [
Llm.GEMINI_3_6_FLASH_HIGH,
Llm.GEMINI_3_6_FLASH_MEDIUM,
Llm.GEMINI_3_6_FLASH_LOW,
Llm.GEMINI_3_6_FLASH_MINIMAL,
]:
return "gemini-3.6-flash"
if model in [
Llm.GEMINI_3_1_PRO_PREVIEW_HIGH,
Llm.GEMINI_3_1_PRO_PREVIEW_MEDIUM,
Llm.GEMINI_3_1_PRO_PREVIEW_LOW,
]:
return "gemini-3.1-pro-preview"
return model.value
def _get_thinking_level_for_model(model: Llm) -> str:
if model in [
Llm.GEMINI_3_FLASH_PREVIEW_HIGH,
Llm.GEMINI_3_1_PRO_PREVIEW_HIGH,
Llm.GEMINI_3_5_FLASH_HIGH,
Llm.GEMINI_3_6_FLASH_HIGH,
]:
return "high"
if model in [
Llm.GEMINI_3_1_PRO_PREVIEW_LOW,
Llm.GEMINI_3_5_FLASH_LOW,
Llm.GEMINI_3_6_FLASH_LOW,
]:
return "low"
if model in [
Llm.GEMINI_3_1_PRO_PREVIEW_MEDIUM,
Llm.GEMINI_3_5_FLASH_MEDIUM,
Llm.GEMINI_3_6_FLASH_MEDIUM,
]:
return "medium"
if model in [
Llm.GEMINI_3_FLASH_PREVIEW_MINIMAL,
Llm.GEMINI_3_5_FLASH_MINIMAL,
Llm.GEMINI_3_6_FLASH_MINIMAL,
]:
return "minimal"
return "high"
def _extract_text_from_content(content: str | List[Dict[str, Any]]) -> str:
if isinstance(content, str):
return content
for content_part in content:
if content_part.get("type") == "text":
return content_part.get("text", "")
return ""
def _detect_mime_type_from_base64(base64_data: str) -> str | None:
try:
decoded = base64.b64decode(base64_data[:32])
if decoded[:8] == b"\x89PNG\r\n\x1a\n":
return "image/png"
if decoded[:2] == b"\xff\xd8":
return "image/jpeg"
if decoded[:6] in (b"GIF87a", b"GIF89a"):
return "image/gif"
if decoded[:4] == b"RIFF" and decoded[8:12] == b"WEBP":
return "image/webp"
if decoded[4:8] == b"ftyp":
return "video/mp4"
if decoded[:4] == b"\x1aE\xdf\xa3":
return "video/webm"
except Exception:
pass
return None
def _extract_images_from_content(content: str | List[Dict[str, Any]]) -> List[Dict[str, str]]:
if isinstance(content, str):
return []
images: List[Dict[str, str]] = []
for content_part in content:
if content_part.get("type") != "image_url":
continue
image_url = content_part["image_url"]["url"]
if image_url.startswith("data:"):
mime_type = image_url.split(";")[0].split(":")[1]
base64_data = image_url.split(",")[1]
if mime_type == "application/octet-stream":
detected_mime = _detect_mime_type_from_base64(base64_data)
if detected_mime:
mime_type = detected_mime
else:
print("Warning: Could not detect MIME type for data URL, skipping")
continue
images.append({"mime_type": mime_type, "data": base64_data})
continue
images.append({"uri": image_url})
return images
def _convert_message_to_gemini_content(
message: ChatCompletionMessageParam,
) -> types.Content:
role = message.get("role", "user")
content = message.get("content", "")
gemini_role = "model" if role == "assistant" else "user"
parts: List[types.Part | Dict[str, str]] = []
text = _extract_text_from_content(content) # type: ignore
image_data_list = _extract_images_from_content(content) # type: ignore
if text:
parts.append({"text": text})
for image_data in image_data_list:
if "data" in image_data:
mime_type = image_data["mime_type"]
media_bytes = base64.b64decode(image_data["data"])
if mime_type.startswith("video/"):
parts.append(
types.Part(
inline_data=types.Blob(data=media_bytes, mime_type=mime_type),
video_metadata=types.VideoMetadata(fps=DEFAULT_VIDEO_FPS),
media_resolution=types.PartMediaResolutionLevel.MEDIA_RESOLUTION_HIGH,
)
)
continue
parts.append(
types.Part.from_bytes(
data=media_bytes,
mime_type=mime_type,
media_resolution=types.PartMediaResolutionLevel.MEDIA_RESOLUTION_ULTRA_HIGH,
)
)
continue
if "uri" in image_data:
parts.append({"file_uri": image_data["uri"]})
return types.Content(role=gemini_role, parts=parts) # type: ignore
@dataclass
class GeminiParseState:
assistant_text: str = ""
tool_calls: List[ToolCall] = field(default_factory=list)
model_parts: List[types.Part] = field(default_factory=list)
model_role: str = "model"
def _extract_usage(chunk: types.GenerateContentResponse) -> TokenUsage | None:
"""Extract unified token usage from a Gemini streaming chunk.
Gemini reports thinking tokens separately; they are folded into ``output``
to match the unified schema used by the other providers.
``prompt_token_count`` *includes* ``cached_content_token_count``, so we
subtract cached tokens to get the non-cached input count (same approach
as the OpenAI provider).
"""
meta = chunk.usage_metadata
if meta is None:
return None
candidates = meta.candidates_token_count or 0
thoughts = meta.thoughts_token_count or 0
prompt_tokens = meta.prompt_token_count or 0
cached_tokens = meta.cached_content_token_count or 0
return TokenUsage(
input=prompt_tokens - cached_tokens,
output=candidates + thoughts,
cache_read=cached_tokens,
cache_write=0,
total=meta.total_token_count or 0,
)
async def _parse_chunk(
chunk: types.GenerateContentResponse,
state: GeminiParseState,
on_event: EventSink,
) -> None:
if not chunk.candidates:
return
candidate_content = chunk.candidates[0].content
if not candidate_content or not candidate_content.parts:
return
if candidate_content.role:
state.model_role = candidate_content.role
for part in candidate_content.parts:
# Preserve each model part as streamed so thought signatures remain attached.
state.model_parts.append(part)
if getattr(part, "thought", False) and part.text:
await on_event(StreamEvent(type="thinking_delta", text=part.text))
continue
if part.function_call:
args = part.function_call.args or {}
tool_id = part.function_call.id or f"tool-{uuid.uuid4().hex[:6]}"
tool_name = part.function_call.name or "unknown_tool"
await on_event(
StreamEvent(
type="tool_call_delta",
tool_call_id=tool_id,
tool_name=tool_name,
tool_arguments=args,
)
)
state.tool_calls.append(
ToolCall(
id=tool_id,
name=tool_name,
arguments=args,
)
)
continue
if part.text:
state.assistant_text += part.text
await on_event(StreamEvent(type="assistant_delta", text=part.text))
class GeminiProviderSession(ProviderSession):
def __init__(
self,
client: genai.Client,
model: Llm,
prompt_messages: List[ChatCompletionMessageParam],
tools: List[types.Tool],
recorder: Optional[AgentRunRecorder] = None,
):
self._client = client
self._model = model
self._tools = tools
self._total_usage = TokenUsage()
self._recorder = recorder
self._prompt_report_logger = PromptReportLogger(
provider="gemini",
model=model,
api_model_name=_get_gemini_api_model_name(model),
)
self._system_prompt = str(prompt_messages[0].get("content", ""))
self._contents: List[types.Content] = [
_convert_message_to_gemini_content(msg) for msg in prompt_messages[1:]
]
async def stream_turn(self, on_event: EventSink) -> ProviderTurn:
thinking_level = _get_thinking_level_for_model(self._model)
api_model_name = _get_gemini_api_model_name(self._model)
config = types.GenerateContentConfig(
temperature=1.0,
max_output_tokens=50000,
system_instruction=self._system_prompt,
thinking_config=types.ThinkingConfig(
thinking_level=cast(Any, thinking_level),
include_thoughts=True,
),
tools=self._tools,
)
request_payload = {
"model": api_model_name,
"contents": self._contents,
"config": config,
}
self._prompt_report_logger.record_request(request_payload)
if self._recorder is not None:
self._recorder.record_llm_request(
"gemini", api_model_name, request_payload
)
stream = await self._client.aio.models.generate_content_stream(
model=api_model_name,
contents=cast(Any, self._contents),
config=config,
)
state = GeminiParseState()
turn_usage: TokenUsage | None = None
async for chunk in stream:
await _parse_chunk(chunk, state, on_event)
chunk_usage = _extract_usage(chunk)
if chunk_usage is not None:
turn_usage = chunk_usage
if turn_usage is not None:
self._prompt_report_logger.record_usage(turn_usage)
self._total_usage.accumulate(turn_usage)
if self._recorder is not None:
self._recorder.record_llm_response(
state.assistant_text, state.tool_calls, turn_usage
)
assistant_turn = (
types.Content(role=state.model_role, parts=state.model_parts)
if state.model_parts
else None
)
return ProviderTurn(
assistant_text=state.assistant_text,
tool_calls=state.tool_calls,
assistant_turn=assistant_turn,
)
def total_cost_usd(self) -> Optional[float]:
pricing = MODEL_PRICING.get(_get_gemini_api_model_name(self._model))
if pricing is None:
return None
return self._total_usage.cost(pricing)
@staticmethod
async def _resolve_part_bytes(part: Any) -> tuple[bytes | None, str]:
"""Gemini only accepts inline bytes, so download public URLs."""
if part.data is not None:
return part.data, part.mime_type
if part.image_url:
try:
async with httpx.AsyncClient(timeout=30) as client:
response = await client.get(part.image_url)
response.raise_for_status()
content_type = (
response.headers.get("content-type", "").split(";")[0].strip()
)
return response.content, content_type or part.mime_type
except Exception as exc:
print(f"[gemini] failed to fetch tool image {part.image_url}: {exc}")
return None, part.mime_type
async def append_tool_results(
self,
turn: ProviderTurn,
executed_tool_calls: list[ExecutedToolCall],
) -> None:
model_content = turn.assistant_turn
if not isinstance(model_content, types.Content) or not model_content.parts:
raise ValueError(
"Gemini step is missing model content. Cannot append tool results without the original model turn."
)
self._contents.append(model_content)
tool_result_parts: List[types.Part] = []
for executed in executed_tool_calls:
multimodal_parts: List[types.FunctionResponsePart] = []
for part in executed.result.multimodal_parts or []:
data, mime_type = await self._resolve_part_bytes(part)
if data is None:
continue
multimodal_parts.append(
types.FunctionResponsePart(
inline_data=types.FunctionResponseBlob(
mime_type=mime_type,
display_name=part.display_name,
data=data,
)
)
)
tool_result_parts.append(
types.Part(
function_response=types.FunctionResponse(
id=executed.tool_call.id,
name=executed.tool_call.name,
response=executed.result.result,
parts=multimodal_parts,
)
)
)
self._contents.append(types.Content(role="user", parts=tool_result_parts))
async def close(self) -> None:
u = self._total_usage
model_name = _get_gemini_api_model_name(self._model)
pricing = MODEL_PRICING.get(model_name)
cost_str = f" cost=${u.cost(pricing):.4f}" if pricing else ""
cache_hit_rate_str = f" cache_hit_rate={u.cache_hit_rate_percent():.2f}%"
print(
f"[TOKEN USAGE] provider=gemini model={model_name} | "
f"input={u.input} output={u.output} "
f"cache_read={u.cache_read} cache_write={u.cache_write} "
f"total={u.total}{cache_hit_rate_str}{cost_str}"
)