Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)
Python
Carry multimodal tool results back to Gemini
Abi Raja815f5c3f7e75
diff --git a/backend/agent/providers/gemini.py b/backend/agent/providers/gemini.pyindex ec967933..30acfd1c 100644--- a/backend/agent/providers/gemini.py+++ b/backend/agent/providers/gemini.py@@ -362,14 +362,28 @@ class GeminiProviderSession(ProviderSession):tool_result_parts: List[types.Part] = []for executed in executed_tool_calls:+ multimodal_parts = [+ types.FunctionResponsePart(+ inline_data=types.FunctionResponseBlob(+ mime_type=part.mime_type,+ display_name=part.display_name,+ data=part.data,+ )+ )+ for part in (executed.result.multimodal_parts or [])+ ]tool_result_parts.append(- types.Part.from_function_response(- name=executed.tool_call.name,- response=executed.result.result,+ types.Part(+ function_response=types.FunctionResponse(+ id=executed.tool_call.id,+ name=executed.tool_call.name,+ response=executed.result.result,+ parts=multimodal_parts,+ )))- self._contents.append(types.Content(role="tool", parts=tool_result_parts))+ self._contents.append(types.Content(role="user", parts=tool_result_parts))async def close(self) -> None:u = self._total_usagediff --git a/backend/agent/tools/types.py b/backend/agent/tools/types.pyindex fc0ee7c4..ecfc7f38 100644--- a/backend/agent/tools/types.py+++ b/backend/agent/tools/types.py@@ -1,5 +1,5 @@from dataclasses import dataclass-from typing import Any, Dict, Optional+from typing import Any, Dict, List, Optional@dataclass(frozen=True)@@ -9,12 +9,20 @@ class ToolCall:arguments: Dict[str, Any]+@dataclass+class ToolMultimodalPart:+ display_name: str+ mime_type: str+ data: bytes++@dataclassclass ToolExecutionResult:ok: boolresult: Dict[str, Any]summary: Dict[str, Any]updated_content: Optional[str] = None+ multimodal_parts: Optional[List[ToolMultimodalPart]] = None@dataclass(frozen=True)diff --git a/backend/tests/test_gemini_provider_session.py b/backend/tests/test_gemini_provider_session.pynew file mode 100644index 00000000..ed3ef2f1--- /dev/null+++ b/backend/tests/test_gemini_provider_session.py@@ -0,0 +1,91 @@+from typing import Any, cast++from google.genai import types++from agent.providers.base import ExecutedToolCall, ProviderTurn+from agent.providers.gemini import GeminiProviderSession+from agent.tools.types import ToolCall, ToolExecutionResult, ToolMultimodalPart+from llm import Llm+++def test_gemini_provider_appends_multimodal_function_response_parts() -> None:+ session = GeminiProviderSession(+ client=object(), # type: ignore[arg-type]+ model=Llm.GEMINI_3_FLASH_PREVIEW_MINIMAL,+ prompt_messages=[+ {"role": "system", "content": "You are helpful."},+ {"role": "user", "content": "Extract the logo."},+ ],+ tools=[],+ )+ model_turn = types.Content(+ role="model",+ parts=[+ types.Part.from_function_call(+ name="extract_assets",+ args={"asset_descriptions": ["logo"]},+ )+ ],+ )++ session.append_tool_results(+ ProviderTurn(+ assistant_text="",+ tool_calls=[],+ assistant_turn=model_turn,+ ),+ [+ ExecutedToolCall(+ tool_call=ToolCall(+ id="call-1",+ name="extract_assets",+ arguments={"asset_descriptions": ["logo"]},+ ),+ result=ToolExecutionResult(+ ok=True,+ result={+ "assets": [+ {+ "description": "logo",+ "image_part_index": 0,+ "image_display_name": "asset_0.png",+ }+ ]+ },+ summary={},+ multimodal_parts=[+ ToolMultimodalPart(+ display_name="asset_0.png",+ mime_type="image/png",+ data=b"logo-image",+ )+ ],+ ),+ )+ ],+ )++ contents: list[types.Content] = session._contents # type: ignore[attr-defined]+ tool_response_content = contents[-1]+ assert tool_response_content.parts is not None+ function_response = tool_response_content.parts[0].function_response+ assert tool_response_content.role == "user"+ assert function_response is not None+ assert function_response.id == "call-1"+ assert function_response.name == "extract_assets"+ assert function_response.response == {+ "assets": [+ {+ "description": "logo",+ "image_part_index": 0,+ "image_display_name": "asset_0.png",+ }+ ]+ }++ response_parts = cast(Any, function_response.parts)+ assert len(response_parts) == 1+ inline_data = response_parts[0].inline_data+ assert inline_data.mime_type == "image/png"+ assert inline_data.display_name == "asset_0.png"+ assert inline_data.data == b"logo-image"