Skip to content

Drop in a screenshot and convert it to clean code (HTML/Tailwind/React/Vue)

Python

Carry multimodal tool results back to Gemini

Abi Raja815f5c3f7e75
diff --git a/backend/agent/providers/gemini.py b/backend/agent/providers/gemini.py
index ec967933..30acfd1c 100644
--- a/backend/agent/providers/gemini.py
+++ b/backend/agent/providers/gemini.py
@@ -362,14 +362,28 @@ class GeminiProviderSession(ProviderSession):
tool_result_parts: List[types.Part] = []
for executed in executed_tool_calls:
+ multimodal_parts = [
+ types.FunctionResponsePart(
+ inline_data=types.FunctionResponseBlob(
+ mime_type=part.mime_type,
+ display_name=part.display_name,
+ data=part.data,
+ )
+ )
+ for part in (executed.result.multimodal_parts or [])
+ ]
tool_result_parts.append(
- types.Part.from_function_response(
- name=executed.tool_call.name,
- response=executed.result.result,
+ types.Part(
+ function_response=types.FunctionResponse(
+ id=executed.tool_call.id,
+ name=executed.tool_call.name,
+ response=executed.result.result,
+ parts=multimodal_parts,
+ )
)
)
- self._contents.append(types.Content(role="tool", parts=tool_result_parts))
+ self._contents.append(types.Content(role="user", parts=tool_result_parts))
async def close(self) -> None:
u = self._total_usage
diff --git a/backend/agent/tools/types.py b/backend/agent/tools/types.py
index fc0ee7c4..ecfc7f38 100644
--- a/backend/agent/tools/types.py
+++ b/backend/agent/tools/types.py
@@ -1,5 +1,5 @@
from dataclasses import dataclass
-from typing import Any, Dict, Optional
+from typing import Any, Dict, List, Optional
@dataclass(frozen=True)
@@ -9,12 +9,20 @@ class ToolCall:
arguments: Dict[str, Any]
+@dataclass
+class ToolMultimodalPart:
+ display_name: str
+ mime_type: str
+ data: bytes
+
+
@dataclass
class ToolExecutionResult:
ok: bool
result: Dict[str, Any]
summary: Dict[str, Any]
updated_content: Optional[str] = None
+ multimodal_parts: Optional[List[ToolMultimodalPart]] = None
@dataclass(frozen=True)
diff --git a/backend/tests/test_gemini_provider_session.py b/backend/tests/test_gemini_provider_session.py
new file mode 100644
index 00000000..ed3ef2f1
--- /dev/null
+++ b/backend/tests/test_gemini_provider_session.py
@@ -0,0 +1,91 @@
+from typing import Any, cast
+
+from google.genai import types
+
+from agent.providers.base import ExecutedToolCall, ProviderTurn
+from agent.providers.gemini import GeminiProviderSession
+from agent.tools.types import ToolCall, ToolExecutionResult, ToolMultimodalPart
+from llm import Llm
+
+
+def test_gemini_provider_appends_multimodal_function_response_parts() -> None:
+ session = GeminiProviderSession(
+ client=object(), # type: ignore[arg-type]
+ model=Llm.GEMINI_3_FLASH_PREVIEW_MINIMAL,
+ prompt_messages=[
+ {"role": "system", "content": "You are helpful."},
+ {"role": "user", "content": "Extract the logo."},
+ ],
+ tools=[],
+ )
+ model_turn = types.Content(
+ role="model",
+ parts=[
+ types.Part.from_function_call(
+ name="extract_assets",
+ args={"asset_descriptions": ["logo"]},
+ )
+ ],
+ )
+
+ session.append_tool_results(
+ ProviderTurn(
+ assistant_text="",
+ tool_calls=[],
+ assistant_turn=model_turn,
+ ),
+ [
+ ExecutedToolCall(
+ tool_call=ToolCall(
+ id="call-1",
+ name="extract_assets",
+ arguments={"asset_descriptions": ["logo"]},
+ ),
+ result=ToolExecutionResult(
+ ok=True,
+ result={
+ "assets": [
+ {
+ "description": "logo",
+ "image_part_index": 0,
+ "image_display_name": "asset_0.png",
+ }
+ ]
+ },
+ summary={},
+ multimodal_parts=[
+ ToolMultimodalPart(
+ display_name="asset_0.png",
+ mime_type="image/png",
+ data=b"logo-image",
+ )
+ ],
+ ),
+ )
+ ],
+ )
+
+ contents: list[types.Content] = session._contents # type: ignore[attr-defined]
+ tool_response_content = contents[-1]
+ assert tool_response_content.parts is not None
+ function_response = tool_response_content.parts[0].function_response
+ assert tool_response_content.role == "user"
+ assert function_response is not None
+ assert function_response.id == "call-1"
+ assert function_response.name == "extract_assets"
+ assert function_response.response == {
+ "assets": [
+ {
+ "description": "logo",
+ "image_part_index": 0,
+ "image_display_name": "asset_0.png",
+ }
+ ]
+ }
+
+ response_parts = cast(Any, function_response.parts)
+ assert len(response_parts) == 1
+ inline_data = response_parts[0].inline_data
+ assert inline_data.mime_type == "image/png"
+ assert inline_data.display_name == "asset_0.png"
+ assert inline_data.data == b"logo-image"