Skip to content

Personal AI, On Personal Devices

Python

fix(cloud): isolate Gemini stream metadata

Elliot Slusky7feda3dad3c4
diff --git a/src/openjarvis/engine/cloud.py b/src/openjarvis/engine/cloud.py
index e4485772..a55e0058 100644
--- a/src/openjarvis/engine/cloud.py
+++ b/src/openjarvis/engine/cloud.py
@@ -9,6 +9,7 @@ import json
import logging
import os
import time
+import uuid
from collections.abc import AsyncIterator, Sequence
from typing import Any, Dict, List, Tuple
@@ -1380,11 +1381,25 @@ class CloudEngine(InferenceEngine):
config.tools = [{"function_declarations": _convert_tools_to_google(tools)}]
tool_call_count = 0
+ stream_id = uuid.uuid4().hex
+ final_usage: Dict[str, Any] | None = None
for chunk in self._google_client.models.generate_content_stream(
model=model,
contents=contents,
config=config,
):
+ usage_metadata = getattr(chunk, "usage_metadata", None)
+ if usage_metadata is not None:
+ prompt_tokens = getattr(usage_metadata, "prompt_token_count", 0) or 0
+ completion_tokens = (
+ getattr(usage_metadata, "candidates_token_count", 0) or 0
+ )
+ final_usage = {
+ "prompt_tokens": prompt_tokens,
+ "completion_tokens": completion_tokens,
+ "total_tokens": prompt_tokens + completion_tokens,
+ }
+
candidates = getattr(chunk, "candidates", None)
parts = []
if candidates:
@@ -1408,7 +1423,11 @@ class CloudEngine(InferenceEngine):
# a distinct invocation. The same function may legitimately
# be called more than once in a parallel response.
tool_index = tool_call_count
- tool_id = f"google_{name}_{tool_index}"
+ # The engine is shared across server requests, and saved
+ # thought signatures are keyed by tool-call ID. Include a
+ # per-stream nonce so concurrent conversations cannot
+ # overwrite each other's signatures.
+ tool_id = f"google_{stream_id}_{tool_index}"
tool_call_count += 1
tool_call = {
"index": tool_index,
@@ -1436,7 +1455,10 @@ class CloudEngine(InferenceEngine):
if text:
yield StreamChunk(content=text)
- yield StreamChunk(finish_reason="tool_calls" if tool_call_count else "stop")
+ yield StreamChunk(
+ finish_reason="tool_calls" if tool_call_count else "stop",
+ usage=final_usage,
+ )
async def _stream_openrouter(
self,
diff --git a/tests/engine/test_cloud_stream_full.py b/tests/engine/test_cloud_stream_full.py
index a57373a0..9cb5071a 100644
--- a/tests/engine/test_cloud_stream_full.py
+++ b/tests/engine/test_cloud_stream_full.py
@@ -70,11 +70,19 @@ class _GoogleConfig:
self.__dict__.update(kwargs)
-def _google_stream_chunk(*parts: Any, text: str | None = None) -> Any:
+def _google_stream_chunk(
+ *parts: Any,
+ text: str | None = None,
+ usage_metadata: Any = None,
+) -> Any:
candidates = []
if parts:
candidates = [SimpleNamespace(content=SimpleNamespace(parts=list(parts)))]
- return SimpleNamespace(text=text, candidates=candidates, usage_metadata=None)
+ return SimpleNamespace(
+ text=text,
+ candidates=candidates,
+ usage_metadata=usage_metadata,
+ )
def _google_types_modules() -> dict[str, ModuleType]:
@@ -504,14 +512,15 @@ async def test_stream_full_google_preserves_tool_calls(monkeypatch: pytest.Monke
]
tool_call = result[0].tool_calls[0]
- assert tool_call == {
- "index": 0,
- "id": "google_get_weather_0",
- "type": "function",
- "function": {"name": "get_weather", "arguments": '{"city": "Berlin"}'},
- "thought_signature": b"sig",
+ assert tool_call["index"] == 0
+ assert tool_call["id"].startswith("google_")
+ assert tool_call["type"] == "function"
+ assert tool_call["function"] == {
+ "name": "get_weather",
+ "arguments": '{"city": "Berlin"}',
}
- assert engine._thought_sigs["google_get_weather_0"] == b"sig"
+ assert tool_call["thought_signature"] == b"sig"
+ assert engine._thought_sigs[tool_call["id"]] == b"sig"
assert result[-1].finish_reason == "tool_calls"
config = client.models.generate_content_stream.call_args.kwargs["config"]
assert config.tools == [
@@ -531,7 +540,7 @@ async def test_stream_full_google_preserves_tool_calls(monkeypatch: pytest.Monke
async def test_stream_full_google_preserves_mixed_and_multiple_calls(
monkeypatch: pytest.MonkeyPatch,
):
- """Google streams retain mixed text and multiple deterministic tool calls."""
+ """Google streams retain mixed text and multiple tool calls."""
weather = SimpleNamespace(name="get_weather", args={"city": "Berlin"})
calendar = SimpleNamespace(name="get_calendar", args={"day": "Monday"})
text_part = SimpleNamespace(text="I'll check.", function_call=None)
@@ -563,28 +572,19 @@ async def test_stream_full_google_preserves_mixed_and_multiple_calls(
]
assert result[0].content == "I'll check."
- assert result[1].tool_calls == [
- {
- "index": 0,
- "id": "google_get_weather_0",
- "type": "function",
- "function": {
- "name": "get_weather",
- "arguments": '{"city": "Berlin"}',
- },
- }
- ]
- assert result[2].tool_calls == [
- {
- "index": 1,
- "id": "google_get_calendar_1",
- "type": "function",
- "function": {
- "name": "get_calendar",
- "arguments": '{"day": "Monday"}',
- },
- },
- ]
+ weather_call = result[1].tool_calls[0]
+ calendar_call = result[2].tool_calls[0]
+ assert weather_call["index"] == 0
+ assert weather_call["function"] == {
+ "name": "get_weather",
+ "arguments": '{"city": "Berlin"}',
+ }
+ assert calendar_call["index"] == 1
+ assert calendar_call["function"] == {
+ "name": "get_calendar",
+ "arguments": '{"day": "Monday"}',
+ }
+ assert weather_call["id"] != calendar_call["id"]
assert result[-1].finish_reason == "tool_calls"
@@ -618,16 +618,95 @@ async def test_stream_full_google_keeps_parallel_same_name_calls_distinct(
]
calls = result[0].tool_calls
- assert [(call["index"], call["id"]) for call in calls] == [
- (0, "google_get_weather_0"),
- (1, "google_get_weather_1"),
- ]
+ assert [call["index"] for call in calls] == [0, 1]
+ assert calls[0]["id"] != calls[1]["id"]
assert [call["function"]["arguments"] for call in calls] == [
'{"city": "Paris"}',
'{"city": "London"}',
]
+async def test_stream_full_google_ids_are_unique_across_requests(
+ monkeypatch: pytest.MonkeyPatch,
+):
+ """Shared engines keep signatures isolated between conversations."""
+ first_part = SimpleNamespace(
+ function_call=SimpleNamespace(name="get_weather", args={"city": "Paris"}),
+ text=None,
+ thought_signature=b"paris-sig",
+ )
+ second_part = SimpleNamespace(
+ function_call=SimpleNamespace(name="get_weather", args={"city": "London"}),
+ text=None,
+ thought_signature=b"london-sig",
+ )
+ client = MagicMock()
+ client.models.generate_content_stream.side_effect = [
+ iter([_google_stream_chunk(first_part)]),
+ iter([_google_stream_chunk(second_part)]),
+ ]
+ engine = _make_cloud_engine(google_client=client)
+ engine._thought_sigs = {}
+
+ with monkeypatch.context() as patch:
+ for name, module in _google_types_modules().items():
+ patch.setitem(sys.modules, name, module)
+ first = [
+ chunk
+ async for chunk in engine.stream_full(
+ [Message(role=Role.USER, content="Weather in Paris")],
+ model="gemini-3-flash-preview",
+ )
+ ]
+ second = [
+ chunk
+ async for chunk in engine.stream_full(
+ [Message(role=Role.USER, content="Weather in London")],
+ model="gemini-3-flash-preview",
+ )
+ ]
+
+ first_id = first[0].tool_calls[0]["id"]
+ second_id = second[0].tool_calls[0]["id"]
+ assert first_id != second_id
+ assert engine._thought_sigs[first_id] == b"paris-sig"
+ assert engine._thought_sigs[second_id] == b"london-sig"
+
+
+async def test_stream_full_google_emits_final_usage(monkeypatch: pytest.MonkeyPatch):
+ """Google's final usage metadata is normalized onto the terminal chunk."""
+ usage = SimpleNamespace(prompt_token_count=12, candidates_token_count=5)
+ client = MagicMock()
+ client.models.generate_content_stream.return_value = iter(
+ [
+ _google_stream_chunk(text="Hello"),
+ _google_stream_chunk(usage_metadata=usage),
+ ]
+ )
+ engine = _make_cloud_engine(google_client=client)
+ engine._thought_sigs = {}
+
+ with monkeypatch.context() as patch:
+ for name, module in _google_types_modules().items():
+ patch.setitem(sys.modules, name, module)
+ result = [
+ chunk
+ async for chunk in engine.stream_full(
+ [Message(role=Role.USER, content="hi")],
+ model="gemini-2.5-flash",
+ )
+ ]
+
+ assert result[-1].finish_reason == "stop"
+ assert result[-1].usage == {
+ "prompt_tokens": 12,
+ "completion_tokens": 5,
+ "total_tokens": 17,
+ }
+
+
@pytest.mark.asyncio
async def test_stream_full_google_replays_signature_on_part(
monkeypatch: pytest.MonkeyPatch,