Build and run agents you can see, understand and trust.
Python agentchatbotlarge-language-modelsllmllm-agentmcpmulti-agentmulti-modalreact-agentrealtime-agentvoice-agentvoice-assistant
fix(gemini): exclude tool-execution tokens from output_tokens (#2406)
DiegoCluv7e8c8d7b9f562
diff --git a/src/agentscope/model/_gemini/_model.py b/src/agentscope/model/_gemini/_model.pyindex a4fee3791..37f0607f4 100644--- a/src/agentscope/model/_gemini/_model.py+++ b/src/agentscope/model/_gemini/_model.py@@ -470,9 +470,30 @@ class GeminiChatModel(ChatModelBase):prompt_tokens = usage_metadata.prompt_token_counttotal_tokens = usage_metadata.total_token_countif prompt_tokens is not None and total_tokens is not None:+ tool_use_tokens = (+ getattr(+ usage_metadata,+ "tool_use_prompt_token_count",+ 0,+ )+ or 0+ )+ input_tokens = prompt_tokens + tool_use_tokens+ candidates_tokens = getattr(+ usage_metadata,+ "candidates_token_count",+ None,+ )+ if candidates_tokens is not None:+ output_tokens = candidates_tokens + (+ getattr(usage_metadata, "thoughts_token_count", 0) or 0+ )+ else:+ # Fallback for SDK versions without the candidate count.+ output_tokens = total_tokens - input_tokensreturn ChatUsage(- input_tokens=prompt_tokens,- output_tokens=total_tokens - prompt_tokens,+ input_tokens=input_tokens,+ output_tokens=output_tokens,time=(datetime.now() - start_datetime).total_seconds(),cache_input_tokens=getattr(usage_metadata,diff --git a/tests/model_gemini_test.py b/tests/model_gemini_test.pyindex 1de053cf4..d0ce125f9 100644--- a/tests/model_gemini_test.py+++ b/tests/model_gemini_test.py@@ -6,6 +6,7 @@ Tests cover both non-streaming and streaming modes.Gemini uses google.genai client with async iterator streaming."""import json+from types import SimpleNamespacefrom typing import Anyimport unittestfrom unittest import IsolatedAsyncioTestCase@@ -68,9 +69,14 @@ def _mock_completion(resp.candidates = [MagicMock()]resp.candidates[0].content = MagicMock()resp.candidates[0].content.parts = parts- resp.usage_metadata = MagicMock()- resp.usage_metadata.prompt_token_count = 10- resp.usage_metadata.candidates_token_count = 5+ resp.usage_metadata = SimpleNamespace(+ prompt_token_count=10,+ candidates_token_count=5,+ tool_use_prompt_token_count=0,+ thoughts_token_count=0,+ total_token_count=15,+ cached_content_token_count=0,+ )return resp@@ -84,9 +90,14 @@ def _make_stream_chunk(chunk.candidates = [MagicMock()]chunk.candidates[0].content = MagicMock()chunk.candidates[0].content.parts = parts- chunk.usage_metadata = MagicMock()- chunk.usage_metadata.prompt_token_count = 10- chunk.usage_metadata.candidates_token_count = 5+ chunk.usage_metadata = SimpleNamespace(+ prompt_token_count=10,+ candidates_token_count=5,+ tool_use_prompt_token_count=0,+ thoughts_token_count=0,+ total_token_count=15,+ cached_content_token_count=0,+ )return chunk@@ -243,6 +254,68 @@ class TestGeminiNonStream(IsolatedAsyncioTestCase):),)+ async def test_usage_classifies_tool_use_tokens_as_input(self) -> None:+ """Usage classifies tool-use tokens as input, not output."""+ parts = [_make_part(text="hello")]+ resp = _mock_completion(parts)+ # Mirror the SDK's documented invariant: total = prompt ++ # candidates + tool_use_prompt + thoughts.+ resp.usage_metadata = SimpleNamespace(+ prompt_token_count=500,+ candidates_token_count=120,+ tool_use_prompt_token_count=300,+ thoughts_token_count=10,+ total_token_count=930,+ cached_content_token_count=50,+ )+ self.mock_client.aio.models.generate_content = AsyncMock(+ return_value=resp,+ )++ result = await self.model([])++ self.assertEqual(+ dict(result.usage),+ {+ "input_tokens": 800,+ "output_tokens": 130,+ "time": result.usage.time,+ "cache_creation_input_tokens": 0,+ "cache_input_tokens": 50,+ "type": "chat",+ "metadata": None,+ },+ )++ async def test_usage_fallback_excludes_tool_use_tokens(self) -> None:+ """Usage fallback excludes tool-use tokens from output."""+ resp = _mock_completion([_make_part(text="hello")])+ resp.usage_metadata = SimpleNamespace(+ prompt_token_count=500,+ tool_use_prompt_token_count=300,+ thoughts_token_count=10,+ total_token_count=810,+ cached_content_token_count=0,+ )+ self.mock_client.aio.models.generate_content = AsyncMock(+ return_value=resp,+ )++ result = await self.model([])++ self.assertEqual(+ dict(result.usage),+ {+ "input_tokens": 800,+ "output_tokens": 10,+ "time": result.usage.time,+ "cache_creation_input_tokens": 0,+ "cache_input_tokens": 0,+ "type": "chat",+ "metadata": None,+ },+ )+# ---------------------------------------------------------------------------# Streaming tests@@ -309,6 +382,40 @@ class TestGeminiStream(IsolatedAsyncioTestCase):],)+ async def test_stream_usage_classifies_tool_use_tokens(self) -> None:+ """Streaming usage classifies tool-use tokens as input."""+ chunk = _make_stream_chunk([_make_part(text="hello")])+ chunk.usage_metadata = SimpleNamespace(+ prompt_token_count=500,+ candidates_token_count=120,+ tool_use_prompt_token_count=300,+ thoughts_token_count=10,+ total_token_count=930,+ cached_content_token_count=50,+ )+ self.mock_client.aio.models.generate_content_stream = AsyncMock(+ return_value=_MockAsyncStream([chunk]),+ )++ gen = await self.model([])+ responses = [r async for r in gen]++ # The delta and the accumulated final response both carry the+ # usage of the last chunk.+ self.assertEqual(+ dict(responses[0].usage),+ {+ "input_tokens": 800,+ "output_tokens": 130,+ "time": responses[0].usage.time,+ "cache_creation_input_tokens": 0,+ "cache_input_tokens": 50,+ "type": "chat",+ "metadata": None,+ },+ )+ self.assertEqual(responses[-1].usage, responses[0].usage)+async def test_stream_thinking_and_text(self) -> None:"""Stream thinking + text yields deltas then accumulated final."""chunks = [