Skip to content

Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.

fix(googlegenai): count tool-use prompt tokens as input tokens (#4594)

alexcastgonze807e6523eea
diff --git a/internal/models/api/googlegenai/client.go b/internal/models/api/googlegenai/client.go
index a7043d8ed..b6c1215b9 100644
--- a/internal/models/api/googlegenai/client.go
+++ b/internal/models/api/googlegenai/client.go
@@ -192,6 +192,7 @@ type rawUsage struct {
PromptTokenCount int `json:"promptTokenCount"`
CandidatesTokenCount int `json:"candidatesTokenCount"`
ThoughtsTokenCount int `json:"thoughtsTokenCount"`
+ ToolUsePromptTokenCount int `json:"toolUsePromptTokenCount"`
TotalTokenCount int `json:"totalTokenCount"`
CachedContentTokenCount *int `json:"cachedContentTokenCount"`
}
@@ -210,8 +211,12 @@ type rawResponse struct {
}
func (c *Client) usage(u rawUsage) types.TokenUsage {
+ // Tokens read by Gemini's built-in tools (URL context, search, code
+ // execution) are billed as input but reported apart from promptTokenCount;
+ // totalTokenCount counts them as "tool-use prompt".
+ promptTokens := u.PromptTokenCount + u.ToolUsePromptTokenCount
out := types.TokenUsage{
- PromptTokens: u.PromptTokenCount,
+ PromptTokens: promptTokens,
CompletionTokens: u.CandidatesTokenCount + u.ThoughtsTokenCount,
TotalTokens: u.TotalTokenCount,
}
@@ -221,7 +226,7 @@ func (c *Client) usage(u rawUsage) types.TokenUsage {
switch {
case u.CachedContentTokenCount != nil:
cached := *u.CachedContentTokenCount
- out.SetPromptCacheUsage(cached, 0, max(0, u.PromptTokenCount-cached), true)
+ out.SetPromptCacheUsage(cached, 0, max(0, promptTokens-cached), true)
case c.cfg.Settings.PromptCacheAcct:
out.SetPromptCacheUsage(0, 0, 0, false)
default:
diff --git a/internal/models/api/googlegenai/googlegenai_test.go b/internal/models/api/googlegenai/googlegenai_test.go
index 034aa4dc0..c96c99b24 100644
--- a/internal/models/api/googlegenai/googlegenai_test.go
+++ b/internal/models/api/googlegenai/googlegenai_test.go
@@ -730,6 +730,42 @@ func TestChatStreamEOFWithoutFinishReasonIsIncomplete(t *testing.T) {
}
}
+// Tokens read by Gemini's built-in tools (URL context, search, code execution)
+// are reported in toolUsePromptTokenCount, apart from promptTokenCount, and
+// totalTokenCount includes them.
+func TestUsageCountsToolUsePromptTokens(t *testing.T) {
+ client := newTestClient(t, "http://example.invalid", nil)
+ cached := 4
+ got := client.usage(rawUsage{
+ PromptTokenCount: 10,
+ ToolUsePromptTokenCount: 6,
+ CandidatesTokenCount: 5,
+ ThoughtsTokenCount: 3,
+ TotalTokenCount: 24,
+ CachedContentTokenCount: &cached,
+ })
+ if got.PromptTokens != 16 {
+ t.Errorf("PromptTokens = %d, want 16", got.PromptTokens)
+ }
+ if got.CompletionTokens != 8 {
+ t.Errorf("CompletionTokens = %d, want 8", got.CompletionTokens)
+ }
+ if got.TotalTokens != 24 {
+ t.Errorf("TotalTokens = %d, want 24", got.TotalTokens)
+ }
+ if got.CacheReadTokens != 4 || got.CacheMissTokens != 12 {
+ t.Errorf("cache read/miss = %d/%d, want 4/12", got.CacheReadTokens, got.CacheMissTokens)
+ }
+}
+
+func TestUsageWithoutTotalSumsToolUsePromptTokens(t *testing.T) {
+ client := newTestClient(t, "http://example.invalid", nil)
+ got := client.usage(rawUsage{PromptTokenCount: 10, ToolUsePromptTokenCount: 6, CandidatesTokenCount: 5})
+ if got.PromptTokens != 16 || got.TotalTokens != 21 {
+ t.Errorf("PromptTokens/TotalTokens = %d/%d, want 16/21", got.PromptTokens, got.TotalTokens)
+ }
+}
+
func TestMapFinishReason(t *testing.T) {
tests := []struct {
reason string