Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(googlegenai): count tool-use prompt tokens as input tokens (#4594)
alexcastgonze807e6523eea
diff --git a/internal/models/api/googlegenai/client.go b/internal/models/api/googlegenai/client.goindex a7043d8ed..b6c1215b9 100644--- a/internal/models/api/googlegenai/client.go+++ b/internal/models/api/googlegenai/client.go@@ -192,6 +192,7 @@ type rawUsage struct {PromptTokenCount int `json:"promptTokenCount"`CandidatesTokenCount int `json:"candidatesTokenCount"`ThoughtsTokenCount int `json:"thoughtsTokenCount"`+ ToolUsePromptTokenCount int `json:"toolUsePromptTokenCount"`TotalTokenCount int `json:"totalTokenCount"`CachedContentTokenCount *int `json:"cachedContentTokenCount"`}@@ -210,8 +211,12 @@ type rawResponse struct {}func (c *Client) usage(u rawUsage) types.TokenUsage {+ // Tokens read by Gemini's built-in tools (URL context, search, code+ // execution) are billed as input but reported apart from promptTokenCount;+ // totalTokenCount counts them as "tool-use prompt".+ promptTokens := u.PromptTokenCount + u.ToolUsePromptTokenCountout := types.TokenUsage{- PromptTokens: u.PromptTokenCount,+ PromptTokens: promptTokens,CompletionTokens: u.CandidatesTokenCount + u.ThoughtsTokenCount,TotalTokens: u.TotalTokenCount,}@@ -221,7 +226,7 @@ func (c *Client) usage(u rawUsage) types.TokenUsage {switch {case u.CachedContentTokenCount != nil:cached := *u.CachedContentTokenCount- out.SetPromptCacheUsage(cached, 0, max(0, u.PromptTokenCount-cached), true)+ out.SetPromptCacheUsage(cached, 0, max(0, promptTokens-cached), true)case c.cfg.Settings.PromptCacheAcct:out.SetPromptCacheUsage(0, 0, 0, false)default:diff --git a/internal/models/api/googlegenai/googlegenai_test.go b/internal/models/api/googlegenai/googlegenai_test.goindex 034aa4dc0..c96c99b24 100644--- a/internal/models/api/googlegenai/googlegenai_test.go+++ b/internal/models/api/googlegenai/googlegenai_test.go@@ -730,6 +730,42 @@ func TestChatStreamEOFWithoutFinishReasonIsIncomplete(t *testing.T) {}}+// Tokens read by Gemini's built-in tools (URL context, search, code execution)+// are reported in toolUsePromptTokenCount, apart from promptTokenCount, and+// totalTokenCount includes them.+func TestUsageCountsToolUsePromptTokens(t *testing.T) {+ client := newTestClient(t, "http://example.invalid", nil)+ cached := 4+ got := client.usage(rawUsage{+ PromptTokenCount: 10,+ ToolUsePromptTokenCount: 6,+ CandidatesTokenCount: 5,+ ThoughtsTokenCount: 3,+ TotalTokenCount: 24,+ CachedContentTokenCount: &cached,+ })+ if got.PromptTokens != 16 {+ t.Errorf("PromptTokens = %d, want 16", got.PromptTokens)+ }+ if got.CompletionTokens != 8 {+ t.Errorf("CompletionTokens = %d, want 8", got.CompletionTokens)+ }+ if got.TotalTokens != 24 {+ t.Errorf("TotalTokens = %d, want 24", got.TotalTokens)+ }+ if got.CacheReadTokens != 4 || got.CacheMissTokens != 12 {+ t.Errorf("cache read/miss = %d/%d, want 4/12", got.CacheReadTokens, got.CacheMissTokens)+ }+}++func TestUsageWithoutTotalSumsToolUsePromptTokens(t *testing.T) {+ client := newTestClient(t, "http://example.invalid", nil)+ got := client.usage(rawUsage{PromptTokenCount: 10, ToolUsePromptTokenCount: 6, CandidatesTokenCount: 5})+ if got.PromptTokens != 16 || got.TotalTokens != 21 {+ t.Errorf("PromptTokens/TotalTokens = %d/%d, want 16/21", got.PromptTokens, got.TotalTokens)+ }+}+func TestMapFinishReason(t *testing.T) {tests := []struct {reason string