Skip to content

Git designed for AI agents.

bench: report the token usage of the model calls

Federico Elia0e106ab06236
diff --git a/tools/graft-agents/llm/llm.go b/tools/graft-agents/llm/llm.go
index f7fd0f4..d365b1b 100644
--- a/tools/graft-agents/llm/llm.go
+++ b/tools/graft-agents/llm/llm.go
@@ -46,6 +46,9 @@ type Request struct {
type Usage struct {
InputTokens int
OutputTokens int
+ // ReasoningTokens are the output tokens the model spent reasoning (part of OutputTokens),
+ // when the provider reports them.
+ ReasoningTokens int
}
// Response is one completion.
diff --git a/tools/graft-agents/llm/openai/openai.go b/tools/graft-agents/llm/openai/openai.go
index af8cfe3..60116c9 100644
--- a/tools/graft-agents/llm/openai/openai.go
+++ b/tools/graft-agents/llm/openai/openai.go
@@ -377,8 +377,11 @@ type chatResponse struct {
FinishReason string `json:"finish_reason"`
} `json:"choices"`
Usage struct {
- PromptTokens int `json:"prompt_tokens"`
- CompletionTokens int `json:"completion_tokens"`
+ PromptTokens int `json:"prompt_tokens"`
+ CompletionTokens int `json:"completion_tokens"`
+ CompletionTokensDetails struct {
+ ReasoningTokens int `json:"reasoning_tokens"`
+ } `json:"completion_tokens_details"`
} `json:"usage"`
}
@@ -415,8 +418,9 @@ func (p *Provider) answer(chat *chatResponse, wantJSON bool) (*llm.Response, err
Text: text,
Model: model,
Usage: llm.Usage{
- InputTokens: chat.Usage.PromptTokens,
- OutputTokens: chat.Usage.CompletionTokens,
+ InputTokens: chat.Usage.PromptTokens,
+ OutputTokens: chat.Usage.CompletionTokens,
+ ReasoningTokens: chat.Usage.CompletionTokensDetails.ReasoningTokens,
},
}
if wantJSON {
diff --git a/tools/graft-agents/llm/openai/openai_test.go b/tools/graft-agents/llm/openai/openai_test.go
index 7c41bab..ac9f431 100644
--- a/tools/graft-agents/llm/openai/openai_test.go
+++ b/tools/graft-agents/llm/openai/openai_test.go
@@ -793,3 +793,24 @@ func TestProviderSatisfiesInterface(t *testing.T) {
var _ llm.Provider = (*Provider)(nil)
}
+
+// TestCompleteReportsReasoningTokens: the reasoning tokens of the usage, part of the output, are
+// reported when the endpoint gives them.
+func TestCompleteReportsReasoningTokens(t *testing.T) {
+ body, _ := json.Marshal(map[string]any{
+ "model": "served-model",
+ "choices": []any{map[string]any{
+ "message": map[string]any{"role": "assistant", "content": "answer"},
+ "finish_reason": "stop",
+ }},
+ "usage": map[string]any{"prompt_tokens": 12, "completion_tokens": 7, "completion_tokens_details": map[string]any{"reasoning_tokens": 5}},
+ })
+ f := newFakeServer(t, func(int, recorded) reply { return reply{status: http.StatusOK, body: string(body)} })
+ resp, err := newProvider(t, f).Complete(context.Background(), llm.Request{Prompt: "p"})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resp.Usage != (llm.Usage{InputTokens: 12, OutputTokens: 7, ReasoningTokens: 5}) {
+ t.Fatalf("Usage = %+v", resp.Usage)
+ }
+}
diff --git a/tools/graft-agents/resolve/bench/r1_test.go b/tools/graft-agents/resolve/bench/r1_test.go
index 958160c..c7ece9b 100644
--- a/tools/graft-agents/resolve/bench/r1_test.go
+++ b/tools/graft-agents/resolve/bench/r1_test.go
@@ -122,7 +122,8 @@ func (benchOursProvider) Complete(_ context.Context, req llm.Request) (*llm.Resp
return nil, fmt.Errorf("no CODE in the request")
}
n := m[1]
- return &llm.Response{Text: "===== BEGIN PATCHED CODE " + n + " =====\n" + m[2] + "===== END PATCHED CODE " + n + " =====\n", Model: "bench-ours"}, nil
+ return &llm.Response{Text: "===== BEGIN PATCHED CODE " + n + " =====\n" + m[2] + "===== END PATCHED CODE " + n + " =====\n", Model: "bench-ours",
+ Usage: llm.Usage{InputTokens: 100, OutputTokens: 40, ReasoningTokens: 10}}, nil
}
func init() {
@@ -159,6 +160,12 @@ func TestRunnerCountsRegionsEqualToOurs(t *testing.T) {
if err := rep.WriteMarkdown(&md); err != nil {
t.Fatal(err)
}
+ if results[0].InputTokens != 100 || results[0].OutputTokens != 40 || results[0].ReasoningTokens != 10 {
+ t.Fatalf("tokens %+v", results[0])
+ }
+ if !strings.Contains(md.String(), "Tokens of the model calls, as the provider reports them: input 100, output 40, of which reasoning 10 (25.0% of the output).") {
+ t.Fatalf("markdown:\n%s", md.String())
+ }
if !strings.Contains(md.String(), "Regions of the accepted model answers answered with OURS' region unchanged: 1 of 1.") {
t.Fatalf("markdown:\n%s", md.String())
}
diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.go
index 74def8b..d31195c 100644
--- a/tools/graft-agents/resolve/bench/report.go
+++ b/tools/graft-agents/resolve/bench/report.go
@@ -33,6 +33,11 @@ type Counts struct {
// those answered with OURS' region unchanged.
Regions int `json:"regions"`
RegionsEqualOurs int `json:"regions_equal_ours"`
+ // InputTokens, OutputTokens and ReasoningTokens (part of the output) sum the token usage of
+ // the model calls.
+ InputTokens int `json:"input_tokens"`
+ OutputTokens int `json:"output_tokens"`
+ ReasoningTokens int `json:"reasoning_tokens"`
}
// Accepted is the number of results that would land: correct and wrong.
@@ -60,6 +65,9 @@ func (c *Counts) add(r Result) {
}
c.Regions += r.Regions
c.RegionsEqualOurs += r.RegionsEqualOurs
+ c.InputTokens += r.InputTokens
+ c.OutputTokens += r.OutputTokens
+ c.ReasoningTokens += r.ReasoningTokens
switch r.Outcome {
case OutcomeCorrect:
c.Correct++
@@ -235,6 +243,14 @@ func (rep Report) WriteMarkdown(w io.Writer) error {
rep.Total.EqualsOurs, rep.Total.Accepted(), rep.equalsOurs(OutcomeCorrect), rep.equalsOurs(OutcomeWrong))
fmt.Fprintf(&b, "\nRegions of the accepted model answers answered with OURS' region unchanged: %d of %d.\n",
rep.Total.RegionsEqualOurs, rep.Total.Regions)
+ if t := rep.Total; t.InputTokens+t.OutputTokens > 0 {
+ share := 0.0
+ if t.OutputTokens > 0 {
+ share = 100 * float64(t.ReasoningTokens) / float64(t.OutputTokens)
+ }
+ fmt.Fprintf(&b, "\nTokens of the model calls, as the provider reports them: input %d, output %d, of which reasoning %d (%.1f%% of the output).\n",
+ t.InputTokens, t.OutputTokens, t.ReasoningTokens, share)
+ }
endings := map[string]Counts{}
for _, r := range rep.Results {
if r.LineEndings == "" {
diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.go
index 75d5088..dea4cbe 100644
--- a/tools/graft-agents/resolve/bench/run.go
+++ b/tools/graft-agents/resolve/bench/run.go
@@ -68,7 +68,12 @@ type Result struct {
// LineEndings are those of the sides of a content conflict (resolve.LLMLineEndings): the llm
// strategy sends a CRLF file as LF, and a mixed one as it is.
LineEndings string `json:"line_endings,omitempty"`
- Strategy string `json:"strategy,omitempty"`
+ // InputTokens, OutputTokens and ReasoningTokens (part of the output) are the token usage of
+ // the model calls of the case, answered or rejected, as the provider reports them.
+ InputTokens int `json:"input_tokens,omitempty"`
+ OutputTokens int `json:"output_tokens,omitempty"`
+ ReasoningTokens int `json:"reasoning_tokens,omitempty"`
+ Strategy string `json:"strategy,omitempty"`
// Answer is the action of an accepted resolution ("write" or "delete"); with
// Runner.AnswersDir its content is stored for audit and Rejudge.
Answer string `json:"answer,omitempty"`
@@ -243,6 +248,8 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, er
regionStats := &resolve.LLMRegionStats{}
resp, err := r.Resolver.Resolve(resolve.WithLLMRegionStats(ctx, regionStats), req)
res.Seconds = time.Since(start).Seconds()
+ usage := regionStats.Usage()
+ res.InputTokens, res.OutputTokens, res.ReasoningTokens = usage.InputTokens, usage.OutputTokens, usage.ReasoningTokens
if err != nil {
var unresolved *resolve.UnresolvedError
if !errors.As(err, &unresolved) || len(unresolved.Failures) != 1 {
diff --git a/tools/graft-agents/resolve/llm_core.go b/tools/graft-agents/resolve/llm_core.go
index 9fe573c..0b07b79 100644
--- a/tools/graft-agents/resolve/llm_core.go
+++ b/tools/graft-agents/resolve/llm_core.go
@@ -136,6 +136,23 @@ type LLMRegionStats struct {
regions, equalOurs int
recorded bool
rejected []string
+ usage llm.Usage
+}
+
+// RecordUsage adds the token usage of one model call, answered or rejected.
+func (s *LLMRegionStats) RecordUsage(u llm.Usage) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.usage.InputTokens += u.InputTokens
+ s.usage.OutputTokens += u.OutputTokens
+ s.usage.ReasoningTokens += u.ReasoningTokens
+}
+
+// Usage returns the token usage of the model calls recorded.
+func (s *LLMRegionStats) Usage() llm.Usage {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return s.usage
}
// RecordRejected keeps the raw text of a model answer that was rejected, with why, for audit.
diff --git a/tools/graft-agents/resolve/llm_patch.go b/tools/graft-agents/resolve/llm_patch.go
index 5f43c9f..3021128 100644
--- a/tools/graft-agents/resolve/llm_patch.go
+++ b/tools/graft-agents/resolve/llm_patch.go
@@ -453,6 +453,9 @@ func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOpti
if err != nil {
return LLMResult{}, err
}
+ if stats := llmRegionStatsFrom(ctx); stats != nil {
+ stats.RecordUsage(resp.Usage)
+ }
rejected := func(err error) error {
err = fmt.Errorf("region %d: %w", k+1, err)
if stats := llmRegionStatsFrom(ctx); stats != nil {