Git designed for AI agents.
bench: report the token usage of the model calls
Federico Elia0e106ab06236
diff --git a/tools/graft-agents/llm/llm.go b/tools/graft-agents/llm/llm.goindex f7fd0f4..d365b1b 100644--- a/tools/graft-agents/llm/llm.go+++ b/tools/graft-agents/llm/llm.go@@ -46,6 +46,9 @@ type Request struct {type Usage struct {InputTokens intOutputTokens int+ // ReasoningTokens are the output tokens the model spent reasoning (part of OutputTokens),+ // when the provider reports them.+ ReasoningTokens int}// Response is one completion.diff --git a/tools/graft-agents/llm/openai/openai.go b/tools/graft-agents/llm/openai/openai.goindex af8cfe3..60116c9 100644--- a/tools/graft-agents/llm/openai/openai.go+++ b/tools/graft-agents/llm/openai/openai.go@@ -377,8 +377,11 @@ type chatResponse struct {FinishReason string `json:"finish_reason"`} `json:"choices"`Usage struct {- PromptTokens int `json:"prompt_tokens"`- CompletionTokens int `json:"completion_tokens"`+ PromptTokens int `json:"prompt_tokens"`+ CompletionTokens int `json:"completion_tokens"`+ CompletionTokensDetails struct {+ ReasoningTokens int `json:"reasoning_tokens"`+ } `json:"completion_tokens_details"`} `json:"usage"`}@@ -415,8 +418,9 @@ func (p *Provider) answer(chat *chatResponse, wantJSON bool) (*llm.Response, errText: text,Model: model,Usage: llm.Usage{- InputTokens: chat.Usage.PromptTokens,- OutputTokens: chat.Usage.CompletionTokens,+ InputTokens: chat.Usage.PromptTokens,+ OutputTokens: chat.Usage.CompletionTokens,+ ReasoningTokens: chat.Usage.CompletionTokensDetails.ReasoningTokens,},}if wantJSON {diff --git a/tools/graft-agents/llm/openai/openai_test.go b/tools/graft-agents/llm/openai/openai_test.goindex 7c41bab..ac9f431 100644--- a/tools/graft-agents/llm/openai/openai_test.go+++ b/tools/graft-agents/llm/openai/openai_test.go@@ -793,3 +793,24 @@ func TestProviderSatisfiesInterface(t *testing.T) {var _ llm.Provider = (*Provider)(nil)}++// TestCompleteReportsReasoningTokens: the reasoning tokens of the usage, part of the output, are+// reported when the endpoint gives them.+func TestCompleteReportsReasoningTokens(t *testing.T) {+ body, _ := json.Marshal(map[string]any{+ "model": "served-model",+ "choices": []any{map[string]any{+ "message": map[string]any{"role": "assistant", "content": "answer"},+ "finish_reason": "stop",+ }},+ "usage": map[string]any{"prompt_tokens": 12, "completion_tokens": 7, "completion_tokens_details": map[string]any{"reasoning_tokens": 5}},+ })+ f := newFakeServer(t, func(int, recorded) reply { return reply{status: http.StatusOK, body: string(body)} })+ resp, err := newProvider(t, f).Complete(context.Background(), llm.Request{Prompt: "p"})+ if err != nil {+ t.Fatal(err)+ }+ if resp.Usage != (llm.Usage{InputTokens: 12, OutputTokens: 7, ReasoningTokens: 5}) {+ t.Fatalf("Usage = %+v", resp.Usage)+ }+}diff --git a/tools/graft-agents/resolve/bench/r1_test.go b/tools/graft-agents/resolve/bench/r1_test.goindex 958160c..c7ece9b 100644--- a/tools/graft-agents/resolve/bench/r1_test.go+++ b/tools/graft-agents/resolve/bench/r1_test.go@@ -122,7 +122,8 @@ func (benchOursProvider) Complete(_ context.Context, req llm.Request) (*llm.Respreturn nil, fmt.Errorf("no CODE in the request")}n := m[1]- return &llm.Response{Text: "===== BEGIN PATCHED CODE " + n + " =====\n" + m[2] + "===== END PATCHED CODE " + n + " =====\n", Model: "bench-ours"}, nil+ return &llm.Response{Text: "===== BEGIN PATCHED CODE " + n + " =====\n" + m[2] + "===== END PATCHED CODE " + n + " =====\n", Model: "bench-ours",+ Usage: llm.Usage{InputTokens: 100, OutputTokens: 40, ReasoningTokens: 10}}, nil}func init() {@@ -159,6 +160,12 @@ func TestRunnerCountsRegionsEqualToOurs(t *testing.T) {if err := rep.WriteMarkdown(&md); err != nil {t.Fatal(err)}+ if results[0].InputTokens != 100 || results[0].OutputTokens != 40 || results[0].ReasoningTokens != 10 {+ t.Fatalf("tokens %+v", results[0])+ }+ if !strings.Contains(md.String(), "Tokens of the model calls, as the provider reports them: input 100, output 40, of which reasoning 10 (25.0% of the output).") {+ t.Fatalf("markdown:\n%s", md.String())+ }if !strings.Contains(md.String(), "Regions of the accepted model answers answered with OURS' region unchanged: 1 of 1.") {t.Fatalf("markdown:\n%s", md.String())}diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.goindex 74def8b..d31195c 100644--- a/tools/graft-agents/resolve/bench/report.go+++ b/tools/graft-agents/resolve/bench/report.go@@ -33,6 +33,11 @@ type Counts struct {// those answered with OURS' region unchanged.Regions int `json:"regions"`RegionsEqualOurs int `json:"regions_equal_ours"`+ // InputTokens, OutputTokens and ReasoningTokens (part of the output) sum the token usage of+ // the model calls.+ InputTokens int `json:"input_tokens"`+ OutputTokens int `json:"output_tokens"`+ ReasoningTokens int `json:"reasoning_tokens"`}// Accepted is the number of results that would land: correct and wrong.@@ -60,6 +65,9 @@ func (c *Counts) add(r Result) {}c.Regions += r.Regionsc.RegionsEqualOurs += r.RegionsEqualOurs+ c.InputTokens += r.InputTokens+ c.OutputTokens += r.OutputTokens+ c.ReasoningTokens += r.ReasoningTokensswitch r.Outcome {case OutcomeCorrect:c.Correct++@@ -235,6 +243,14 @@ func (rep Report) WriteMarkdown(w io.Writer) error {rep.Total.EqualsOurs, rep.Total.Accepted(), rep.equalsOurs(OutcomeCorrect), rep.equalsOurs(OutcomeWrong))fmt.Fprintf(&b, "\nRegions of the accepted model answers answered with OURS' region unchanged: %d of %d.\n",rep.Total.RegionsEqualOurs, rep.Total.Regions)+ if t := rep.Total; t.InputTokens+t.OutputTokens > 0 {+ share := 0.0+ if t.OutputTokens > 0 {+ share = 100 * float64(t.ReasoningTokens) / float64(t.OutputTokens)+ }+ fmt.Fprintf(&b, "\nTokens of the model calls, as the provider reports them: input %d, output %d, of which reasoning %d (%.1f%% of the output).\n",+ t.InputTokens, t.OutputTokens, t.ReasoningTokens, share)+ }endings := map[string]Counts{}for _, r := range rep.Results {if r.LineEndings == "" {diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.goindex 75d5088..dea4cbe 100644--- a/tools/graft-agents/resolve/bench/run.go+++ b/tools/graft-agents/resolve/bench/run.go@@ -68,7 +68,12 @@ type Result struct {// LineEndings are those of the sides of a content conflict (resolve.LLMLineEndings): the llm// strategy sends a CRLF file as LF, and a mixed one as it is.LineEndings string `json:"line_endings,omitempty"`- Strategy string `json:"strategy,omitempty"`+ // InputTokens, OutputTokens and ReasoningTokens (part of the output) are the token usage of+ // the model calls of the case, answered or rejected, as the provider reports them.+ InputTokens int `json:"input_tokens,omitempty"`+ OutputTokens int `json:"output_tokens,omitempty"`+ ReasoningTokens int `json:"reasoning_tokens,omitempty"`+ Strategy string `json:"strategy,omitempty"`// Answer is the action of an accepted resolution ("write" or "delete"); with// Runner.AnswersDir its content is stored for audit and Rejudge.Answer string `json:"answer,omitempty"`@@ -243,6 +248,8 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, erregionStats := &resolve.LLMRegionStats{}resp, err := r.Resolver.Resolve(resolve.WithLLMRegionStats(ctx, regionStats), req)res.Seconds = time.Since(start).Seconds()+ usage := regionStats.Usage()+ res.InputTokens, res.OutputTokens, res.ReasoningTokens = usage.InputTokens, usage.OutputTokens, usage.ReasoningTokensif err != nil {var unresolved *resolve.UnresolvedErrorif !errors.As(err, &unresolved) || len(unresolved.Failures) != 1 {diff --git a/tools/graft-agents/resolve/llm_core.go b/tools/graft-agents/resolve/llm_core.goindex 9fe573c..0b07b79 100644--- a/tools/graft-agents/resolve/llm_core.go+++ b/tools/graft-agents/resolve/llm_core.go@@ -136,6 +136,23 @@ type LLMRegionStats struct {regions, equalOurs intrecorded boolrejected []string+ usage llm.Usage+}++// RecordUsage adds the token usage of one model call, answered or rejected.+func (s *LLMRegionStats) RecordUsage(u llm.Usage) {+ s.mu.Lock()+ defer s.mu.Unlock()+ s.usage.InputTokens += u.InputTokens+ s.usage.OutputTokens += u.OutputTokens+ s.usage.ReasoningTokens += u.ReasoningTokens+}++// Usage returns the token usage of the model calls recorded.+func (s *LLMRegionStats) Usage() llm.Usage {+ s.mu.Lock()+ defer s.mu.Unlock()+ return s.usage}// RecordRejected keeps the raw text of a model answer that was rejected, with why, for audit.diff --git a/tools/graft-agents/resolve/llm_patch.go b/tools/graft-agents/resolve/llm_patch.goindex 5f43c9f..3021128 100644--- a/tools/graft-agents/resolve/llm_patch.go+++ b/tools/graft-agents/resolve/llm_patch.go@@ -453,6 +453,9 @@ func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOptiif err != nil {return LLMResult{}, err}+ if stats := llmRegionStatsFrom(ctx); stats != nil {+ stats.RecordUsage(resp.Usage)+ }rejected := func(err error) error {err = fmt.Errorf("region %d: %w", k+1, err)if stats := llmRegionStatsFrom(ctx); stats != nil {