Git designed for AI agents.
graft: the provider that served each model call is recorded (bench report, per case)
Federico Eliafb0118334d22
diff --git a/tools/graft-agents/llm/llm.go b/tools/graft-agents/llm/llm.goindex d365b1b..737450f 100644--- a/tools/graft-agents/llm/llm.go+++ b/tools/graft-agents/llm/llm.go@@ -62,6 +62,9 @@ type Response struct {Model string// Usage is the token usage, if known.Usage Usage+ // Provider is the provider that served the call, when the endpoint says (OpenRouter's "provider"+ // field of the answer): with routing pinned to a provider it is the proof that the call stayed there.+ Provider string}// Provider is a language-model backend.diff --git a/tools/graft-agents/llm/openai/openai.go b/tools/graft-agents/llm/openai/openai.goindex 60116c9..ccf7d69 100644--- a/tools/graft-agents/llm/openai/openai.go+++ b/tools/graft-agents/llm/openai/openai.go@@ -368,7 +368,9 @@ func (p *Provider) body(system, prompt string, maxTokens int, format any) []byte}type chatResponse struct {- Model string `json:"model"`+ Model string `json:"model"`+ // Provider is the provider that served the call, as OpenRouter reports it.+ Provider string `json:"provider"`Choices []struct {Message struct {Content *string `json:"content"`@@ -415,8 +417,9 @@ func (p *Provider) answer(chat *chatResponse, wantJSON bool) (*llm.Response, errmodel = p.model}resp := &llm.Response{- Text: text,- Model: model,+ Text: text,+ Model: model,+ Provider: chat.Provider,Usage: llm.Usage{InputTokens: chat.Usage.PromptTokens,OutputTokens: chat.Usage.CompletionTokens,diff --git a/tools/graft-agents/llm/openai/openai_test.go b/tools/graft-agents/llm/openai/openai_test.goindex ac9f431..8dbc803 100644--- a/tools/graft-agents/llm/openai/openai_test.go+++ b/tools/graft-agents/llm/openai/openai_test.go@@ -794,6 +794,29 @@ func TestProviderSatisfiesInterface(t *testing.T) {var _ llm.Provider = (*Provider)(nil)}+// TestCompleteReportsTheServingProvider: OpenRouter says which provider served the call; the response+// carries it, and is empty when the endpoint does not say.+func TestCompleteReportsTheServingProvider(t *testing.T) {+ body, _ := json.Marshal(map[string]any{+ "model": "served-model",+ "provider": "Z.AI",+ "choices": []any{map[string]any{+ "message": map[string]any{"role": "assistant", "content": "answer"},+ "finish_reason": "stop",+ }},+ "usage": map[string]any{"prompt_tokens": 1, "completion_tokens": 1},+ })+ f := newFakeServer(t, func(int, recorded) reply { return reply{status: http.StatusOK, body: string(body)} })+ resp, err := newProvider(t, f).Complete(context.Background(), llm.Request{Prompt: "p"})+ if err != nil || resp.Provider != "Z.AI" {+ t.Fatalf("provider %q, err %v", resp.Provider, err)+ }+ f = newFakeServer(t, func(int, recorded) reply { return chatOK("answer") })+ if resp, err = newProvider(t, f).Complete(context.Background(), llm.Request{Prompt: "p"}); err != nil || resp.Provider != "" {+ t.Fatalf("provider %q, err %v", resp.Provider, err)+ }+}+// TestCompleteReportsReasoningTokens: the reasoning tokens of the usage, part of the output, are// reported when the endpoint gives them.func TestCompleteReportsReasoningTokens(t *testing.T) {diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.goindex f3a09bc..c451d86 100644--- a/tools/graft-agents/resolve/bench/report.go+++ b/tools/graft-agents/resolve/bench/report.go@@ -257,6 +257,24 @@ func (rep Report) WriteMarkdown(w io.Writer) error {fmt.Fprintf(&b, "\nModel calls: %d, of which %d repair rounds driven by deterministic findings; the model abstained on %d cases (the two changes cannot both hold; each reached the agent with its reason).\n",t.Calls, t.Repairs, t.ModelAbstained)}+ providers := map[string]int{}+ for _, r := range rep.Results {+ for name, n := range r.Providers {+ providers[name] += n+ }+ }+ if len(providers) > 0 {+ names := make([]string, 0, len(providers))+ for name := range providers {+ names = append(names, name)+ }+ sort.Strings(names)+ parts := make([]string, 0, len(names))+ for _, name := range names {+ parts = append(parts, fmt.Sprintf("%s %d", name, providers[name]))+ }+ fmt.Fprintf(&b, "\nModel calls by the provider that served them, as the endpoint reports it: %s.\n", strings.Join(parts, ", "))+ }if t := rep.Total; t.InputTokens+t.OutputTokens > 0 {share := 0.0if t.OutputTokens > 0 {diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.goindex 0de537f..4595982 100644--- a/tools/graft-agents/resolve/bench/run.go+++ b/tools/graft-agents/resolve/bench/run.go@@ -71,6 +71,9 @@ type Result struct {// composition report of an accepted answer (PLAN10 15.3).ModelAbstained bool `json:"model_abstained,omitempty"`Calls int `json:"calls,omitempty"`+ // Providers counts the model calls of the case by the provider that served them, as the endpoint+ // reports it (a run with routing pinned to a provider shows here that it stayed there).+ Providers map[string]int `json:"providers,omitempty"`Repairs int `json:"repairs,omitempty"`Report []string `json:"report,omitempty"`// LineEndings are those of the sides of a content conflict (resolve.LLMLineEndings): the llm@@ -260,6 +263,7 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, erusage := regionStats.Usage()res.InputTokens, res.OutputTokens, res.ReasoningTokens = usage.InputTokens, usage.OutputTokens, usage.ReasoningTokensres.Calls, res.Repairs = regionStats.Calls()+ res.Providers = regionStats.Providers()if err != nil {var unresolved *resolve.UnresolvedErrorif !errors.As(err, &unresolved) || len(unresolved.Failures) != 1 {diff --git a/tools/graft-agents/resolve/llm_core.go b/tools/graft-agents/resolve/llm_core.goindex 914ff1b..95ba381 100644--- a/tools/graft-agents/resolve/llm_core.go+++ b/tools/graft-agents/resolve/llm_core.go@@ -174,6 +174,31 @@ type LLMRegionStats struct {calls, repairs intabstained stringreport []string+ providers map[string]int+}++// RecordProvider counts a call served by the named provider (empty when the endpoint did not say).+func (s *LLMRegionStats) RecordProvider(name string) {+ if name == "" {+ name = "(not reported)"+ }+ s.mu.Lock()+ defer s.mu.Unlock()+ if s.providers == nil {+ s.providers = map[string]int{}+ }+ s.providers[name]+++}++// Providers returns, by provider, the calls recorded.+func (s *LLMRegionStats) Providers() map[string]int {+ s.mu.Lock()+ defer s.mu.Unlock()+ out := make(map[string]int, len(s.providers))+ for k, v := range s.providers {+ out[k] = v+ }+ return out}// RecordCalls adds the model calls of a file and those of them that were repair rounds.diff --git a/tools/graft-agents/resolve/llm_patch.go b/tools/graft-agents/resolve/llm_patch.goindex 5963a0c..8af2e7a 100644--- a/tools/graft-agents/resolve/llm_patch.go+++ b/tools/graft-agents/resolve/llm_patch.go@@ -620,6 +620,7 @@ func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOpticalls++if stats != nil {stats.RecordUsage(resp.Usage)+ stats.RecordProvider(resp.Provider)}model = llmModel(resp, p)return resp, nil