Skip to content

Git designed for AI agents.

graft: the provider that served each model call is recorded (bench report, per case)

Federico Eliafb0118334d22
diff --git a/tools/graft-agents/llm/llm.go b/tools/graft-agents/llm/llm.go
index d365b1b..737450f 100644
--- a/tools/graft-agents/llm/llm.go
+++ b/tools/graft-agents/llm/llm.go
@@ -62,6 +62,9 @@ type Response struct {
Model string
// Usage is the token usage, if known.
Usage Usage
+ // Provider is the provider that served the call, when the endpoint says (OpenRouter's "provider"
+ // field of the answer): with routing pinned to a provider it is the proof that the call stayed there.
+ Provider string
}
// Provider is a language-model backend.
diff --git a/tools/graft-agents/llm/openai/openai.go b/tools/graft-agents/llm/openai/openai.go
index 60116c9..ccf7d69 100644
--- a/tools/graft-agents/llm/openai/openai.go
+++ b/tools/graft-agents/llm/openai/openai.go
@@ -368,7 +368,9 @@ func (p *Provider) body(system, prompt string, maxTokens int, format any) []byte
}
type chatResponse struct {
- Model string `json:"model"`
+ Model string `json:"model"`
+ // Provider is the provider that served the call, as OpenRouter reports it.
+ Provider string `json:"provider"`
Choices []struct {
Message struct {
Content *string `json:"content"`
@@ -415,8 +417,9 @@ func (p *Provider) answer(chat *chatResponse, wantJSON bool) (*llm.Response, err
model = p.model
}
resp := &llm.Response{
- Text: text,
- Model: model,
+ Text: text,
+ Model: model,
+ Provider: chat.Provider,
Usage: llm.Usage{
InputTokens: chat.Usage.PromptTokens,
OutputTokens: chat.Usage.CompletionTokens,
diff --git a/tools/graft-agents/llm/openai/openai_test.go b/tools/graft-agents/llm/openai/openai_test.go
index ac9f431..8dbc803 100644
--- a/tools/graft-agents/llm/openai/openai_test.go
+++ b/tools/graft-agents/llm/openai/openai_test.go
@@ -794,6 +794,29 @@ func TestProviderSatisfiesInterface(t *testing.T) {
var _ llm.Provider = (*Provider)(nil)
}
+// TestCompleteReportsTheServingProvider: OpenRouter says which provider served the call; the response
+// carries it, and is empty when the endpoint does not say.
+func TestCompleteReportsTheServingProvider(t *testing.T) {
+ body, _ := json.Marshal(map[string]any{
+ "model": "served-model",
+ "provider": "Z.AI",
+ "choices": []any{map[string]any{
+ "message": map[string]any{"role": "assistant", "content": "answer"},
+ "finish_reason": "stop",
+ }},
+ "usage": map[string]any{"prompt_tokens": 1, "completion_tokens": 1},
+ })
+ f := newFakeServer(t, func(int, recorded) reply { return reply{status: http.StatusOK, body: string(body)} })
+ resp, err := newProvider(t, f).Complete(context.Background(), llm.Request{Prompt: "p"})
+ if err != nil || resp.Provider != "Z.AI" {
+ t.Fatalf("provider %q, err %v", resp.Provider, err)
+ }
+ f = newFakeServer(t, func(int, recorded) reply { return chatOK("answer") })
+ if resp, err = newProvider(t, f).Complete(context.Background(), llm.Request{Prompt: "p"}); err != nil || resp.Provider != "" {
+ t.Fatalf("provider %q, err %v", resp.Provider, err)
+ }
+}
+
// TestCompleteReportsReasoningTokens: the reasoning tokens of the usage, part of the output, are
// reported when the endpoint gives them.
func TestCompleteReportsReasoningTokens(t *testing.T) {
diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.go
index f3a09bc..c451d86 100644
--- a/tools/graft-agents/resolve/bench/report.go
+++ b/tools/graft-agents/resolve/bench/report.go
@@ -257,6 +257,24 @@ func (rep Report) WriteMarkdown(w io.Writer) error {
fmt.Fprintf(&b, "\nModel calls: %d, of which %d repair rounds driven by deterministic findings; the model abstained on %d cases (the two changes cannot both hold; each reached the agent with its reason).\n",
t.Calls, t.Repairs, t.ModelAbstained)
}
+ providers := map[string]int{}
+ for _, r := range rep.Results {
+ for name, n := range r.Providers {
+ providers[name] += n
+ }
+ }
+ if len(providers) > 0 {
+ names := make([]string, 0, len(providers))
+ for name := range providers {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ parts := make([]string, 0, len(names))
+ for _, name := range names {
+ parts = append(parts, fmt.Sprintf("%s %d", name, providers[name]))
+ }
+ fmt.Fprintf(&b, "\nModel calls by the provider that served them, as the endpoint reports it: %s.\n", strings.Join(parts, ", "))
+ }
if t := rep.Total; t.InputTokens+t.OutputTokens > 0 {
share := 0.0
if t.OutputTokens > 0 {
diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.go
index 0de537f..4595982 100644
--- a/tools/graft-agents/resolve/bench/run.go
+++ b/tools/graft-agents/resolve/bench/run.go
@@ -71,6 +71,9 @@ type Result struct {
// composition report of an accepted answer (PLAN10 15.3).
ModelAbstained bool `json:"model_abstained,omitempty"`
Calls int `json:"calls,omitempty"`
+ // Providers counts the model calls of the case by the provider that served them, as the endpoint
+ // reports it (a run with routing pinned to a provider shows here that it stayed there).
+ Providers map[string]int `json:"providers,omitempty"`
Repairs int `json:"repairs,omitempty"`
Report []string `json:"report,omitempty"`
// LineEndings are those of the sides of a content conflict (resolve.LLMLineEndings): the llm
@@ -260,6 +263,7 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, er
usage := regionStats.Usage()
res.InputTokens, res.OutputTokens, res.ReasoningTokens = usage.InputTokens, usage.OutputTokens, usage.ReasoningTokens
res.Calls, res.Repairs = regionStats.Calls()
+ res.Providers = regionStats.Providers()
if err != nil {
var unresolved *resolve.UnresolvedError
if !errors.As(err, &unresolved) || len(unresolved.Failures) != 1 {
diff --git a/tools/graft-agents/resolve/llm_core.go b/tools/graft-agents/resolve/llm_core.go
index 914ff1b..95ba381 100644
--- a/tools/graft-agents/resolve/llm_core.go
+++ b/tools/graft-agents/resolve/llm_core.go
@@ -174,6 +174,31 @@ type LLMRegionStats struct {
calls, repairs int
abstained string
report []string
+ providers map[string]int
+}
+
+// RecordProvider counts a call served by the named provider (empty when the endpoint did not say).
+func (s *LLMRegionStats) RecordProvider(name string) {
+ if name == "" {
+ name = "(not reported)"
+ }
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.providers == nil {
+ s.providers = map[string]int{}
+ }
+ s.providers[name]++
+}
+
+// Providers returns, by provider, the calls recorded.
+func (s *LLMRegionStats) Providers() map[string]int {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ out := make(map[string]int, len(s.providers))
+ for k, v := range s.providers {
+ out[k] = v
+ }
+ return out
}
// RecordCalls adds the model calls of a file and those of them that were repair rounds.
diff --git a/tools/graft-agents/resolve/llm_patch.go b/tools/graft-agents/resolve/llm_patch.go
index 5963a0c..8af2e7a 100644
--- a/tools/graft-agents/resolve/llm_patch.go
+++ b/tools/graft-agents/resolve/llm_patch.go
@@ -620,6 +620,7 @@ func llmResolvePatch(ctx context.Context, p llm.Provider, f LLMFile, opt LLMOpti
calls++
if stats != nil {
stats.RecordUsage(resp.Usage)
+ stats.RecordProvider(resp.Provider)
}
model = llmModel(resp, p)
return resp, nil