Skip to content

Git designed for AI agents.

graft-resolve bench: read OpenRouter's real pricing shape

Federico Eliaca18987ec8b9
diff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.go
index 2fefbe3..b353f4d 100644
--- a/tools/graft-agents/cmd/graft-resolve/bench.go
+++ b/tools/graft-agents/cmd/graft-resolve/bench.go
@@ -177,7 +177,7 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int
for m := range distinct {
model = m
}
- p, err := bench.OpenRouterPricing(ctx, &http.Client{Timeout: 30 * time.Second}, *openRouterAPI, model)
+ p, err := bench.OpenRouterPricing(ctx, &http.Client{Timeout: 30 * time.Second}, *openRouterAPI, model, res.LLMMaxPromptTokens())
if err != nil {
fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
return 1
diff --git a/tools/graft-agents/resolve/bench/pricing.go b/tools/graft-agents/resolve/bench/pricing.go
index 9590d28..aab96f2 100644
--- a/tools/graft-agents/resolve/bench/pricing.go
+++ b/tools/graft-agents/resolve/bench/pricing.go
@@ -49,17 +49,33 @@ type openRouterEndpoints struct {
Data struct {
ID string `json:"id"`
Endpoints []struct {
- Name string `json:"name"`
- Pricing map[string]string `json:"pricing"`
+ Name string `json:"name"`
+ Pricing map[string]json.RawMessage `json:"pricing"`
} `json:"endpoints"`
} `json:"data"`
}
+// openRouterPrice reads one price, given as a JSON string (OpenRouter's usual form) or number.
+func openRouterPrice(raw json.RawMessage) (float64, error) {
+ var s string
+ if err := json.Unmarshal(raw, &s); err == nil {
+ return strconv.ParseFloat(s, 64)
+ }
+ var f float64
+ if err := json.Unmarshal(raw, &f); err != nil {
+ return 0, fmt.Errorf("%s is neither a string nor a number", raw)
+ }
+ return f, nil
+}
+
// OpenRouterPricing reads the prices of model from OpenRouter's endpoints API
// (GET <api>/models/<model>/endpoints, public). It fails when the model is unknown, has no
// endpoint, or an endpoint lacks a prompt or completion price, or lists one that is not a
// non-negative number: the run must not start on a price it cannot read.
-func OpenRouterPricing(ctx context.Context, client *http.Client, api, model string) (Pricing, error) {
+//
+// maxPromptTokens is an upper bound of the prompt of one call; an override tier whose
+// min_prompt_tokens is above it is left out, since it cannot apply. 0 counts every tier.
+func OpenRouterPricing(ctx context.Context, client *http.Client, api, model string, maxPromptTokens int) (Pricing, error) {
parts := strings.Split(model, "/")
if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
return Pricing{}, fmt.Errorf("pricing: model %q is not author/slug", model)
@@ -89,31 +105,62 @@ func OpenRouterPricing(ctx context.Context, client *http.Client, api, model stri
return Pricing{}, fmt.Errorf("pricing: %s lists no endpoint for %s", source, model)
}
+ // The prices used are prompt, completion, internal_reasoning and request. "overrides" lists
+ // tiers (for very long prompts, say) with prices of their own: those that can apply count
+ // towards the highest price too. A discount is ignored, which can only overstate the cost, and so are the prices of
+ // what the run never uses (web search, images, cache reads and writes).
p := Pricing{Model: model, Source: source, FetchedAt: time.Now().UTC(), Endpoints: len(doc.Data.Endpoints)}
- for _, e := range doc.Data.Endpoints {
+ take := func(endpoint string, prices map[string]json.RawMessage, required bool) error {
for _, f := range []struct {
key string
required bool
dst *float64
}{
- {"prompt", true, &p.Prompt},
- {"completion", true, &p.Completion},
+ {"prompt", required, &p.Prompt},
+ {"completion", required, &p.Completion},
{"internal_reasoning", false, &p.Reasoning},
{"request", false, &p.Request},
} {
- raw, ok := e.Pricing[f.key]
+ raw, ok := prices[f.key]
if !ok {
if f.required {
- return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s has no %s price", e.Name, model, f.key)
+ return fmt.Errorf("pricing: endpoint %q of %s has no %s price", endpoint, model, f.key)
}
continue
}
- v, err := strconv.ParseFloat(raw, 64)
+ v, err := openRouterPrice(raw)
if err != nil || v < 0 || math.IsNaN(v) || math.IsInf(v, 0) {
- return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s has the %s price %q, not a non-negative number", e.Name, model, f.key, raw)
+ return fmt.Errorf("pricing: endpoint %q of %s has the %s price %s, not a non-negative number", endpoint, model, f.key, raw)
}
*f.dst = math.Max(*f.dst, v)
}
+ return nil
+ }
+ for _, e := range doc.Data.Endpoints {
+ if err := take(e.Name, e.Pricing, true); err != nil {
+ return Pricing{}, err
+ }
+ if raw, ok := e.Pricing["overrides"]; ok {
+ var tiers []map[string]json.RawMessage
+ if err := json.Unmarshal(raw, &tiers); err != nil {
+ return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s: overrides: %w", e.Name, model, err)
+ }
+ for _, tier := range tiers {
+ // A tier that starts above the largest possible prompt never applies.
+ if raw, ok := tier["min_prompt_tokens"]; ok && maxPromptTokens > 0 {
+ min, err := openRouterPrice(raw)
+ if err != nil {
+ return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s: min_prompt_tokens %s: %w", e.Name, model, raw, err)
+ }
+ if min > float64(maxPromptTokens) {
+ continue
+ }
+ }
+ if err := take(e.Name+" (override)", tier, false); err != nil {
+ return Pricing{}, err
+ }
+ }
+ }
}
if p.Prompt == 0 && p.Completion == 0 {
return Pricing{}, errors.New("pricing: " + model + " is listed as free; refusing to measure on a price of zero")
diff --git a/tools/graft-agents/resolve/bench/pricing_test.go b/tools/graft-agents/resolve/bench/pricing_test.go
index aa74fda..31c43fd 100644
--- a/tools/graft-agents/resolve/bench/pricing_test.go
+++ b/tools/graft-agents/resolve/bench/pricing_test.go
@@ -18,6 +18,13 @@ func TestOpenRouterPricing(t *testing.T) {
"/models/a/variable/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"-1","completion":"-1"}}]}}`,
"/models/a/free/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"0","completion":"0"}}]}}`,
"/models/a/junk/endpoints": `<html>`,
+ // The shape OpenRouter serves (2026-10): a numeric discount, prices of features the run
+ // does not use, and override tiers for long prompts.
+ "/models/a/real/endpoints": `{"data":{"id":"a/real","endpoints":[
+ {"name":"p1","pricing":{"prompt":"0.0000001","completion":"0.0000005","web_search":"0.01","input_cache_read":"0.00000001","discount":0,
+ "overrides":[{"min_prompt_tokens":272000,"prompt":"0.000003","completion":"0.000004"}]}},
+ {"name":"p2","pricing":{"prompt":"0.0000002","completion":0.000001,"discount":0.5}}]}}`,
+ "/models/a/badtier/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"0.1","completion":"0.1","overrides":[{"prompt":"-3"}]}}]}}`,
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, ok := bodies[r.URL.Path]
@@ -29,7 +36,7 @@ func TestOpenRouterPricing(t *testing.T) {
}))
defer srv.Close()
- p, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL+"/", "a/ok")
+ p, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL+"/", "a/ok", 0)
if err != nil {
t.Fatal(err)
}
@@ -41,7 +48,27 @@ func TestOpenRouterPricing(t *testing.T) {
t.Fatalf("prices %+v", prices)
}
+ // Prompts of at most 100k tokens never reach p1's tier (from 272k tokens): the highest prices
+ // are p2's, prompt 2e-7 and completion 1e-6 (a number); the discount and the unused prices
+ // change nothing.
+ real, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL, "a/real", 100_000)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if real.Prompt != 0.0000002 || real.Completion != 0.000001 || real.Reasoning != 0 || real.Request != 0 {
+ t.Fatalf("pricing %+v", real)
+ }
+ // Without a bound every tier counts.
+ all, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL, "a/real", 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if all.Prompt != 0.000003 || all.Completion != 0.000004 {
+ t.Fatalf("pricing with every tier %+v", all)
+ }
+
for model, want := range map[string]string{
+ "a/badtier": "(override)\" of a/badtier has the prompt price",
"a/none": "no endpoint",
"a/nocomp": "no completion price",
"a/variable": "not a non-negative number",
@@ -50,7 +77,7 @@ func TestOpenRouterPricing(t *testing.T) {
"a/missing": "HTTP 404",
"no-slash": "not author/slug",
} {
- if _, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL, model); err == nil || !strings.Contains(err.Error(), want) {
+ if _, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL, model, 0); err == nil || !strings.Contains(err.Error(), want) {
t.Errorf("%s: err = %v, want %q", model, err, want)
}
}
diff --git a/tools/graft-agents/resolve/budget_strategy.go b/tools/graft-agents/resolve/budget_strategy.go
index 2bc03fd..88e08c2 100644
--- a/tools/graft-agents/resolve/budget_strategy.go
+++ b/tools/graft-agents/resolve/budget_strategy.go
@@ -121,6 +121,20 @@ func (r *Resolver) LLMModels() map[string]string {
return out
}
+// LLMMaxPromptTokens returns an upper bound of the input tokens of one model call of r's llm
+// strategies, counted like the budget's worst case: one token per byte. A prompt holds file data
+// within max_prompt_bytes (whole file or one region), the two commit messages, the system prompt,
+// the schema and the framing; 64 KiB covers all but the file data.
+func (r *Resolver) LLMMaxPromptTokens() int {
+ n := 0
+ for _, bd := range r.bindings {
+ if s, ok := bd.Strategy.(*llmStrategy); ok {
+ n = max(n, s.opt.MaxPromptBytes+2*llmMaxMessageBytes+64<<10)
+ }
+ }
+ return n
+}
+
// Budgets returns the budgets of r's strategies, by strategy name.
func (r *Resolver) Budgets() map[string]*Budget {
out := map[string]*Budget{}
diff --git a/tools/graft-agents/resolve/budget_test.go b/tools/graft-agents/resolve/budget_test.go
index 1a801c7..c4ac6eb 100644
--- a/tools/graft-agents/resolve/budget_test.go
+++ b/tools/graft-agents/resolve/budget_test.go
@@ -332,6 +332,20 @@ func TestLLMStrategyBudgetConfigRejects(t *testing.T) {
}
}
+func TestResolverLLMMaxPromptTokens(t *testing.T) {
+ r := wp19Build(t, wp19LLMConfig)
+ if got, want := r.LLMMaxPromptTokens(), defaultLLMMaxPromptBytes+2*llmMaxMessageBytes+64<<10; got != want {
+ t.Fatalf("bound = %d, want %d", got, want)
+ }
+ r = wp19Build(t, strings.Replace(wp19LLMConfig, "type = \"llm\"\n", "type = \"llm\"\nmax_prompt_bytes = 1000\n", 1))
+ if got := r.LLMMaxPromptTokens(); got != 1000+2*llmMaxMessageBytes+64<<10 {
+ t.Fatalf("bound with max_prompt_bytes 1000 = %d", got)
+ }
+ if n := wp19Build(t, "[[strategy]]\ntype = \"union\"\npaths = [\"*.txt\"]\n").LLMMaxPromptTokens(); n != 0 {
+ t.Fatalf("bound without llm = %d", n)
+ }
+}
+
func TestResolverUseBudget(t *testing.T) {
wp19SetAnswer(func(context.Context, llm.Request) (*llm.Response, error) {
res := wp19Answer("merged\n")