Git designed for AI agents.
graft-resolve bench: read OpenRouter's real pricing shape
Federico Eliaca18987ec8b9
diff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.goindex 2fefbe3..b353f4d 100644--- a/tools/graft-agents/cmd/graft-resolve/bench.go+++ b/tools/graft-agents/cmd/graft-resolve/bench.go@@ -177,7 +177,7 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) intfor m := range distinct {model = m}- p, err := bench.OpenRouterPricing(ctx, &http.Client{Timeout: 30 * time.Second}, *openRouterAPI, model)+ p, err := bench.OpenRouterPricing(ctx, &http.Client{Timeout: 30 * time.Second}, *openRouterAPI, model, res.LLMMaxPromptTokens())if err != nil {fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)return 1diff --git a/tools/graft-agents/resolve/bench/pricing.go b/tools/graft-agents/resolve/bench/pricing.goindex 9590d28..aab96f2 100644--- a/tools/graft-agents/resolve/bench/pricing.go+++ b/tools/graft-agents/resolve/bench/pricing.go@@ -49,17 +49,33 @@ type openRouterEndpoints struct {Data struct {ID string `json:"id"`Endpoints []struct {- Name string `json:"name"`- Pricing map[string]string `json:"pricing"`+ Name string `json:"name"`+ Pricing map[string]json.RawMessage `json:"pricing"`} `json:"endpoints"`} `json:"data"`}+// openRouterPrice reads one price, given as a JSON string (OpenRouter's usual form) or number.+func openRouterPrice(raw json.RawMessage) (float64, error) {+ var s string+ if err := json.Unmarshal(raw, &s); err == nil {+ return strconv.ParseFloat(s, 64)+ }+ var f float64+ if err := json.Unmarshal(raw, &f); err != nil {+ return 0, fmt.Errorf("%s is neither a string nor a number", raw)+ }+ return f, nil+}+// OpenRouterPricing reads the prices of model from OpenRouter's endpoints API// (GET <api>/models/<model>/endpoints, public). It fails when the model is unknown, has no// endpoint, or an endpoint lacks a prompt or completion price, or lists one that is not a// non-negative number: the run must not start on a price it cannot read.-func OpenRouterPricing(ctx context.Context, client *http.Client, api, model string) (Pricing, error) {+//+// maxPromptTokens is an upper bound of the prompt of one call; an override tier whose+// min_prompt_tokens is above it is left out, since it cannot apply. 0 counts every tier.+func OpenRouterPricing(ctx context.Context, client *http.Client, api, model string, maxPromptTokens int) (Pricing, error) {parts := strings.Split(model, "/")if len(parts) != 2 || parts[0] == "" || parts[1] == "" {return Pricing{}, fmt.Errorf("pricing: model %q is not author/slug", model)@@ -89,31 +105,62 @@ func OpenRouterPricing(ctx context.Context, client *http.Client, api, model strireturn Pricing{}, fmt.Errorf("pricing: %s lists no endpoint for %s", source, model)}+ // The prices used are prompt, completion, internal_reasoning and request. "overrides" lists+ // tiers (for very long prompts, say) with prices of their own: those that can apply count+ // towards the highest price too. A discount is ignored, which can only overstate the cost, and so are the prices of+ // what the run never uses (web search, images, cache reads and writes).p := Pricing{Model: model, Source: source, FetchedAt: time.Now().UTC(), Endpoints: len(doc.Data.Endpoints)}- for _, e := range doc.Data.Endpoints {+ take := func(endpoint string, prices map[string]json.RawMessage, required bool) error {for _, f := range []struct {key stringrequired booldst *float64}{- {"prompt", true, &p.Prompt},- {"completion", true, &p.Completion},+ {"prompt", required, &p.Prompt},+ {"completion", required, &p.Completion},{"internal_reasoning", false, &p.Reasoning},{"request", false, &p.Request},} {- raw, ok := e.Pricing[f.key]+ raw, ok := prices[f.key]if !ok {if f.required {- return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s has no %s price", e.Name, model, f.key)+ return fmt.Errorf("pricing: endpoint %q of %s has no %s price", endpoint, model, f.key)}continue}- v, err := strconv.ParseFloat(raw, 64)+ v, err := openRouterPrice(raw)if err != nil || v < 0 || math.IsNaN(v) || math.IsInf(v, 0) {- return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s has the %s price %q, not a non-negative number", e.Name, model, f.key, raw)+ return fmt.Errorf("pricing: endpoint %q of %s has the %s price %s, not a non-negative number", endpoint, model, f.key, raw)}*f.dst = math.Max(*f.dst, v)}+ return nil+ }+ for _, e := range doc.Data.Endpoints {+ if err := take(e.Name, e.Pricing, true); err != nil {+ return Pricing{}, err+ }+ if raw, ok := e.Pricing["overrides"]; ok {+ var tiers []map[string]json.RawMessage+ if err := json.Unmarshal(raw, &tiers); err != nil {+ return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s: overrides: %w", e.Name, model, err)+ }+ for _, tier := range tiers {+ // A tier that starts above the largest possible prompt never applies.+ if raw, ok := tier["min_prompt_tokens"]; ok && maxPromptTokens > 0 {+ min, err := openRouterPrice(raw)+ if err != nil {+ return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s: min_prompt_tokens %s: %w", e.Name, model, raw, err)+ }+ if min > float64(maxPromptTokens) {+ continue+ }+ }+ if err := take(e.Name+" (override)", tier, false); err != nil {+ return Pricing{}, err+ }+ }+ }}if p.Prompt == 0 && p.Completion == 0 {return Pricing{}, errors.New("pricing: " + model + " is listed as free; refusing to measure on a price of zero")diff --git a/tools/graft-agents/resolve/bench/pricing_test.go b/tools/graft-agents/resolve/bench/pricing_test.goindex aa74fda..31c43fd 100644--- a/tools/graft-agents/resolve/bench/pricing_test.go+++ b/tools/graft-agents/resolve/bench/pricing_test.go@@ -18,6 +18,13 @@ func TestOpenRouterPricing(t *testing.T) {"/models/a/variable/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"-1","completion":"-1"}}]}}`,"/models/a/free/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"0","completion":"0"}}]}}`,"/models/a/junk/endpoints": `<html>`,+ // The shape OpenRouter serves (2026-10): a numeric discount, prices of features the run+ // does not use, and override tiers for long prompts.+ "/models/a/real/endpoints": `{"data":{"id":"a/real","endpoints":[+ {"name":"p1","pricing":{"prompt":"0.0000001","completion":"0.0000005","web_search":"0.01","input_cache_read":"0.00000001","discount":0,+ "overrides":[{"min_prompt_tokens":272000,"prompt":"0.000003","completion":"0.000004"}]}},+ {"name":"p2","pricing":{"prompt":"0.0000002","completion":0.000001,"discount":0.5}}]}}`,+ "/models/a/badtier/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"0.1","completion":"0.1","overrides":[{"prompt":"-3"}]}}]}}`,}srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {body, ok := bodies[r.URL.Path]@@ -29,7 +36,7 @@ func TestOpenRouterPricing(t *testing.T) {}))defer srv.Close()- p, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL+"/", "a/ok")+ p, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL+"/", "a/ok", 0)if err != nil {t.Fatal(err)}@@ -41,7 +48,27 @@ func TestOpenRouterPricing(t *testing.T) {t.Fatalf("prices %+v", prices)}+ // Prompts of at most 100k tokens never reach p1's tier (from 272k tokens): the highest prices+ // are p2's, prompt 2e-7 and completion 1e-6 (a number); the discount and the unused prices+ // change nothing.+ real, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL, "a/real", 100_000)+ if err != nil {+ t.Fatal(err)+ }+ if real.Prompt != 0.0000002 || real.Completion != 0.000001 || real.Reasoning != 0 || real.Request != 0 {+ t.Fatalf("pricing %+v", real)+ }+ // Without a bound every tier counts.+ all, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL, "a/real", 0)+ if err != nil {+ t.Fatal(err)+ }+ if all.Prompt != 0.000003 || all.Completion != 0.000004 {+ t.Fatalf("pricing with every tier %+v", all)+ }+for model, want := range map[string]string{+ "a/badtier": "(override)\" of a/badtier has the prompt price","a/none": "no endpoint","a/nocomp": "no completion price","a/variable": "not a non-negative number",@@ -50,7 +77,7 @@ func TestOpenRouterPricing(t *testing.T) {"a/missing": "HTTP 404","no-slash": "not author/slug",} {- if _, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL, model); err == nil || !strings.Contains(err.Error(), want) {+ if _, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL, model, 0); err == nil || !strings.Contains(err.Error(), want) {t.Errorf("%s: err = %v, want %q", model, err, want)}}diff --git a/tools/graft-agents/resolve/budget_strategy.go b/tools/graft-agents/resolve/budget_strategy.goindex 2bc03fd..88e08c2 100644--- a/tools/graft-agents/resolve/budget_strategy.go+++ b/tools/graft-agents/resolve/budget_strategy.go@@ -121,6 +121,20 @@ func (r *Resolver) LLMModels() map[string]string {return out}+// LLMMaxPromptTokens returns an upper bound of the input tokens of one model call of r's llm+// strategies, counted like the budget's worst case: one token per byte. A prompt holds file data+// within max_prompt_bytes (whole file or one region), the two commit messages, the system prompt,+// the schema and the framing; 64 KiB covers all but the file data.+func (r *Resolver) LLMMaxPromptTokens() int {+ n := 0+ for _, bd := range r.bindings {+ if s, ok := bd.Strategy.(*llmStrategy); ok {+ n = max(n, s.opt.MaxPromptBytes+2*llmMaxMessageBytes+64<<10)+ }+ }+ return n+}+// Budgets returns the budgets of r's strategies, by strategy name.func (r *Resolver) Budgets() map[string]*Budget {out := map[string]*Budget{}diff --git a/tools/graft-agents/resolve/budget_test.go b/tools/graft-agents/resolve/budget_test.goindex 1a801c7..c4ac6eb 100644--- a/tools/graft-agents/resolve/budget_test.go+++ b/tools/graft-agents/resolve/budget_test.go@@ -332,6 +332,20 @@ func TestLLMStrategyBudgetConfigRejects(t *testing.T) {}}+func TestResolverLLMMaxPromptTokens(t *testing.T) {+ r := wp19Build(t, wp19LLMConfig)+ if got, want := r.LLMMaxPromptTokens(), defaultLLMMaxPromptBytes+2*llmMaxMessageBytes+64<<10; got != want {+ t.Fatalf("bound = %d, want %d", got, want)+ }+ r = wp19Build(t, strings.Replace(wp19LLMConfig, "type = \"llm\"\n", "type = \"llm\"\nmax_prompt_bytes = 1000\n", 1))+ if got := r.LLMMaxPromptTokens(); got != 1000+2*llmMaxMessageBytes+64<<10 {+ t.Fatalf("bound with max_prompt_bytes 1000 = %d", got)+ }+ if n := wp19Build(t, "[[strategy]]\ntype = \"union\"\npaths = [\"*.txt\"]\n").LLMMaxPromptTokens(); n != 0 {+ t.Fatalf("bound without llm = %d", n)+ }+}+func TestResolverUseBudget(t *testing.T) {wp19SetAnswer(func(context.Context, llm.Request) (*llm.Response, error) {res := wp19Answer("merged\n")