Git designed for AI agents.
graft-resolve: monthly AI spending cap, and the benchmark's hard cap (WP4)
Federico Elia297499dbaddd
diff --git a/doc/graft/RESOLVE.md b/doc/graft/RESOLVE.mdindex 66982aa..181d3a0 100644--- a/doc/graft/RESOLVE.md+++ b/doc/graft/RESOLVE.md@@ -303,6 +303,55 @@ Turn AI on only if you accept that the resolution is produced by a model.Every AI resolution carries the `Graft-Resolved-By` trailer, so it can beaudited with `graft log` and reverted with `graft-verify` if it breaks tests.+### Monthly AI spending cap++An `llm` strategy can carry a monthly cap:++```toml+[strategy.llm]+type = "openai"+base_url = "https://openrouter.ai/api/v1"+api_key_env = "OPENROUTER_API_KEY"+model = "openai/gpt-6.1-sol"+max_tokens = 8000 # required with a budget: it bounds a call's output++[strategy.llm.budget]+monthly_usd = 50 # the cap; 0 or absent: none, the spend is still tracked+input_usd_per_million = 2.0 # the model's prices; keep them current+output_usd_per_million = 10.0 # output tokens, reasoning included+request_usd = 0 # optional: a fixed price per call+state = "/var/lib/graft/resolve-budget.json"+```++Before each model call graft-resolve books the call's worst case: its input+counted as one token per byte of the prompt (an upper bound), plus a margin,+and `max_tokens` of output. If that would take the month's spend past the+cap, the call is not made. The file then fails with the reason "AI budget+exhausted", which the server puts in the conflict package for the agent.+After the call the worst case is replaced by the cost of the usage the+provider reports. A provider that reports no usage, or a call that fails, is+left at its worst case. So the recorded spend is never below the real one.+A new calendar month (UTC) starts at zero.++The spend is kept in the `state` file (`{"month": "2026-10", "spent_usd":+...}`). Every update takes an exclusive `flock` on `<state>.lock` and+rewrites the file atomically (temporary file, fsync, rename), so the spend+survives restarts and is shared by every graft-resolve process that uses the+same file. The state file needs a platform with `flock` (Linux, macOS, the+BSDs). The prices are configuration values: graft-resolve does not look them+up, and they should be set from the provider's price page.++The spend is reported in three places:++- In HTTP mode, `GET /metrics` (with the same bearer token) serves+ `graft_resolve_ai_spend_usd` (gauge, the month's spend),+ `graft_resolve_ai_budget_usd` (gauge, the cap) and+ `graft_resolve_ai_budget_exhausted_total` (counter, the calls not made),+ labelled by strategy.+- In command mode, the spend is in the state file.+- In both modes, the answer's `explanation` gives the spend of each strategy+ that answered.+## Bounds and failure modes| Limit | Default | Where |diff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.goindex 3bea1bc..2fefbe3 100644--- a/tools/graft-agents/cmd/graft-resolve/bench.go+++ b/tools/graft-agents/cmd/graft-resolve/bench.go@@ -7,12 +7,15 @@ import ("flag""fmt""io"+ "net/http""os""os/signal""path/filepath""strings""syscall"+ "time"+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve""github.com/feder-cr/git-ai/tools/graft-agents/resolve/bench""github.com/feder-cr/git-ai/tools/graft-agents/resolve/syntax")@@ -20,6 +23,8 @@ import (// runBench runs the benchmark subcommands://// graft-resolve bench mine --corpus FILE --clones DIR --manifest OUT [--max-cases-per-repo N]+// graft-resolve bench run ... --max-usd USD [--pricing openrouter|fixed]+// [--input-usd-per-million P --output-usd-per-million P]// graft-resolve bench run --config FILE --manifest FILE --clones DIR [--sample N] [--seed S]// [--label L] [--json-out FILE] [--md-out FILE]func runBench(args []string, stdout, stderr io.Writer) int {@@ -120,6 +125,11 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) intlabel := fs.String("label", "", "name of the run in the report")jsonOut := fs.String("json-out", "", "report as JSON")mdOut := fs.String("md-out", "", "report as Markdown")+ maxUSD := fs.Float64("max-usd", 0, "hard cap on the AI spend of the run in USD (required when the chain has an llm strategy)")+ pricingSource := fs.String("pricing", "openrouter", "where the model's price comes from: openrouter (read at run time) or fixed (the two price flags)")+ inputPrice := fs.Float64("input-usd-per-million", 0, "input price, with --pricing fixed")+ outputPrice := fs.Float64("output-usd-per-million", 0, "output price, reasoning included, with --pricing fixed")+ openRouterAPI := fs.String("openrouter-api", bench.DefaultOpenRouterAPI, "OpenRouter API base, for --pricing openrouter")if err := fs.Parse(args); err != nil {return 2}@@ -143,6 +153,54 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) intreturn 1}+ // A chain with a model runs under a hard cap: every call reserves its worst case first+ // (resolve.Budget), and the calls that would pass the cap are not made.+ var budget *resolve.Budget+ var pricing *bench.Pricing+ if models := res.LLMModels(); len(models) > 0 {+ if *maxUSD <= 0 {+ fmt.Fprintln(stderr, "graft-resolve bench run: the chain calls a model, so --max-usd is required")+ return 2+ }+ var prices resolve.Prices+ switch *pricingSource {+ case "openrouter":+ distinct := map[string]bool{}+ for _, m := range models {+ distinct[m] = true+ }+ if len(distinct) != 1 {+ fmt.Fprintf(stderr, "graft-resolve bench run: --pricing openrouter prices one model, the chain has %v\n", models)+ return 2+ }+ var model string+ for m := range distinct {+ model = m+ }+ p, err := bench.OpenRouterPricing(ctx, &http.Client{Timeout: 30 * time.Second}, *openRouterAPI, model)+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }+ pricing, prices = &p, p.Prices()+ case "fixed":+ prices = resolve.Prices{InputPerMillion: *inputPrice, OutputPerMillion: *outputPrice}+ default:+ fmt.Fprintf(stderr, "graft-resolve bench run: unknown --pricing %q, want openrouter or fixed\n", *pricingSource)+ return 2+ }+ if budget, err = resolve.NewBudget(*maxUSD, prices, ""); err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }+ if _, err := res.UseBudget(budget); err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }+ fmt.Fprintf(stderr, "cap $%.2f; $%g per million input tokens, $%g per million output tokens, $%g per request\n",+ *maxUSD, prices.InputPerMillion, prices.OutputPerMillion, prices.PerRequest)+ }+cases := bench.Sample(manifest.Cases, *sample, *seed)runner := bench.Runner{Resolver: res,@@ -153,6 +211,14 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int}results, runErr := runner.Run(ctx, cases)report := bench.NewReport(*label, results)+ if budget != nil {+ snap, err := budget.Snapshot()+ if err != nil {+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)+ return 1+ }+ report.SpendUSD, report.MaxUSD, report.Pricing = snap.SpentUSD, *maxUSD, pricing+ }if *jsonOut != "" {data, err := json.MarshalIndent(report, "", " ")diff --git a/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go b/tools/graft-agents/cmd/graft-resolve/bench_budget_test.gonew file mode 100644index 0000000..a2fd51e--- /dev/null+++ b/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go@@ -0,0 +1,143 @@+//go:build cgo++package main++import (+ "bytes"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/feder-cr/git-ai/tools/graft-agents/resolve/bench"+)++// modelConfig writes a resolve.toml whose llm strategy runs a program that answers the fixture's+// conflict with the human resolution, and reports no usage.+func modelConfig(t *testing.T, work string) string {+ t.Helper()+ answer := filepath.Join(work, "answer.sh")+ script := "#!/bin/sh\ncat >/dev/null\nprintf '{\"content\":\"x = 2 + 3\\\\n\",\"explanation\":\"bench\"}'\n"+ if err := os.WriteFile(answer, []byte(script), 0o755); err != nil {+ t.Fatal(err)+ }+ cfg := filepath.Join(work, "resolve.toml")+ toml := "[[strategy]]\ntype = \"llm\"\n\n[strategy.llm]\ntype = \"command\"\ncommand = [\"" + answer + "\"]\nmodel = \"openai/gpt-test\"\nmax_tokens = 1000\n"+ if err := os.WriteFile(cfg, []byte(toml), 0o644); err != nil {+ t.Fatal(err)+ }+ return cfg+}++// openRouterFake serves the endpoints of openai/gpt-test, two of them at different prices.+func openRouterFake(t *testing.T) *httptest.Server {+ t.Helper()+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/models/openai/gpt-test/endpoints" {+ http.NotFound(w, r)+ return+ }+ _, _ = w.Write([]byte(`{"data":{"id":"openai/gpt-test","endpoints":[+ {"name":"a","pricing":{"prompt":"0.000002","completion":"0.00001","request":"0"}},+ {"name":"b","pricing":{"prompt":"0.000003","completion":"0.000008","internal_reasoning":"0.000001"}}]}}`))+ }))+}++func runBenchJSON(t *testing.T, args ...string) (int, bench.Report, string) {+ t.Helper()+ var stdout, stderr bytes.Buffer+ jsonOut := filepath.Join(t.TempDir(), "report.json")+ code := run(append([]string{"bench", "run", "--json-out", jsonOut}, args...), nil, &stdout, &stderr)+ var rep bench.Report+ if code == 0 {+ raw, err := os.ReadFile(jsonOut)+ if err != nil {+ t.Fatal(err)+ }+ if err := json.Unmarshal(raw, &rep); err != nil {+ t.Fatal(err)+ }+ }+ return code, rep, stdout.String() + stderr.String()+}++func TestBenchRunWithModelNeedsCap(t *testing.T) {+ work, _, clones, manifest := mineFixture(t)+ cfg := modelConfig(t, work)+ code, _, out := runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones)+ if code != 2 || !strings.Contains(out, "--max-usd is required") {+ t.Fatalf("exit %d: %s", code, out)+ }+}++func TestBenchRunPricesFromOpenRouterAndCaps(t *testing.T) {+ work, _, clones, manifest := mineFixture(t)+ cfg := modelConfig(t, work)+ api := openRouterFake(t)+ defer api.Close()++ code, rep, out := runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones,+ "--max-usd", "1", "--openrouter-api", api.URL, "--label", "model")+ if code != 0 {+ t.Fatalf("exit %d: %s", code, out)+ }+ if rep.Total.Correct != 1 || rep.MaxUSD != 1 || rep.SpendUSD <= 0 || rep.SpendUSD > 1 {+ t.Fatalf("report %+v", rep)+ }+ // The worst case of every kind over the endpoints: prompt 3e-6, completion 1e-5,+ // internal reasoning 1e-6.+ p := rep.Pricing+ if p == nil || p.Model != "openai/gpt-test" || p.Endpoints != 2 || p.Prompt != 0.000003 || p.Completion != 0.00001 || p.Reasoning != 0.000001 {+ t.Fatalf("pricing %+v", p)+ }+ // No usage reported: the spend is the reservation, the input bytes plus 1000 output tokens+ // at $11 per million.+ if rep.SpendUSD < 1000*11e-6 {+ t.Fatalf("spend %v is below the output worst case", rep.SpendUSD)+ }++ // A cap below one call's worst case: the call is not made.+ code, rep, out = runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones,+ "--max-usd", "0.001", "--openrouter-api", api.URL)+ if code != 0 {+ t.Fatalf("exit %d: %s", code, out)+ }+ if rep.Total.NotRun != 1 || rep.SpendUSD != 0 {+ t.Fatalf("report %+v", rep)+ }+ if !strings.Contains(out, "1 not run: the cap was reached") {+ t.Fatalf("markdown: %s", out)+ }+}++func TestBenchRunRefusesUnreadablePrice(t *testing.T) {+ work, _, clones, manifest := mineFixture(t)+ cfg := modelConfig(t, work)+ api := openRouterFake(t)+ defer api.Close()+ toml, _ := os.ReadFile(cfg)+ if err := os.WriteFile(cfg, []byte(strings.Replace(string(toml), "openai/gpt-test", "openai/unknown", 1)), 0o644); err != nil {+ t.Fatal(err)+ }+ code, _, out := runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones, "--max-usd", "1", "--openrouter-api", api.URL)+ if code != 1 || !strings.Contains(out, "pricing:") || !strings.Contains(out, "HTTP 404") {+ t.Fatalf("exit %d: %s", code, out)+ }+}++func TestBenchRunFixedPricing(t *testing.T) {+ work, _, clones, manifest := mineFixture(t)+ cfg := modelConfig(t, work)+ code, rep, out := runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones,+ "--max-usd", "1", "--pricing", "fixed", "--input-usd-per-million", "1", "--output-usd-per-million", "1")+ if code != 0 || rep.Pricing != nil || rep.Total.Correct != 1 {+ t.Fatalf("exit %d, report %+v: %s", code, rep, out)+ }+ code, _, out = runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones, "--max-usd", "1", "--pricing", "fixed")+ if code != 1 || !strings.Contains(out, "no price is set") {+ t.Fatalf("fixed pricing without prices: exit %d: %s", code, out)+ }+}diff --git a/tools/graft-agents/cmd/graft-resolve/bench_test.go b/tools/graft-agents/cmd/graft-resolve/bench_test.goindex 34afbc9..ec6aefc 100644--- a/tools/graft-agents/cmd/graft-resolve/bench_test.go+++ b/tools/graft-agents/cmd/graft-resolve/bench_test.go@@ -26,9 +26,10 @@ func gitIn(t *testing.T, dir string, args ...string) string {return strings.TrimSpace(string(out))}-// TestBenchMineAndRun mines a repository with one hand-resolved conflict through a file:// clone-// and runs the default chain (trivial only) on it, which abstains.-func TestBenchMineAndRun(t *testing.T) {+// mineFixture mines a repository with one hand-resolved conflict (a.py, resolved "x = 2 + 3")+// through a file:// clone and returns the work directory, the corpus, the clones and the manifest.+func mineFixture(t *testing.T) (work, corpus, clones, manifest string) {+ t.Helper()if _, err := exec.LookPath("git"); err != nil {t.Skip("git is not installed")}@@ -53,13 +54,13 @@ func TestBenchMineAndRun(t *testing.T) {gitIn(t, src, "commit", "-q", "-m", "merge feature")head := gitIn(t, src, "rev-parse", "HEAD")- work := t.TempDir()- corpus := filepath.Join(work, "corpus.json")+ work = t.TempDir()+ corpus = filepath.Join(work, "corpus.json")data, _ := json.Marshal(bench.Corpus{Repos: []bench.Repo{{Name: "local/fixture", URL: "file://" + src, Commit: head, License: "MIT"}}})if err := os.WriteFile(corpus, data, 0o644); err != nil {t.Fatal(err)}- clones, manifest := filepath.Join(work, "clones"), filepath.Join(work, "manifest.json")+ clones, manifest = filepath.Join(work, "clones"), filepath.Join(work, "manifest.json")var stdout, stderr bytes.Bufferif code := run([]string{"bench", "mine", "--corpus", corpus, "--clones", clones, "--manifest", manifest}, nil, &stdout, &stderr); code != 0 {@@ -68,6 +69,14 @@ func TestBenchMineAndRun(t *testing.T) {if !strings.Contains(stdout.String(), "1 cases in") {t.Fatalf("mine output: %s", stdout.String())}+ return work, corpus, clones, manifest+}++// TestBenchMineAndRun mines the fixture, resumes the mining from its part, and runs the default+// chain (trivial only) on it, which abstains.+func TestBenchMineAndRun(t *testing.T) {+ work, corpus, clones, manifest := mineFixture(t)+ var stdout, stderr bytes.Buffer// A second run takes the repository from its part, without the clone.moved := filepath.Join(work, "moved")diff --git a/tools/graft-agents/cmd/graft-resolve/metrics_test.go b/tools/graft-agents/cmd/graft-resolve/metrics_test.gonew file mode 100644index 0000000..0cb1b89--- /dev/null+++ b/tools/graft-agents/cmd/graft-resolve/metrics_test.go@@ -0,0 +1,84 @@+package main++import (+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"+)++func TestServeMetrics(t *testing.T) {+ state := filepath.Join(t.TempDir(), "budget.json")+ month := time.Now().UTC().Format("2006-01")+ if err := os.WriteFile(state, []byte(`{"month":"`+month+`","spent_usd":1.25}`), 0o600); err != nil {+ t.Fatal(err)+ }+ cfg := `+[[strategy]]+type = "llm"+name = "ai"++[strategy.llm]+type = "command"+command = ["true"]+max_tokens = 1000++[strategy.llm.budget]+monthly_usd = 20+input_usd_per_million = 2+output_usd_per_million = 10+state = "` + state + `"+`+ srv := httptest.NewServer(newTestHandler(t, testToken, cfg))+ defer srv.Close()++ get := func(token string) (*http.Response, string) {+ req, err := http.NewRequest(http.MethodGet, srv.URL+"/metrics", nil)+ if err != nil {+ t.Fatal(err)+ }+ if token != "" {+ req.Header.Set("Authorization", "Bearer "+token)+ }+ resp, err := http.DefaultClient.Do(req)+ if err != nil {+ t.Fatal(err)+ }+ return resp, string(readAll(t, resp))+ }++ if resp, _ := get(""); resp.StatusCode != http.StatusUnauthorized {+ t.Fatalf("metrics without the token: status %d", resp.StatusCode)+ }+ resp, body := get(testToken)+ if resp.StatusCode != http.StatusOK || !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/plain") {+ t.Fatalf("status %d, content type %q", resp.StatusCode, resp.Header.Get("Content-Type"))+ }+ for _, want := range []string{+ "# TYPE graft_resolve_ai_spend_usd gauge\n",+ `graft_resolve_ai_spend_usd{strategy="ai"} 1.25` + "\n",+ `graft_resolve_ai_budget_usd{strategy="ai"} 20` + "\n",+ "# TYPE graft_resolve_ai_budget_exhausted_total counter\n",+ `graft_resolve_ai_budget_exhausted_total{strategy="ai"} 0` + "\n",+ } {+ if !strings.Contains(body, want) {+ t.Fatalf("metrics lack %q:\n%s", want, body)+ }+ }+}++func TestServeMetricsWithoutBudget(t *testing.T) {+ srv := httptest.NewServer(newTestHandler(t, "", ""))+ defer srv.Close()+ resp, err := http.Get(srv.URL + "/metrics")+ if err != nil {+ t.Fatal(err)+ }+ body := string(readAll(t, resp))+ if resp.StatusCode != http.StatusOK || !strings.Contains(body, "# TYPE graft_resolve_ai_spend_usd gauge") || strings.Contains(body, "{strategy=") {+ t.Fatalf("status %d:\n%s", resp.StatusCode, body)+ }+}diff --git a/tools/graft-agents/cmd/graft-resolve/serve.go b/tools/graft-agents/cmd/graft-resolve/serve.goindex 5600c54..1a3de45 100644--- a/tools/graft-agents/cmd/graft-resolve/serve.go+++ b/tools/graft-agents/cmd/graft-resolve/serve.go@@ -5,8 +5,11 @@ import ("crypto/subtle""encoding/json""errors"+ "fmt""io""net/http"+ "sort"+ "strconv""strings""time"@@ -34,8 +37,20 @@ func newServer(listen string, res *resolve.Resolver, token string) (*http.Server// that the server can be configured with any URL. When token is non-empty the// request must carry "Authorization: Bearer <token>", compared in constant// time.+//+// GET /metrics, with the same token, answers the AI budget metrics in the+// Prometheus text format (writeMetrics).func newHandler(res *resolve.Resolver, token string) http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Method == http.MethodGet && r.URL.Path == "/metrics" {+ if token != "" && !authorized(r, token) {+ w.Header().Set("WWW-Authenticate", `Bearer realm="graft-resolve"`)+ writeError(w, http.StatusUnauthorized, "unauthorized")+ return+ }+ writeMetrics(w, res)+ return+ }if r.Method != http.MethodPost {w.Header().Set("Allow", http.MethodPost)writeError(w, http.StatusMethodNotAllowed, "method not allowed")@@ -83,6 +98,50 @@ func newHandler(res *resolve.Resolver, token string) http.Handler {})}+// writeMetrics writes the AI budget of every llm strategy that has one, by+// strategy name: the month's spend (read from the state file, so it includes+// the other processes that share it), the cap (0: none), and the calls this+// process did not make because of the cap.+func writeMetrics(w http.ResponseWriter, res *resolve.Resolver) {+ budgets := res.Budgets()+ names := make([]string, 0, len(budgets))+ for name := range budgets {+ names = append(names, name)+ }+ sort.Strings(names)++ var b strings.Builder+ metrics := []struct {+ name, typ, help string+ value func(resolve.BudgetSnapshot) string+ }{+ {"graft_resolve_ai_spend_usd", "gauge", "AI spend of the current month (UTC) in USD.",+ func(s resolve.BudgetSnapshot) string { return strconv.FormatFloat(s.SpentUSD, 'f', -1, 64) }},+ {"graft_resolve_ai_budget_usd", "gauge", "Monthly AI spending cap in USD (0: none).",+ func(s resolve.BudgetSnapshot) string { return strconv.FormatFloat(s.MonthlyUSD, 'f', -1, 64) }},+ {"graft_resolve_ai_budget_exhausted_total", "counter", "Model calls not made because the monthly cap was reached.",+ func(s resolve.BudgetSnapshot) string { return strconv.FormatInt(s.Exhausted, 10) }},+ }+ snaps := make(map[string]resolve.BudgetSnapshot, len(names))+ for _, name := range names {+ s, err := budgets[name].Snapshot()+ if err != nil {+ writeError(w, http.StatusInternalServerError, err.Error())+ return+ }+ snaps[name] = s+ }+ for _, m := range metrics {+ fmt.Fprintf(&b, "# HELP %s %s\n# TYPE %s %s\n", m.name, m.help, m.name, m.typ)+ for _, name := range names {+ fmt.Fprintf(&b, "%s{strategy=%q} %s\n", m.name, name, m.value(snaps[name]))+ }+ }+ w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")+ w.WriteHeader(http.StatusOK)+ _, _ = io.WriteString(w, b.String())+}+func authorized(r *http.Request, token string) bool {const prefix = "Bearer "auth := r.Header.Get("Authorization")diff --git a/tools/graft-agents/resolve/bench/pricing.go b/tools/graft-agents/resolve/bench/pricing.gonew file mode 100644index 0000000..9590d28--- /dev/null+++ b/tools/graft-agents/resolve/bench/pricing.go@@ -0,0 +1,122 @@+package bench++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io"+ "math"+ "net/http"+ "net/url"+ "strconv"+ "strings"+ "time"++ "github.com/feder-cr/git-ai/tools/graft-agents/resolve"+)++// DefaultOpenRouterAPI is the base of OpenRouter's API.+const DefaultOpenRouterAPI = "https://openrouter.ai/api/v1"++// Pricing is the price of the measured model, as read at run time (never from memory, PLAN10 12):+// the highest price of each kind over the model's endpoints, in USD per token, since OpenRouter+// may route a call to any of them.+type Pricing struct {+ Model string `json:"model"`+ Source string `json:"source"`+ FetchedAt time.Time `json:"fetched_at"`+ Endpoints int `json:"endpoints"`+ // Prompt, Completion and Reasoning are per token; Request is per call.+ Prompt float64 `json:"prompt_usd_per_token"`+ Completion float64 `json:"completion_usd_per_token"`+ Reasoning float64 `json:"internal_reasoning_usd_per_token"`+ Request float64 `json:"request_usd"`+}++// Prices returns the budget prices of p. Every output token is priced at the completion price+// plus the internal-reasoning price: a reasoning token is reported as output, and charging both+// prices on every output token can only overstate the cost.+func (p Pricing) Prices() resolve.Prices {+ return resolve.Prices{+ InputPerMillion: p.Prompt * 1e6,+ OutputPerMillion: (p.Completion + p.Reasoning) * 1e6,+ PerRequest: p.Request,+ }+}++type openRouterEndpoints struct {+ Data struct {+ ID string `json:"id"`+ Endpoints []struct {+ Name string `json:"name"`+ Pricing map[string]string `json:"pricing"`+ } `json:"endpoints"`+ } `json:"data"`+}++// OpenRouterPricing reads the prices of model from OpenRouter's endpoints API+// (GET <api>/models/<model>/endpoints, public). It fails when the model is unknown, has no+// endpoint, or an endpoint lacks a prompt or completion price, or lists one that is not a+// non-negative number: the run must not start on a price it cannot read.+func OpenRouterPricing(ctx context.Context, client *http.Client, api, model string) (Pricing, error) {+ parts := strings.Split(model, "/")+ if len(parts) != 2 || parts[0] == "" || parts[1] == "" {+ return Pricing{}, fmt.Errorf("pricing: model %q is not author/slug", model)+ }+ source := strings.TrimRight(api, "/") + "/models/" + url.PathEscape(parts[0]) + "/" + url.PathEscape(parts[1]) + "/endpoints"+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)+ if err != nil {+ return Pricing{}, err+ }+ resp, err := client.Do(req)+ if err != nil {+ return Pricing{}, fmt.Errorf("pricing: %w", err)+ }+ defer resp.Body.Close()+ body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))+ if err != nil {+ return Pricing{}, fmt.Errorf("pricing: %w", err)+ }+ if resp.StatusCode != http.StatusOK {+ return Pricing{}, fmt.Errorf("pricing: %s: HTTP %d: %.200s", source, resp.StatusCode, body)+ }+ var doc openRouterEndpoints+ if err := json.Unmarshal(body, &doc); err != nil {+ return Pricing{}, fmt.Errorf("pricing: %s: %w", source, err)+ }+ if len(doc.Data.Endpoints) == 0 {+ return Pricing{}, fmt.Errorf("pricing: %s lists no endpoint for %s", source, model)+ }++ p := Pricing{Model: model, Source: source, FetchedAt: time.Now().UTC(), Endpoints: len(doc.Data.Endpoints)}+ for _, e := range doc.Data.Endpoints {+ for _, f := range []struct {+ key string+ required bool+ dst *float64+ }{+ {"prompt", true, &p.Prompt},+ {"completion", true, &p.Completion},+ {"internal_reasoning", false, &p.Reasoning},+ {"request", false, &p.Request},+ } {+ raw, ok := e.Pricing[f.key]+ if !ok {+ if f.required {+ return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s has no %s price", e.Name, model, f.key)+ }+ continue+ }+ v, err := strconv.ParseFloat(raw, 64)+ if err != nil || v < 0 || math.IsNaN(v) || math.IsInf(v, 0) {+ return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s has the %s price %q, not a non-negative number", e.Name, model, f.key, raw)+ }+ *f.dst = math.Max(*f.dst, v)+ }+ }+ if p.Prompt == 0 && p.Completion == 0 {+ return Pricing{}, errors.New("pricing: " + model + " is listed as free; refusing to measure on a price of zero")+ }+ return p, nil+}diff --git a/tools/graft-agents/resolve/bench/pricing_test.go b/tools/graft-agents/resolve/bench/pricing_test.gonew file mode 100644index 0000000..aa74fda--- /dev/null+++ b/tools/graft-agents/resolve/bench/pricing_test.go@@ -0,0 +1,57 @@+package bench++import (+ "context"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"+)++func TestOpenRouterPricing(t *testing.T) {+ bodies := map[string]string{+ "/models/a/ok/endpoints": `{"data":{"endpoints":[+ {"name":"x","pricing":{"prompt":"0.000001","completion":"0.000004"}},+ {"name":"y","pricing":{"prompt":"0.0000015","completion":"0.000002","request":"0.0001","internal_reasoning":"0.000003"}}]}}`,+ "/models/a/none/endpoints": `{"data":{"endpoints":[]}}`,+ "/models/a/nocomp/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"0.000001"}}]}}`,+ "/models/a/variable/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"-1","completion":"-1"}}]}}`,+ "/models/a/free/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"0","completion":"0"}}]}}`,+ "/models/a/junk/endpoints": `<html>`,+ }+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ body, ok := bodies[r.URL.Path]+ if !ok {+ http.NotFound(w, r)+ return+ }+ _, _ = w.Write([]byte(body))+ }))+ defer srv.Close()++ p, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL+"/", "a/ok")+ if err != nil {+ t.Fatal(err)+ }+ if p.Prompt != 0.0000015 || p.Completion != 0.000004 || p.Reasoning != 0.000003 || p.Request != 0.0001 || p.Endpoints != 2 {+ t.Fatalf("pricing %+v, want the highest of each price", p)+ }+ prices := p.Prices()+ if prices.InputPerMillion != 1.5 || prices.OutputPerMillion != 7 || prices.PerRequest != 0.0001 {+ t.Fatalf("prices %+v", prices)+ }++ for model, want := range map[string]string{+ "a/none": "no endpoint",+ "a/nocomp": "no completion price",+ "a/variable": "not a non-negative number",+ "a/free": "listed as free",+ "a/junk": "invalid character",+ "a/missing": "HTTP 404",+ "no-slash": "not author/slug",+ } {+ if _, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL, model); err == nil || !strings.Contains(err.Error(), want) {+ t.Errorf("%s: err = %v, want %q", model, err, want)+ }+ }+}diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.goindex f8b4070..3c68209 100644--- a/tools/graft-agents/resolve/bench/report.go+++ b/tools/graft-agents/resolve/bench/report.go@@ -5,6 +5,7 @@ import ("io""sort""strings"+ "time")// Counts aggregates the outcomes of a set of cases.@@ -68,9 +69,13 @@ type Report struct {Families map[string]Counts `json:"families"`// NotSent counts the cases that were not sent, by reason.NotSent map[string]int `json:"not_sent"`- // Spend is the AI spend of the run in USD, when a budget was used.- SpendUSD float64 `json:"spend_usd,omitempty"`- Results []Result `json:"results"`+ // SpendUSD is the AI spend of the run in USD, when a budget was used: the usage the provider+ // reported, and the worst case of the calls whose usage was not reported. MaxUSD is the cap.+ SpendUSD float64 `json:"spend_usd,omitempty"`+ MaxUSD float64 `json:"max_usd,omitempty"`+ // Pricing is the price the spend was computed with, as read at run time.+ Pricing *Pricing `json:"pricing,omitempty"`+ Results []Result `json:"results"`}// NewReport aggregates results.@@ -92,11 +97,19 @@ func NewReport(label string, results []Result) Report {func (rep Report) WriteMarkdown(w io.Writer) error {var b strings.Builderfmt.Fprintf(&b, "## graft-resolve bench: %s\n\n", rep.Label)- fmt.Fprintf(&b, "Cases run: %d", rep.Total.Cases)- if rep.SpendUSD > 0 {- fmt.Fprintf(&b, ", AI spend $%.2f", rep.SpendUSD)+ fmt.Fprintf(&b, "Cases sampled: %d; run: %d", rep.Total.Cases, rep.Total.Cases-rep.Total.NotRun)+ if rep.Total.NotRun > 0 {+ fmt.Fprintf(&b, " (%d not run: the cap was reached)", rep.Total.NotRun)+ }+ if rep.MaxUSD > 0 {+ fmt.Fprintf(&b, "; AI spend $%.4f of the $%.2f cap", rep.SpendUSD, rep.MaxUSD)}b.WriteString(".\n\n")+ if p := rep.Pricing; p != nil {+ fmt.Fprintf(&b, "Model %s, priced at run time (%s, %s, highest of %d endpoints): $%g per million input tokens, $%g per million output tokens (completion $%g + internal reasoning $%g), $%g per request.\n\n",+ p.Model, p.Source, p.FetchedAt.Format(time.RFC3339), p.Endpoints,+ p.Prompt*1e6, (p.Completion+p.Reasoning)*1e6, p.Completion*1e6, p.Reasoning*1e6, p.Request)+ }b.WriteString("| family | cases | correct (exact) | wrong | rejected by checks | not sent | model error | not run (cap) | abstained | error rate |\n")b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")families := make([]string, 0, len(rep.Families))diff --git a/tools/graft-agents/resolve/budget.go b/tools/graft-agents/resolve/budget.gonew file mode 100644index 0000000..8414de2--- /dev/null+++ b/tools/graft-agents/resolve/budget.go@@ -0,0 +1,306 @@+package resolve++import (+ "context"+ "encoding/json"+ "errors"+ "fmt"+ "io/fs"+ "math"+ "os"+ "path/filepath"+ "strconv"+ "sync"+ "sync/atomic"+ "time"++ "github.com/feder-cr/git-ai/tools/graft-agents/llm"+)++// Prices are the prices of a model in USD. OutputPerMillion applies to every output token,+// reasoning included (providers count reasoning tokens as output).+type Prices struct {+ InputPerMillion float64+ OutputPerMillion float64+ // PerRequest is a fixed price per call, when the provider charges one.+ PerRequest float64+}++func (p Prices) cost(inputTokens, outputTokens int) float64 {+ return float64(inputTokens)*p.InputPerMillion/1e6 + float64(outputTokens)*p.OutputPerMillion/1e6 + p.PerRequest+}++func (p Prices) validate() error {+ for _, v := range []float64{p.InputPerMillion, p.OutputPerMillion, p.PerRequest} {+ if v < 0 || math.IsNaN(v) || math.IsInf(v, 0) {+ return fmt.Errorf("invalid price %v", v)+ }+ }+ if p.InputPerMillion == 0 && p.OutputPerMillion == 0 && p.PerRequest == 0 {+ return errors.New("no price is set")+ }+ return nil+}++// Budget tracks the AI spend of a calendar month (UTC) and caps it (PLAN10 3.5). Before every+// model call the worst case of the call is reserved: the spend is raised by it, and the call is+// not made when that would exceed the cap. After the call the reservation is replaced by the+// cost of the usage the provider reports. So the recorded spend is never below the real one, and+// a process that dies between the two steps leaves its reservation booked.+//+// With a state file the spend survives restarts and is shared by every graft-resolve process+// that uses the file: each update holds an exclusive lock (flock on state + ".lock"), reads the+// file, and writes it again atomically (temporary file, fsync, rename, fsync of the directory).+// Without a state file the spend lives in memory (the benchmark).+type Budget struct {+ // MonthlyUSD is the cap; 0 means no cap (the spend is still tracked).+ MonthlyUSD float64+ Prices Prices++ state string+ now func() time.Time++ mu sync.Mutex+ mem budgetState+ calls atomic.Int64+ // exhausted counts the calls refused for the cap.+ exhausted atomic.Int64+}++type budgetState struct {+ Month string `json:"month"`+ SpentUSD float64 `json:"spent_usd"`+}++// NewBudget returns a budget with the cap monthlyUSD (0: none) and the prices of the model,+// kept in stateFile ("" keeps it in memory).+func NewBudget(monthlyUSD float64, prices Prices, stateFile string) (*Budget, error) {+ if monthlyUSD < 0 || math.IsNaN(monthlyUSD) || math.IsInf(monthlyUSD, 0) {+ return nil, fmt.Errorf("budget: invalid monthly cap %v", monthlyUSD)+ }+ if err := prices.validate(); err != nil {+ return nil, fmt.Errorf("budget: %w", err)+ }+ if stateFile != "" {+ if !filepath.IsAbs(stateFile) {+ return nil, fmt.Errorf("budget: state file %q must be an absolute path", stateFile)+ }+ if err := checkFlock(); err != nil {+ return nil, err+ }+ }+ return &Budget{MonthlyUSD: monthlyUSD, Prices: prices, state: stateFile, now: time.Now}, nil+}++func (b *Budget) month() string {+ return b.now().UTC().Format("2006-01")+}++// update applies fn to the state of the current month, under the lock, and stores the result+// when fn succeeds.+func (b *Budget) update(fn func(st *budgetState) error) (budgetState, error) {+ b.mu.Lock()+ defer b.mu.Unlock()+ month := b.month()++ if b.state == "" {+ st := b.mem+ if st.Month != month {+ st = budgetState{Month: month}+ }+ if err := fn(&st); err != nil {+ return st, err+ }+ b.mem = st+ return st, nil+ }++ unlock, err := lockFile(b.state + ".lock")+ if err != nil {+ return budgetState{}, err+ }+ defer unlock()+ st, err := readBudgetState(b.state)+ if err != nil {+ return st, err+ }+ if st.Month != month {+ st = budgetState{Month: month}+ }+ if err := fn(&st); err != nil {+ return st, err+ }+ return st, writeBudgetState(b.state, st)+}++func readBudgetState(file string) (budgetState, error) {+ var st budgetState+ data, err := os.ReadFile(file)+ if errors.Is(err, fs.ErrNotExist) {+ return st, nil+ }+ if err != nil {+ return st, fmt.Errorf("budget: %w", err)+ }+ if err := json.Unmarshal(data, &st); err != nil {+ return st, fmt.Errorf("budget: state file %s: %w", file, err)+ }+ if st.SpentUSD < 0 || math.IsNaN(st.SpentUSD) || math.IsInf(st.SpentUSD, 0) {+ return st, fmt.Errorf("budget: state file %s: invalid spend %v", file, st.SpentUSD)+ }+ return st, nil+}++func writeBudgetState(file string, st budgetState) error {+ data, err := json.Marshal(st)+ if err != nil {+ return err+ }+ dir := filepath.Dir(file)+ tmp, err := os.CreateTemp(dir, filepath.Base(file)+".tmp-*")+ if err != nil {+ return fmt.Errorf("budget: %w", err)+ }+ defer func() { _ = os.Remove(tmp.Name()) }()+ if _, err := tmp.Write(append(data, '\n')); err != nil {+ _ = tmp.Close()+ return fmt.Errorf("budget: %w", err)+ }+ if err := tmp.Sync(); err != nil {+ _ = tmp.Close()+ return fmt.Errorf("budget: %w", err)+ }+ if err := tmp.Close(); err != nil {+ return fmt.Errorf("budget: %w", err)+ }+ if err := os.Rename(tmp.Name(), file); err != nil {+ return fmt.Errorf("budget: %w", err)+ }+ d, err := os.Open(dir)+ if err != nil {+ return fmt.Errorf("budget: %w", err)+ }+ defer d.Close()+ if err := d.Sync(); err != nil {+ return fmt.Errorf("budget: %w", err)+ }+ return nil+}++// Reservation is the worst case of one call, booked until the call's usage replaces it.+type Reservation struct {+ b *Budget+ month string+ usd float64+}++// Reserve books the worst case of a call with at most inputTokens of input and maxOutputTokens of+// output. It fails with ErrBudgetExhausted, without booking anything, when the month's spend plus+// the worst case would exceed the cap.+func (b *Budget) Reserve(inputTokens, maxOutputTokens int) (*Reservation, error) {+ worst := b.Prices.cost(inputTokens, maxOutputTokens)+ st, err := b.update(func(st *budgetState) error {+ // The margin absorbs the rounding of sums of float prices (49 calls of $0.01 add up+ // to a little more than $0.49); it lets the spend pass the cap by a nano-dollar at most.+ if b.MonthlyUSD > 0 && st.SpentUSD+worst > b.MonthlyUSD+1e-9 {+ return ErrBudgetExhausted+ }+ st.SpentUSD += worst+ return nil+ })+ if errors.Is(err, ErrBudgetExhausted) {+ b.exhausted.Add(1)+ return nil, fmt.Errorf("%w: $%.4f spent of $%.2f this month, the next call may cost up to $%.4f", ErrBudgetExhausted, st.SpentUSD, b.MonthlyUSD, worst)+ }+ if err != nil {+ return nil, err+ }+ b.calls.Add(1)+ return &Reservation{b: b, month: st.Month, usd: worst}, nil+}++// Book replaces the reservation with the cost of the usage the provider reported. A usage with no+// tokens (the provider reports none, or the call failed) keeps the reservation. A reservation of+// a month that has ended stays in that month.+func (r *Reservation) Book(u llm.Usage) error {+ if r == nil || (u.InputTokens == 0 && u.OutputTokens == 0) {+ return nil+ }+ actual := r.b.Prices.cost(u.InputTokens, u.OutputTokens)+ _, err := r.b.update(func(st *budgetState) error {+ if st.Month != r.month {+ return nil+ }+ st.SpentUSD = math.Max(0, st.SpentUSD+actual-r.usd)+ return nil+ })+ return err+}++// BudgetSnapshot is the state of a budget, for metrics and reports.+type BudgetSnapshot struct {+ Month string+ SpentUSD float64+ MonthlyUSD float64+ Calls int64+ Exhausted int64+}++// Snapshot returns the spend of the current month.+func (b *Budget) Snapshot() (BudgetSnapshot, error) {+ st, err := b.update(func(*budgetState) error { return errReadOnly })+ if err != nil && !errors.Is(err, errReadOnly) {+ return BudgetSnapshot{}, err+ }+ return BudgetSnapshot{Month: st.Month, SpentUSD: st.SpentUSD, MonthlyUSD: b.MonthlyUSD, Calls: b.calls.Load(), Exhausted: b.exhausted.Load()}, nil+}++var errReadOnly = errors.New("read only")++// String describes the spend for an explanation.+func (s BudgetSnapshot) String() string {+ limit := "no cap"+ if s.MonthlyUSD > 0 {+ limit = "cap $" + strconv.FormatFloat(s.MonthlyUSD, 'f', 2, 64)+ }+ return fmt.Sprintf("AI spend %s: $%.4f (%s)", s.Month, s.SpentUSD, limit)+}++// budgetProvider makes every call of p go through the budget.+type budgetProvider struct {+ p llm.Provider+ b *Budget+}++func (bp budgetProvider) Name() string { return bp.p.Name() }++// Complete reserves the worst case of the call, makes it, and books the usage. The input is+// bounded by its size in bytes (a token of the providers' byte-level tokenizers covers at least+// one byte), plus the schema, which a provider may also put into the prompt, and a margin for+// the message framing. The output is bounded by req.MaxTokens, which a budget requires.+func (bp budgetProvider) Complete(ctx context.Context, req llm.Request) (*llm.Response, error) {+ if req.MaxTokens <= 0 {+ return nil, errors.New("budget: a call without max_tokens has no worst case")+ }+ input := len(req.System) + len(req.Prompt) + 256+ if req.JSONSchema != nil {+ schema, err := json.Marshal(req.JSONSchema)+ if err != nil {+ return nil, err+ }+ input += 2 * len(schema)+ }+ res, err := bp.b.Reserve(input, req.MaxTokens)+ if err != nil {+ return nil, err+ }+ resp, err := bp.p.Complete(ctx, req)+ var usage llm.Usage+ if resp != nil {+ usage = resp.Usage+ }+ if bookErr := res.Book(usage); bookErr != nil && err == nil {+ return nil, bookErr+ }+ return resp, err+}diff --git a/tools/graft-agents/resolve/budget_flock.go b/tools/graft-agents/resolve/budget_flock.gonew file mode 100644index 0000000..8a8e31c--- /dev/null+++ b/tools/graft-agents/resolve/budget_flock.go@@ -0,0 +1,33 @@+//go:build linux || darwin || freebsd || netbsd || openbsd || dragonfly++package resolve++import (+ "fmt"+ "os"+ "syscall"+)++func checkFlock() error { return nil }++// lockFile takes an exclusive flock on file, creating it, and returns the unlock function.+func lockFile(file string) (func(), error) {+ f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0o600)+ if err != nil {+ return nil, fmt.Errorf("budget: %w", err)+ }+ for {+ err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX)+ if err != syscall.EINTR {+ break+ }+ }+ if err != nil {+ _ = f.Close()+ return nil, fmt.Errorf("budget: locking %s: %w", file, err)+ }+ return func() {+ _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)+ _ = f.Close()+ }, nil+}diff --git a/tools/graft-agents/resolve/budget_noflock.go b/tools/graft-agents/resolve/budget_noflock.gonew file mode 100644index 0000000..7f062a1--- /dev/null+++ b/tools/graft-agents/resolve/budget_noflock.go@@ -0,0 +1,11 @@+//go:build !(linux || darwin || freebsd || netbsd || openbsd || dragonfly)++package resolve++import "errors"++var errNoFlock = errors.New("budget: a state file needs flock, which this platform lacks")++func checkFlock() error { return errNoFlock }++func lockFile(string) (func(), error) { return nil, errNoFlock }diff --git a/tools/graft-agents/resolve/budget_strategy.go b/tools/graft-agents/resolve/budget_strategy.gonew file mode 100644index 0000000..2bc03fd--- /dev/null+++ b/tools/graft-agents/resolve/budget_strategy.go@@ -0,0 +1,133 @@+package resolve++import (+ "errors"+ "fmt"+ "sort"+)++// The llm strategy's budget is the [strategy.llm.budget] table:+//+// [strategy.llm.budget]+// monthly_usd = 50 # the cap; 0 or absent: none (the spend is still tracked)+// input_usd_per_million = 2.0 # the model's prices, kept current by the operator+// output_usd_per_million = 10.0 # output tokens, reasoning included+// request_usd = 0 # optional: a fixed price per call+// state = "/var/lib/graft/resolve-budget.json"+//+// The table needs max_tokens in [strategy.llm], which bounds the worst case of a call. When the+// month's spend plus the worst case of a call would exceed the cap, the call is not made and the+// file fails with "AI budget exhausted", which the server puts into the conflict package.++// llmBudgetConfig decodes the optional budget table of the [strategy.llm] table.+func llmBudgetConfig(spec StrategySpec, tbl map[string]any) (*Budget, error) {+ raw, ok := tbl["budget"]+ if !ok {+ return nil, nil+ }+ bt, ok := raw.(map[string]any)+ if !ok {+ return nil, fmt.Errorf("strategy %q: llm: budget must be a table", spec.Name)+ }+ var unknown []string+ for k := range bt {+ switch k {+ case "monthly_usd", "input_usd_per_million", "output_usd_per_million", "request_usd", "state":+ default:+ unknown = append(unknown, k)+ }+ }+ if len(unknown) > 0 {+ sort.Strings(unknown)+ return nil, fmt.Errorf("strategy %q: llm.budget: unknown keys %v", spec.Name, unknown)+ }++ number := func(key string) (float64, error) {+ switch v := bt[key].(type) {+ case nil:+ return 0, nil+ case int64:+ if v >= 0 {+ return float64(v), nil+ }+ case float64:+ if v >= 0 {+ return v, nil+ }+ }+ return 0, fmt.Errorf("strategy %q: llm.budget: %s must be a non-negative number", spec.Name, key)+ }+ var prices Prices+ var monthly float64+ var err error+ for _, f := range []struct {+ key string+ dst *float64+ }{+ {"monthly_usd", &monthly},+ {"input_usd_per_million", &prices.InputPerMillion},+ {"output_usd_per_million", &prices.OutputPerMillion},+ {"request_usd", &prices.PerRequest},+ } {+ if *f.dst, err = number(f.key); err != nil {+ return nil, err+ }+ }+ state, err := stringField(bt, "state")+ if err != nil {+ return nil, fmt.Errorf("strategy %q: llm.budget: %w", spec.Name, err)+ }+ if state == "" {+ return nil, fmt.Errorf("strategy %q: llm.budget: state is required: the spend must survive restarts", spec.Name)+ }+ b, err := NewBudget(monthly, prices, state)+ if err != nil {+ return nil, fmt.Errorf("strategy %q: llm.%w", spec.Name, err)+ }+ return b, nil+}++// UseBudget makes every llm strategy of r go through b, in place of a configured budget, and+// returns how many there are. It must be called before the first Resolve. Each llm strategy+// needs max_tokens.+func (r *Resolver) UseBudget(b *Budget) (int, error) {+ if b == nil {+ return 0, errors.New("resolve: nil budget")+ }+ n := 0+ for _, bd := range r.bindings {+ s, ok := bd.Strategy.(*llmStrategy)+ if !ok {+ continue+ }+ if s.opt.MaxTokens <= 0 {+ return 0, fmt.Errorf("strategy %q: a budget needs max_tokens in [strategy.llm]", bd.Name)+ }+ s.budget = b+ s.provider = budgetProvider{p: s.base, b: b}+ n+++ }+ return n, nil+}++// LLMModels returns the configured model of each llm strategy of r, by strategy name.+func (r *Resolver) LLMModels() map[string]string {+ out := map[string]string{}+ for _, bd := range r.bindings {+ if s, ok := bd.Strategy.(*llmStrategy); ok {+ out[bd.Name] = s.model+ }+ }+ return out+}++// Budgets returns the budgets of r's strategies, by strategy name.+func (r *Resolver) Budgets() map[string]*Budget {+ out := map[string]*Budget{}+ for _, bd := range r.bindings {+ if s, ok := bd.Strategy.(*llmStrategy); ok && s.budget != nil {+ out[bd.Name] = s.budget+ }+ }+ return out+}diff --git a/tools/graft-agents/resolve/budget_test.go b/tools/graft-agents/resolve/budget_test.gonew file mode 100644index 0000000..12ab0aa--- /dev/null+++ b/tools/graft-agents/resolve/budget_test.go@@ -0,0 +1,356 @@+package resolve++import (+ "context"+ "encoding/json"+ "errors"+ "math"+ "os"+ "path/filepath"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/feder-cr/git-ai/tools/graft-agents/llm"+)++// wp4Prices costs $1 per million input tokens and $10 per million output tokens.+var wp4Prices = Prices{InputPerMillion: 1, OutputPerMillion: 10}++func wp4Near(a, b float64) bool { return math.Abs(a-b) < 1e-9 }++func wp4Spent(t *testing.T, b *Budget) float64 {+ t.Helper()+ s, err := b.Snapshot()+ if err != nil {+ t.Fatal(err)+ }+ return s.SpentUSD+}++func TestBudgetReserveAndBook(t *testing.T) {+ b, err := NewBudget(1, wp4Prices, "")+ if err != nil {+ t.Fatal(err)+ }+ // Worst case: 100k input ($0.10) + 10k output ($0.10).+ r, err := b.Reserve(100_000, 10_000)+ if err != nil {+ t.Fatal(err)+ }+ if got := wp4Spent(t, b); !wp4Near(got, 0.2) {+ t.Fatalf("spent after reserve = %v, want the worst case 0.2", got)+ }+ if err := r.Book(llm.Usage{InputTokens: 30_000, OutputTokens: 2_000}); err != nil {+ t.Fatal(err)+ }+ if got := wp4Spent(t, b); !wp4Near(got, 0.05) {+ t.Fatalf("spent after book = %v, want the real cost 0.05", got)+ }+}++func TestBudgetRefusesBeyondCap(t *testing.T) {+ b, _ := NewBudget(0.25, wp4Prices, "")+ if _, err := b.Reserve(100_000, 10_000); err != nil { // 0.20+ t.Fatal(err)+ }+ _, err := b.Reserve(100_000, 10_000) // 0.40 > 0.25+ if !errors.Is(err, ErrBudgetExhausted) {+ t.Fatalf("err = %v, want ErrBudgetExhausted", err)+ }+ if !strings.Contains(err.Error(), "AI budget exhausted") {+ t.Fatalf("message %q", err)+ }+ s, _ := b.Snapshot()+ if !wp4Near(s.SpentUSD, 0.2) || s.Exhausted != 1 || s.Calls != 1 {+ t.Fatalf("snapshot %+v: a refused call must book nothing", s)+ }+ // A smaller call still fits.+ if _, err := b.Reserve(10_000, 1_000); err != nil {+ t.Fatalf("small call refused: %v", err)+ }+}++func TestBudgetUnknownUsageKeepsReservation(t *testing.T) {+ b, _ := NewBudget(0, wp4Prices, "")+ r, _ := b.Reserve(100_000, 10_000)+ if err := r.Book(llm.Usage{}); err != nil {+ t.Fatal(err)+ }+ if got := wp4Spent(t, b); !wp4Near(got, 0.2) {+ t.Fatalf("spent = %v, want the reservation kept", got)+ }+}++func TestBudgetNoCapTracksSpend(t *testing.T) {+ b, _ := NewBudget(0, wp4Prices, "")+ for i := 0; i < 100; i++ {+ if _, err := b.Reserve(1_000_000, 0); err != nil {+ t.Fatalf("call %d refused without a cap: %v", i, err)+ }+ }+ if got := wp4Spent(t, b); !wp4Near(got, 100) {+ t.Fatalf("spent = %v", got)+ }+}++func TestBudgetStateFileSurvivesRestartAndIsShared(t *testing.T) {+ state := filepath.Join(t.TempDir(), "budget.json")+ a, err := NewBudget(1, wp4Prices, state)+ if err != nil {+ t.Fatal(err)+ }+ r, _ := a.Reserve(100_000, 10_000)+ _ = r.Book(llm.Usage{InputTokens: 100_000, OutputTokens: 0}) // 0.10++ // Another process (or a restart) sees the spend.+ b, _ := NewBudget(1, wp4Prices, state)+ if got := wp4Spent(t, b); !wp4Near(got, 0.1) {+ t.Fatalf("restarted budget spent = %v, want 0.1", got)+ }+ if _, err := b.Reserve(800_000, 0); err != nil { // 0.90 total+ t.Fatal(err)+ }+ if _, err := a.Reserve(200_000, 0); !errors.Is(err, ErrBudgetExhausted) { // 1.10 > 1+ t.Fatalf("err = %v: the other process's spend was not seen", err)+ }++ var st budgetState+ data, err := os.ReadFile(state)+ if err != nil {+ t.Fatal(err)+ }+ if err := json.Unmarshal(data, &st); err != nil || st.Month != time.Now().UTC().Format("2006-01") || !wp4Near(st.SpentUSD, 0.9) {+ t.Fatalf("state %s (%v)", data, err)+ }+ if matches, _ := filepath.Glob(state + ".tmp-*"); len(matches) != 0 {+ t.Fatalf("temporary files left: %v", matches)+ }+}++func TestBudgetConcurrentReservationsNeverExceedCap(t *testing.T) {+ state := filepath.Join(t.TempDir(), "budget.json")+ // Two budgets on one file stand for two processes; each call's worst case is $0.01.+ budgets := make([]*Budget, 2)+ for i := range budgets {+ var err error+ if budgets[i], err = NewBudget(0.5, wp4Prices, state); err != nil {+ t.Fatal(err)+ }+ }+ var wg sync.WaitGroup+ var mu sync.Mutex+ granted := 0+ for i := 0; i < 80; i++ {+ wg.Add(1)+ go func(b *Budget) {+ defer wg.Done()+ if _, err := b.Reserve(10_000, 0); err == nil {+ mu.Lock()+ granted+++ mu.Unlock()+ }+ }(budgets[i%2])+ }+ wg.Wait()+ if granted != 50 {+ t.Fatalf("granted %d calls of $0.01 under a $0.50 cap, want 50", granted)+ }+ if got := wp4Spent(t, budgets[0]); !wp4Near(got, 0.5) {+ t.Fatalf("spent = %v", got)+ }+}++func TestBudgetNewMonthStartsAtZero(t *testing.T) {+ b, _ := NewBudget(1, wp4Prices, filepath.Join(t.TempDir(), "budget.json"))+ now := time.Date(2026, 10, 31, 23, 0, 0, 0, time.UTC)+ b.now = func() time.Time { return now }+ r, err := b.Reserve(900_000, 0)+ if err != nil {+ t.Fatal(err)+ }+ now = time.Date(2026, 11, 1, 0, 30, 0, 0, time.UTC)+ if got := wp4Spent(t, b); got != 0 {+ t.Fatalf("new month spent = %v, want 0", got)+ }+ // A call reserved in October and booked in November leaves November alone.+ if err := r.Book(llm.Usage{InputTokens: 1, OutputTokens: 1}); err != nil {+ t.Fatal(err)+ }+ if got := wp4Spent(t, b); got != 0 {+ t.Fatalf("November spent = %v after booking an October call", got)+ }+}++func TestNewBudgetRejects(t *testing.T) {+ for name, tc := range map[string]struct {+ monthly float64+ prices Prices+ state string+ }{+ "negative cap": {monthly: -1, prices: wp4Prices},+ "no price": {monthly: 1},+ "negative price": {monthly: 1, prices: Prices{InputPerMillion: -1, OutputPerMillion: 1}},+ "relative state": {monthly: 1, prices: wp4Prices, state: "budget.json"},+ } {+ if _, err := NewBudget(tc.monthly, tc.prices, tc.state); err == nil {+ t.Errorf("%s accepted", name)+ }+ }+}++// wp4Provider answers with a fixed usage and counts its calls.+type wp4Provider struct {+ mu sync.Mutex+ calls int+ usage llm.Usage+}++func (p *wp4Provider) Name() string { return "wp4:fake" }++func (p *wp4Provider) Complete(context.Context, llm.Request) (*llm.Response, error) {+ p.mu.Lock()+ p.calls+++ p.mu.Unlock()+ b, _ := json.Marshal(map[string]string{"content": "x\n", "explanation": "wp4"})+ return &llm.Response{Text: string(b), JSON: b, Usage: p.usage}, nil+}++func TestBudgetProviderReservesInputBytesAndMaxTokens(t *testing.T) {+ b, _ := NewBudget(0, Prices{InputPerMillion: 1e6}, "") // $1 per input token: the cost counts the tokens+ p := &wp4Provider{}+ bp := budgetProvider{p: p, b: b}+ if _, err := bp.Complete(context.Background(), llm.Request{Prompt: "abc"}); err == nil {+ t.Fatal("call without max_tokens accepted")+ }+ if _, err := bp.Complete(context.Background(), llm.Request{System: "s", Prompt: strings.Repeat("p", 1000), MaxTokens: 10}); err != nil {+ t.Fatal(err)+ }+ // No usage reported: the reservation stays, 1 + 1000 bytes + 256 of framing.+ if got := wp4Spent(t, b); got != 1257 {+ t.Fatalf("reserved %v input tokens, want 1257", got)+ }+}++func TestBudgetProviderDoesNotCallWhenExhausted(t *testing.T) {+ b, _ := NewBudget(0.001, wp4Prices, "")+ p := &wp4Provider{usage: llm.Usage{InputTokens: 10, OutputTokens: 10}}+ bp := budgetProvider{p: p, b: b}+ _, err := bp.Complete(context.Background(), llm.Request{Prompt: "x", MaxTokens: 1_000_000})+ if !errors.Is(err, ErrBudgetExhausted) || p.calls != 0 {+ t.Fatalf("err = %v, calls = %d: want no call", err, p.calls)+ }+}++const wp4Config = `+[[strategy]]+type = "llm"+paths = ["*.txt"]++[strategy.llm]+type = "wp19-resolve-fake"+max_tokens = 1000++[strategy.llm.budget]+monthly_usd = 5+input_usd_per_million = 2+output_usd_per_million = 10.5+state = "STATE"+`++func TestLLMStrategyBudgetConfig(t *testing.T) {+ state := filepath.Join(t.TempDir(), "budget.json")+ wp19SetAnswer(func(context.Context, llm.Request) (*llm.Response, error) {+ r := wp19Answer("merged\n")+ r.Usage = llm.Usage{InputTokens: 1000, OutputTokens: 100}+ return r, nil+ })+ r := wp19Build(t, strings.Replace(wp4Config, "STATE", state, 1))+ budgets := r.Budgets()+ b := budgets["llm"]+ if b == nil || b.MonthlyUSD != 5 || b.Prices != (Prices{InputPerMillion: 2, OutputPerMillion: 10.5}) {+ t.Fatalf("budgets %+v", budgets)+ }+ resp, err := r.Resolve(context.Background(), wp19Request(wp19Conflict("a.txt")))+ if err != nil {+ t.Fatal(err)+ }+ // 1000 × $2/M + 100 × $10.5/M = $0.00305.+ if got := wp4Spent(t, b); !wp4Near(got, 0.00305) {+ t.Fatalf("spent = %v", got)+ }+ if !strings.Contains(resp.Explanation, "llm: AI spend ") || !strings.Contains(resp.Explanation, "cap $5.00") {+ t.Fatalf("explanation %q lacks the spend", resp.Explanation)+ }+}++func TestLLMStrategyBudgetExhaustedIsTheReason(t *testing.T) {+ state := filepath.Join(t.TempDir(), "budget.json")+ if err := os.WriteFile(state, []byte(`{"month":"`+time.Now().UTC().Format("2006-01")+`","spent_usd":4.9999}`), 0o600); err != nil {+ t.Fatal(err)+ }+ fake := wp19SetAnswer(func(context.Context, llm.Request) (*llm.Response, error) {+ return wp19Answer("merged\n"), nil+ })+ r := wp19Build(t, strings.Replace(wp4Config, "STATE", state, 1))+ _, err := r.Resolve(context.Background(), wp19Request(wp19Conflict("a.txt")))+ var unresolved *UnresolvedError+ if !errors.As(err, &unresolved) {+ t.Fatalf("err = %v", err)+ }+ attempts := unresolved.Failures[0].Attempts+ if len(attempts) != 1 || !errors.Is(attempts[0].Err, ErrBudgetExhausted) {+ t.Fatalf("attempts %+v", attempts)+ }+ if !strings.Contains(strings.Join(unresolved.Failures[0].Reasons, " "), "AI budget exhausted") {+ t.Fatalf("reasons %v", unresolved.Failures[0].Reasons)+ }+ if fake.calls() != 0 {+ t.Fatalf("model called %d times past the cap", fake.calls())+ }+}++func TestLLMStrategyBudgetConfigRejects(t *testing.T) {+ state := filepath.Join(t.TempDir(), "budget.json")+ base := strings.Replace(wp4Config, "STATE", state, 1)+ for name, tc := range map[string]struct{ config, want string }{+ "no max_tokens": {strings.Replace(base, "max_tokens = 1000\n", "", 1), "needs max_tokens"},+ "no state": {strings.Replace(base, `state = "`+state+`"`+"\n", "", 1), "state is required"},+ "unknown key": {base + "currency = \"eur\"\n", "unknown keys [currency]"},+ "negative price": {strings.Replace(base, "input_usd_per_million = 2", "input_usd_per_million = -2", 1), "non-negative number"},+ "string cap": {strings.Replace(base, "monthly_usd = 5", `monthly_usd = "5"`, 1), "non-negative number"},+ } {+ t.Run(name, func(t *testing.T) {+ _, err := wp19BuildErr(tc.config)+ if err == nil || !strings.Contains(err.Error(), tc.want) {+ t.Fatalf("err = %v, want %q", err, tc.want)+ }+ })+ }+}++func TestResolverUseBudget(t *testing.T) {+ wp19SetAnswer(func(context.Context, llm.Request) (*llm.Response, error) {+ res := wp19Answer("merged\n")+ res.Usage = llm.Usage{InputTokens: 1_000_000}+ return res, nil+ })+ r := wp19Build(t, wp19LLMConfig) // max_tokens = 512+ b, _ := NewBudget(0, wp4Prices, "")+ n, err := r.UseBudget(b)+ if err != nil || n != 1 {+ t.Fatalf("UseBudget = %d, %v", n, err)+ }+ if _, err := r.Resolve(context.Background(), wp19Request(wp19Conflict("a.txt"))); err != nil {+ t.Fatal(err)+ }+ if got := wp4Spent(t, b); !wp4Near(got, 1) {+ t.Fatalf("spent = %v, want $1", got)+ }++ noLimit := wp19Build(t, "[[strategy]]\ntype = \"llm\"\n[strategy.llm]\ntype = \"wp19-resolve-fake\"\n")+ if _, err := noLimit.UseBudget(b); err == nil || !strings.Contains(err.Error(), "max_tokens") {+ t.Fatalf("UseBudget without max_tokens: %v", err)+ }+}diff --git a/tools/graft-agents/resolve/llm_strategy.go b/tools/graft-agents/resolve/llm_strategy.goindex 844816c..54db40d 100644--- a/tools/graft-agents/resolve/llm_strategy.go+++ b/tools/graft-agents/resolve/llm_strategy.go@@ -34,6 +34,9 @@ import (// [strategy.llm.options] # optional, provider specific// ...//+// [strategy.llm.budget] # optional: monthly spending cap (budget_strategy.go)+// ...+//// The strategy declines (the chain continues, no model call is made) for files// it cannot merge: a side that is absent, a symlink, a mode conflict, binary// content, a diff too large to split (the LCS table of the changed middle of@@ -54,9 +57,14 @@ func init() {// llmStrategy is immutable after construction. Resolve may run concurrently// (HTTP mode), so the provider must be safe for concurrent use.type llmStrategy struct {+ // provider is base, or base behind the budget.provider llm.Provider- opt LLMOptions- timeout time.Duration+ base llm.Provider+ budget *Budget+ // model is the configured model name.+ model string+ opt LLMOptions+ timeout time.Duration}func newLLMStrategy(spec StrategySpec) (Strategy, error) {@@ -104,7 +112,20 @@ func newLLMStrategy(spec StrategySpec) (Strategy, error) {if err != nil {return nil, fmt.Errorf("strategy %q: %w", spec.Name, err)}- return &llmStrategy{provider: p, opt: opt, timeout: timeout}, nil+ s := &llmStrategy{provider: p, base: p, model: cfg.Model, opt: opt, timeout: timeout}++ budget, err := llmBudgetConfig(spec, spec.Options["llm"].(map[string]any))+ if err != nil {+ return nil, err+ }+ if budget != nil {+ if opt.MaxTokens <= 0 {+ return nil, fmt.Errorf("strategy %q: llm.budget needs max_tokens in [strategy.llm]: it bounds the worst case of a call", spec.Name)+ }+ s.budget = budget+ s.provider = budgetProvider{p: p, b: budget}+ }+ return s, nil}// Resolve implements Strategy.@@ -166,7 +187,7 @@ func llmProviderConfig(spec StrategySpec) (llm.Config, error) {var unknown []stringfor k := range tbl {switch k {- case "type", "model", "base_url", "api_key_env", "command", "timeout", "max_tokens", "effort", "options":+ case "type", "model", "base_url", "api_key_env", "command", "timeout", "max_tokens", "effort", "options", "budget":default:unknown = append(unknown, k)}diff --git a/tools/graft-agents/resolve/resolve.go b/tools/graft-agents/resolve/resolve.goindex fecbfff..558a0b6 100644--- a/tools/graft-agents/resolve/resolve.go+++ b/tools/graft-agents/resolve/resolve.go@@ -299,6 +299,15 @@ func (r *Resolver) Resolve(ctx context.Context, req protocol.Request) (protocol.if len(notes) > 0 {resp.Explanation = "resolved " + strings.Join(notes, ", ")}+ // The month's AI spend of the strategies that answered, so that command mode reports it too.+ budgets := r.Budgets()+ for _, name := range used {+ if b := budgets[name]; b != nil {+ if snap, err := b.Snapshot(); err == nil {+ resp.Explanation += "; " + name + ": " + snap.String()+ }+ }+ }return resp, nil}