Skip to content

Git designed for AI agents.

graft-resolve: monthly AI spending cap, and the benchmark's hard cap (WP4)

Federico Elia297499dbaddd
diff --git a/doc/graft/RESOLVE.md b/doc/graft/RESOLVE.md
index 66982aa..181d3a0 100644
--- a/doc/graft/RESOLVE.md
+++ b/doc/graft/RESOLVE.md
@@ -303,6 +303,55 @@ Turn AI on only if you accept that the resolution is produced by a model.
Every AI resolution carries the `Graft-Resolved-By` trailer, so it can be
audited with `graft log` and reverted with `graft-verify` if it breaks tests.
+### Monthly AI spending cap
+
+An `llm` strategy can carry a monthly cap:
+
+```toml
+[strategy.llm]
+type = "openai"
+base_url = "https://openrouter.ai/api/v1"
+api_key_env = "OPENROUTER_API_KEY"
+model = "openai/gpt-6.1-sol"
+max_tokens = 8000 # required with a budget: it bounds a call's output
+
+[strategy.llm.budget]
+monthly_usd = 50 # the cap; 0 or absent: none, the spend is still tracked
+input_usd_per_million = 2.0 # the model's prices; keep them current
+output_usd_per_million = 10.0 # output tokens, reasoning included
+request_usd = 0 # optional: a fixed price per call
+state = "/var/lib/graft/resolve-budget.json"
+```
+
+Before each model call graft-resolve books the call's worst case: its input
+counted as one token per byte of the prompt (an upper bound), plus a margin,
+and `max_tokens` of output. If that would take the month's spend past the
+cap, the call is not made. The file then fails with the reason "AI budget
+exhausted", which the server puts in the conflict package for the agent.
+After the call the worst case is replaced by the cost of the usage the
+provider reports. A provider that reports no usage, or a call that fails, is
+left at its worst case. So the recorded spend is never below the real one.
+A new calendar month (UTC) starts at zero.
+
+The spend is kept in the `state` file (`{"month": "2026-10", "spent_usd":
+...}`). Every update takes an exclusive `flock` on `<state>.lock` and
+rewrites the file atomically (temporary file, fsync, rename), so the spend
+survives restarts and is shared by every graft-resolve process that uses the
+same file. The state file needs a platform with `flock` (Linux, macOS, the
+BSDs). The prices are configuration values: graft-resolve does not look them
+up, and they should be set from the provider's price page.
+
+The spend is reported in three places:
+
+- In HTTP mode, `GET /metrics` (with the same bearer token) serves
+ `graft_resolve_ai_spend_usd` (gauge, the month's spend),
+ `graft_resolve_ai_budget_usd` (gauge, the cap) and
+ `graft_resolve_ai_budget_exhausted_total` (counter, the calls not made),
+ labelled by strategy.
+- In command mode, the spend is in the state file.
+- In both modes, the answer's `explanation` gives the spend of each strategy
+ that answered.
+
## Bounds and failure modes
| Limit | Default | Where |
diff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.go
index 3bea1bc..2fefbe3 100644
--- a/tools/graft-agents/cmd/graft-resolve/bench.go
+++ b/tools/graft-agents/cmd/graft-resolve/bench.go
@@ -7,12 +7,15 @@ import (
"flag"
"fmt"
"io"
+ "net/http"
"os"
"os/signal"
"path/filepath"
"strings"
"syscall"
+ "time"
+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve"
"github.com/feder-cr/git-ai/tools/graft-agents/resolve/bench"
"github.com/feder-cr/git-ai/tools/graft-agents/resolve/syntax"
)
@@ -20,6 +23,8 @@ import (
// runBench runs the benchmark subcommands:
//
// graft-resolve bench mine --corpus FILE --clones DIR --manifest OUT [--max-cases-per-repo N]
+// graft-resolve bench run ... --max-usd USD [--pricing openrouter|fixed]
+// [--input-usd-per-million P --output-usd-per-million P]
// graft-resolve bench run --config FILE --manifest FILE --clones DIR [--sample N] [--seed S]
// [--label L] [--json-out FILE] [--md-out FILE]
func runBench(args []string, stdout, stderr io.Writer) int {
@@ -120,6 +125,11 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int
label := fs.String("label", "", "name of the run in the report")
jsonOut := fs.String("json-out", "", "report as JSON")
mdOut := fs.String("md-out", "", "report as Markdown")
+ maxUSD := fs.Float64("max-usd", 0, "hard cap on the AI spend of the run in USD (required when the chain has an llm strategy)")
+ pricingSource := fs.String("pricing", "openrouter", "where the model's price comes from: openrouter (read at run time) or fixed (the two price flags)")
+ inputPrice := fs.Float64("input-usd-per-million", 0, "input price, with --pricing fixed")
+ outputPrice := fs.Float64("output-usd-per-million", 0, "output price, reasoning included, with --pricing fixed")
+ openRouterAPI := fs.String("openrouter-api", bench.DefaultOpenRouterAPI, "OpenRouter API base, for --pricing openrouter")
if err := fs.Parse(args); err != nil {
return 2
}
@@ -143,6 +153,54 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int
return 1
}
+ // A chain with a model runs under a hard cap: every call reserves its worst case first
+ // (resolve.Budget), and the calls that would pass the cap are not made.
+ var budget *resolve.Budget
+ var pricing *bench.Pricing
+ if models := res.LLMModels(); len(models) > 0 {
+ if *maxUSD <= 0 {
+ fmt.Fprintln(stderr, "graft-resolve bench run: the chain calls a model, so --max-usd is required")
+ return 2
+ }
+ var prices resolve.Prices
+ switch *pricingSource {
+ case "openrouter":
+ distinct := map[string]bool{}
+ for _, m := range models {
+ distinct[m] = true
+ }
+ if len(distinct) != 1 {
+ fmt.Fprintf(stderr, "graft-resolve bench run: --pricing openrouter prices one model, the chain has %v\n", models)
+ return 2
+ }
+ var model string
+ for m := range distinct {
+ model = m
+ }
+ p, err := bench.OpenRouterPricing(ctx, &http.Client{Timeout: 30 * time.Second}, *openRouterAPI, model)
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+ pricing, prices = &p, p.Prices()
+ case "fixed":
+ prices = resolve.Prices{InputPerMillion: *inputPrice, OutputPerMillion: *outputPrice}
+ default:
+ fmt.Fprintf(stderr, "graft-resolve bench run: unknown --pricing %q, want openrouter or fixed\n", *pricingSource)
+ return 2
+ }
+ if budget, err = resolve.NewBudget(*maxUSD, prices, ""); err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+ if _, err := res.UseBudget(budget); err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+ fmt.Fprintf(stderr, "cap $%.2f; $%g per million input tokens, $%g per million output tokens, $%g per request\n",
+ *maxUSD, prices.InputPerMillion, prices.OutputPerMillion, prices.PerRequest)
+ }
+
cases := bench.Sample(manifest.Cases, *sample, *seed)
runner := bench.Runner{
Resolver: res,
@@ -153,6 +211,14 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) int
}
results, runErr := runner.Run(ctx, cases)
report := bench.NewReport(*label, results)
+ if budget != nil {
+ snap, err := budget.Snapshot()
+ if err != nil {
+ fmt.Fprintf(stderr, "graft-resolve bench run: %v\n", err)
+ return 1
+ }
+ report.SpendUSD, report.MaxUSD, report.Pricing = snap.SpentUSD, *maxUSD, pricing
+ }
if *jsonOut != "" {
data, err := json.MarshalIndent(report, "", " ")
diff --git a/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go b/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go
new file mode 100644
index 0000000..a2fd51e
--- /dev/null
+++ b/tools/graft-agents/cmd/graft-resolve/bench_budget_test.go
@@ -0,0 +1,143 @@
+//go:build cgo
+
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve/bench"
+)
+
+// modelConfig writes a resolve.toml whose llm strategy runs a program that answers the fixture's
+// conflict with the human resolution, and reports no usage.
+func modelConfig(t *testing.T, work string) string {
+ t.Helper()
+ answer := filepath.Join(work, "answer.sh")
+ script := "#!/bin/sh\ncat >/dev/null\nprintf '{\"content\":\"x = 2 + 3\\\\n\",\"explanation\":\"bench\"}'\n"
+ if err := os.WriteFile(answer, []byte(script), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ cfg := filepath.Join(work, "resolve.toml")
+ toml := "[[strategy]]\ntype = \"llm\"\n\n[strategy.llm]\ntype = \"command\"\ncommand = [\"" + answer + "\"]\nmodel = \"openai/gpt-test\"\nmax_tokens = 1000\n"
+ if err := os.WriteFile(cfg, []byte(toml), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ return cfg
+}
+
+// openRouterFake serves the endpoints of openai/gpt-test, two of them at different prices.
+func openRouterFake(t *testing.T) *httptest.Server {
+ t.Helper()
+ return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/models/openai/gpt-test/endpoints" {
+ http.NotFound(w, r)
+ return
+ }
+ _, _ = w.Write([]byte(`{"data":{"id":"openai/gpt-test","endpoints":[
+ {"name":"a","pricing":{"prompt":"0.000002","completion":"0.00001","request":"0"}},
+ {"name":"b","pricing":{"prompt":"0.000003","completion":"0.000008","internal_reasoning":"0.000001"}}]}}`))
+ }))
+}
+
+func runBenchJSON(t *testing.T, args ...string) (int, bench.Report, string) {
+ t.Helper()
+ var stdout, stderr bytes.Buffer
+ jsonOut := filepath.Join(t.TempDir(), "report.json")
+ code := run(append([]string{"bench", "run", "--json-out", jsonOut}, args...), nil, &stdout, &stderr)
+ var rep bench.Report
+ if code == 0 {
+ raw, err := os.ReadFile(jsonOut)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := json.Unmarshal(raw, &rep); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return code, rep, stdout.String() + stderr.String()
+}
+
+func TestBenchRunWithModelNeedsCap(t *testing.T) {
+ work, _, clones, manifest := mineFixture(t)
+ cfg := modelConfig(t, work)
+ code, _, out := runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones)
+ if code != 2 || !strings.Contains(out, "--max-usd is required") {
+ t.Fatalf("exit %d: %s", code, out)
+ }
+}
+
+func TestBenchRunPricesFromOpenRouterAndCaps(t *testing.T) {
+ work, _, clones, manifest := mineFixture(t)
+ cfg := modelConfig(t, work)
+ api := openRouterFake(t)
+ defer api.Close()
+
+ code, rep, out := runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones,
+ "--max-usd", "1", "--openrouter-api", api.URL, "--label", "model")
+ if code != 0 {
+ t.Fatalf("exit %d: %s", code, out)
+ }
+ if rep.Total.Correct != 1 || rep.MaxUSD != 1 || rep.SpendUSD <= 0 || rep.SpendUSD > 1 {
+ t.Fatalf("report %+v", rep)
+ }
+ // The worst case of every kind over the endpoints: prompt 3e-6, completion 1e-5,
+ // internal reasoning 1e-6.
+ p := rep.Pricing
+ if p == nil || p.Model != "openai/gpt-test" || p.Endpoints != 2 || p.Prompt != 0.000003 || p.Completion != 0.00001 || p.Reasoning != 0.000001 {
+ t.Fatalf("pricing %+v", p)
+ }
+ // No usage reported: the spend is the reservation, the input bytes plus 1000 output tokens
+ // at $11 per million.
+ if rep.SpendUSD < 1000*11e-6 {
+ t.Fatalf("spend %v is below the output worst case", rep.SpendUSD)
+ }
+
+ // A cap below one call's worst case: the call is not made.
+ code, rep, out = runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones,
+ "--max-usd", "0.001", "--openrouter-api", api.URL)
+ if code != 0 {
+ t.Fatalf("exit %d: %s", code, out)
+ }
+ if rep.Total.NotRun != 1 || rep.SpendUSD != 0 {
+ t.Fatalf("report %+v", rep)
+ }
+ if !strings.Contains(out, "1 not run: the cap was reached") {
+ t.Fatalf("markdown: %s", out)
+ }
+}
+
+func TestBenchRunRefusesUnreadablePrice(t *testing.T) {
+ work, _, clones, manifest := mineFixture(t)
+ cfg := modelConfig(t, work)
+ api := openRouterFake(t)
+ defer api.Close()
+ toml, _ := os.ReadFile(cfg)
+ if err := os.WriteFile(cfg, []byte(strings.Replace(string(toml), "openai/gpt-test", "openai/unknown", 1)), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ code, _, out := runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones, "--max-usd", "1", "--openrouter-api", api.URL)
+ if code != 1 || !strings.Contains(out, "pricing:") || !strings.Contains(out, "HTTP 404") {
+ t.Fatalf("exit %d: %s", code, out)
+ }
+}
+
+func TestBenchRunFixedPricing(t *testing.T) {
+ work, _, clones, manifest := mineFixture(t)
+ cfg := modelConfig(t, work)
+ code, rep, out := runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones,
+ "--max-usd", "1", "--pricing", "fixed", "--input-usd-per-million", "1", "--output-usd-per-million", "1")
+ if code != 0 || rep.Pricing != nil || rep.Total.Correct != 1 {
+ t.Fatalf("exit %d, report %+v: %s", code, rep, out)
+ }
+ code, _, out = runBenchJSON(t, "--config", cfg, "--manifest", manifest, "--clones", clones, "--max-usd", "1", "--pricing", "fixed")
+ if code != 1 || !strings.Contains(out, "no price is set") {
+ t.Fatalf("fixed pricing without prices: exit %d: %s", code, out)
+ }
+}
diff --git a/tools/graft-agents/cmd/graft-resolve/bench_test.go b/tools/graft-agents/cmd/graft-resolve/bench_test.go
index 34afbc9..ec6aefc 100644
--- a/tools/graft-agents/cmd/graft-resolve/bench_test.go
+++ b/tools/graft-agents/cmd/graft-resolve/bench_test.go
@@ -26,9 +26,10 @@ func gitIn(t *testing.T, dir string, args ...string) string {
return strings.TrimSpace(string(out))
}
-// TestBenchMineAndRun mines a repository with one hand-resolved conflict through a file:// clone
-// and runs the default chain (trivial only) on it, which abstains.
-func TestBenchMineAndRun(t *testing.T) {
+// mineFixture mines a repository with one hand-resolved conflict (a.py, resolved "x = 2 + 3")
+// through a file:// clone and returns the work directory, the corpus, the clones and the manifest.
+func mineFixture(t *testing.T) (work, corpus, clones, manifest string) {
+ t.Helper()
if _, err := exec.LookPath("git"); err != nil {
t.Skip("git is not installed")
}
@@ -53,13 +54,13 @@ func TestBenchMineAndRun(t *testing.T) {
gitIn(t, src, "commit", "-q", "-m", "merge feature")
head := gitIn(t, src, "rev-parse", "HEAD")
- work := t.TempDir()
- corpus := filepath.Join(work, "corpus.json")
+ work = t.TempDir()
+ corpus = filepath.Join(work, "corpus.json")
data, _ := json.Marshal(bench.Corpus{Repos: []bench.Repo{{Name: "local/fixture", URL: "file://" + src, Commit: head, License: "MIT"}}})
if err := os.WriteFile(corpus, data, 0o644); err != nil {
t.Fatal(err)
}
- clones, manifest := filepath.Join(work, "clones"), filepath.Join(work, "manifest.json")
+ clones, manifest = filepath.Join(work, "clones"), filepath.Join(work, "manifest.json")
var stdout, stderr bytes.Buffer
if code := run([]string{"bench", "mine", "--corpus", corpus, "--clones", clones, "--manifest", manifest}, nil, &stdout, &stderr); code != 0 {
@@ -68,6 +69,14 @@ func TestBenchMineAndRun(t *testing.T) {
if !strings.Contains(stdout.String(), "1 cases in") {
t.Fatalf("mine output: %s", stdout.String())
}
+ return work, corpus, clones, manifest
+}
+
+// TestBenchMineAndRun mines the fixture, resumes the mining from its part, and runs the default
+// chain (trivial only) on it, which abstains.
+func TestBenchMineAndRun(t *testing.T) {
+ work, corpus, clones, manifest := mineFixture(t)
+ var stdout, stderr bytes.Buffer
// A second run takes the repository from its part, without the clone.
moved := filepath.Join(work, "moved")
diff --git a/tools/graft-agents/cmd/graft-resolve/metrics_test.go b/tools/graft-agents/cmd/graft-resolve/metrics_test.go
new file mode 100644
index 0000000..0cb1b89
--- /dev/null
+++ b/tools/graft-agents/cmd/graft-resolve/metrics_test.go
@@ -0,0 +1,84 @@
+package main
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestServeMetrics(t *testing.T) {
+ state := filepath.Join(t.TempDir(), "budget.json")
+ month := time.Now().UTC().Format("2006-01")
+ if err := os.WriteFile(state, []byte(`{"month":"`+month+`","spent_usd":1.25}`), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ cfg := `
+[[strategy]]
+type = "llm"
+name = "ai"
+
+[strategy.llm]
+type = "command"
+command = ["true"]
+max_tokens = 1000
+
+[strategy.llm.budget]
+monthly_usd = 20
+input_usd_per_million = 2
+output_usd_per_million = 10
+state = "` + state + `"
+`
+ srv := httptest.NewServer(newTestHandler(t, testToken, cfg))
+ defer srv.Close()
+
+ get := func(token string) (*http.Response, string) {
+ req, err := http.NewRequest(http.MethodGet, srv.URL+"/metrics", nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if token != "" {
+ req.Header.Set("Authorization", "Bearer "+token)
+ }
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return resp, string(readAll(t, resp))
+ }
+
+ if resp, _ := get(""); resp.StatusCode != http.StatusUnauthorized {
+ t.Fatalf("metrics without the token: status %d", resp.StatusCode)
+ }
+ resp, body := get(testToken)
+ if resp.StatusCode != http.StatusOK || !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/plain") {
+ t.Fatalf("status %d, content type %q", resp.StatusCode, resp.Header.Get("Content-Type"))
+ }
+ for _, want := range []string{
+ "# TYPE graft_resolve_ai_spend_usd gauge\n",
+ `graft_resolve_ai_spend_usd{strategy="ai"} 1.25` + "\n",
+ `graft_resolve_ai_budget_usd{strategy="ai"} 20` + "\n",
+ "# TYPE graft_resolve_ai_budget_exhausted_total counter\n",
+ `graft_resolve_ai_budget_exhausted_total{strategy="ai"} 0` + "\n",
+ } {
+ if !strings.Contains(body, want) {
+ t.Fatalf("metrics lack %q:\n%s", want, body)
+ }
+ }
+}
+
+func TestServeMetricsWithoutBudget(t *testing.T) {
+ srv := httptest.NewServer(newTestHandler(t, "", ""))
+ defer srv.Close()
+ resp, err := http.Get(srv.URL + "/metrics")
+ if err != nil {
+ t.Fatal(err)
+ }
+ body := string(readAll(t, resp))
+ if resp.StatusCode != http.StatusOK || !strings.Contains(body, "# TYPE graft_resolve_ai_spend_usd gauge") || strings.Contains(body, "{strategy=") {
+ t.Fatalf("status %d:\n%s", resp.StatusCode, body)
+ }
+}
diff --git a/tools/graft-agents/cmd/graft-resolve/serve.go b/tools/graft-agents/cmd/graft-resolve/serve.go
index 5600c54..1a3de45 100644
--- a/tools/graft-agents/cmd/graft-resolve/serve.go
+++ b/tools/graft-agents/cmd/graft-resolve/serve.go
@@ -5,8 +5,11 @@ import (
"crypto/subtle"
"encoding/json"
"errors"
+ "fmt"
"io"
"net/http"
+ "sort"
+ "strconv"
"strings"
"time"
@@ -34,8 +37,20 @@ func newServer(listen string, res *resolve.Resolver, token string) (*http.Server
// that the server can be configured with any URL. When token is non-empty the
// request must carry "Authorization: Bearer <token>", compared in constant
// time.
+//
+// GET /metrics, with the same token, answers the AI budget metrics in the
+// Prometheus text format (writeMetrics).
func newHandler(res *resolve.Resolver, token string) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Method == http.MethodGet && r.URL.Path == "/metrics" {
+ if token != "" && !authorized(r, token) {
+ w.Header().Set("WWW-Authenticate", `Bearer realm="graft-resolve"`)
+ writeError(w, http.StatusUnauthorized, "unauthorized")
+ return
+ }
+ writeMetrics(w, res)
+ return
+ }
if r.Method != http.MethodPost {
w.Header().Set("Allow", http.MethodPost)
writeError(w, http.StatusMethodNotAllowed, "method not allowed")
@@ -83,6 +98,50 @@ func newHandler(res *resolve.Resolver, token string) http.Handler {
})
}
+// writeMetrics writes the AI budget of every llm strategy that has one, by
+// strategy name: the month's spend (read from the state file, so it includes
+// the other processes that share it), the cap (0: none), and the calls this
+// process did not make because of the cap.
+func writeMetrics(w http.ResponseWriter, res *resolve.Resolver) {
+ budgets := res.Budgets()
+ names := make([]string, 0, len(budgets))
+ for name := range budgets {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+
+ var b strings.Builder
+ metrics := []struct {
+ name, typ, help string
+ value func(resolve.BudgetSnapshot) string
+ }{
+ {"graft_resolve_ai_spend_usd", "gauge", "AI spend of the current month (UTC) in USD.",
+ func(s resolve.BudgetSnapshot) string { return strconv.FormatFloat(s.SpentUSD, 'f', -1, 64) }},
+ {"graft_resolve_ai_budget_usd", "gauge", "Monthly AI spending cap in USD (0: none).",
+ func(s resolve.BudgetSnapshot) string { return strconv.FormatFloat(s.MonthlyUSD, 'f', -1, 64) }},
+ {"graft_resolve_ai_budget_exhausted_total", "counter", "Model calls not made because the monthly cap was reached.",
+ func(s resolve.BudgetSnapshot) string { return strconv.FormatInt(s.Exhausted, 10) }},
+ }
+ snaps := make(map[string]resolve.BudgetSnapshot, len(names))
+ for _, name := range names {
+ s, err := budgets[name].Snapshot()
+ if err != nil {
+ writeError(w, http.StatusInternalServerError, err.Error())
+ return
+ }
+ snaps[name] = s
+ }
+ for _, m := range metrics {
+ fmt.Fprintf(&b, "# HELP %s %s\n# TYPE %s %s\n", m.name, m.help, m.name, m.typ)
+ for _, name := range names {
+ fmt.Fprintf(&b, "%s{strategy=%q} %s\n", m.name, name, m.value(snaps[name]))
+ }
+ }
+ w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
+ w.WriteHeader(http.StatusOK)
+ _, _ = io.WriteString(w, b.String())
+}
+
func authorized(r *http.Request, token string) bool {
const prefix = "Bearer "
auth := r.Header.Get("Authorization")
diff --git a/tools/graft-agents/resolve/bench/pricing.go b/tools/graft-agents/resolve/bench/pricing.go
new file mode 100644
index 0000000..9590d28
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/pricing.go
@@ -0,0 +1,122 @@
+package bench
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/resolve"
+)
+
+// DefaultOpenRouterAPI is the base of OpenRouter's API.
+const DefaultOpenRouterAPI = "https://openrouter.ai/api/v1"
+
+// Pricing is the price of the measured model, as read at run time (never from memory, PLAN10 12):
+// the highest price of each kind over the model's endpoints, in USD per token, since OpenRouter
+// may route a call to any of them.
+type Pricing struct {
+ Model string `json:"model"`
+ Source string `json:"source"`
+ FetchedAt time.Time `json:"fetched_at"`
+ Endpoints int `json:"endpoints"`
+ // Prompt, Completion and Reasoning are per token; Request is per call.
+ Prompt float64 `json:"prompt_usd_per_token"`
+ Completion float64 `json:"completion_usd_per_token"`
+ Reasoning float64 `json:"internal_reasoning_usd_per_token"`
+ Request float64 `json:"request_usd"`
+}
+
+// Prices returns the budget prices of p. Every output token is priced at the completion price
+// plus the internal-reasoning price: a reasoning token is reported as output, and charging both
+// prices on every output token can only overstate the cost.
+func (p Pricing) Prices() resolve.Prices {
+ return resolve.Prices{
+ InputPerMillion: p.Prompt * 1e6,
+ OutputPerMillion: (p.Completion + p.Reasoning) * 1e6,
+ PerRequest: p.Request,
+ }
+}
+
+type openRouterEndpoints struct {
+ Data struct {
+ ID string `json:"id"`
+ Endpoints []struct {
+ Name string `json:"name"`
+ Pricing map[string]string `json:"pricing"`
+ } `json:"endpoints"`
+ } `json:"data"`
+}
+
+// OpenRouterPricing reads the prices of model from OpenRouter's endpoints API
+// (GET <api>/models/<model>/endpoints, public). It fails when the model is unknown, has no
+// endpoint, or an endpoint lacks a prompt or completion price, or lists one that is not a
+// non-negative number: the run must not start on a price it cannot read.
+func OpenRouterPricing(ctx context.Context, client *http.Client, api, model string) (Pricing, error) {
+ parts := strings.Split(model, "/")
+ if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
+ return Pricing{}, fmt.Errorf("pricing: model %q is not author/slug", model)
+ }
+ source := strings.TrimRight(api, "/") + "/models/" + url.PathEscape(parts[0]) + "/" + url.PathEscape(parts[1]) + "/endpoints"
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)
+ if err != nil {
+ return Pricing{}, err
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return Pricing{}, fmt.Errorf("pricing: %w", err)
+ }
+ defer resp.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
+ if err != nil {
+ return Pricing{}, fmt.Errorf("pricing: %w", err)
+ }
+ if resp.StatusCode != http.StatusOK {
+ return Pricing{}, fmt.Errorf("pricing: %s: HTTP %d: %.200s", source, resp.StatusCode, body)
+ }
+ var doc openRouterEndpoints
+ if err := json.Unmarshal(body, &doc); err != nil {
+ return Pricing{}, fmt.Errorf("pricing: %s: %w", source, err)
+ }
+ if len(doc.Data.Endpoints) == 0 {
+ return Pricing{}, fmt.Errorf("pricing: %s lists no endpoint for %s", source, model)
+ }
+
+ p := Pricing{Model: model, Source: source, FetchedAt: time.Now().UTC(), Endpoints: len(doc.Data.Endpoints)}
+ for _, e := range doc.Data.Endpoints {
+ for _, f := range []struct {
+ key string
+ required bool
+ dst *float64
+ }{
+ {"prompt", true, &p.Prompt},
+ {"completion", true, &p.Completion},
+ {"internal_reasoning", false, &p.Reasoning},
+ {"request", false, &p.Request},
+ } {
+ raw, ok := e.Pricing[f.key]
+ if !ok {
+ if f.required {
+ return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s has no %s price", e.Name, model, f.key)
+ }
+ continue
+ }
+ v, err := strconv.ParseFloat(raw, 64)
+ if err != nil || v < 0 || math.IsNaN(v) || math.IsInf(v, 0) {
+ return Pricing{}, fmt.Errorf("pricing: endpoint %q of %s has the %s price %q, not a non-negative number", e.Name, model, f.key, raw)
+ }
+ *f.dst = math.Max(*f.dst, v)
+ }
+ }
+ if p.Prompt == 0 && p.Completion == 0 {
+ return Pricing{}, errors.New("pricing: " + model + " is listed as free; refusing to measure on a price of zero")
+ }
+ return p, nil
+}
diff --git a/tools/graft-agents/resolve/bench/pricing_test.go b/tools/graft-agents/resolve/bench/pricing_test.go
new file mode 100644
index 0000000..aa74fda
--- /dev/null
+++ b/tools/graft-agents/resolve/bench/pricing_test.go
@@ -0,0 +1,57 @@
+package bench
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func TestOpenRouterPricing(t *testing.T) {
+ bodies := map[string]string{
+ "/models/a/ok/endpoints": `{"data":{"endpoints":[
+ {"name":"x","pricing":{"prompt":"0.000001","completion":"0.000004"}},
+ {"name":"y","pricing":{"prompt":"0.0000015","completion":"0.000002","request":"0.0001","internal_reasoning":"0.000003"}}]}}`,
+ "/models/a/none/endpoints": `{"data":{"endpoints":[]}}`,
+ "/models/a/nocomp/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"0.000001"}}]}}`,
+ "/models/a/variable/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"-1","completion":"-1"}}]}}`,
+ "/models/a/free/endpoints": `{"data":{"endpoints":[{"name":"x","pricing":{"prompt":"0","completion":"0"}}]}}`,
+ "/models/a/junk/endpoints": `<html>`,
+ }
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ body, ok := bodies[r.URL.Path]
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+ _, _ = w.Write([]byte(body))
+ }))
+ defer srv.Close()
+
+ p, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL+"/", "a/ok")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if p.Prompt != 0.0000015 || p.Completion != 0.000004 || p.Reasoning != 0.000003 || p.Request != 0.0001 || p.Endpoints != 2 {
+ t.Fatalf("pricing %+v, want the highest of each price", p)
+ }
+ prices := p.Prices()
+ if prices.InputPerMillion != 1.5 || prices.OutputPerMillion != 7 || prices.PerRequest != 0.0001 {
+ t.Fatalf("prices %+v", prices)
+ }
+
+ for model, want := range map[string]string{
+ "a/none": "no endpoint",
+ "a/nocomp": "no completion price",
+ "a/variable": "not a non-negative number",
+ "a/free": "listed as free",
+ "a/junk": "invalid character",
+ "a/missing": "HTTP 404",
+ "no-slash": "not author/slug",
+ } {
+ if _, err := OpenRouterPricing(context.Background(), srv.Client(), srv.URL, model); err == nil || !strings.Contains(err.Error(), want) {
+ t.Errorf("%s: err = %v, want %q", model, err, want)
+ }
+ }
+}
diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.go
index f8b4070..3c68209 100644
--- a/tools/graft-agents/resolve/bench/report.go
+++ b/tools/graft-agents/resolve/bench/report.go
@@ -5,6 +5,7 @@ import (
"io"
"sort"
"strings"
+ "time"
)
// Counts aggregates the outcomes of a set of cases.
@@ -68,9 +69,13 @@ type Report struct {
Families map[string]Counts `json:"families"`
// NotSent counts the cases that were not sent, by reason.
NotSent map[string]int `json:"not_sent"`
- // Spend is the AI spend of the run in USD, when a budget was used.
- SpendUSD float64 `json:"spend_usd,omitempty"`
- Results []Result `json:"results"`
+ // SpendUSD is the AI spend of the run in USD, when a budget was used: the usage the provider
+ // reported, and the worst case of the calls whose usage was not reported. MaxUSD is the cap.
+ SpendUSD float64 `json:"spend_usd,omitempty"`
+ MaxUSD float64 `json:"max_usd,omitempty"`
+ // Pricing is the price the spend was computed with, as read at run time.
+ Pricing *Pricing `json:"pricing,omitempty"`
+ Results []Result `json:"results"`
}
// NewReport aggregates results.
@@ -92,11 +97,19 @@ func NewReport(label string, results []Result) Report {
func (rep Report) WriteMarkdown(w io.Writer) error {
var b strings.Builder
fmt.Fprintf(&b, "## graft-resolve bench: %s\n\n", rep.Label)
- fmt.Fprintf(&b, "Cases run: %d", rep.Total.Cases)
- if rep.SpendUSD > 0 {
- fmt.Fprintf(&b, ", AI spend $%.2f", rep.SpendUSD)
+ fmt.Fprintf(&b, "Cases sampled: %d; run: %d", rep.Total.Cases, rep.Total.Cases-rep.Total.NotRun)
+ if rep.Total.NotRun > 0 {
+ fmt.Fprintf(&b, " (%d not run: the cap was reached)", rep.Total.NotRun)
+ }
+ if rep.MaxUSD > 0 {
+ fmt.Fprintf(&b, "; AI spend $%.4f of the $%.2f cap", rep.SpendUSD, rep.MaxUSD)
}
b.WriteString(".\n\n")
+ if p := rep.Pricing; p != nil {
+ fmt.Fprintf(&b, "Model %s, priced at run time (%s, %s, highest of %d endpoints): $%g per million input tokens, $%g per million output tokens (completion $%g + internal reasoning $%g), $%g per request.\n\n",
+ p.Model, p.Source, p.FetchedAt.Format(time.RFC3339), p.Endpoints,
+ p.Prompt*1e6, (p.Completion+p.Reasoning)*1e6, p.Completion*1e6, p.Reasoning*1e6, p.Request)
+ }
b.WriteString("| family | cases | correct (exact) | wrong | rejected by checks | not sent | model error | not run (cap) | abstained | error rate |\n")
b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")
families := make([]string, 0, len(rep.Families))
diff --git a/tools/graft-agents/resolve/budget.go b/tools/graft-agents/resolve/budget.go
new file mode 100644
index 0000000..8414de2
--- /dev/null
+++ b/tools/graft-agents/resolve/budget.go
@@ -0,0 +1,306 @@
+package resolve
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io/fs"
+ "math"
+ "os"
+ "path/filepath"
+ "strconv"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/llm"
+)
+
+// Prices are the prices of a model in USD. OutputPerMillion applies to every output token,
+// reasoning included (providers count reasoning tokens as output).
+type Prices struct {
+ InputPerMillion float64
+ OutputPerMillion float64
+ // PerRequest is a fixed price per call, when the provider charges one.
+ PerRequest float64
+}
+
+func (p Prices) cost(inputTokens, outputTokens int) float64 {
+ return float64(inputTokens)*p.InputPerMillion/1e6 + float64(outputTokens)*p.OutputPerMillion/1e6 + p.PerRequest
+}
+
+func (p Prices) validate() error {
+ for _, v := range []float64{p.InputPerMillion, p.OutputPerMillion, p.PerRequest} {
+ if v < 0 || math.IsNaN(v) || math.IsInf(v, 0) {
+ return fmt.Errorf("invalid price %v", v)
+ }
+ }
+ if p.InputPerMillion == 0 && p.OutputPerMillion == 0 && p.PerRequest == 0 {
+ return errors.New("no price is set")
+ }
+ return nil
+}
+
+// Budget tracks the AI spend of a calendar month (UTC) and caps it (PLAN10 3.5). Before every
+// model call the worst case of the call is reserved: the spend is raised by it, and the call is
+// not made when that would exceed the cap. After the call the reservation is replaced by the
+// cost of the usage the provider reports. So the recorded spend is never below the real one, and
+// a process that dies between the two steps leaves its reservation booked.
+//
+// With a state file the spend survives restarts and is shared by every graft-resolve process
+// that uses the file: each update holds an exclusive lock (flock on state + ".lock"), reads the
+// file, and writes it again atomically (temporary file, fsync, rename, fsync of the directory).
+// Without a state file the spend lives in memory (the benchmark).
+type Budget struct {
+ // MonthlyUSD is the cap; 0 means no cap (the spend is still tracked).
+ MonthlyUSD float64
+ Prices Prices
+
+ state string
+ now func() time.Time
+
+ mu sync.Mutex
+ mem budgetState
+ calls atomic.Int64
+ // exhausted counts the calls refused for the cap.
+ exhausted atomic.Int64
+}
+
+type budgetState struct {
+ Month string `json:"month"`
+ SpentUSD float64 `json:"spent_usd"`
+}
+
+// NewBudget returns a budget with the cap monthlyUSD (0: none) and the prices of the model,
+// kept in stateFile ("" keeps it in memory).
+func NewBudget(monthlyUSD float64, prices Prices, stateFile string) (*Budget, error) {
+ if monthlyUSD < 0 || math.IsNaN(monthlyUSD) || math.IsInf(monthlyUSD, 0) {
+ return nil, fmt.Errorf("budget: invalid monthly cap %v", monthlyUSD)
+ }
+ if err := prices.validate(); err != nil {
+ return nil, fmt.Errorf("budget: %w", err)
+ }
+ if stateFile != "" {
+ if !filepath.IsAbs(stateFile) {
+ return nil, fmt.Errorf("budget: state file %q must be an absolute path", stateFile)
+ }
+ if err := checkFlock(); err != nil {
+ return nil, err
+ }
+ }
+ return &Budget{MonthlyUSD: monthlyUSD, Prices: prices, state: stateFile, now: time.Now}, nil
+}
+
+func (b *Budget) month() string {
+ return b.now().UTC().Format("2006-01")
+}
+
+// update applies fn to the state of the current month, under the lock, and stores the result
+// when fn succeeds.
+func (b *Budget) update(fn func(st *budgetState) error) (budgetState, error) {
+ b.mu.Lock()
+ defer b.mu.Unlock()
+ month := b.month()
+
+ if b.state == "" {
+ st := b.mem
+ if st.Month != month {
+ st = budgetState{Month: month}
+ }
+ if err := fn(&st); err != nil {
+ return st, err
+ }
+ b.mem = st
+ return st, nil
+ }
+
+ unlock, err := lockFile(b.state + ".lock")
+ if err != nil {
+ return budgetState{}, err
+ }
+ defer unlock()
+ st, err := readBudgetState(b.state)
+ if err != nil {
+ return st, err
+ }
+ if st.Month != month {
+ st = budgetState{Month: month}
+ }
+ if err := fn(&st); err != nil {
+ return st, err
+ }
+ return st, writeBudgetState(b.state, st)
+}
+
+func readBudgetState(file string) (budgetState, error) {
+ var st budgetState
+ data, err := os.ReadFile(file)
+ if errors.Is(err, fs.ErrNotExist) {
+ return st, nil
+ }
+ if err != nil {
+ return st, fmt.Errorf("budget: %w", err)
+ }
+ if err := json.Unmarshal(data, &st); err != nil {
+ return st, fmt.Errorf("budget: state file %s: %w", file, err)
+ }
+ if st.SpentUSD < 0 || math.IsNaN(st.SpentUSD) || math.IsInf(st.SpentUSD, 0) {
+ return st, fmt.Errorf("budget: state file %s: invalid spend %v", file, st.SpentUSD)
+ }
+ return st, nil
+}
+
+func writeBudgetState(file string, st budgetState) error {
+ data, err := json.Marshal(st)
+ if err != nil {
+ return err
+ }
+ dir := filepath.Dir(file)
+ tmp, err := os.CreateTemp(dir, filepath.Base(file)+".tmp-*")
+ if err != nil {
+ return fmt.Errorf("budget: %w", err)
+ }
+ defer func() { _ = os.Remove(tmp.Name()) }()
+ if _, err := tmp.Write(append(data, '\n')); err != nil {
+ _ = tmp.Close()
+ return fmt.Errorf("budget: %w", err)
+ }
+ if err := tmp.Sync(); err != nil {
+ _ = tmp.Close()
+ return fmt.Errorf("budget: %w", err)
+ }
+ if err := tmp.Close(); err != nil {
+ return fmt.Errorf("budget: %w", err)
+ }
+ if err := os.Rename(tmp.Name(), file); err != nil {
+ return fmt.Errorf("budget: %w", err)
+ }
+ d, err := os.Open(dir)
+ if err != nil {
+ return fmt.Errorf("budget: %w", err)
+ }
+ defer d.Close()
+ if err := d.Sync(); err != nil {
+ return fmt.Errorf("budget: %w", err)
+ }
+ return nil
+}
+
+// Reservation is the worst case of one call, booked until the call's usage replaces it.
+type Reservation struct {
+ b *Budget
+ month string
+ usd float64
+}
+
+// Reserve books the worst case of a call with at most inputTokens of input and maxOutputTokens of
+// output. It fails with ErrBudgetExhausted, without booking anything, when the month's spend plus
+// the worst case would exceed the cap.
+func (b *Budget) Reserve(inputTokens, maxOutputTokens int) (*Reservation, error) {
+ worst := b.Prices.cost(inputTokens, maxOutputTokens)
+ st, err := b.update(func(st *budgetState) error {
+ // The margin absorbs the rounding of sums of float prices (49 calls of $0.01 add up
+ // to a little more than $0.49); it lets the spend pass the cap by a nano-dollar at most.
+ if b.MonthlyUSD > 0 && st.SpentUSD+worst > b.MonthlyUSD+1e-9 {
+ return ErrBudgetExhausted
+ }
+ st.SpentUSD += worst
+ return nil
+ })
+ if errors.Is(err, ErrBudgetExhausted) {
+ b.exhausted.Add(1)
+ return nil, fmt.Errorf("%w: $%.4f spent of $%.2f this month, the next call may cost up to $%.4f", ErrBudgetExhausted, st.SpentUSD, b.MonthlyUSD, worst)
+ }
+ if err != nil {
+ return nil, err
+ }
+ b.calls.Add(1)
+ return &Reservation{b: b, month: st.Month, usd: worst}, nil
+}
+
+// Book replaces the reservation with the cost of the usage the provider reported. A usage with no
+// tokens (the provider reports none, or the call failed) keeps the reservation. A reservation of
+// a month that has ended stays in that month.
+func (r *Reservation) Book(u llm.Usage) error {
+ if r == nil || (u.InputTokens == 0 && u.OutputTokens == 0) {
+ return nil
+ }
+ actual := r.b.Prices.cost(u.InputTokens, u.OutputTokens)
+ _, err := r.b.update(func(st *budgetState) error {
+ if st.Month != r.month {
+ return nil
+ }
+ st.SpentUSD = math.Max(0, st.SpentUSD+actual-r.usd)
+ return nil
+ })
+ return err
+}
+
+// BudgetSnapshot is the state of a budget, for metrics and reports.
+type BudgetSnapshot struct {
+ Month string
+ SpentUSD float64
+ MonthlyUSD float64
+ Calls int64
+ Exhausted int64
+}
+
+// Snapshot returns the spend of the current month.
+func (b *Budget) Snapshot() (BudgetSnapshot, error) {
+ st, err := b.update(func(*budgetState) error { return errReadOnly })
+ if err != nil && !errors.Is(err, errReadOnly) {
+ return BudgetSnapshot{}, err
+ }
+ return BudgetSnapshot{Month: st.Month, SpentUSD: st.SpentUSD, MonthlyUSD: b.MonthlyUSD, Calls: b.calls.Load(), Exhausted: b.exhausted.Load()}, nil
+}
+
+var errReadOnly = errors.New("read only")
+
+// String describes the spend for an explanation.
+func (s BudgetSnapshot) String() string {
+ limit := "no cap"
+ if s.MonthlyUSD > 0 {
+ limit = "cap $" + strconv.FormatFloat(s.MonthlyUSD, 'f', 2, 64)
+ }
+ return fmt.Sprintf("AI spend %s: $%.4f (%s)", s.Month, s.SpentUSD, limit)
+}
+
+// budgetProvider makes every call of p go through the budget.
+type budgetProvider struct {
+ p llm.Provider
+ b *Budget
+}
+
+func (bp budgetProvider) Name() string { return bp.p.Name() }
+
+// Complete reserves the worst case of the call, makes it, and books the usage. The input is
+// bounded by its size in bytes (a token of the providers' byte-level tokenizers covers at least
+// one byte), plus the schema, which a provider may also put into the prompt, and a margin for
+// the message framing. The output is bounded by req.MaxTokens, which a budget requires.
+func (bp budgetProvider) Complete(ctx context.Context, req llm.Request) (*llm.Response, error) {
+ if req.MaxTokens <= 0 {
+ return nil, errors.New("budget: a call without max_tokens has no worst case")
+ }
+ input := len(req.System) + len(req.Prompt) + 256
+ if req.JSONSchema != nil {
+ schema, err := json.Marshal(req.JSONSchema)
+ if err != nil {
+ return nil, err
+ }
+ input += 2 * len(schema)
+ }
+ res, err := bp.b.Reserve(input, req.MaxTokens)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := bp.p.Complete(ctx, req)
+ var usage llm.Usage
+ if resp != nil {
+ usage = resp.Usage
+ }
+ if bookErr := res.Book(usage); bookErr != nil && err == nil {
+ return nil, bookErr
+ }
+ return resp, err
+}
diff --git a/tools/graft-agents/resolve/budget_flock.go b/tools/graft-agents/resolve/budget_flock.go
new file mode 100644
index 0000000..8a8e31c
--- /dev/null
+++ b/tools/graft-agents/resolve/budget_flock.go
@@ -0,0 +1,33 @@
+//go:build linux || darwin || freebsd || netbsd || openbsd || dragonfly
+
+package resolve
+
+import (
+ "fmt"
+ "os"
+ "syscall"
+)
+
+func checkFlock() error { return nil }
+
+// lockFile takes an exclusive flock on file, creating it, and returns the unlock function.
+func lockFile(file string) (func(), error) {
+ f, err := os.OpenFile(file, os.O_RDWR|os.O_CREATE, 0o600)
+ if err != nil {
+ return nil, fmt.Errorf("budget: %w", err)
+ }
+ for {
+ err = syscall.Flock(int(f.Fd()), syscall.LOCK_EX)
+ if err != syscall.EINTR {
+ break
+ }
+ }
+ if err != nil {
+ _ = f.Close()
+ return nil, fmt.Errorf("budget: locking %s: %w", file, err)
+ }
+ return func() {
+ _ = syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
+ _ = f.Close()
+ }, nil
+}
diff --git a/tools/graft-agents/resolve/budget_noflock.go b/tools/graft-agents/resolve/budget_noflock.go
new file mode 100644
index 0000000..7f062a1
--- /dev/null
+++ b/tools/graft-agents/resolve/budget_noflock.go
@@ -0,0 +1,11 @@
+//go:build !(linux || darwin || freebsd || netbsd || openbsd || dragonfly)
+
+package resolve
+
+import "errors"
+
+var errNoFlock = errors.New("budget: a state file needs flock, which this platform lacks")
+
+func checkFlock() error { return errNoFlock }
+
+func lockFile(string) (func(), error) { return nil, errNoFlock }
diff --git a/tools/graft-agents/resolve/budget_strategy.go b/tools/graft-agents/resolve/budget_strategy.go
new file mode 100644
index 0000000..2bc03fd
--- /dev/null
+++ b/tools/graft-agents/resolve/budget_strategy.go
@@ -0,0 +1,133 @@
+package resolve
+
+import (
+ "errors"
+ "fmt"
+ "sort"
+)
+
+// The llm strategy's budget is the [strategy.llm.budget] table:
+//
+// [strategy.llm.budget]
+// monthly_usd = 50 # the cap; 0 or absent: none (the spend is still tracked)
+// input_usd_per_million = 2.0 # the model's prices, kept current by the operator
+// output_usd_per_million = 10.0 # output tokens, reasoning included
+// request_usd = 0 # optional: a fixed price per call
+// state = "/var/lib/graft/resolve-budget.json"
+//
+// The table needs max_tokens in [strategy.llm], which bounds the worst case of a call. When the
+// month's spend plus the worst case of a call would exceed the cap, the call is not made and the
+// file fails with "AI budget exhausted", which the server puts into the conflict package.
+
+// llmBudgetConfig decodes the optional budget table of the [strategy.llm] table.
+func llmBudgetConfig(spec StrategySpec, tbl map[string]any) (*Budget, error) {
+ raw, ok := tbl["budget"]
+ if !ok {
+ return nil, nil
+ }
+ bt, ok := raw.(map[string]any)
+ if !ok {
+ return nil, fmt.Errorf("strategy %q: llm: budget must be a table", spec.Name)
+ }
+ var unknown []string
+ for k := range bt {
+ switch k {
+ case "monthly_usd", "input_usd_per_million", "output_usd_per_million", "request_usd", "state":
+ default:
+ unknown = append(unknown, k)
+ }
+ }
+ if len(unknown) > 0 {
+ sort.Strings(unknown)
+ return nil, fmt.Errorf("strategy %q: llm.budget: unknown keys %v", spec.Name, unknown)
+ }
+
+ number := func(key string) (float64, error) {
+ switch v := bt[key].(type) {
+ case nil:
+ return 0, nil
+ case int64:
+ if v >= 0 {
+ return float64(v), nil
+ }
+ case float64:
+ if v >= 0 {
+ return v, nil
+ }
+ }
+ return 0, fmt.Errorf("strategy %q: llm.budget: %s must be a non-negative number", spec.Name, key)
+ }
+ var prices Prices
+ var monthly float64
+ var err error
+ for _, f := range []struct {
+ key string
+ dst *float64
+ }{
+ {"monthly_usd", &monthly},
+ {"input_usd_per_million", &prices.InputPerMillion},
+ {"output_usd_per_million", &prices.OutputPerMillion},
+ {"request_usd", &prices.PerRequest},
+ } {
+ if *f.dst, err = number(f.key); err != nil {
+ return nil, err
+ }
+ }
+ state, err := stringField(bt, "state")
+ if err != nil {
+ return nil, fmt.Errorf("strategy %q: llm.budget: %w", spec.Name, err)
+ }
+ if state == "" {
+ return nil, fmt.Errorf("strategy %q: llm.budget: state is required: the spend must survive restarts", spec.Name)
+ }
+ b, err := NewBudget(monthly, prices, state)
+ if err != nil {
+ return nil, fmt.Errorf("strategy %q: llm.%w", spec.Name, err)
+ }
+ return b, nil
+}
+
+// UseBudget makes every llm strategy of r go through b, in place of a configured budget, and
+// returns how many there are. It must be called before the first Resolve. Each llm strategy
+// needs max_tokens.
+func (r *Resolver) UseBudget(b *Budget) (int, error) {
+ if b == nil {
+ return 0, errors.New("resolve: nil budget")
+ }
+ n := 0
+ for _, bd := range r.bindings {
+ s, ok := bd.Strategy.(*llmStrategy)
+ if !ok {
+ continue
+ }
+ if s.opt.MaxTokens <= 0 {
+ return 0, fmt.Errorf("strategy %q: a budget needs max_tokens in [strategy.llm]", bd.Name)
+ }
+ s.budget = b
+ s.provider = budgetProvider{p: s.base, b: b}
+ n++
+ }
+ return n, nil
+}
+
+// LLMModels returns the configured model of each llm strategy of r, by strategy name.
+func (r *Resolver) LLMModels() map[string]string {
+ out := map[string]string{}
+ for _, bd := range r.bindings {
+ if s, ok := bd.Strategy.(*llmStrategy); ok {
+ out[bd.Name] = s.model
+ }
+ }
+ return out
+}
+
+// Budgets returns the budgets of r's strategies, by strategy name.
+func (r *Resolver) Budgets() map[string]*Budget {
+ out := map[string]*Budget{}
+ for _, bd := range r.bindings {
+ if s, ok := bd.Strategy.(*llmStrategy); ok && s.budget != nil {
+ out[bd.Name] = s.budget
+ }
+ }
+ return out
+}
diff --git a/tools/graft-agents/resolve/budget_test.go b/tools/graft-agents/resolve/budget_test.go
new file mode 100644
index 0000000..12ab0aa
--- /dev/null
+++ b/tools/graft-agents/resolve/budget_test.go
@@ -0,0 +1,356 @@
+package resolve
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "math"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/llm"
+)
+
+// wp4Prices costs $1 per million input tokens and $10 per million output tokens.
+var wp4Prices = Prices{InputPerMillion: 1, OutputPerMillion: 10}
+
+func wp4Near(a, b float64) bool { return math.Abs(a-b) < 1e-9 }
+
+func wp4Spent(t *testing.T, b *Budget) float64 {
+ t.Helper()
+ s, err := b.Snapshot()
+ if err != nil {
+ t.Fatal(err)
+ }
+ return s.SpentUSD
+}
+
+func TestBudgetReserveAndBook(t *testing.T) {
+ b, err := NewBudget(1, wp4Prices, "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Worst case: 100k input ($0.10) + 10k output ($0.10).
+ r, err := b.Reserve(100_000, 10_000)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := wp4Spent(t, b); !wp4Near(got, 0.2) {
+ t.Fatalf("spent after reserve = %v, want the worst case 0.2", got)
+ }
+ if err := r.Book(llm.Usage{InputTokens: 30_000, OutputTokens: 2_000}); err != nil {
+ t.Fatal(err)
+ }
+ if got := wp4Spent(t, b); !wp4Near(got, 0.05) {
+ t.Fatalf("spent after book = %v, want the real cost 0.05", got)
+ }
+}
+
+func TestBudgetRefusesBeyondCap(t *testing.T) {
+ b, _ := NewBudget(0.25, wp4Prices, "")
+ if _, err := b.Reserve(100_000, 10_000); err != nil { // 0.20
+ t.Fatal(err)
+ }
+ _, err := b.Reserve(100_000, 10_000) // 0.40 > 0.25
+ if !errors.Is(err, ErrBudgetExhausted) {
+ t.Fatalf("err = %v, want ErrBudgetExhausted", err)
+ }
+ if !strings.Contains(err.Error(), "AI budget exhausted") {
+ t.Fatalf("message %q", err)
+ }
+ s, _ := b.Snapshot()
+ if !wp4Near(s.SpentUSD, 0.2) || s.Exhausted != 1 || s.Calls != 1 {
+ t.Fatalf("snapshot %+v: a refused call must book nothing", s)
+ }
+ // A smaller call still fits.
+ if _, err := b.Reserve(10_000, 1_000); err != nil {
+ t.Fatalf("small call refused: %v", err)
+ }
+}
+
+func TestBudgetUnknownUsageKeepsReservation(t *testing.T) {
+ b, _ := NewBudget(0, wp4Prices, "")
+ r, _ := b.Reserve(100_000, 10_000)
+ if err := r.Book(llm.Usage{}); err != nil {
+ t.Fatal(err)
+ }
+ if got := wp4Spent(t, b); !wp4Near(got, 0.2) {
+ t.Fatalf("spent = %v, want the reservation kept", got)
+ }
+}
+
+func TestBudgetNoCapTracksSpend(t *testing.T) {
+ b, _ := NewBudget(0, wp4Prices, "")
+ for i := 0; i < 100; i++ {
+ if _, err := b.Reserve(1_000_000, 0); err != nil {
+ t.Fatalf("call %d refused without a cap: %v", i, err)
+ }
+ }
+ if got := wp4Spent(t, b); !wp4Near(got, 100) {
+ t.Fatalf("spent = %v", got)
+ }
+}
+
+func TestBudgetStateFileSurvivesRestartAndIsShared(t *testing.T) {
+ state := filepath.Join(t.TempDir(), "budget.json")
+ a, err := NewBudget(1, wp4Prices, state)
+ if err != nil {
+ t.Fatal(err)
+ }
+ r, _ := a.Reserve(100_000, 10_000)
+ _ = r.Book(llm.Usage{InputTokens: 100_000, OutputTokens: 0}) // 0.10
+
+ // Another process (or a restart) sees the spend.
+ b, _ := NewBudget(1, wp4Prices, state)
+ if got := wp4Spent(t, b); !wp4Near(got, 0.1) {
+ t.Fatalf("restarted budget spent = %v, want 0.1", got)
+ }
+ if _, err := b.Reserve(800_000, 0); err != nil { // 0.90 total
+ t.Fatal(err)
+ }
+ if _, err := a.Reserve(200_000, 0); !errors.Is(err, ErrBudgetExhausted) { // 1.10 > 1
+ t.Fatalf("err = %v: the other process's spend was not seen", err)
+ }
+
+ var st budgetState
+ data, err := os.ReadFile(state)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := json.Unmarshal(data, &st); err != nil || st.Month != time.Now().UTC().Format("2006-01") || !wp4Near(st.SpentUSD, 0.9) {
+ t.Fatalf("state %s (%v)", data, err)
+ }
+ if matches, _ := filepath.Glob(state + ".tmp-*"); len(matches) != 0 {
+ t.Fatalf("temporary files left: %v", matches)
+ }
+}
+
+func TestBudgetConcurrentReservationsNeverExceedCap(t *testing.T) {
+ state := filepath.Join(t.TempDir(), "budget.json")
+ // Two budgets on one file stand for two processes; each call's worst case is $0.01.
+ budgets := make([]*Budget, 2)
+ for i := range budgets {
+ var err error
+ if budgets[i], err = NewBudget(0.5, wp4Prices, state); err != nil {
+ t.Fatal(err)
+ }
+ }
+ var wg sync.WaitGroup
+ var mu sync.Mutex
+ granted := 0
+ for i := 0; i < 80; i++ {
+ wg.Add(1)
+ go func(b *Budget) {
+ defer wg.Done()
+ if _, err := b.Reserve(10_000, 0); err == nil {
+ mu.Lock()
+ granted++
+ mu.Unlock()
+ }
+ }(budgets[i%2])
+ }
+ wg.Wait()
+ if granted != 50 {
+ t.Fatalf("granted %d calls of $0.01 under a $0.50 cap, want 50", granted)
+ }
+ if got := wp4Spent(t, budgets[0]); !wp4Near(got, 0.5) {
+ t.Fatalf("spent = %v", got)
+ }
+}
+
+func TestBudgetNewMonthStartsAtZero(t *testing.T) {
+ b, _ := NewBudget(1, wp4Prices, filepath.Join(t.TempDir(), "budget.json"))
+ now := time.Date(2026, 10, 31, 23, 0, 0, 0, time.UTC)
+ b.now = func() time.Time { return now }
+ r, err := b.Reserve(900_000, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ now = time.Date(2026, 11, 1, 0, 30, 0, 0, time.UTC)
+ if got := wp4Spent(t, b); got != 0 {
+ t.Fatalf("new month spent = %v, want 0", got)
+ }
+ // A call reserved in October and booked in November leaves November alone.
+ if err := r.Book(llm.Usage{InputTokens: 1, OutputTokens: 1}); err != nil {
+ t.Fatal(err)
+ }
+ if got := wp4Spent(t, b); got != 0 {
+ t.Fatalf("November spent = %v after booking an October call", got)
+ }
+}
+
+func TestNewBudgetRejects(t *testing.T) {
+ for name, tc := range map[string]struct {
+ monthly float64
+ prices Prices
+ state string
+ }{
+ "negative cap": {monthly: -1, prices: wp4Prices},
+ "no price": {monthly: 1},
+ "negative price": {monthly: 1, prices: Prices{InputPerMillion: -1, OutputPerMillion: 1}},
+ "relative state": {monthly: 1, prices: wp4Prices, state: "budget.json"},
+ } {
+ if _, err := NewBudget(tc.monthly, tc.prices, tc.state); err == nil {
+ t.Errorf("%s accepted", name)
+ }
+ }
+}
+
+// wp4Provider answers with a fixed usage and counts its calls.
+type wp4Provider struct {
+ mu sync.Mutex
+ calls int
+ usage llm.Usage
+}
+
+func (p *wp4Provider) Name() string { return "wp4:fake" }
+
+func (p *wp4Provider) Complete(context.Context, llm.Request) (*llm.Response, error) {
+ p.mu.Lock()
+ p.calls++
+ p.mu.Unlock()
+ b, _ := json.Marshal(map[string]string{"content": "x\n", "explanation": "wp4"})
+ return &llm.Response{Text: string(b), JSON: b, Usage: p.usage}, nil
+}
+
+func TestBudgetProviderReservesInputBytesAndMaxTokens(t *testing.T) {
+ b, _ := NewBudget(0, Prices{InputPerMillion: 1e6}, "") // $1 per input token: the cost counts the tokens
+ p := &wp4Provider{}
+ bp := budgetProvider{p: p, b: b}
+ if _, err := bp.Complete(context.Background(), llm.Request{Prompt: "abc"}); err == nil {
+ t.Fatal("call without max_tokens accepted")
+ }
+ if _, err := bp.Complete(context.Background(), llm.Request{System: "s", Prompt: strings.Repeat("p", 1000), MaxTokens: 10}); err != nil {
+ t.Fatal(err)
+ }
+ // No usage reported: the reservation stays, 1 + 1000 bytes + 256 of framing.
+ if got := wp4Spent(t, b); got != 1257 {
+ t.Fatalf("reserved %v input tokens, want 1257", got)
+ }
+}
+
+func TestBudgetProviderDoesNotCallWhenExhausted(t *testing.T) {
+ b, _ := NewBudget(0.001, wp4Prices, "")
+ p := &wp4Provider{usage: llm.Usage{InputTokens: 10, OutputTokens: 10}}
+ bp := budgetProvider{p: p, b: b}
+ _, err := bp.Complete(context.Background(), llm.Request{Prompt: "x", MaxTokens: 1_000_000})
+ if !errors.Is(err, ErrBudgetExhausted) || p.calls != 0 {
+ t.Fatalf("err = %v, calls = %d: want no call", err, p.calls)
+ }
+}
+
+const wp4Config = `
+[[strategy]]
+type = "llm"
+paths = ["*.txt"]
+
+[strategy.llm]
+type = "wp19-resolve-fake"
+max_tokens = 1000
+
+[strategy.llm.budget]
+monthly_usd = 5
+input_usd_per_million = 2
+output_usd_per_million = 10.5
+state = "STATE"
+`
+
+func TestLLMStrategyBudgetConfig(t *testing.T) {
+ state := filepath.Join(t.TempDir(), "budget.json")
+ wp19SetAnswer(func(context.Context, llm.Request) (*llm.Response, error) {
+ r := wp19Answer("merged\n")
+ r.Usage = llm.Usage{InputTokens: 1000, OutputTokens: 100}
+ return r, nil
+ })
+ r := wp19Build(t, strings.Replace(wp4Config, "STATE", state, 1))
+ budgets := r.Budgets()
+ b := budgets["llm"]
+ if b == nil || b.MonthlyUSD != 5 || b.Prices != (Prices{InputPerMillion: 2, OutputPerMillion: 10.5}) {
+ t.Fatalf("budgets %+v", budgets)
+ }
+ resp, err := r.Resolve(context.Background(), wp19Request(wp19Conflict("a.txt")))
+ if err != nil {
+ t.Fatal(err)
+ }
+ // 1000 × $2/M + 100 × $10.5/M = $0.00305.
+ if got := wp4Spent(t, b); !wp4Near(got, 0.00305) {
+ t.Fatalf("spent = %v", got)
+ }
+ if !strings.Contains(resp.Explanation, "llm: AI spend ") || !strings.Contains(resp.Explanation, "cap $5.00") {
+ t.Fatalf("explanation %q lacks the spend", resp.Explanation)
+ }
+}
+
+func TestLLMStrategyBudgetExhaustedIsTheReason(t *testing.T) {
+ state := filepath.Join(t.TempDir(), "budget.json")
+ if err := os.WriteFile(state, []byte(`{"month":"`+time.Now().UTC().Format("2006-01")+`","spent_usd":4.9999}`), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ fake := wp19SetAnswer(func(context.Context, llm.Request) (*llm.Response, error) {
+ return wp19Answer("merged\n"), nil
+ })
+ r := wp19Build(t, strings.Replace(wp4Config, "STATE", state, 1))
+ _, err := r.Resolve(context.Background(), wp19Request(wp19Conflict("a.txt")))
+ var unresolved *UnresolvedError
+ if !errors.As(err, &unresolved) {
+ t.Fatalf("err = %v", err)
+ }
+ attempts := unresolved.Failures[0].Attempts
+ if len(attempts) != 1 || !errors.Is(attempts[0].Err, ErrBudgetExhausted) {
+ t.Fatalf("attempts %+v", attempts)
+ }
+ if !strings.Contains(strings.Join(unresolved.Failures[0].Reasons, " "), "AI budget exhausted") {
+ t.Fatalf("reasons %v", unresolved.Failures[0].Reasons)
+ }
+ if fake.calls() != 0 {
+ t.Fatalf("model called %d times past the cap", fake.calls())
+ }
+}
+
+func TestLLMStrategyBudgetConfigRejects(t *testing.T) {
+ state := filepath.Join(t.TempDir(), "budget.json")
+ base := strings.Replace(wp4Config, "STATE", state, 1)
+ for name, tc := range map[string]struct{ config, want string }{
+ "no max_tokens": {strings.Replace(base, "max_tokens = 1000\n", "", 1), "needs max_tokens"},
+ "no state": {strings.Replace(base, `state = "`+state+`"`+"\n", "", 1), "state is required"},
+ "unknown key": {base + "currency = \"eur\"\n", "unknown keys [currency]"},
+ "negative price": {strings.Replace(base, "input_usd_per_million = 2", "input_usd_per_million = -2", 1), "non-negative number"},
+ "string cap": {strings.Replace(base, "monthly_usd = 5", `monthly_usd = "5"`, 1), "non-negative number"},
+ } {
+ t.Run(name, func(t *testing.T) {
+ _, err := wp19BuildErr(tc.config)
+ if err == nil || !strings.Contains(err.Error(), tc.want) {
+ t.Fatalf("err = %v, want %q", err, tc.want)
+ }
+ })
+ }
+}
+
+func TestResolverUseBudget(t *testing.T) {
+ wp19SetAnswer(func(context.Context, llm.Request) (*llm.Response, error) {
+ res := wp19Answer("merged\n")
+ res.Usage = llm.Usage{InputTokens: 1_000_000}
+ return res, nil
+ })
+ r := wp19Build(t, wp19LLMConfig) // max_tokens = 512
+ b, _ := NewBudget(0, wp4Prices, "")
+ n, err := r.UseBudget(b)
+ if err != nil || n != 1 {
+ t.Fatalf("UseBudget = %d, %v", n, err)
+ }
+ if _, err := r.Resolve(context.Background(), wp19Request(wp19Conflict("a.txt"))); err != nil {
+ t.Fatal(err)
+ }
+ if got := wp4Spent(t, b); !wp4Near(got, 1) {
+ t.Fatalf("spent = %v, want $1", got)
+ }
+
+ noLimit := wp19Build(t, "[[strategy]]\ntype = \"llm\"\n[strategy.llm]\ntype = \"wp19-resolve-fake\"\n")
+ if _, err := noLimit.UseBudget(b); err == nil || !strings.Contains(err.Error(), "max_tokens") {
+ t.Fatalf("UseBudget without max_tokens: %v", err)
+ }
+}
diff --git a/tools/graft-agents/resolve/llm_strategy.go b/tools/graft-agents/resolve/llm_strategy.go
index 844816c..54db40d 100644
--- a/tools/graft-agents/resolve/llm_strategy.go
+++ b/tools/graft-agents/resolve/llm_strategy.go
@@ -34,6 +34,9 @@ import (
// [strategy.llm.options] # optional, provider specific
// ...
//
+// [strategy.llm.budget] # optional: monthly spending cap (budget_strategy.go)
+// ...
+//
// The strategy declines (the chain continues, no model call is made) for files
// it cannot merge: a side that is absent, a symlink, a mode conflict, binary
// content, a diff too large to split (the LCS table of the changed middle of
@@ -54,9 +57,14 @@ func init() {
// llmStrategy is immutable after construction. Resolve may run concurrently
// (HTTP mode), so the provider must be safe for concurrent use.
type llmStrategy struct {
+ // provider is base, or base behind the budget.
provider llm.Provider
- opt LLMOptions
- timeout time.Duration
+ base llm.Provider
+ budget *Budget
+ // model is the configured model name.
+ model string
+ opt LLMOptions
+ timeout time.Duration
}
func newLLMStrategy(spec StrategySpec) (Strategy, error) {
@@ -104,7 +112,20 @@ func newLLMStrategy(spec StrategySpec) (Strategy, error) {
if err != nil {
return nil, fmt.Errorf("strategy %q: %w", spec.Name, err)
}
- return &llmStrategy{provider: p, opt: opt, timeout: timeout}, nil
+ s := &llmStrategy{provider: p, base: p, model: cfg.Model, opt: opt, timeout: timeout}
+
+ budget, err := llmBudgetConfig(spec, spec.Options["llm"].(map[string]any))
+ if err != nil {
+ return nil, err
+ }
+ if budget != nil {
+ if opt.MaxTokens <= 0 {
+ return nil, fmt.Errorf("strategy %q: llm.budget needs max_tokens in [strategy.llm]: it bounds the worst case of a call", spec.Name)
+ }
+ s.budget = budget
+ s.provider = budgetProvider{p: p, b: budget}
+ }
+ return s, nil
}
// Resolve implements Strategy.
@@ -166,7 +187,7 @@ func llmProviderConfig(spec StrategySpec) (llm.Config, error) {
var unknown []string
for k := range tbl {
switch k {
- case "type", "model", "base_url", "api_key_env", "command", "timeout", "max_tokens", "effort", "options":
+ case "type", "model", "base_url", "api_key_env", "command", "timeout", "max_tokens", "effort", "options", "budget":
default:
unknown = append(unknown, k)
}
diff --git a/tools/graft-agents/resolve/resolve.go b/tools/graft-agents/resolve/resolve.go
index fecbfff..558a0b6 100644
--- a/tools/graft-agents/resolve/resolve.go
+++ b/tools/graft-agents/resolve/resolve.go
@@ -299,6 +299,15 @@ func (r *Resolver) Resolve(ctx context.Context, req protocol.Request) (protocol.
if len(notes) > 0 {
resp.Explanation = "resolved " + strings.Join(notes, ", ")
}
+ // The month's AI spend of the strategies that answered, so that command mode reports it too.
+ budgets := r.Budgets()
+ for _, name := range used {
+ if b := budgets[name]; b != nil {
+ if snap, err := b.Snapshot(); err == nil {
+ resp.Explanation += "; " + name + ": " + snap.String()
+ }
+ }
+ }
return resp, nil
}