Git designed for AI agents.
graft-resolve: release unbilled refusals, stop a bench at the first 402
Federico Elia844f7235f5e2
diff --git a/doc/graft/RESOLVE.md b/doc/graft/RESOLVE.mdindex fb1ba23..1d48abc 100644--- a/doc/graft/RESOLVE.md+++ b/doc/graft/RESOLVE.md@@ -624,8 +624,12 @@ and `max_tokens` of output. If that would take the month's spend past thecap, the call is not made. The file then fails with the reason "AI budgetexhausted", which the server puts in the conflict package for the agent.After the call the worst case is replaced by the cost of the usage the-provider reports. A provider that reports no usage, or a call that fails, is-left at its worst case. So the recorded spend is never below the real one.+provider reports. A call that the provider refused before generating anything,+with HTTP 400, 401, 402 or 403 (a bad request, no or a wrong key, no credit,+forbidden), is not billed: its worst case is released. A provider that reports+no usage, and every other failure (a 429 or 5xx answer, a timeout, a broken+connection, an error in the middle of a stream), is left at its worst case, as+it may have been billed. So the recorded spend is never below the real one.A new calendar month (UTC) starts at zero.The spend is kept in the `state` file (`{"month": "2026-10", "spent_usd":@@ -676,7 +680,9 @@ resolved in real repositories. It needs the grammars (a cgo build).against the human file. The sample is stratified by family, with Rust andJava apart, so a language with few cases is taken whole. A chain with amodel needs `--max-usd`: every call books its worst case first, and a call- that would pass the cap is not made. The model's price is read at run time+ that would pass the cap is not made. The first HTTP 402 of the provider (no+ credit) stops the run: the remaining cases are reported as stopped, with the+ reason, apart from those the cap left out. The model's price is read at run timefrom OpenRouter's endpoints API (`--pricing openrouter`, the highest priceof the model's endpoints, or of those the routing allows), or given as flags (`--pricing fixed`). Theanswers are kept in `--answers-dir`.diff --git a/tools/graft-agents/cmd/graft-resolve/bench.go b/tools/graft-agents/cmd/graft-resolve/bench.goindex 1523b3a..0952633 100644--- a/tools/graft-agents/cmd/graft-resolve/bench.go+++ b/tools/graft-agents/cmd/graft-resolve/bench.go@@ -240,6 +240,16 @@ func benchRun(ctx context.Context, args []string, stdout, stderr io.Writer) intfmt.Fprintf(stderr, "[%d/%d] %s %s %s: %s %s\n", done, total, r.Family, r.Repo, r.Path, r.Outcome, r.Reason)},}+ // The first HTTP 402 of the provider (no credit) stops the run: every call after it fails the+ // same way, and a run that goes on only measures the refusals.+ if budget != nil {+ runner.Stop = func() string {+ if n := budget.PaymentRefusals(); n > 0 {+ return "the provider refused a call with HTTP 402 (no credit)"+ }+ return ""+ }+ }results, runErr := runner.Run(ctx, cases)report := bench.NewReport(*label, results)if budget != nil {diff --git a/tools/graft-agents/llm/anthropic/anthropic.go b/tools/graft-agents/llm/anthropic/anthropic.goindex e23ce7a..a1f94dc 100644--- a/tools/graft-agents/llm/anthropic/anthropic.go+++ b/tools/graft-agents/llm/anthropic/anthropic.go@@ -213,12 +213,19 @@ func (p *Provider) Complete(ctx context.Context, req llm.Request) (*llm.Responsemsg *anthropicsdk.Messageerr error)+ // started is set once a stream has delivered an event: an error after it came in the middle of+ // the generation. A request that is not streamed has no such error.+ started := falseif maxTokens > streamingThreshold {- msg, err = p.stream(ctx, params)+ msg, err = p.stream(ctx, params, &started)} else {msg, err = p.client.Messages.New(ctx, params)}if err != nil {+ var apiErr *anthropicsdk.Error+ if errors.As(err, &apiErr) {+ err = &statusError{err: err, status: apiErr.StatusCode, beforeGeneration: !started}+ }return nil, fmt.Errorf("anthropic: %w", err)}@@ -266,13 +273,27 @@ func (p *Provider) Complete(ctx context.Context, req llm.Request) (*llm.Responsereturn resp, nil}-// stream runs the request as a stream and accumulates the final message.-func (p *Provider) stream(ctx context.Context, params anthropicsdk.MessageNewParams) (*anthropicsdk.Message, error) {+// statusError is an error answer of the API with its status (llm.StatusError).+type statusError struct {+ err error+ status int+ beforeGeneration bool+}++func (e *statusError) Error() string { return e.err.Error() }+func (e *statusError) Unwrap() error { return e.err }+func (e *statusError) HTTPStatus() int { return e.status }+func (e *statusError) BeforeGeneration() bool { return e.beforeGeneration }++// stream runs the request as a stream and accumulates the final message. It sets *started once+// the stream delivered an event.+func (p *Provider) stream(ctx context.Context, params anthropicsdk.MessageNewParams, started *bool) (*anthropicsdk.Message, error) {stream := p.client.Messages.NewStreaming(ctx, params)defer stream.Close()var msg anthropicsdk.Messagefor stream.Next() {+ *started = trueif err := msg.Accumulate(stream.Current()); err != nil {return nil, err}diff --git a/tools/graft-agents/llm/anthropic/status_test.go b/tools/graft-agents/llm/anthropic/status_test.gonew file mode 100644index 0000000..58cd16c--- /dev/null+++ b/tools/graft-agents/llm/anthropic/status_test.go@@ -0,0 +1,48 @@+package anthropic++import (+ "context"+ "net/http"+ "strings"+ "testing"++ "github.com/feder-cr/git-ai/tools/graft-agents/llm"+)++const errorBody = `{"type":"error","error":{"type":"billing_error","message":"no credit"}}`++// TestErrorStatusIsReported: an error status of the API reaches the caller as an llm.StatusError,+// unbilled when the API refused the request before generating anything, as a refusal of a+// streamed request is too; an error in the middle of a stream is not.+func TestErrorStatusIsReported(t *testing.T) {+ for _, tc := range []struct {+ name string+ maxTokens int+ respond func(int, map[string]any) (int, string)+ status int+ unbilled bool+ }{+ {"refused request", 1000, func(int, map[string]any) (int, string) { return http.StatusPaymentRequired, errorBody }, 402, true},+ {"refused streamed request", 20000, func(int, map[string]any) (int, string) { return http.StatusForbidden, errorBody }, 403, true},+ {"error in the middle of a stream", 20000, func(int, map[string]any) (int, string) {+ stream := sse([]string{"partial"}, "end_turn")+ cut := strings.Index(stream, "event: content_block_stop")+ return http.StatusOK, stream[:cut] + "event: error\ndata: " + errorBody + "\n\n"+ }, 0, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ f := newFakeAPI(t, tc.respond)+ p := newProvider(t, f)+ _, err := p.Complete(context.Background(), llm.Request{Prompt: "x", MaxTokens: tc.maxTokens})+ if err == nil {+ t.Fatal("expected an error")+ }+ if status, ok := llm.HTTPStatus(err); tc.status != 0 && (!ok || status != tc.status) {+ t.Fatalf("status %d (%v), want %d: %v", status, ok, tc.status, err)+ }+ if got := llm.Unbilled(err); got != tc.unbilled {+ t.Fatalf("unbilled %v, want %v: %v", got, tc.unbilled, err)+ }+ })+ }+}diff --git a/tools/graft-agents/llm/openai/openai.go b/tools/graft-agents/llm/openai/openai.goindex 95c597f..af8cfe3 100644--- a/tools/graft-agents/llm/openai/openai.go+++ b/tools/graft-agents/llm/openai/openai.go@@ -84,6 +84,13 @@ func (e *APIError) Error() string {return fmt.Sprintf("openai: HTTP %d: %s", e.StatusCode, e.Message)}+// HTTPStatus implements llm.StatusError.+func (e *APIError) HTTPStatus() int { return e.StatusCode }++// BeforeGeneration implements llm.StatusError: the provider does not stream, so an error status+// is the status of the response, before anything was generated.+func (e *APIError) BeforeGeneration() bool { return true }+// Provider is an OpenAI-compatible Chat Completions client. It is safe for// concurrent use.type Provider struct {diff --git a/tools/graft-agents/llm/openai/status_test.go b/tools/graft-agents/llm/openai/status_test.gonew file mode 100644index 0000000..84acf65--- /dev/null+++ b/tools/graft-agents/llm/openai/status_test.go@@ -0,0 +1,27 @@+package openai++import (+ "context"+ "testing"++ "github.com/feder-cr/git-ai/tools/graft-agents/llm"+)++// TestErrorStatusIsReported: the provider does not stream, so an error status is a refusal before+// generation; it is unbilled for 400 to 403, and not for a 5xx that outlasted the retries.+func TestErrorStatusIsReported(t *testing.T) {+ for _, tc := range []struct {+ status int+ unbilled bool+ }{{400, true}, {401, true}, {402, true}, {403, true}, {404, false}, {429, false}, {500, false}} {+ f := newFakeServer(t, scripted(reply{status: tc.status, body: `{"error":{"message":"no"}}`}))+ p := newProvider(t, f)+ _, err := p.Complete(context.Background(), llm.Request{Prompt: "x"})+ if status, ok := llm.HTTPStatus(err); !ok || status != tc.status {+ t.Fatalf("%d: status %d (%v): %v", tc.status, status, ok, err)+ }+ if got := llm.Unbilled(err); got != tc.unbilled {+ t.Fatalf("%d: unbilled %v, want %v", tc.status, got, tc.unbilled)+ }+ }+}diff --git a/tools/graft-agents/llm/status.go b/tools/graft-agents/llm/status.gonew file mode 100644index 0000000..67603f1--- /dev/null+++ b/tools/graft-agents/llm/status.go@@ -0,0 +1,40 @@+package llm++import "errors"++// StatusError is implemented by the errors of providers for an answer of an HTTP API with an+// error status.+type StatusError interface {+ error+ // HTTPStatus is the status of the answer.+ HTTPStatus() int+ // BeforeGeneration reports whether the answer came before the model generated anything: the+ // status of the response to the request, not an error in the middle of a stream.+ BeforeGeneration() bool+}++// HTTPStatus returns the status of the provider's answer that err carries, if it carries one.+func HTTPStatus(err error) (int, bool) {+ var se StatusError+ if !errors.As(err, &se) {+ return 0, false+ }+ return se.HTTPStatus(), true+}++// Unbilled reports whether err is a refusal of the call that the provider does not bill: an+// answer before generation with the status 400 (a bad request), 401 (no or a wrong key), 402 (no+// credit) or 403 (forbidden). Every other failure may have been billed: 429 and 5xx answers, a+// timeout, a broken connection, an error in the middle of a stream, or a provider that reports no+// status (such as a command).+func Unbilled(err error) bool {+ var se StatusError+ if !errors.As(err, &se) || !se.BeforeGeneration() {+ return false+ }+ switch se.HTTPStatus() {+ case 400, 401, 402, 403:+ return true+ }+ return false+}diff --git a/tools/graft-agents/resolve/bench/mine_test.go b/tools/graft-agents/resolve/bench/mine_test.goindex 6fe7aff..b52917b 100644--- a/tools/graft-agents/resolve/bench/mine_test.go+++ b/tools/graft-agents/resolve/bench/mine_test.go@@ -5,6 +5,7 @@ package benchimport ("context""errors"+ "fmt""os""os/exec""path/filepath"@@ -186,13 +187,19 @@ func TestMineSkipsUnrelatedHistories(t *testing.T) {type fixedStrategy struct {content []byteerr error+ // mode is the mode of the answer; empty is a regular file.+ mode string}func (s fixedStrategy) Resolve(_ context.Context, _ resolve.File, _ protocol.Request) (resolve.Resolution, bool, error) {if s.err != nil {return resolve.Resolution{}, false, s.err}- return resolve.Resolution{Action: protocol.ActionWrite, Mode: protocol.ModeFile, Content: s.content}, true, nil+ mode := s.mode+ if mode == "" {+ mode = protocol.ModeFile+ }+ return resolve.Resolution{Action: protocol.ActionWrite, Mode: mode, Content: s.content}, true, nil}func TestRunnerOutcomes(t *testing.T) {@@ -231,6 +238,49 @@ func TestRunnerOutcomes(t *testing.T) {}})}++}++// TestRunnerSyntaxUnchecked: an accepted Go answer whose sides the grammar cannot read counts as+// having no syntax verdict, while a symbolic link, which has no syntax, never does.+func TestRunnerSyntaxUnchecked(t *testing.T) {+ const broken = "package a\n\nfunc A( int {\n\treturn %d\n}\n"+ f := newFixture(t)+ f.commit("base", map[string]string{"a.go": fmt.Sprintf(broken, 1)})+ f.git("checkout", "-q", "-b", "feature")+ f.commit("feature", map[string]string{"a.go": fmt.Sprintf(broken, 3)})+ f.git("checkout", "-q", "main")+ f.commit("main", map[string]string{"a.go": fmt.Sprintf(broken, 2)})+ f.conflictMerge("feature", map[string]string{"a.go": fmt.Sprintf(broken, 5)})+ head := f.git("rev-parse", "HEAD")+ cases, _, err := Mine(context.Background(), Repo{Name: "fixture", Commit: head, License: "MIT"}, f.dir, MineOptions{})+ if err != nil {+ t.Fatal(err)+ }+ if len(cases) != 1 {+ t.Fatalf("%d cases, want 1", len(cases))+ }++ for _, tc := range []struct {+ name string+ mode string+ unchecked bool+ }{+ {"a regular file the grammar cannot check", protocol.ModeFile, true},+ {"a symbolic link", protocol.ModeSymlink, false},+ } {+ res, err := resolve.NewResolver([]resolve.Binding{{Name: "fake", Strategy: fixedStrategy{content: []byte("target\n"), mode: tc.mode}}}, nil, time.Minute)+ if err != nil {+ t.Fatal(err)+ }+ results, err := Runner{Resolver: res, Clone: func(string) string { return f.dir }}.Run(context.Background(), cases)+ if err != nil {+ t.Fatal(err)+ }+ if len(results) != 1 || results[0].Outcome != OutcomeWrong || results[0].SyntaxUnchecked != tc.unchecked {+ t.Fatalf("%s: results %+v, want an accepted answer with syntax_unchecked %v", tc.name, results, tc.unchecked)+ }+ }}// TestMineLeavesOutInvalidReferences mines two conflicting merges whose human resolutions are not@@ -282,3 +332,52 @@ func TestMineLeavesOutInvalidReferences(t *testing.T) {t.Fatalf("kept %d, stats %v", len(kept), fstats)}}++// TestRunnerStops: once Stop gives a reason, that case and the ones after it are not run; they+// are counted apart from the cap, and the report says why.+func TestRunnerStops(t *testing.T) {+ f := newFixture(t)+ buildHistory(f)+ head := f.git("rev-parse", "HEAD")+ cases, _, err := Mine(context.Background(), Repo{Name: "fixture", Commit: head, License: "MIT"}, f.dir, MineOptions{})+ if err != nil {+ t.Fatal(err)+ }+ cases = append(cases, cases[0], cases[0])++ res, err := resolve.NewResolver([]resolve.Binding{{Name: "fake", Strategy: fixedStrategy{content: []byte(goHuman)}}}, nil, time.Minute)+ if err != nil {+ t.Fatal(err)+ }+ asked := 0+ stop := func() string {+ asked+++ if asked > 1 {+ return "the provider refused a call with HTTP 402 (no credit)"+ }+ return ""+ }+ results, err := Runner{Resolver: res, Clone: func(string) string { return f.dir }, Stop: stop}.Run(context.Background(), cases)+ if err != nil {+ t.Fatal(err)+ }+ if len(results) != len(cases) || results[0].Outcome != OutcomeCorrect {+ t.Fatalf("results %+v, want the first case run", results)+ }+ for _, r := range results[1:] {+ if r.Outcome != OutcomeStopped || !strings.Contains(r.Reason, "HTTP 402") {+ t.Fatalf("result %+v, want stopped", r)+ }+ }+ rep := NewReport("stop", results)+ if rep.Total.Stopped != len(cases)-1 || rep.Total.NotRun != 0 || !strings.Contains(rep.StopReason, "HTTP 402") {+ t.Fatalf("report totals %+v, reason %q", rep.Total, rep.StopReason)+ }+ var md strings.Builder+ if err := rep.WriteMarkdown(&md); err != nil {+ t.Fatal(err)+ }+ if !strings.Contains(md.String(), "not run: the run stopped: the provider refused a call with HTTP 402") {+ t.Fatalf("markdown does not say why the run stopped:\n%s", md.String())+ }+}diff --git a/tools/graft-agents/resolve/bench/report.go b/tools/graft-agents/resolve/bench/report.goindex b48c6ba..8ad2c21 100644--- a/tools/graft-agents/resolve/bench/report.go+++ b/tools/graft-agents/resolve/bench/report.go@@ -18,6 +18,7 @@ type Counts struct {NotSent int `json:"not_sent"`Model int `json:"model_error"`NotRun int `json:"not_run_cap"`+ Stopped int `json:"not_run_stopped"`Abstained int `json:"abstained"`// BlankLinesOnly counts the wrong results that differ from the human resolution in blank// lines only (part of Wrong); SyntaxUnchecked the accepted results the syntax check gave no@@ -62,6 +63,8 @@ func (c *Counts) add(r Result) {c.Model++case OutcomeNotRun:c.NotRun+++ case OutcomeStopped:+ c.Stopped++case OutcomeAbstained:c.Abstained++}@@ -86,7 +89,9 @@ type Report struct {MaxUSD float64 `json:"max_usd,omitempty"`// Pricing is the price the spend was computed with, as read at run time.Pricing *Pricing `json:"pricing,omitempty"`- Results []Result `json:"results"`+ // StopReason is why the run stopped before its last cases (OutcomeStopped), if it did.+ StopReason string `json:"stop_reason,omitempty"`+ Results []Result `json:"results"`}// LowN is the number of cases below which a row is marked as low-n in the Markdown report.@@ -109,6 +114,9 @@ func NewReport(label string, results []Result) Report {if r.Outcome == OutcomeNotSent {rep.NotSent[r.Reason]++}+ if r.Outcome == OutcomeStopped && rep.StopReason == "" {+ rep.StopReason = r.Reason+ }}return rep}@@ -117,10 +125,13 @@ func NewReport(label string, results []Result) Report {func (rep Report) WriteMarkdown(w io.Writer) error {var b strings.Builderfmt.Fprintf(&b, "## graft-resolve bench: %s\n\n", rep.Label)- fmt.Fprintf(&b, "Cases sampled: %d; run: %d", rep.Total.Cases, rep.Total.Cases-rep.Total.NotRun)+ fmt.Fprintf(&b, "Cases sampled: %d; run: %d", rep.Total.Cases, rep.Total.Cases-rep.Total.NotRun-rep.Total.Stopped)if rep.Total.NotRun > 0 {fmt.Fprintf(&b, " (%d not run: the cap was reached)", rep.Total.NotRun)}+ if rep.Total.Stopped > 0 {+ fmt.Fprintf(&b, " (%d not run: the run stopped: %s)", rep.Total.Stopped, rep.StopReason)+ }if rep.MaxUSD > 0 {fmt.Fprintf(&b, "; AI spend $%.4f of the $%.2f cap", rep.SpendUSD, rep.MaxUSD)}@@ -134,16 +145,16 @@ func (rep Report) WriteMarkdown(w io.Writer) error {p.Model, p.Source, p.FetchedAt.Format(time.RFC3339), over,PerMillion(p.Prompt), PerMillion(p.Completion, p.Reasoning), PerMillion(p.Completion), PerMillion(p.Reasoning), p.Request)}- b.WriteString("| family | cases | correct (exact) | wrong | rejected by checks | not sent | model error | not run (cap) | abstained | error rate |\n")- b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")+ b.WriteString("| family | cases | correct (exact) | wrong | rejected by checks | not sent | model error | not run (cap) | stopped | abstained | error rate |\n")+ b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")families := make([]string, 0, len(rep.Families))for f := range rep.Families {families = append(families, f)}sort.Strings(families)row := func(name string, c Counts) {- fmt.Fprintf(&b, "| %s | %d | %d (%d) | %d | %d | %d | %d | %d | %d | %.1f%% |\n",- name, c.Cases, c.Correct, c.Exact, c.Wrong, c.Rejected, c.NotSent, c.Model, c.NotRun, c.Abstained, 100*c.ErrorRate())+ fmt.Fprintf(&b, "| %s | %d | %d (%d) | %d | %d | %d | %d | %d | %d | %d | %.1f%% |\n",+ name, c.Cases, c.Correct, c.Exact, c.Wrong, c.Rejected, c.NotSent, c.Model, c.NotRun, c.Stopped, c.Abstained, 100*c.ErrorRate())}for _, f := range families {row(f, rep.Families[f])@@ -158,8 +169,8 @@ func (rep Report) WriteMarkdown(w io.Writer) error {if len(group.m) == 0 {continue}- fmt.Fprintf(&b, "\n| %s | cases | correct (exact) | wrong | rejected by checks | not sent | model error | not run (cap) | abstained | error rate |\n", group.title)- b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")+ fmt.Fprintf(&b, "\n| %s | cases | correct (exact) | wrong | rejected by checks | not sent | model error | not run (cap) | stopped | abstained | error rate |\n", group.title)+ b.WriteString("|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|\n")keys := make([]string, 0, len(group.m))for k := range group.m {keys = append(keys, k)diff --git a/tools/graft-agents/resolve/bench/run.go b/tools/graft-agents/resolve/bench/run.goindex 131c451..931b4cc 100644--- a/tools/graft-agents/resolve/bench/run.go+++ b/tools/graft-agents/resolve/bench/run.go@@ -33,6 +33,9 @@ const (OutcomeModelError = "model_error"// OutcomeNotRun: the AI budget of the run was spent before the case.OutcomeNotRun = "not_run_cap"+ // OutcomeStopped: the run stopped before the case (Runner.Stop), for example because the+ // provider refused a call for lack of credit; the reason is the result's.+ OutcomeStopped = "not_run_stopped"// OutcomeAbstained: no strategy applied to the file.OutcomeAbstained = "abstained")@@ -121,6 +124,9 @@ type Runner struct {// same answers judged again (Rejudge) without new model calls. The directory holds// third-party code and stays local.AnswersDir string+ // Stop, when set, is asked before each case whether the run must stop. A reason stops it:+ // that case and every case after it are not run, with outcome stopped and that reason.+ Stop func() string}// Run replays every case and returns the results in the order of cases.@@ -135,6 +141,19 @@ func (r Runner) Run(ctx context.Context, cases []Case) ([]Result, error) {if err := ctx.Err(); err != nil {return results, err}+ if r.Stop != nil {+ if reason := r.Stop(); reason != "" {+ for j, rest := range cases[i:] {+ res := Result{ID: rest.ID, Repo: rest.Repo, Family: rest.Family, Language: LanguageOf(rest.Path), Kind: rest.Kind, Path: rest.Path,+ Outcome: OutcomeStopped, Reason: reason}+ results = append(results, res)+ if r.Progress != nil {+ r.Progress(i+j+1, len(cases), res)+ }+ }+ return results, nil+ }+ }res, err := r.runCase(ctx, c, maxBytes)if err != nil {return results, fmt.Errorf("case %s (%s %s): %w", c.ID, c.Repo, c.Path, err)@@ -225,7 +244,9 @@ func (r Runner) runCase(ctx context.Context, c Case, maxBytes int64) (Result, erreturn res, err}}- if f.Action == protocol.ActionWrite {+ // A symbolic link has no syntax: its verdict is none, as in the check operation, never+ // unchecked.+ if f.Action == protocol.ActionWrite && f.Mode != protocol.ModeSymlink {if _, ok := syntax.LanguageOf(c.Path); ok {checked, err := syntax.CheckMerge(ctx, c.Path, f.Content, sideBytes(sides["ours"]), sideBytes(sides["theirs"]))if err != nil && ctx.Err() != nil {diff --git a/tools/graft-agents/resolve/budget.go b/tools/graft-agents/resolve/budget.goindex 55e4185..93cf368 100644--- a/tools/graft-agents/resolve/budget.go+++ b/tools/graft-agents/resolve/budget.go@@ -7,6 +7,7 @@ import ("errors""fmt""io/fs"+ "net/http""os""path/filepath""sync"@@ -90,6 +91,8 @@ type Budget struct {calls atomic.Int64// exhausted counts the calls refused for the cap.exhausted atomic.Int64+ // paymentRefusals counts the calls the provider refused with HTTP 402.+ paymentRefusals atomic.Int64}// budgetState is the content of the state file: {"month": "2026-10", "spent_usd": 1.250000000},@@ -301,6 +304,23 @@ func (r *Reservation) Book(u llm.Usage) error {return err}+// Release removes the reservation of a call that the provider refused before generating anything+// with a status it does not bill (llm.Unbilled): it cost nothing. A reservation of a month that+// has ended stays in that month.+func (r *Reservation) Release() error {+ if r == nil {+ return nil+ }+ _, err := r.b.update(func(st *budgetState) error {+ if st.Month != r.month {+ return nil+ }+ st.Spent -= min(st.Spent, r.worst)+ return nil+ })+ return err+}+// BudgetSnapshot is the state of a budget, for metrics and reports.type BudgetSnapshot struct {Month string@@ -364,6 +384,17 @@ func (bp budgetProvider) Complete(ctx context.Context, req llm.Request) (*llm.Rereturn nil, err}resp, err := bp.p.Complete(ctx, req)+ if status, ok := llm.HTTPStatus(err); ok && status == http.StatusPaymentRequired {+ bp.b.paymentRefusals.Add(1)+ }+ // A refusal before generation that the provider does not bill releases the worst case; any+ // other failure keeps it, as it may have been billed.+ if err != nil && llm.Unbilled(err) {+ if relErr := res.Release(); relErr != nil {+ return nil, errors.Join(err, relErr)+ }+ return nil, err+ }var usage llm.Usageif resp != nil {usage = resp.Usage@@ -373,3 +404,9 @@ func (bp budgetProvider) Complete(ctx context.Context, req llm.Request) (*llm.Re}return resp, err}++// PaymentRefusals returns the number of calls the provider refused with HTTP 402 (no credit), of+// any kind. A run that meets one should stop: the calls after it fail the same way.+func (b *Budget) PaymentRefusals() int64 {+ return b.paymentRefusals.Load()+}diff --git a/tools/graft-agents/resolve/budget_refusal_test.go b/tools/graft-agents/resolve/budget_refusal_test.gonew file mode 100644index 0000000..e8efc7f--- /dev/null+++ b/tools/graft-agents/resolve/budget_refusal_test.go@@ -0,0 +1,81 @@+//go:build cgo++package resolve++import (+ "context"+ "errors"+ "fmt"+ "testing"++ "github.com/feder-cr/git-ai/tools/graft-agents/llm"+)++// refusingProvider fails every call with err.+type refusingProvider struct{ err error }++func (p refusingProvider) Name() string { return "refusing" }++func (p refusingProvider) Complete(context.Context, llm.Request) (*llm.Response, error) {+ return nil, p.err+}++// statusErr is an llm.StatusError.+type statusErr struct {+ status int+ before bool+}++func (e statusErr) Error() string { return fmt.Sprintf("HTTP %d", e.status) }+func (e statusErr) HTTPStatus() int { return e.status }+func (e statusErr) BeforeGeneration() bool { return e.before }++// TestBudgetProviderBooksFailuresByClass: a call the provider refused before generating anything+// with a status it does not bill (400, 401, 402, 403) costs nothing, and its worst case is+// released; every other failure keeps the worst case, as it may have been billed. Every 402 is+// counted, so that a run can stop at the first.+func TestBudgetProviderBooksFailuresByClass(t *testing.T) {+ for _, tc := range []struct {+ name string+ err error+ released bool+ payment bool+ }{+ {"400 before generation", statusErr{400, true}, true, false},+ {"401 before generation", statusErr{401, true}, true, false},+ {"402 before generation", statusErr{402, true}, true, true},+ {"403 before generation", statusErr{403, true}, true, false},+ {"wrapped 402", fmt.Errorf("llm: %w", statusErr{402, true}), true, true},+ {"402 in the middle of a stream", statusErr{402, false}, false, true},+ {"400 in the middle of a stream", statusErr{400, false}, false, false},+ {"429", statusErr{429, true}, false, false},+ {"500", statusErr{500, true}, false, false},+ {"503", statusErr{503, true}, false, false},+ {"timeout", context.DeadlineExceeded, false, false},+ {"transport error", errors.New("connection reset by peer"), false, false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ b, err := NewBudget(0, wp4Prices, "")+ if err != nil {+ t.Fatal(err)+ }+ bp := budgetProvider{p: refusingProvider{err: tc.err}, b: b}+ _, callErr := bp.Complete(context.Background(), llm.Request{Prompt: "x", MaxTokens: 1000})+ if !errors.Is(callErr, tc.err) {+ t.Fatalf("err = %v, want the provider's %v", callErr, tc.err)+ }+ // The worst case: (1 + 256) input tokens at $1 per million and 1000 output tokens at $10.+ worst := wp4USD(t, "0.010257")+ want := worst+ if tc.released {+ want = 0+ }+ if got := wp4Spent(t, b); got != want {+ t.Fatalf("spent %v, want %v", got, want)+ }+ if got := b.PaymentRefusals() > 0; got != tc.payment {+ t.Fatalf("payment refusals counted: %v, want %v", got, tc.payment)+ }+ })+ }+}