Git designed for AI agents.
llm: every provider honours each setting or rejects it
Federico Elia40d7c566ce6d
diff --git a/doc/graft/RESOLVE.md b/doc/graft/RESOLVE.mdindex 7340f9f..a0fb01f 100644--- a/doc/graft/RESOLVE.md+++ b/doc/graft/RESOLVE.md@@ -296,11 +296,32 @@ configured, with a message naming that requirement. The syntax check itselffails closed in such a build (it rejects every file that has a grammar), and`graft-resolve bench run` refuses to start.-With the `openai` provider, `effort` (`minimal`, `low`, `medium`, `high` or-`xhigh`) is sent as OpenRouter's `reasoning` parameter (`{"effort": ...}`) on-every call; configure it only for an endpoint that takes that parameter. The-provider takes no `options`, and a configuration that sets any, or an effort-it does not know, fails at startup. No setting is dropped silently. An llm+Every setting of `[strategy.llm]` is either honoured by the provider or an+error at startup. None is dropped silently:++| setting | `openai` | `anthropic` | `command` |+|---|---|---|---|+| `model` | sent | sent (default `claude-opus-5-5`) | `GRAFT_LLM_MODEL`, and the reported model |+| `base_url` | the endpoint | the endpoint | error |+| `api_key_env` | the key's variable (default `OPENAI_API_KEY`) | the key's variable | error |+| `command` | error | error | the program (argv, no shell) |+| `timeout` | per call | per call | per call, the program is killed |+| `max_tokens` | `max_tokens` | `max_tokens` (default 16000) | `GRAFT_LLM_MAX_TOKENS` |+| `effort` | OpenRouter's `reasoning: {"effort": ...}`; `minimal`, `low`, `medium`, `high`, `xhigh` | `output_config.effort`; `low` to `max` (default `high`) | `GRAFT_LLM_EFFORT`, verbatim |+| `options` | error | error | error |++The `openai` provider's `reasoning` parameter is OpenRouter's. Set `effort`+only for an endpoint that takes it.++The `command` program inherits graft-resolve's environment, and gets three+extra variables for its settings: `GRAFT_LLM_MODEL`, `GRAFT_LLM_EFFORT` and+`GRAFT_LLM_MAX_TOKENS`. The token limit is the call's, else the configured+default. Each variable is set only when its setting is, and inherited+variables of those names are removed. The program reads its own endpoint and+credentials, which is why `base_url` and `api_key_env` are errors for it. It+is expected to honour the variables. In particular, the AI budget counts a+call's output as at most `GRAFT_LLM_MAX_TOKENS`, and a command reports no+usage, so each of its calls is booked at that worst case. An llmstrategy also refuses to start in a graft-resolve built without cgo (seeabove).diff --git a/tools/graft-agents/llm/anthropic/anthropic.go b/tools/graft-agents/llm/anthropic/anthropic.goindex 3de31c9..473a41a 100644--- a/tools/graft-agents/llm/anthropic/anthropic.go+++ b/tools/graft-agents/llm/anthropic/anthropic.go@@ -14,6 +14,7 @@ import ("fmt""net/url""os"+ "sort""strings""time"@@ -78,6 +79,20 @@ func New(cfg llm.Config) (*Provider, error) {model = DefaultModel}+ // No setting is dropped silently: the provider runs no command and supports+ // no options.+ if len(cfg.Command) > 0 {+ return nil, errors.New("anthropic: command is for the command provider; this provider takes none")+ }+ if len(cfg.Options) > 0 {+ keys := make([]string, 0, len(cfg.Options))+ for k := range cfg.Options {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ return nil, fmt.Errorf("anthropic: unsupported options %v: this provider takes none", keys)+ }+effortName := strings.ToLower(strings.TrimSpace(cfg.Effort))if effortName == "" {effortName = DefaultEffortdiff --git a/tools/graft-agents/llm/anthropic/settings_test.go b/tools/graft-agents/llm/anthropic/settings_test.gonew file mode 100644index 0000000..a0f87b8--- /dev/null+++ b/tools/graft-agents/llm/anthropic/settings_test.go@@ -0,0 +1,26 @@+package anthropic++import (+ "strings"+ "testing"++ "github.com/feder-cr/git-ai/tools/graft-agents/llm"+)++// TestNewRejectsSettingsItCannotTake checks that no setting is dropped silently. (Effort and+// max_tokens reach the request body: TestCompleteRequestShape and+// TestRequestMaxTokensOverridesDefault assert them.)+func TestNewRejectsSettingsItCannotTake(t *testing.T) {+ for name, tc := range map[string]struct {+ cfg llm.Config+ want string+ }{+ "command": {llm.Config{Type: "anthropic", Command: []string{"claude"}}, "command is for the command provider"},+ "options": {llm.Config{Type: "anthropic", Options: map[string]string{"z": "1", "a": "2"}}, "unsupported options [a z]"},+ } {+ _, err := New(tc.cfg)+ if err == nil || !strings.Contains(err.Error(), tc.want) {+ t.Errorf("%s: err = %v, want %q", name, err, tc.want)+ }+ }+}diff --git a/tools/graft-agents/llm/command/command.go b/tools/graft-agents/llm/command/command.goindex c7fc2a5..2755d60 100644--- a/tools/graft-agents/llm/command/command.go+++ b/tools/graft-agents/llm/command/command.go@@ -10,6 +10,11 @@// object in prose or a Markdown code fence. A non-zero exit status is an// error; its stderr (bounded) is included in the error message.//+// The program inherits graft-resolve's environment, plus GRAFT_LLM_MODEL,+// GRAFT_LLM_EFFORT and GRAFT_LLM_MAX_TOKENS when model, effort and a token+// limit are set (inherited variables of those names are removed). base_url, api_key_env and options are rejected: the program+// reads its own endpoint and credentials.+//// This backend is the escape hatch for any model or service that has a// command-line client: the operator decides what runs, Graft only speaks the// stdin/stdout contract.@@ -21,8 +26,11 @@ import ("encoding/json""errors""fmt"+ "os""os/exec""path/filepath"+ "sort"+ "strconv""strings""time"@@ -58,10 +66,23 @@ type Provider struct {argv []stringlabel stringmodel string+ effort string+ maxTokens inttimeout time.DurationmaxOutputBytes int}+// Environment variables the program receives, set only when the setting is:+// the model name, the reasoning effort (verbatim), and the answer token limit+// of the call (the request's, else the configured default). The program is+// expected to honour them; the AI budget counts a call's output as at most+// GRAFT_LLM_MAX_TOKENS.+const (+ EnvModel = "GRAFT_LLM_MODEL"+ EnvEffort = "GRAFT_LLM_EFFORT"+ EnvMaxTokens = "GRAFT_LLM_MAX_TOKENS"+)+// stdinPayload is the request written to the program's stdin.type stdinPayload struct {System string `json:"system"`@@ -85,6 +106,32 @@ func New(cfg llm.Config) (*Provider, error) {if cfg.MaxTokens < 0 {return nil, errors.New("command: max_tokens must not be negative")}+ // No setting is dropped silently. The program reads its own endpoint and+ // credentials from its environment, so base_url, api_key_env and options+ // are errors here.+ var unsupported []string+ if strings.TrimSpace(cfg.BaseURL) != "" {+ unsupported = append(unsupported, "base_url")+ }+ if strings.TrimSpace(cfg.APIKeyEnv) != "" {+ unsupported = append(unsupported, "api_key_env")+ }+ if len(cfg.Options) > 0 {+ keys := make([]string, 0, len(cfg.Options))+ for k := range cfg.Options {+ keys = append(keys, k)+ }+ sort.Strings(keys)+ unsupported = append(unsupported, fmt.Sprintf("options %v", keys))+ }+ if len(unsupported) > 0 {+ return nil, fmt.Errorf("command: unsupported settings %s: the program reads its endpoint and credentials from its own environment", strings.Join(unsupported, ", "))+ }+ for name, v := range map[string]string{"model": cfg.Model, "effort": cfg.Effort} {+ if strings.ContainsRune(v, 0) {+ return nil, fmt.Errorf("command: %s contains a NUL byte", name)+ }+ }timeout, err := cfg.TimeoutOrDefault()if err != nil {return nil, err@@ -94,7 +141,9 @@ func New(cfg llm.Config) (*Provider, error) {return &Provider{argv: argv,label: "command:" + filepath.Base(argv[0]),- model: cfg.Model,+ model: strings.TrimSpace(cfg.Model),+ effort: strings.TrimSpace(cfg.Effort),+ maxTokens: cfg.MaxTokens,timeout: timeout,maxOutputBytes: DefaultMaxOutputBytes,}, nil@@ -133,6 +182,26 @@ func (p *Provider) Complete(ctx context.Context, req llm.Request) (*llm.Responsestderr := &limitedBuffer{limit: maxStderrBytes}cmd := exec.CommandContext(ctx, p.argv[0], p.argv[1:]...)+ // The GRAFT_LLM_ variables are the provider's settings only: inherited ones are removed.+ for _, kv := range os.Environ() {+ if k, _, _ := strings.Cut(kv, "="); k == EnvModel || k == EnvEffort || k == EnvMaxTokens {+ continue+ }+ cmd.Env = append(cmd.Env, kv)+ }+ if p.model != "" {+ cmd.Env = append(cmd.Env, EnvModel+"="+p.model)+ }+ if p.effort != "" {+ cmd.Env = append(cmd.Env, EnvEffort+"="+p.effort)+ }+ maxTokens := req.MaxTokens+ if maxTokens == 0 {+ maxTokens = p.maxTokens+ }+ if maxTokens > 0 {+ cmd.Env = append(cmd.Env, EnvMaxTokens+"="+strconv.Itoa(maxTokens))+ }cmd.Stdin = bytes.NewReader(stdin)cmd.Stdout = stdoutcmd.Stderr = stderrdiff --git a/tools/graft-agents/llm/command/command_test.go b/tools/graft-agents/llm/command/command_test.goindex 21f0851..399c1ad 100644--- a/tools/graft-agents/llm/command/command_test.go+++ b/tools/graft-agents/llm/command/command_test.go@@ -69,6 +69,17 @@ func runHelper(mode string, args []string) int {case "sleep":time.Sleep(30 * time.Second)return 0+ case "env":+ // Write the GRAFT_LLM_ variables as a JSON object; absent ones are left out.+ env := map[string]string{}+ for _, k := range []string{EnvModel, EnvEffort, EnvMaxTokens} {+ if v, ok := os.LookupEnv(k); ok {+ env[k] = v+ }+ }+ b, _ := json.Marshal(env)+ _, _ = os.Stdout.Write(b)+ return 0case "args":b, _ := json.Marshal(args)_, _ = os.Stdout.Write(b)diff --git a/tools/graft-agents/llm/command/settings_test.go b/tools/graft-agents/llm/command/settings_test.gonew file mode 100644index 0000000..2498079--- /dev/null+++ b/tools/graft-agents/llm/command/settings_test.go@@ -0,0 +1,77 @@+package command++import (+ "context"+ "encoding/json"+ "strings"+ "testing"++ "github.com/feder-cr/git-ai/tools/graft-agents/llm"+)++func helperEnvOf(t *testing.T, cfg llm.Config, req llm.Request) map[string]string {+ t.Helper()+ resp, err := newProvider(t, cfg).Complete(context.Background(), req)+ if err != nil {+ t.Fatal(err)+ }+ var env map[string]string+ if err := json.Unmarshal([]byte(resp.Text), &env); err != nil {+ t.Fatalf("helper output %q: %v", resp.Text, err)+ }+ return env+}++func TestSettingsReachTheProgramAsEnvironment(t *testing.T) {+ cfg := helperConfig(t, "env")+ cfg.Effort = " low "+ cfg.MaxTokens = 900++ env := helperEnvOf(t, cfg, llm.Request{Prompt: "hi"})+ want := map[string]string{EnvModel: "helper-model", EnvEffort: "low", EnvMaxTokens: "900"}+ if len(env) != len(want) {+ t.Fatalf("env %v, want %v", env, want)+ }+ for k, v := range want {+ if env[k] != v {+ t.Fatalf("env %v, want %v", env, want)+ }+ }++ // The request's limit takes precedence over the configured default.+ if env := helperEnvOf(t, cfg, llm.Request{Prompt: "hi", MaxTokens: 50}); env[EnvMaxTokens] != "50" {+ t.Fatalf("max tokens = %q, want the request's 50", env[EnvMaxTokens])+ }+}++func TestUnsetSettingsAreNotInTheEnvironment(t *testing.T) {+ // Values inherited from graft-resolve's own environment are removed: the variables carry+ // the provider's settings and nothing else.+ t.Setenv(EnvEffort, "inherited")+ t.Setenv(EnvMaxTokens, "123")+ cfg := helperConfig(t, "env")+ cfg.Model = ""+ env := helperEnvOf(t, cfg, llm.Request{Prompt: "hi"})+ if len(env) != 0 {+ t.Fatalf("env %v, want no GRAFT_LLM_ variables", env)+ }+}++func TestNewRejectsSettingsTheProgramCannotTake(t *testing.T) {+ for name, tc := range map[string]struct {+ mutate func(*llm.Config)+ want string+ }{+ "base_url": {func(c *llm.Config) { c.BaseURL = "https://example.com/v1" }, "base_url"},+ "api_key_env": {func(c *llm.Config) { c.APIKeyEnv = "KEY" }, "api_key_env"},+ "options": {func(c *llm.Config) { c.Options = map[string]string{"b": "1", "a": "2"} }, "options [a b]"},+ "nul effort": {func(c *llm.Config) { c.Effort = "lo\x00w" }, "effort contains a NUL byte"},+ } {+ cfg := helperConfig(t, "env")+ tc.mutate(&cfg)+ _, err := New(cfg)+ if err == nil || !strings.Contains(err.Error(), tc.want) {+ t.Errorf("%s: err = %v, want %q", name, err, tc.want)+ }+ }+}diff --git a/tools/graft-agents/llm/llm.go b/tools/graft-agents/llm/llm.goindex fdc35f4..1657582 100644--- a/tools/graft-agents/llm/llm.go+++ b/tools/graft-agents/llm/llm.go@@ -73,6 +73,9 @@ type Provider interface {// Config selects and configures a provider. It is embedded in the TOML// configuration of the Graft agents.+//+// A provider honours every field it is given or fails at construction: no+// setting is dropped silently.type Config struct {// Type is "anthropic", "openai", "command" or "none".Type string `toml:"type" json:"type"`@@ -92,8 +95,8 @@ type Config struct {Timeout string `toml:"timeout" json:"timeout"`// MaxTokens is the default Request.MaxTokens.MaxTokens int `toml:"max_tokens" json:"max_tokens"`- // Effort is a provider-specific reasoning-effort hint ("low", "medium",- // "high", ...); providers that do not support it ignore it.+ // Effort is a provider-specific reasoning effort ("low", "medium",+ // "high", ...); a provider that cannot send it rejects it.Effort string `toml:"effort" json:"effort"`// Options holds extra provider-specific settings.Options map[string]string `toml:"options" json:"options"`diff --git a/tools/graft-agents/llm/openai/effort_test.go b/tools/graft-agents/llm/openai/effort_test.goindex 2867699..7db80b4 100644--- a/tools/graft-agents/llm/openai/effort_test.go+++ b/tools/graft-agents/llm/openai/effort_test.go@@ -59,6 +59,7 @@ func TestNewRejectsUnsupportedSettings(t *testing.T) {}{"unknown effort": {llm.Config{Type: "openai", Model: "m", Effort: "turbo"}, `invalid effort "turbo"`},"options": {llm.Config{Type: "openai", Model: "m", Options: map[string]string{"region": "eu", "a": "b"}}, "unsupported options [a region]"},+ "command": {llm.Config{Type: "openai", Model: "m", Command: []string{"x"}}, "command is for the command provider"},} {_, err := New(tc.cfg)if err == nil || !strings.Contains(err.Error(), tc.want) {diff --git a/tools/graft-agents/llm/openai/openai.go b/tools/graft-agents/llm/openai/openai.goindex 2910fed..4604209 100644--- a/tools/graft-agents/llm/openai/openai.go+++ b/tools/graft-agents/llm/openai/openai.go@@ -111,8 +111,11 @@ func New(cfg llm.Config) (*Provider, error) {if cfg.MaxTokens < 0 {return nil, errors.New("openai: max_tokens must not be negative")}- // No setting is dropped silently: the provider supports no options, and an- // effort it cannot send is an error.+ // No setting is dropped silently: the provider runs no command and supports+ // no options, and an effort it cannot send is an error.+ if len(cfg.Command) > 0 {+ return nil, errors.New("openai: command is for the command provider; this provider takes none")+ }if len(cfg.Options) > 0 {keys := make([]string, 0, len(cfg.Options))for k := range cfg.Options {