Skip to content

Git designed for AI agents.

llm: every provider honours each setting or rejects it

Federico Elia40d7c566ce6d
diff --git a/doc/graft/RESOLVE.md b/doc/graft/RESOLVE.md
index 7340f9f..a0fb01f 100644
--- a/doc/graft/RESOLVE.md
+++ b/doc/graft/RESOLVE.md
@@ -296,11 +296,32 @@ configured, with a message naming that requirement. The syntax check itself
fails closed in such a build (it rejects every file that has a grammar), and
`graft-resolve bench run` refuses to start.
-With the `openai` provider, `effort` (`minimal`, `low`, `medium`, `high` or
-`xhigh`) is sent as OpenRouter's `reasoning` parameter (`{"effort": ...}`) on
-every call; configure it only for an endpoint that takes that parameter. The
-provider takes no `options`, and a configuration that sets any, or an effort
-it does not know, fails at startup. No setting is dropped silently. An llm
+Every setting of `[strategy.llm]` is either honoured by the provider or an
+error at startup. None is dropped silently:
+
+| setting | `openai` | `anthropic` | `command` |
+|---|---|---|---|
+| `model` | sent | sent (default `claude-opus-5-5`) | `GRAFT_LLM_MODEL`, and the reported model |
+| `base_url` | the endpoint | the endpoint | error |
+| `api_key_env` | the key's variable (default `OPENAI_API_KEY`) | the key's variable | error |
+| `command` | error | error | the program (argv, no shell) |
+| `timeout` | per call | per call | per call, the program is killed |
+| `max_tokens` | `max_tokens` | `max_tokens` (default 16000) | `GRAFT_LLM_MAX_TOKENS` |
+| `effort` | OpenRouter's `reasoning: {"effort": ...}`; `minimal`, `low`, `medium`, `high`, `xhigh` | `output_config.effort`; `low` to `max` (default `high`) | `GRAFT_LLM_EFFORT`, verbatim |
+| `options` | error | error | error |
+
+The `openai` provider's `reasoning` parameter is OpenRouter's. Set `effort`
+only for an endpoint that takes it.
+
+The `command` program inherits graft-resolve's environment, and gets three
+extra variables for its settings: `GRAFT_LLM_MODEL`, `GRAFT_LLM_EFFORT` and
+`GRAFT_LLM_MAX_TOKENS`. The token limit is the call's, else the configured
+default. Each variable is set only when its setting is, and inherited
+variables of those names are removed. The program reads its own endpoint and
+credentials, which is why `base_url` and `api_key_env` are errors for it. It
+is expected to honour the variables. In particular, the AI budget counts a
+call's output as at most `GRAFT_LLM_MAX_TOKENS`, and a command reports no
+usage, so each of its calls is booked at that worst case. An llm
strategy also refuses to start in a graft-resolve built without cgo (see
above).
diff --git a/tools/graft-agents/llm/anthropic/anthropic.go b/tools/graft-agents/llm/anthropic/anthropic.go
index 3de31c9..473a41a 100644
--- a/tools/graft-agents/llm/anthropic/anthropic.go
+++ b/tools/graft-agents/llm/anthropic/anthropic.go
@@ -14,6 +14,7 @@ import (
"fmt"
"net/url"
"os"
+ "sort"
"strings"
"time"
@@ -78,6 +79,20 @@ func New(cfg llm.Config) (*Provider, error) {
model = DefaultModel
}
+ // No setting is dropped silently: the provider runs no command and supports
+ // no options.
+ if len(cfg.Command) > 0 {
+ return nil, errors.New("anthropic: command is for the command provider; this provider takes none")
+ }
+ if len(cfg.Options) > 0 {
+ keys := make([]string, 0, len(cfg.Options))
+ for k := range cfg.Options {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ return nil, fmt.Errorf("anthropic: unsupported options %v: this provider takes none", keys)
+ }
+
effortName := strings.ToLower(strings.TrimSpace(cfg.Effort))
if effortName == "" {
effortName = DefaultEffort
diff --git a/tools/graft-agents/llm/anthropic/settings_test.go b/tools/graft-agents/llm/anthropic/settings_test.go
new file mode 100644
index 0000000..a0f87b8
--- /dev/null
+++ b/tools/graft-agents/llm/anthropic/settings_test.go
@@ -0,0 +1,26 @@
+package anthropic
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/llm"
+)
+
+// TestNewRejectsSettingsItCannotTake checks that no setting is dropped silently. (Effort and
+// max_tokens reach the request body: TestCompleteRequestShape and
+// TestRequestMaxTokensOverridesDefault assert them.)
+func TestNewRejectsSettingsItCannotTake(t *testing.T) {
+ for name, tc := range map[string]struct {
+ cfg llm.Config
+ want string
+ }{
+ "command": {llm.Config{Type: "anthropic", Command: []string{"claude"}}, "command is for the command provider"},
+ "options": {llm.Config{Type: "anthropic", Options: map[string]string{"z": "1", "a": "2"}}, "unsupported options [a z]"},
+ } {
+ _, err := New(tc.cfg)
+ if err == nil || !strings.Contains(err.Error(), tc.want) {
+ t.Errorf("%s: err = %v, want %q", name, err, tc.want)
+ }
+ }
+}
diff --git a/tools/graft-agents/llm/command/command.go b/tools/graft-agents/llm/command/command.go
index c7fc2a5..2755d60 100644
--- a/tools/graft-agents/llm/command/command.go
+++ b/tools/graft-agents/llm/command/command.go
@@ -10,6 +10,11 @@
// object in prose or a Markdown code fence. A non-zero exit status is an
// error; its stderr (bounded) is included in the error message.
//
+// The program inherits graft-resolve's environment, plus GRAFT_LLM_MODEL,
+// GRAFT_LLM_EFFORT and GRAFT_LLM_MAX_TOKENS when model, effort and a token
+// limit are set (inherited variables of those names are removed). base_url, api_key_env and options are rejected: the program
+// reads its own endpoint and credentials.
+//
// This backend is the escape hatch for any model or service that has a
// command-line client: the operator decides what runs, Graft only speaks the
// stdin/stdout contract.
@@ -21,8 +26,11 @@ import (
"encoding/json"
"errors"
"fmt"
+ "os"
"os/exec"
"path/filepath"
+ "sort"
+ "strconv"
"strings"
"time"
@@ -58,10 +66,23 @@ type Provider struct {
argv []string
label string
model string
+ effort string
+ maxTokens int
timeout time.Duration
maxOutputBytes int
}
+// Environment variables the program receives, set only when the setting is:
+// the model name, the reasoning effort (verbatim), and the answer token limit
+// of the call (the request's, else the configured default). The program is
+// expected to honour them; the AI budget counts a call's output as at most
+// GRAFT_LLM_MAX_TOKENS.
+const (
+ EnvModel = "GRAFT_LLM_MODEL"
+ EnvEffort = "GRAFT_LLM_EFFORT"
+ EnvMaxTokens = "GRAFT_LLM_MAX_TOKENS"
+)
+
// stdinPayload is the request written to the program's stdin.
type stdinPayload struct {
System string `json:"system"`
@@ -85,6 +106,32 @@ func New(cfg llm.Config) (*Provider, error) {
if cfg.MaxTokens < 0 {
return nil, errors.New("command: max_tokens must not be negative")
}
+ // No setting is dropped silently. The program reads its own endpoint and
+ // credentials from its environment, so base_url, api_key_env and options
+ // are errors here.
+ var unsupported []string
+ if strings.TrimSpace(cfg.BaseURL) != "" {
+ unsupported = append(unsupported, "base_url")
+ }
+ if strings.TrimSpace(cfg.APIKeyEnv) != "" {
+ unsupported = append(unsupported, "api_key_env")
+ }
+ if len(cfg.Options) > 0 {
+ keys := make([]string, 0, len(cfg.Options))
+ for k := range cfg.Options {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+ unsupported = append(unsupported, fmt.Sprintf("options %v", keys))
+ }
+ if len(unsupported) > 0 {
+ return nil, fmt.Errorf("command: unsupported settings %s: the program reads its endpoint and credentials from its own environment", strings.Join(unsupported, ", "))
+ }
+ for name, v := range map[string]string{"model": cfg.Model, "effort": cfg.Effort} {
+ if strings.ContainsRune(v, 0) {
+ return nil, fmt.Errorf("command: %s contains a NUL byte", name)
+ }
+ }
timeout, err := cfg.TimeoutOrDefault()
if err != nil {
return nil, err
@@ -94,7 +141,9 @@ func New(cfg llm.Config) (*Provider, error) {
return &Provider{
argv: argv,
label: "command:" + filepath.Base(argv[0]),
- model: cfg.Model,
+ model: strings.TrimSpace(cfg.Model),
+ effort: strings.TrimSpace(cfg.Effort),
+ maxTokens: cfg.MaxTokens,
timeout: timeout,
maxOutputBytes: DefaultMaxOutputBytes,
}, nil
@@ -133,6 +182,26 @@ func (p *Provider) Complete(ctx context.Context, req llm.Request) (*llm.Response
stderr := &limitedBuffer{limit: maxStderrBytes}
cmd := exec.CommandContext(ctx, p.argv[0], p.argv[1:]...)
+ // The GRAFT_LLM_ variables are the provider's settings only: inherited ones are removed.
+ for _, kv := range os.Environ() {
+ if k, _, _ := strings.Cut(kv, "="); k == EnvModel || k == EnvEffort || k == EnvMaxTokens {
+ continue
+ }
+ cmd.Env = append(cmd.Env, kv)
+ }
+ if p.model != "" {
+ cmd.Env = append(cmd.Env, EnvModel+"="+p.model)
+ }
+ if p.effort != "" {
+ cmd.Env = append(cmd.Env, EnvEffort+"="+p.effort)
+ }
+ maxTokens := req.MaxTokens
+ if maxTokens == 0 {
+ maxTokens = p.maxTokens
+ }
+ if maxTokens > 0 {
+ cmd.Env = append(cmd.Env, EnvMaxTokens+"="+strconv.Itoa(maxTokens))
+ }
cmd.Stdin = bytes.NewReader(stdin)
cmd.Stdout = stdout
cmd.Stderr = stderr
diff --git a/tools/graft-agents/llm/command/command_test.go b/tools/graft-agents/llm/command/command_test.go
index 21f0851..399c1ad 100644
--- a/tools/graft-agents/llm/command/command_test.go
+++ b/tools/graft-agents/llm/command/command_test.go
@@ -69,6 +69,17 @@ func runHelper(mode string, args []string) int {
case "sleep":
time.Sleep(30 * time.Second)
return 0
+ case "env":
+ // Write the GRAFT_LLM_ variables as a JSON object; absent ones are left out.
+ env := map[string]string{}
+ for _, k := range []string{EnvModel, EnvEffort, EnvMaxTokens} {
+ if v, ok := os.LookupEnv(k); ok {
+ env[k] = v
+ }
+ }
+ b, _ := json.Marshal(env)
+ _, _ = os.Stdout.Write(b)
+ return 0
case "args":
b, _ := json.Marshal(args)
_, _ = os.Stdout.Write(b)
diff --git a/tools/graft-agents/llm/command/settings_test.go b/tools/graft-agents/llm/command/settings_test.go
new file mode 100644
index 0000000..2498079
--- /dev/null
+++ b/tools/graft-agents/llm/command/settings_test.go
@@ -0,0 +1,77 @@
+package command
+
+import (
+ "context"
+ "encoding/json"
+ "strings"
+ "testing"
+
+ "github.com/feder-cr/git-ai/tools/graft-agents/llm"
+)
+
+func helperEnvOf(t *testing.T, cfg llm.Config, req llm.Request) map[string]string {
+ t.Helper()
+ resp, err := newProvider(t, cfg).Complete(context.Background(), req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var env map[string]string
+ if err := json.Unmarshal([]byte(resp.Text), &env); err != nil {
+ t.Fatalf("helper output %q: %v", resp.Text, err)
+ }
+ return env
+}
+
+func TestSettingsReachTheProgramAsEnvironment(t *testing.T) {
+ cfg := helperConfig(t, "env")
+ cfg.Effort = " low "
+ cfg.MaxTokens = 900
+
+ env := helperEnvOf(t, cfg, llm.Request{Prompt: "hi"})
+ want := map[string]string{EnvModel: "helper-model", EnvEffort: "low", EnvMaxTokens: "900"}
+ if len(env) != len(want) {
+ t.Fatalf("env %v, want %v", env, want)
+ }
+ for k, v := range want {
+ if env[k] != v {
+ t.Fatalf("env %v, want %v", env, want)
+ }
+ }
+
+ // The request's limit takes precedence over the configured default.
+ if env := helperEnvOf(t, cfg, llm.Request{Prompt: "hi", MaxTokens: 50}); env[EnvMaxTokens] != "50" {
+ t.Fatalf("max tokens = %q, want the request's 50", env[EnvMaxTokens])
+ }
+}
+
+func TestUnsetSettingsAreNotInTheEnvironment(t *testing.T) {
+ // Values inherited from graft-resolve's own environment are removed: the variables carry
+ // the provider's settings and nothing else.
+ t.Setenv(EnvEffort, "inherited")
+ t.Setenv(EnvMaxTokens, "123")
+ cfg := helperConfig(t, "env")
+ cfg.Model = ""
+ env := helperEnvOf(t, cfg, llm.Request{Prompt: "hi"})
+ if len(env) != 0 {
+ t.Fatalf("env %v, want no GRAFT_LLM_ variables", env)
+ }
+}
+
+func TestNewRejectsSettingsTheProgramCannotTake(t *testing.T) {
+ for name, tc := range map[string]struct {
+ mutate func(*llm.Config)
+ want string
+ }{
+ "base_url": {func(c *llm.Config) { c.BaseURL = "https://example.com/v1" }, "base_url"},
+ "api_key_env": {func(c *llm.Config) { c.APIKeyEnv = "KEY" }, "api_key_env"},
+ "options": {func(c *llm.Config) { c.Options = map[string]string{"b": "1", "a": "2"} }, "options [a b]"},
+ "nul effort": {func(c *llm.Config) { c.Effort = "lo\x00w" }, "effort contains a NUL byte"},
+ } {
+ cfg := helperConfig(t, "env")
+ tc.mutate(&cfg)
+ _, err := New(cfg)
+ if err == nil || !strings.Contains(err.Error(), tc.want) {
+ t.Errorf("%s: err = %v, want %q", name, err, tc.want)
+ }
+ }
+}
diff --git a/tools/graft-agents/llm/llm.go b/tools/graft-agents/llm/llm.go
index fdc35f4..1657582 100644
--- a/tools/graft-agents/llm/llm.go
+++ b/tools/graft-agents/llm/llm.go
@@ -73,6 +73,9 @@ type Provider interface {
// Config selects and configures a provider. It is embedded in the TOML
// configuration of the Graft agents.
+//
+// A provider honours every field it is given or fails at construction: no
+// setting is dropped silently.
type Config struct {
// Type is "anthropic", "openai", "command" or "none".
Type string `toml:"type" json:"type"`
@@ -92,8 +95,8 @@ type Config struct {
Timeout string `toml:"timeout" json:"timeout"`
// MaxTokens is the default Request.MaxTokens.
MaxTokens int `toml:"max_tokens" json:"max_tokens"`
- // Effort is a provider-specific reasoning-effort hint ("low", "medium",
- // "high", ...); providers that do not support it ignore it.
+ // Effort is a provider-specific reasoning effort ("low", "medium",
+ // "high", ...); a provider that cannot send it rejects it.
Effort string `toml:"effort" json:"effort"`
// Options holds extra provider-specific settings.
Options map[string]string `toml:"options" json:"options"`
diff --git a/tools/graft-agents/llm/openai/effort_test.go b/tools/graft-agents/llm/openai/effort_test.go
index 2867699..7db80b4 100644
--- a/tools/graft-agents/llm/openai/effort_test.go
+++ b/tools/graft-agents/llm/openai/effort_test.go
@@ -59,6 +59,7 @@ func TestNewRejectsUnsupportedSettings(t *testing.T) {
}{
"unknown effort": {llm.Config{Type: "openai", Model: "m", Effort: "turbo"}, `invalid effort "turbo"`},
"options": {llm.Config{Type: "openai", Model: "m", Options: map[string]string{"region": "eu", "a": "b"}}, "unsupported options [a region]"},
+ "command": {llm.Config{Type: "openai", Model: "m", Command: []string{"x"}}, "command is for the command provider"},
} {
_, err := New(tc.cfg)
if err == nil || !strings.Contains(err.Error(), tc.want) {
diff --git a/tools/graft-agents/llm/openai/openai.go b/tools/graft-agents/llm/openai/openai.go
index 2910fed..4604209 100644
--- a/tools/graft-agents/llm/openai/openai.go
+++ b/tools/graft-agents/llm/openai/openai.go
@@ -111,8 +111,11 @@ func New(cfg llm.Config) (*Provider, error) {
if cfg.MaxTokens < 0 {
return nil, errors.New("openai: max_tokens must not be negative")
}
- // No setting is dropped silently: the provider supports no options, and an
- // effort it cannot send is an error.
+ // No setting is dropped silently: the provider runs no command and supports
+ // no options, and an effort it cannot send is an error.
+ if len(cfg.Command) > 0 {
+ return nil, errors.New("openai: command is for the command provider; this provider takes none")
+ }
if len(cfg.Options) > 0 {
keys := make([]string, 0, len(cfg.Options))
for k := range cfg.Options {