Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Honour BEACON_JEV_ENDPOINT and BEACON_JEV_MODEL in memory evaluations run

Lawrence Jones5f6aeffe5289
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.go
index fb4c77ce..caec1016 100644
--- a/cli/beacon/cmd/memory.go
+++ b/cli/beacon/cmd/memory.go
@@ -191,9 +191,9 @@ func init() {
memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.since, "since", "", "RFC3339 lower time bound, inclusive")
memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.until, "until", "", "RFC3339 upper time bound, inclusive")
memoryEvaluationsRunCmd.Flags().BoolVar(&memoryOpts.dryRun, "dry-run", false, "Preview selected traces, Jev calls, and estimated cost without writing evaluations")
- memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevEndpoint, "jev-endpoint", learning.DefaultJevEndpoint, "Jev System One endpoint")
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevEndpoint, "jev-endpoint", "", "Jev System One endpoint (defaults to BEACON_JEV_ENDPOINT, then "+learning.DefaultJevEndpoint+")")
memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevAPIKey, "jev-api-key", "", "Jev API key (defaults to TYPESAFE_API_KEY, then BEACON_JEV_API_KEY)")
- memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevModel, "jev-model", learning.DefaultJevModel, "Jev model name, such as jev-latest or a pinned Jev version")
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevModel, "jev-model", "", "Jev model name, such as jev-latest or a pinned Jev version (defaults to BEACON_JEV_MODEL, then "+learning.DefaultJevModel+")")
memoryEvaluationsRunCmd.Flags().Float64Var(&memoryOpts.jevCost, "jev-cost-per-trace", learning.DefaultCostPerTrace, "Estimated Jev cost per trace in USD")
memoryEvaluationsRunCmd.Flags().DurationVar(&memoryOpts.timeout, "timeout", 10*time.Second, "Jev request timeout")
}
@@ -208,13 +208,7 @@ func runMemoryEvaluationsRun(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
- evaluatorOpts := learning.EvaluatorOptions{
- Endpoint: firstNonEmpty(memoryOpts.jevEndpoint, os.Getenv("BEACON_JEV_ENDPOINT"), learning.DefaultJevEndpoint),
- APIKey: firstNonEmpty(memoryOpts.jevAPIKey, os.Getenv("TYPESAFE_API_KEY"), os.Getenv("BEACON_JEV_API_KEY")),
- Model: firstNonEmpty(memoryOpts.jevModel, os.Getenv("BEACON_JEV_MODEL"), learning.DefaultJevModel),
- CostPerTrace: memoryOpts.jevCost,
- Timeout: memoryOpts.timeout,
- }
+ evaluatorOpts := jevEvaluatorOptions()
result := evaluationRunResult{
DryRun: memoryOpts.dryRun,
Project: project,
@@ -506,6 +500,19 @@ func parseMemoryTime(name, value string) (time.Time, error) {
return parsed, nil
}
+// jevEvaluatorOptions resolves each Jev setting from its flag, then the environment,
+// then the built-in default. The flags must not carry the defaults themselves, or the
+// environment variables documented for them would never be consulted.
+func jevEvaluatorOptions() learning.EvaluatorOptions {
+ return learning.EvaluatorOptions{
+ Endpoint: firstNonEmpty(memoryOpts.jevEndpoint, os.Getenv("BEACON_JEV_ENDPOINT"), learning.DefaultJevEndpoint),
+ APIKey: firstNonEmpty(memoryOpts.jevAPIKey, os.Getenv("TYPESAFE_API_KEY"), os.Getenv("BEACON_JEV_API_KEY")),
+ Model: firstNonEmpty(memoryOpts.jevModel, os.Getenv("BEACON_JEV_MODEL"), learning.DefaultJevModel),
+ CostPerTrace: memoryOpts.jevCost,
+ Timeout: memoryOpts.timeout,
+ }
+}
+
func firstNonEmpty(values ...string) string {
for _, value := range values {
if strings.TrimSpace(value) != "" {
diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.go
index f5fb9786..2ca8a8a6 100644
--- a/cli/beacon/cmd/memory_test.go
+++ b/cli/beacon/cmd/memory_test.go
@@ -241,7 +241,7 @@ func resetMemoryOpts(t *testing.T) {
reason string
replacement string
force bool
- }{userMode: true, limit: 25, page: 1, jevEndpoint: learning.DefaultJevEndpoint, jevModel: learning.DefaultJevModel, jevCost: learning.DefaultCostPerTrace}
+ }{userMode: true, limit: 25, page: 1, jevCost: learning.DefaultCostPerTrace}
t.Cleanup(func() { memoryOpts = previous })
}
@@ -334,3 +334,122 @@ func TestMemoryEvaluationsShowHidesPlaceholderReason(t *testing.T) {
t.Fatalf("show dropped a real reason:\n%s", out.String())
}
}
+
+// The --jev-endpoint and --jev-model flags used to default to the hosted TypeSafe
+// values, so BEACON_JEV_ENDPOINT and BEACON_JEV_MODEL were never consulted even
+// though the docs and the --jev-api-key help text promise them. Leaving the flags
+// unset must fall through to the environment, then to the built-in defaults.
+func TestMemoryEvaluationsRunHonoursJevEnvironment(t *testing.T) {
+ logPath, project := writeMemoryCommandFixture(t)
+ var gotModel string
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "Bearer env-key" {
+ t.Errorf("Authorization = %q, want Bearer env-key", r.Header.Get("Authorization"))
+ }
+ var req struct {
+ Model string `json:"model"`
+ }
+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+ t.Errorf("decode request: %v", err)
+ }
+ gotModel = req.Model
+ _, _ = w.Write([]byte(`{"model":"typesafe-ai/jev","answers":{"task_success":{"type":"noul","noul":0.9},"reusable_correction":{"type":"noul","noul":0.8},"evidence_supported":{"type":"noul","noul":0.7}}}`))
+ }))
+ defer server.Close()
+
+ t.Setenv("BEACON_JEV_ENDPOINT", server.URL)
+ t.Setenv("BEACON_JEV_MODEL", "typesafe-ai/jev")
+ t.Setenv("TYPESAFE_API_KEY", "env-key")
+ t.Setenv("BEACON_JEV_API_KEY", "")
+
+ resetMemoryOpts(t)
+ memoryOpts.userMode = true
+ memoryOpts.logPath = logPath
+ memoryOpts.projectPath = project
+ memoryOpts.traceID = "session:cursor:s1"
+ memoryOpts.jsonOutput = true
+
+ var out bytes.Buffer
+ cmd := &cobra.Command{}
+ cmd.SetOut(&out)
+ if err := runMemoryEvaluationsRun(cmd, nil); err != nil {
+ t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)
+ }
+ if gotModel != "typesafe-ai/jev" {
+ t.Fatalf("request model = %q, want typesafe-ai/jev", gotModel)
+ }
+ var result evaluationRunResult
+ if err := json.Unmarshal(out.Bytes(), &result); err != nil {
+ t.Fatalf("decode result: %v\n%s", err, out.String())
+ }
+ if len(result.Evaluations) != 1 {
+ t.Fatalf("evaluations = %#v", result.Evaluations)
+ }
+ if got := result.Evaluations[0].Evaluator; !strings.Contains(got, server.URL) {
+ t.Fatalf("evaluator endpoint = %q, want it to reference %s", got, server.URL)
+ }
+}
+
+func TestJevEvaluatorOptionsPrecedence(t *testing.T) {
+ cases := []struct {
+ name string
+ flagEndpoint string
+ flagModel string
+ flagKey string
+ env map[string]string
+ wantEndpoint string
+ wantModel string
+ wantKey string
+ }{
+ {
+ name: "defaults when nothing is set",
+ env: map[string]string{"BEACON_JEV_ENDPOINT": "", "BEACON_JEV_MODEL": "", "TYPESAFE_API_KEY": "", "BEACON_JEV_API_KEY": ""},
+ wantEndpoint: learning.DefaultJevEndpoint,
+ wantModel: learning.DefaultJevModel,
+ },
+ {
+ name: "environment overrides defaults",
+ env: map[string]string{
+ "BEACON_JEV_ENDPOINT": "https://gateway.example/typesafe/v1/systemone",
+ "BEACON_JEV_MODEL": "typesafe-ai/jev",
+ "TYPESAFE_API_KEY": "",
+ "BEACON_JEV_API_KEY": "fallback-key",
+ },
+ wantEndpoint: "https://gateway.example/typesafe/v1/systemone",
+ wantModel: "typesafe-ai/jev",
+ wantKey: "fallback-key",
+ },
+ {
+ name: "flags override environment",
+ flagEndpoint: "https://flag.example/v1/systemone",
+ flagModel: "jev-1.13.0",
+ flagKey: "flag-key",
+ env: map[string]string{
+ "BEACON_JEV_ENDPOINT": "https://gateway.example/typesafe/v1/systemone",
+ "BEACON_JEV_MODEL": "typesafe-ai/jev",
+ "TYPESAFE_API_KEY": "env-key",
+ "BEACON_JEV_API_KEY": "fallback-key",
+ },
+ wantEndpoint: "https://flag.example/v1/systemone",
+ wantModel: "jev-1.13.0",
+ wantKey: "flag-key",
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ for k, v := range tc.env {
+ t.Setenv(k, v)
+ }
+ resetMemoryOpts(t)
+ memoryOpts.jevEndpoint = tc.flagEndpoint
+ memoryOpts.jevModel = tc.flagModel
+ memoryOpts.jevAPIKey = tc.flagKey
+
+ got := jevEvaluatorOptions()
+ if got.Endpoint != tc.wantEndpoint || got.Model != tc.wantModel || got.APIKey != tc.wantKey {
+ t.Fatalf("jevEvaluatorOptions() = endpoint %q model %q key %q, want %q %q %q",
+ got.Endpoint, got.Model, got.APIKey, tc.wantEndpoint, tc.wantModel, tc.wantKey)
+ }
+ })
+ }
+}
diff --git a/docs/cli/memory.mdx b/docs/cli/memory.mdx
index c4907c97..13625933 100644
--- a/docs/cli/memory.mdx
+++ b/docs/cli/memory.mdx
@@ -59,9 +59,23 @@ Useful flags:
- `--timeout <duration>` sets the Jev request timeout.
- `--json` prints machine-readable output.
-`--jev-api-key` defaults to `TYPESAFE_API_KEY`, then `BEACON_JEV_API_KEY`.
-`--jev-endpoint` can also come from `BEACON_JEV_ENDPOINT`, and the model can come
-from `BEACON_JEV_MODEL`.
+Each Jev setting is resolved from the flag first, then the environment, then the
+built-in default:
+
+- `--jev-api-key` defaults to `TYPESAFE_API_KEY`, then `BEACON_JEV_API_KEY`.
+- `--jev-endpoint` defaults to `BEACON_JEV_ENDPOINT`, then the hosted TypeSafe
+ endpoint `https://api.typesafe.ai/v1/systemone`.
+- `--jev-model` defaults to `BEACON_JEV_MODEL`, then `jev-latest`.
+
+This is how you point Beacon at a TypeSafe-compatible gateway. For example, to
+evaluate through Vercel AI Gateway with an AI Gateway API key:
+
+```bash
+export TYPESAFE_API_KEY=<AI Gateway API key>
+export BEACON_JEV_ENDPOINT=https://ai-gateway.vercel.sh/typesafe/v1/systemone
+export BEACON_JEV_MODEL=typesafe-ai/jev
+beacon memory evaluations run --limit 10
+```
List and inspect stored evaluations: