The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Honour BEACON_JEV_ENDPOINT and BEACON_JEV_MODEL in memory evaluations run
Lawrence Jones5f6aeffe5289
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.goindex fb4c77ce..caec1016 100644--- a/cli/beacon/cmd/memory.go+++ b/cli/beacon/cmd/memory.go@@ -191,9 +191,9 @@ func init() {memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.since, "since", "", "RFC3339 lower time bound, inclusive")memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.until, "until", "", "RFC3339 upper time bound, inclusive")memoryEvaluationsRunCmd.Flags().BoolVar(&memoryOpts.dryRun, "dry-run", false, "Preview selected traces, Jev calls, and estimated cost without writing evaluations")- memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevEndpoint, "jev-endpoint", learning.DefaultJevEndpoint, "Jev System One endpoint")+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevEndpoint, "jev-endpoint", "", "Jev System One endpoint (defaults to BEACON_JEV_ENDPOINT, then "+learning.DefaultJevEndpoint+")")memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevAPIKey, "jev-api-key", "", "Jev API key (defaults to TYPESAFE_API_KEY, then BEACON_JEV_API_KEY)")- memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevModel, "jev-model", learning.DefaultJevModel, "Jev model name, such as jev-latest or a pinned Jev version")+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevModel, "jev-model", "", "Jev model name, such as jev-latest or a pinned Jev version (defaults to BEACON_JEV_MODEL, then "+learning.DefaultJevModel+")")memoryEvaluationsRunCmd.Flags().Float64Var(&memoryOpts.jevCost, "jev-cost-per-trace", learning.DefaultCostPerTrace, "Estimated Jev cost per trace in USD")memoryEvaluationsRunCmd.Flags().DurationVar(&memoryOpts.timeout, "timeout", 10*time.Second, "Jev request timeout")}@@ -208,13 +208,7 @@ func runMemoryEvaluationsRun(cmd *cobra.Command, args []string) error {if err != nil {return err}- evaluatorOpts := learning.EvaluatorOptions{- Endpoint: firstNonEmpty(memoryOpts.jevEndpoint, os.Getenv("BEACON_JEV_ENDPOINT"), learning.DefaultJevEndpoint),- APIKey: firstNonEmpty(memoryOpts.jevAPIKey, os.Getenv("TYPESAFE_API_KEY"), os.Getenv("BEACON_JEV_API_KEY")),- Model: firstNonEmpty(memoryOpts.jevModel, os.Getenv("BEACON_JEV_MODEL"), learning.DefaultJevModel),- CostPerTrace: memoryOpts.jevCost,- Timeout: memoryOpts.timeout,- }+ evaluatorOpts := jevEvaluatorOptions()result := evaluationRunResult{DryRun: memoryOpts.dryRun,Project: project,@@ -506,6 +500,19 @@ func parseMemoryTime(name, value string) (time.Time, error) {return parsed, nil}+// jevEvaluatorOptions resolves each Jev setting from its flag, then the environment,+// then the built-in default. The flags must not carry the defaults themselves, or the+// environment variables documented for them would never be consulted.+func jevEvaluatorOptions() learning.EvaluatorOptions {+ return learning.EvaluatorOptions{+ Endpoint: firstNonEmpty(memoryOpts.jevEndpoint, os.Getenv("BEACON_JEV_ENDPOINT"), learning.DefaultJevEndpoint),+ APIKey: firstNonEmpty(memoryOpts.jevAPIKey, os.Getenv("TYPESAFE_API_KEY"), os.Getenv("BEACON_JEV_API_KEY")),+ Model: firstNonEmpty(memoryOpts.jevModel, os.Getenv("BEACON_JEV_MODEL"), learning.DefaultJevModel),+ CostPerTrace: memoryOpts.jevCost,+ Timeout: memoryOpts.timeout,+ }+}+func firstNonEmpty(values ...string) string {for _, value := range values {if strings.TrimSpace(value) != "" {diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.goindex f5fb9786..2ca8a8a6 100644--- a/cli/beacon/cmd/memory_test.go+++ b/cli/beacon/cmd/memory_test.go@@ -241,7 +241,7 @@ func resetMemoryOpts(t *testing.T) {reason stringreplacement stringforce bool- }{userMode: true, limit: 25, page: 1, jevEndpoint: learning.DefaultJevEndpoint, jevModel: learning.DefaultJevModel, jevCost: learning.DefaultCostPerTrace}+ }{userMode: true, limit: 25, page: 1, jevCost: learning.DefaultCostPerTrace}t.Cleanup(func() { memoryOpts = previous })}@@ -334,3 +334,122 @@ func TestMemoryEvaluationsShowHidesPlaceholderReason(t *testing.T) {t.Fatalf("show dropped a real reason:\n%s", out.String())}}++// The --jev-endpoint and --jev-model flags used to default to the hosted TypeSafe+// values, so BEACON_JEV_ENDPOINT and BEACON_JEV_MODEL were never consulted even+// though the docs and the --jev-api-key help text promise them. Leaving the flags+// unset must fall through to the environment, then to the built-in defaults.+func TestMemoryEvaluationsRunHonoursJevEnvironment(t *testing.T) {+ logPath, project := writeMemoryCommandFixture(t)+ var gotModel string+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Authorization") != "Bearer env-key" {+ t.Errorf("Authorization = %q, want Bearer env-key", r.Header.Get("Authorization"))+ }+ var req struct {+ Model string `json:"model"`+ }+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {+ t.Errorf("decode request: %v", err)+ }+ gotModel = req.Model+ _, _ = w.Write([]byte(`{"model":"typesafe-ai/jev","answers":{"task_success":{"type":"noul","noul":0.9},"reusable_correction":{"type":"noul","noul":0.8},"evidence_supported":{"type":"noul","noul":0.7}}}`))+ }))+ defer server.Close()++ t.Setenv("BEACON_JEV_ENDPOINT", server.URL)+ t.Setenv("BEACON_JEV_MODEL", "typesafe-ai/jev")+ t.Setenv("TYPESAFE_API_KEY", "env-key")+ t.Setenv("BEACON_JEV_API_KEY", "")++ resetMemoryOpts(t)+ memoryOpts.userMode = true+ memoryOpts.logPath = logPath+ memoryOpts.projectPath = project+ memoryOpts.traceID = "session:cursor:s1"+ memoryOpts.jsonOutput = true++ var out bytes.Buffer+ cmd := &cobra.Command{}+ cmd.SetOut(&out)+ if err := runMemoryEvaluationsRun(cmd, nil); err != nil {+ t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)+ }+ if gotModel != "typesafe-ai/jev" {+ t.Fatalf("request model = %q, want typesafe-ai/jev", gotModel)+ }+ var result evaluationRunResult+ if err := json.Unmarshal(out.Bytes(), &result); err != nil {+ t.Fatalf("decode result: %v\n%s", err, out.String())+ }+ if len(result.Evaluations) != 1 {+ t.Fatalf("evaluations = %#v", result.Evaluations)+ }+ if got := result.Evaluations[0].Evaluator; !strings.Contains(got, server.URL) {+ t.Fatalf("evaluator endpoint = %q, want it to reference %s", got, server.URL)+ }+}++func TestJevEvaluatorOptionsPrecedence(t *testing.T) {+ cases := []struct {+ name string+ flagEndpoint string+ flagModel string+ flagKey string+ env map[string]string+ wantEndpoint string+ wantModel string+ wantKey string+ }{+ {+ name: "defaults when nothing is set",+ env: map[string]string{"BEACON_JEV_ENDPOINT": "", "BEACON_JEV_MODEL": "", "TYPESAFE_API_KEY": "", "BEACON_JEV_API_KEY": ""},+ wantEndpoint: learning.DefaultJevEndpoint,+ wantModel: learning.DefaultJevModel,+ },+ {+ name: "environment overrides defaults",+ env: map[string]string{+ "BEACON_JEV_ENDPOINT": "https://gateway.example/typesafe/v1/systemone",+ "BEACON_JEV_MODEL": "typesafe-ai/jev",+ "TYPESAFE_API_KEY": "",+ "BEACON_JEV_API_KEY": "fallback-key",+ },+ wantEndpoint: "https://gateway.example/typesafe/v1/systemone",+ wantModel: "typesafe-ai/jev",+ wantKey: "fallback-key",+ },+ {+ name: "flags override environment",+ flagEndpoint: "https://flag.example/v1/systemone",+ flagModel: "jev-1.13.0",+ flagKey: "flag-key",+ env: map[string]string{+ "BEACON_JEV_ENDPOINT": "https://gateway.example/typesafe/v1/systemone",+ "BEACON_JEV_MODEL": "typesafe-ai/jev",+ "TYPESAFE_API_KEY": "env-key",+ "BEACON_JEV_API_KEY": "fallback-key",+ },+ wantEndpoint: "https://flag.example/v1/systemone",+ wantModel: "jev-1.13.0",+ wantKey: "flag-key",+ },+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ for k, v := range tc.env {+ t.Setenv(k, v)+ }+ resetMemoryOpts(t)+ memoryOpts.jevEndpoint = tc.flagEndpoint+ memoryOpts.jevModel = tc.flagModel+ memoryOpts.jevAPIKey = tc.flagKey++ got := jevEvaluatorOptions()+ if got.Endpoint != tc.wantEndpoint || got.Model != tc.wantModel || got.APIKey != tc.wantKey {+ t.Fatalf("jevEvaluatorOptions() = endpoint %q model %q key %q, want %q %q %q",+ got.Endpoint, got.Model, got.APIKey, tc.wantEndpoint, tc.wantModel, tc.wantKey)+ }+ })+ }+}diff --git a/docs/cli/memory.mdx b/docs/cli/memory.mdxindex c4907c97..13625933 100644--- a/docs/cli/memory.mdx+++ b/docs/cli/memory.mdx@@ -59,9 +59,23 @@ Useful flags:- `--timeout <duration>` sets the Jev request timeout.- `--json` prints machine-readable output.-`--jev-api-key` defaults to `TYPESAFE_API_KEY`, then `BEACON_JEV_API_KEY`.-`--jev-endpoint` can also come from `BEACON_JEV_ENDPOINT`, and the model can come-from `BEACON_JEV_MODEL`.+Each Jev setting is resolved from the flag first, then the environment, then the+built-in default:++- `--jev-api-key` defaults to `TYPESAFE_API_KEY`, then `BEACON_JEV_API_KEY`.+- `--jev-endpoint` defaults to `BEACON_JEV_ENDPOINT`, then the hosted TypeSafe+ endpoint `https://api.typesafe.ai/v1/systemone`.+- `--jev-model` defaults to `BEACON_JEV_MODEL`, then `jev-latest`.++This is how you point Beacon at a TypeSafe-compatible gateway. For example, to+evaluate through Vercel AI Gateway with an AI Gateway API key:++```bash+export TYPESAFE_API_KEY=<AI Gateway API key>+export BEACON_JEV_ENDPOINT=https://ai-gateway.vercel.sh/typesafe/v1/systemone+export BEACON_JEV_MODEL=typesafe-ai/jev+beacon memory evaluations run --limit 10+```List and inspect stored evaluations: