Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Add Jev trace evaluation

Justin D'Souza579dfdfde9d2
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.go
new file mode 100644
index 00000000..53075ad6
--- /dev/null
+++ b/cli/beacon/cmd/memory.go
@@ -0,0 +1,297 @@
+package cmd
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "strings"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/dashboard"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/learning"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+var memoryOpts struct {
+ userMode bool
+ systemMode bool
+ logPath string
+ jsonOutput bool
+ projectPath string
+ limit int
+ page int
+ query string
+ harness string
+ traceID string
+ since string
+ until string
+ dryRun bool
+ jevEndpoint string
+ jevAPIKey string
+ jevCost float64
+ timeout time.Duration
+}
+
+var memoryCmd = &cobra.Command{
+ Use: "memory",
+ Short: "Evaluate and review cross-harness agent memory",
+}
+
+var memoryEvaluationsCmd = &cobra.Command{
+ Use: "evaluations",
+ Short: "Run and inspect local trace evaluations",
+}
+
+var memoryEvaluationsRunCmd = &cobra.Command{
+ Use: "run",
+ Short: "Evaluate local traces with Jev",
+ SilenceUsage: true,
+ RunE: runMemoryEvaluationsRun,
+}
+
+var memoryEvaluationsListCmd = &cobra.Command{
+ Use: "list",
+ Short: "List persisted trace evaluations",
+ SilenceUsage: true,
+ RunE: runMemoryEvaluationsList,
+}
+
+var memoryEvaluationsShowCmd = &cobra.Command{
+ Use: "show <evaluation-id>",
+ Short: "Show one persisted trace evaluation",
+ Args: cobra.ExactArgs(1),
+ SilenceUsage: true,
+ RunE: runMemoryEvaluationsShow,
+}
+
+type evaluationRunResult struct {
+ DryRun bool `json:"dry_run"`
+ Project asymptoteobserve.LearningProjectV1 `json:"project"`
+ StorePath string `json:"store_path"`
+ TraceCount int `json:"trace_count"`
+ EstimatedCalls int `json:"estimated_calls"`
+ EstimatedCostUSD float64 `json:"estimated_cost_usd"`
+ Previews []learning.EvaluationPreview `json:"previews,omitempty"`
+ Evaluations []asymptoteobserve.LearningEvaluationV1 `json:"evaluations,omitempty"`
+}
+
+func init() {
+ rootCmd.AddCommand(memoryCmd)
+ memoryCmd.AddCommand(memoryEvaluationsCmd)
+ memoryEvaluationsCmd.AddCommand(memoryEvaluationsRunCmd)
+ memoryEvaluationsCmd.AddCommand(memoryEvaluationsListCmd)
+ memoryEvaluationsCmd.AddCommand(memoryEvaluationsShowCmd)
+
+ for _, c := range []*cobra.Command{memoryEvaluationsRunCmd, memoryEvaluationsListCmd, memoryEvaluationsShowCmd} {
+ c.Flags().BoolVar(&memoryOpts.userMode, "user", true, "Use per-user endpoint paths")
+ c.Flags().BoolVar(&memoryOpts.systemMode, "system", false, "Use system endpoint paths")
+ c.Flags().StringVar(&memoryOpts.logPath, "log-path", "", "Runtime JSONL log path")
+ c.Flags().BoolVar(&memoryOpts.jsonOutput, "json", false, "Print machine-readable JSON")
+ c.Flags().StringVar(&memoryOpts.projectPath, "project", "", "Project path for memory scoping (defaults to current directory)")
+ }
+ for _, c := range []*cobra.Command{memoryEvaluationsRunCmd, memoryEvaluationsListCmd} {
+ c.Flags().IntVar(&memoryOpts.limit, "limit", 25, "Limit returned traces or evaluations")
+ c.Flags().IntVar(&memoryOpts.page, "page", 1, "Page number for collection output")
+ c.Flags().StringVarP(&memoryOpts.query, "query", "q", "", "Free-text query")
+ }
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.harness, "harness", "", "Filter traces by harness")
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.traceID, "trace", "", "Evaluate a single trace ID")
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.since, "since", "", "RFC3339 lower time bound, inclusive")
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.until, "until", "", "RFC3339 upper time bound, inclusive")
+ memoryEvaluationsRunCmd.Flags().BoolVar(&memoryOpts.dryRun, "dry-run", false, "Preview selected traces, Jev calls, and estimated cost without writing evaluations")
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevEndpoint, "jev-endpoint", "", "Jev evaluator endpoint (defaults to BEACON_JEV_ENDPOINT)")
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevAPIKey, "jev-api-key", "", "Jev API key (defaults to BEACON_JEV_API_KEY)")
+ memoryEvaluationsRunCmd.Flags().Float64Var(&memoryOpts.jevCost, "jev-cost-per-trace", learning.DefaultCostPerTrace, "Estimated Jev cost per trace in USD")
+ memoryEvaluationsRunCmd.Flags().DurationVar(&memoryOpts.timeout, "timeout", 10*time.Second, "Jev request timeout")
+}
+
+func runMemoryEvaluationsRun(cmd *cobra.Command, args []string) error {
+ project, err := learning.ResolveProject(memoryOpts.projectPath)
+ if err != nil {
+ return err
+ }
+ store := memoryStore()
+ inputs, err := selectedEvaluationInputs(project)
+ if err != nil {
+ return err
+ }
+ evaluatorOpts := learning.EvaluatorOptions{
+ Endpoint: firstNonEmpty(memoryOpts.jevEndpoint, os.Getenv("BEACON_JEV_ENDPOINT")),
+ APIKey: firstNonEmpty(memoryOpts.jevAPIKey, os.Getenv("BEACON_JEV_API_KEY")),
+ CostPerTrace: memoryOpts.jevCost,
+ Timeout: memoryOpts.timeout,
+ }
+ result := evaluationRunResult{
+ DryRun: memoryOpts.dryRun,
+ Project: project,
+ StorePath: store.Path(),
+ TraceCount: len(inputs),
+ EstimatedCalls: len(inputs),
+ EstimatedCostUSD: float64(len(inputs)) * memoryOpts.jevCost,
+ }
+ for _, input := range inputs {
+ input.DryRun = memoryOpts.dryRun
+ if memoryOpts.dryRun {
+ result.Previews = append(result.Previews, learning.Preview(input, evaluatorOpts))
+ continue
+ }
+ ctx := cmd.Context()
+ if ctx == nil {
+ ctx = context.Background()
+ }
+ eval, err := learning.Evaluate(ctx, evaluatorOpts, input)
+ if err != nil {
+ return err
+ }
+ if err := store.PutEvaluation(eval); err != nil {
+ return err
+ }
+ result.Evaluations = append(result.Evaluations, eval)
+ }
+ if memoryOpts.jsonOutput {
+ return json.NewEncoder(cmd.OutOrStdout()).Encode(result)
+ }
+ if result.DryRun {
+ fmt.Fprintf(cmd.OutOrStdout(), "Selected %d trace(s); estimated Jev calls: %d; estimated cost: $%.5f\n", result.TraceCount, result.EstimatedCalls, result.EstimatedCostUSD)
+ for _, preview := range result.Previews {
+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t$%.5f\n", preview.TraceID, preview.Title, preview.EstimatedCostUSD)
+ }
+ return nil
+ }
+ fmt.Fprintf(cmd.OutOrStdout(), "Evaluated %d trace(s); wrote results to %s\n", len(result.Evaluations), store.Path())
+ for _, eval := range result.Evaluations {
+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%.2f\t%s\n", eval.ID, eval.Trace.ID, eval.Score, eval.Status)
+ }
+ return nil
+}
+
+func runMemoryEvaluationsList(cmd *cobra.Command, args []string) error {
+ query, err := memoryQuery()
+ if err != nil {
+ return err
+ }
+ evals, err := memoryStore().ListEvaluations(query)
+ if err != nil {
+ return err
+ }
+ if memoryOpts.jsonOutput {
+ return json.NewEncoder(cmd.OutOrStdout()).Encode(evals)
+ }
+ for _, eval := range evals {
+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%s\t%.2f\t%s\n", eval.ID, eval.Trace.ID, eval.Status, eval.Score, eval.Trace.Title)
+ }
+ return nil
+}
+
+func runMemoryEvaluationsShow(cmd *cobra.Command, args []string) error {
+ eval, ok, err := memoryStore().GetEvaluation(args[0])
+ if err != nil {
+ return err
+ }
+ if !ok {
+ return fmt.Errorf("evaluation not found: %s", args[0])
+ }
+ if memoryOpts.jsonOutput {
+ return json.NewEncoder(cmd.OutOrStdout()).Encode(eval)
+ }
+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%s\t%.2f\n", eval.ID, eval.Trace.ID, eval.Status, eval.Score)
+ for _, question := range eval.Questions {
+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%.2f\t%.2f\t%s\n", question.ID, question.Probability, question.Confidence, question.Reason)
+ }
+ return nil
+}
+
+func selectedEvaluationInputs(project asymptoteobserve.LearningProjectV1) ([]learning.EvaluationInput, error) {
+ logPath := memoryLogPath()
+ if memoryOpts.traceID != "" {
+ show, ok, err := dashboard.ShowTrace(logPath, memoryOpts.traceID, dashboard.TraceQuery{Limit: 2000})
+ if err != nil {
+ return nil, err
+ }
+ if !ok {
+ return nil, fmt.Errorf("trace not found: %s", memoryOpts.traceID)
+ }
+ return []learning.EvaluationInput{{Project: project, Trace: show}}, nil
+ }
+ query := dashboard.TraceQuery{
+ EventQuery: dashboard.EventQuery{
+ Q: memoryOpts.query,
+ Harness: memoryOpts.harness,
+ },
+ Limit: memoryOpts.limit,
+ Page: memoryOpts.page,
+ }
+ var err error
+ if query.Since, err = parseMemoryTime("since", memoryOpts.since); err != nil {
+ return nil, err
+ }
+ if query.Until, err = parseMemoryTime("until", memoryOpts.until); err != nil {
+ return nil, err
+ }
+ traces, err := dashboard.ReadTraceList(logPath, query)
+ if err != nil {
+ return nil, err
+ }
+ inputs := make([]learning.EvaluationInput, 0, len(traces.Traces))
+ for _, summary := range traces.Traces {
+ show, ok, err := dashboard.ShowTrace(logPath, summary.ID, dashboard.TraceQuery{Limit: 2000})
+ if err != nil {
+ return nil, err
+ }
+ if !ok {
+ continue
+ }
+ inputs = append(inputs, learning.EvaluationInput{Project: project, Trace: show})
+ }
+ return inputs, nil
+}
+
+func memoryQuery() (learning.Query, error) {
+ project, err := learning.ResolveProject(memoryOpts.projectPath)
+ if err != nil {
+ return learning.Query{}, err
+ }
+ return learning.Query{
+ ProjectID: project.ID,
+ Q: memoryOpts.query,
+ Limit: memoryOpts.limit,
+ Page: memoryOpts.page,
+ }, nil
+}
+
+func memoryStore() *learning.Store {
+ return learning.Open(learning.PathForRuntimeLog(memoryLogPath()))
+}
+
+func memoryLogPath() string {
+ runtimeLog := lifecycle.ResolveRuntimeLog(memoryUserMode(), memoryOpts.logPath)
+ return runtimeLog.EffectiveLogPath
+}
+
+func memoryUserMode() bool {
+ return memoryOpts.userMode && !memoryOpts.systemMode
+}
+
+func parseMemoryTime(name, value string) (time.Time, error) {
+ if strings.TrimSpace(value) == "" {
+ return time.Time{}, nil
+ }
+ parsed, err := time.Parse(time.RFC3339, value)
+ if err != nil {
+ return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", name, err)
+ }
+ return parsed, nil
+}
+
+func firstNonEmpty(values ...string) string {
+ for _, value := range values {
+ if strings.TrimSpace(value) != "" {
+ return strings.TrimSpace(value)
+ }
+ }
+ return ""
+}
diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.go
new file mode 100644
index 00000000..b5e9a0b1
--- /dev/null
+++ b/cli/beacon/cmd/memory_test.go
@@ -0,0 +1,167 @@
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/learning"
+)
+
+func TestMemoryEvaluationCommandsRegistered(t *testing.T) {
+ for _, path := range [][]string{
+ {"memory", "evaluations", "run"},
+ {"memory", "evaluations", "list"},
+ {"memory", "evaluations", "show"},
+ } {
+ cmd, _, err := rootCmd.Find(path)
+ if err != nil || cmd == nil {
+ t.Fatalf("command %v not registered: %v", path, err)
+ }
+ }
+}
+
+func TestMemoryEvaluationsRunDryRunPreviewsCost(t *testing.T) {
+ logPath, project := writeMemoryCommandFixture(t)
+ resetMemoryOpts(t)
+ memoryOpts.userMode = true
+ memoryOpts.logPath = logPath
+ memoryOpts.projectPath = project
+ memoryOpts.traceID = "session:cursor:s1"
+ memoryOpts.dryRun = true
+ memoryOpts.jsonOutput = true
+ memoryOpts.jevCost = 0.01
+
+ var out bytes.Buffer
+ cmd := &cobra.Command{}
+ cmd.SetOut(&out)
+ if err := runMemoryEvaluationsRun(cmd, nil); err != nil {
+ t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)
+ }
+ var result evaluationRunResult
+ if err := json.Unmarshal(out.Bytes(), &result); err != nil {
+ t.Fatalf("decode result: %v\n%s", err, out.String())
+ }
+ if !result.DryRun || result.TraceCount != 1 || result.EstimatedCostUSD != 0.01 {
+ t.Fatalf("result = %#v", result)
+ }
+ if _, err := os.Stat(learning.PathForRuntimeLog(logPath)); !os.IsNotExist(err) {
+ t.Fatalf("dry-run should not create memory store, stat err=%v", err)
+ }
+}
+
+func TestMemoryEvaluationsRunPersistsMockedJevResult(t *testing.T) {
+ logPath, project := writeMemoryCommandFixture(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") != "Bearer test-key" {
+ t.Fatalf("missing auth header: %s", r.Header.Get("Authorization"))
+ }
+ _, _ = w.Write([]byte(`{"questions":[{"id":"task_success","probability":0.9},{"id":"reusable_correction","probability":0.8},{"id":"evidence_supported","probability":0.7}]}`))
+ }))
+ defer server.Close()
+
+ resetMemoryOpts(t)
+ memoryOpts.userMode = true
+ memoryOpts.logPath = logPath
+ memoryOpts.projectPath = project
+ memoryOpts.traceID = "session:cursor:s1"
+ memoryOpts.jsonOutput = true
+ memoryOpts.jevEndpoint = server.URL
+ memoryOpts.jevAPIKey = "test-key"
+ memoryOpts.jevCost = learning.DefaultCostPerTrace
+
+ var out bytes.Buffer
+ cmd := &cobra.Command{}
+ cmd.SetOut(&out)
+ if err := runMemoryEvaluationsRun(cmd, nil); err != nil {
+ t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)
+ }
+ var result evaluationRunResult
+ if err := json.Unmarshal(out.Bytes(), &result); err != nil {
+ t.Fatalf("decode result: %v\n%s", err, out.String())
+ }
+ if len(result.Evaluations) != 1 || result.Evaluations[0].Score < 0.79 || result.Evaluations[0].Score > 0.81 {
+ t.Fatalf("evaluations = %#v", result.Evaluations)
+ }
+
+ memoryOpts.query = "session:cursor:s1"
+ list, err := memoryStore().ListEvaluations(learning.Query{ProjectID: result.Project.ID, Q: "cursor"})
+ if err != nil {
+ t.Fatalf("ListEvaluations: %v", err)
+ }
+ if len(list) != 1 || !strings.HasPrefix(list[0].ID, "eval_") {
+ t.Fatalf("persisted evaluations = %#v", list)
+ }
+}
+
+func writeMemoryCommandFixture(t *testing.T) (string, string) {
+ t.Helper()
+ dir := t.TempDir()
+ project := filepath.Join(dir, "project")
+ if err := os.MkdirAll(filepath.Join(project, ".git"), 0755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(project, ".git", "HEAD"), []byte("ref: refs/heads/main\n"), 0644); err != nil {
+ t.Fatal(err)
+ }
+ logPath := filepath.Join(dir, "endpoint", "logs", "runtime.jsonl")
+ if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil {
+ t.Fatal(err)
+ }
+ event := schema.Event{
+ Timestamp: "2026-09-21T01:00:00Z",
+ Vendor: schema.Vendor,
+ Product: schema.Product,
+ SchemaVersion: schema.SchemaVersion,
+ Event: schema.EventInfo{ID: "event-1", Kind: "agent_runtime", Action: "prompt.submitted", Category: "prompt"},
+ Severity: schema.SeverityInfo,
+ Endpoint: schema.EndpointInfo{OS: "darwin"},
+ Harness: schema.HarnessInfo{Name: "cursor"},
+ Session: &schema.SessionInfo{ID: "s1", WorkingDirectory: project},
+ Prompt: &schema.PromptInfo{Text: "Fix flaky package smoke"},
+ Message: "Fix flaky package smoke",
+ Repository: project,
+ }
+ line, err := json.Marshal(event)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(logPath, append(line, '\n'), 0644); err != nil {
+ t.Fatal(err)
+ }
+ return logPath, project
+}
+
+func resetMemoryOpts(t *testing.T) {
+ t.Helper()
+ previous := memoryOpts
+ memoryOpts = struct {
+ userMode bool
+ systemMode bool
+ logPath string
+ jsonOutput bool
+ projectPath string
+ limit int
+ page int
+ query string
+ harness string
+ traceID string
+ since string
+ until string
+ dryRun bool
+ jevEndpoint string
+ jevAPIKey string
+ jevCost float64
+ timeout time.Duration
+ }{userMode: true, limit: 25, page: 1, jevCost: learning.DefaultCostPerTrace}
+ t.Cleanup(func() { memoryOpts = previous })
+}
diff --git a/cli/beacon/internal/learning/evaluator.go b/cli/beacon/internal/learning/evaluator.go
new file mode 100644
index 00000000..703b0e65
--- /dev/null
+++ b/cli/beacon/internal/learning/evaluator.go
@@ -0,0 +1,317 @@
+package learning
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+const (
+ RubricVersion = "beacon.learning.rubric.v1"
+ DefaultCostPerTrace = 0.00035
+ maxProjectionEvents = 80
+ maxProjectionText = 1200
+)
+
+var RubricQuestions = []asymptoteobserve.LearningEvaluationQuestionV1{
+ {ID: "task_success", Prompt: "Did the trace complete the user's engineering task successfully?"},
+ {ID: "reusable_correction", Prompt: "Does the trace contain a correction or debugging pattern that future agents should reuse?"},
+ {ID: "evidence_supported", Prompt: "Is the reusable lesson supported by concrete events in the trace?"},
+}
+
+type EvaluatorOptions struct {
+ Endpoint string
+ APIKey string
+ HTTPClient *http.Client
+ CostPerTrace float64
+ Timeout time.Duration
+}
+
+type EvaluationInput struct {
+ Project asymptoteobserve.LearningProjectV1
+ Trace asymptoteobserve.TraceShowResultV1
+ DryRun bool
+}
+
+type EvaluationPreview struct {
+ TraceID string `json:"trace_id"`
+ Title string `json:"title,omitempty"`
+ EventCount int `json:"event_count"`
+ EstimatedCalls int `json:"estimated_calls"`
+ EstimatedCostUSD float64 `json:"estimated_cost_usd"`
+ Reason string `json:"reason,omitempty"`
+}
+
+type Projection struct {
+ TraceID string `json:"trace_id"`
+ Title string `json:"title,omitempty"`
+ Harness string `json:"harness,omitempty"`
+ Repository string `json:"repository,omitempty"`
+ Events []ProjectedEvent `json:"events"`
+ Questions []ProjectedQuestion `json:"questions"`
+}
+
+type ProjectedEvent struct {
+ Number int `json:"number"`
+ Type string `json:"type"`
+ Action string `json:"action"`
+ Title string `json:"title,omitempty"`
+ Summary string `json:"summary,omitempty"`
+ Content string `json:"content,omitempty"`
+}
+
+type ProjectedQuestion struct {
+ ID string `json:"id"`
+ Prompt string `json:"prompt"`
+}
+
+type jevRequest struct {
+ RubricVersion string `json:"rubric_version"`
+ RubricHash string `json:"rubric_hash"`
+ Trace Projection `json:"trace"`
+}
+
+type jevResponse struct {
+ Questions []asymptoteobserve.LearningEvaluationQuestionV1 `json:"questions"`
+ Results []asymptoteobserve.LearningEvaluationQuestionV1 `json:"results"`
+ Score *float64 `json:"score,omitempty"`
+ CostEstimateUSD *float64 `json:"cost_estimate_usd,omitempty"`
+}
+
+func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput) (asymptoteobserve.LearningEvaluationV1, error) {
+ projection := BuildProjection(input.Trace)
+ evaluation := baseEvaluation(opts, input, projection)
+ if input.DryRun {
+ evaluation.Status = asymptoteobserve.LearningEvaluationStatusDryRun
+ evaluation.CostEstimateUSD = costPerTrace(opts)
+ evaluation.ID = EvaluationID(evaluation)
+ return evaluation, nil
+ }
+ if strings.TrimSpace(opts.Endpoint) == "" {
+ return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("Jev endpoint is required; pass --jev-endpoint or set BEACON_JEV_ENDPOINT")
+ }
+ resp, err := callEvaluator(ctx, opts, projection)
+ if err != nil {
+ evaluation.Status = asymptoteobserve.LearningEvaluationStatusFailed
+ evaluation.Error = err.Error()
+ evaluation.ID = EvaluationID(evaluation)
+ return evaluation, err
+ }
+ questions := resp.Questions
+ if len(questions) == 0 {
+ questions = resp.Results
+ }
+ if len(questions) == 0 {
+ return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("Jev response did not include questions or results")
+ }
+ evaluation.Questions = normalizeQuestionResults(questions)
+ evaluation.Score = evaluationScore(evaluation.Questions)
+ if resp.Score != nil {
+ evaluation.Score = *resp.Score
+ }
+ if resp.CostEstimateUSD != nil {
+ evaluation.CostEstimateUSD = *resp.CostEstimateUSD
+ } else {
+ evaluation.CostEstimateUSD = costPerTrace(opts)
+ }
+ evaluation.Status = asymptoteobserve.LearningEvaluationStatusCompleted
+ evaluation.ID = EvaluationID(evaluation)
+ return evaluation, nil
+}
+
+func Preview(input EvaluationInput, opts EvaluatorOptions) EvaluationPreview {
+ trace := input.Trace.Trace
+ return EvaluationPreview{
+ TraceID: trace.ID,
+ Title: trace.Title,
+ EventCount: input.Trace.Range.TotalEvents,
+ EstimatedCalls: 1,
+ EstimatedCostUSD: costPerTrace(opts),
+ Reason: "explicit evaluation command; hooks and collectors do not call Jev",
+ }
+}
+
+func BuildProjection(show asymptoteobserve.TraceShowResultV1) Projection {
+ projection := Projection{
+ TraceID: show.Trace.ID,
+ Title: show.Trace.Title,
+ Harness: show.Trace.Harness.Name,
+ }
+ if show.Trace.Repository != nil {
+ projection.Repository = firstNonEmpty(show.Trace.Repository.RemoteURL, show.Trace.Repository.Path)
+ }
+ for _, question := range RubricQuestions {
+ projection.Questions = append(projection.Questions, ProjectedQuestion{ID: question.ID, Prompt: question.Prompt})
+ }
+ events := show.Events
+ if len(events) > maxProjectionEvents {
+ events = events[:maxProjectionEvents]
+ }
+ for _, event := range events {
+ projected := ProjectedEvent{
+ Number: event.Number,
+ Type: event.Type,
+ Action: event.Action,
+ Title: cleanText(event.Title),
+ Summary: cleanText(event.Summary),
+ }
+ if event.Content != nil {
+ projected.Content = cleanText(event.Content.Text)
+ }
+ if event.Command != nil && projected.Content == "" {
+ projected.Content = cleanText(event.Command.Command)
+ }
+ projection.Events = append(projection.Events, projected)
+ }
+ return projection
+}
+
+func RubricHash() string {
+ data, _ := json.Marshal(RubricQuestions)
+ sum := sha256.Sum256(data)
+ return "sha256:" + hex.EncodeToString(sum[:])
+}
+
+func baseEvaluation(opts EvaluatorOptions, input EvaluationInput, projection Projection) asymptoteobserve.LearningEvaluationV1 {
+ trace := input.Trace.Trace
+ ref := asymptoteobserve.LearningTraceRefV1{
+ ID: trace.ID,
+ Title: trace.Title,
+ Harness: trace.Harness,
+ Session: trace.Session,
+ Repo: trace.Repository,
+ }
+ for _, event := range input.Trace.Events {
+ ref.EventIDs = append(ref.EventIDs, event.ID)
+ }
+ return asymptoteobserve.LearningEvaluationV1{
+ SchemaVersion: asymptoteobserve.LearningSchemaVersion,
+ Status: asymptoteobserve.LearningEvaluationStatusCompleted,
+ Project: input.Project,
+ RubricVersion: RubricVersion,
+ RubricHash: RubricHash(),
+ Evaluator: evaluatorName(opts),
+ DryRun: input.DryRun,
+ Trace: ref,
+ CostEstimateUSD: costPerTrace(opts),
+ Questions: projectionQuestionDefaults(projection),
+ }
+}
+
+func callEvaluator(ctx context.Context, opts EvaluatorOptions, projection Projection) (jevResponse, error) {
+ client := opts.HTTPClient
+ if client == nil {
+ client = http.DefaultClient
+ }
+ timeout := opts.Timeout
+ if timeout <= 0 {
+ timeout = 10 * time.Second
+ }
+ ctx, cancel := context.WithTimeout(ctx, timeout)
+ defer cancel()
+ body, err := json.Marshal(jevRequest{RubricVersion: RubricVersion, RubricHash: RubricHash(), Trace: projection})
+ if err != nil {
+ return jevResponse{}, err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, opts.Endpoint, bytes.NewReader(body))
+ if err != nil {
+ return jevResponse{}, err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ if opts.APIKey != "" {
+ req.Header.Set("Authorization", "Bearer "+opts.APIKey)
+ }
+ resp, err := client.Do(req)
+ if err != nil {
+ return jevResponse{}, err
+ }
+ defer resp.Body.Close()
+ data, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
+ if err != nil {
+ return jevResponse{}, err
+ }
+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
+ return jevResponse{}, fmt.Errorf("Jev endpoint returned %s: %s", resp.Status, strings.TrimSpace(string(data)))
+ }
+ var parsed jevResponse
+ if err := json.Unmarshal(data, &parsed); err != nil {
+ return jevResponse{}, err
+ }
+ return parsed, nil
+}
+
+func normalizeQuestionResults(results []asymptoteobserve.LearningEvaluationQuestionV1) []asymptoteobserve.LearningEvaluationQuestionV1 {
+ byID := map[string]asymptoteobserve.LearningEvaluationQuestionV1{}
+ for _, result := range results {
+ byID[result.ID] = result
+ }
+ out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(RubricQuestions))
+ for _, question := range RubricQuestions {
+ result := byID[question.ID]
+ if result.ID == "" {
+ result.ID = question.ID
+ }
+ if result.Prompt == "" {
+ result.Prompt = question.Prompt
+ }
+ if result.Probability < 0 {
+ result.Probability = 0
+ }
+ if result.Probability > 1 {
+ result.Probability = 1
+ }
+ out = append(out, result)
+ }
+ return out
+}
+
+func projectionQuestionDefaults(projection Projection) []asymptoteobserve.LearningEvaluationQuestionV1 {
+ out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(projection.Questions))
+ for _, question := range projection.Questions {
+ out = append(out, asymptoteobserve.LearningEvaluationQuestionV1{ID: question.ID, Prompt: question.Prompt})
+ }
+ return out
+}
+
+func evaluationScore(results []asymptoteobserve.LearningEvaluationQuestionV1) float64 {
+ if len(results) == 0 {
+ return 0
+ }
+ var sum float64
+ for _, result := range results {
+ sum += result.Probability
+ }
+ return sum / float64(len(results))
+}
+
+func evaluatorName(opts EvaluatorOptions) string {
+ if opts.Endpoint == "" {
+ return "jev"
+ }
+ return "jev:" + opts.Endpoint
+}
+
+func costPerTrace(opts EvaluatorOptions) float64 {
+ if opts.CostPerTrace > 0 {
+ return opts.CostPerTrace
+ }
+ return DefaultCostPerTrace
+}
+
+func cleanText(value string) string {
+ value = strings.TrimSpace(value)
+ if value == "" {
+ return ""
+ }
+ value = asymptoteobserve.CleanString(value, maxProjectionText, true)
+ return value
+}
diff --git a/cli/beacon/internal/learning/evaluator_test.go b/cli/beacon/internal/learning/evaluator_test.go
new file mode 100644
index 00000000..1a553fbf
--- /dev/null
+++ b/cli/beacon/internal/learning/evaluator_test.go
@@ -0,0 +1,111 @@
+package learning
+
+import (
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+func TestEvaluateDryRunDoesNotNeedEndpoint(t *testing.T) {
+ show := testTraceShow()
+ project := asymptoteobserve.LearningProjectV1{ID: "project-1"}
+ eval, err := Evaluate(t.Context(), EvaluatorOptions{}, EvaluationInput{Project: project, Trace: show, DryRun: true})
+ if err != nil {
+ t.Fatalf("Evaluate dry-run returned error: %v", err)
+ }
+ if eval.Status != asymptoteobserve.LearningEvaluationStatusDryRun {
+ t.Fatalf("status = %s", eval.Status)
+ }
+ if eval.ID == "" || eval.RubricHash == "" || eval.CostEstimateUSD == 0 {
+ t.Fatalf("dry-run missing identifiers/cost: %#v", eval)
+ }
+}
+
+func TestEvaluatePostsProjectionToMockableEndpoint(t *testing.T) {
+ var sawAuth bool
+ var sawTrace bool
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.Header.Get("Authorization") == "Bearer secret" {
+ sawAuth = true
+ }
+ var req jevRequest
+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
+ t.Fatalf("decode request: %v", err)
+ }
+ sawTrace = req.Trace.TraceID == "trace-1" && len(req.Trace.Questions) == len(RubricQuestions)
+ _ = json.NewEncoder(w).Encode(jevResponse{
+ Questions: []asymptoteobserve.LearningEvaluationQuestionV1{
+ {ID: "task_success", Probability: 0.9, Confidence: 0.8},
+ {ID: "reusable_correction", Probability: 0.7, Confidence: 0.6},
+ {ID: "evidence_supported", Probability: 0.8, Confidence: 0.9},
+ },
+ })
+ }))
+ defer server.Close()
+
+ eval, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL, APIKey: "secret"}, EvaluationInput{
+ Project: asymptoteobserve.LearningProjectV1{ID: "project-1"},
+ Trace: testTraceShow(),
+ })
+ if err != nil {
+ t.Fatalf("Evaluate returned error: %v", err)
+ }
+ if !sawAuth || !sawTrace {
+ t.Fatalf("endpoint did not see expected request auth=%v trace=%v", sawAuth, sawTrace)
+ }
+ if eval.Status != asymptoteobserve.LearningEvaluationStatusCompleted {
+ t.Fatalf("status = %s", eval.Status)
+ }
+ if eval.Score <= 0.79 || eval.Score >= 0.81 {
+ t.Fatalf("score = %f", eval.Score)
+ }
+}
+
+func TestProjectionRedactsAndBoundsContent(t *testing.T) {
+ show := testTraceShow()
+ show.Events[0].Content = &asymptoteobserve.TraceContentV1{Text: "token=supersecretvalue " + strings.Repeat("a", maxProjectionText*2)}
+ projection := BuildProjection(show)
+ if len(projection.Events) != 1 {
+ t.Fatalf("events = %d", len(projection.Events))
+ }
+ content := projection.Events[0].Content
+ if strings.Contains(content, "supersecretvalue") {
+ t.Fatalf("projection leaked secret: %s", content)
+ }
+ if len(content) > maxProjectionText+32 {
+ t.Fatalf("projection content not bounded: %d", len(content))
+ }
+}
+
+func TestEvaluateRequiresEndpointOutsideDryRun(t *testing.T) {
+ _, err := Evaluate(t.Context(), EvaluatorOptions{}, EvaluationInput{Project: asymptoteobserve.LearningProjectV1{ID: "p"}, Trace: testTraceShow()})
+ if err == nil || !strings.Contains(err.Error(), "Jev endpoint is required") {
+ t.Fatalf("error = %v", err)
+ }
+}
+
+func testTraceShow() asymptoteobserve.TraceShowResultV1 {
+ return asymptoteobserve.TraceShowResultV1{
+ Trace: asymptoteobserve.TraceSummaryV1{
+ ID: "trace-1",
+ Title: "Fix flaky package smoke",
+ Harness: asymptoteobserve.TraceHarnessV1{Name: "claude_code"},
+ },
+ Events: []asymptoteobserve.TraceEventV1{
+ {
+ ID: "event-1",
+ Number: 1,
+ Type: "command",
+ Action: "command.executed",
+ Title: "Run package smoke",
+ Summary: "Package smoke failed once, then passed after retry.",
+ Content: &asymptoteobserve.TraceContentV1{Text: "npm test passed"},
+ },
+ },
+ Range: asymptoteobserve.TraceRangeV1{TotalEvents: 1, ReturnedEvents: 1},
+ }
+}