The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Add Jev trace evaluation
Justin D'Souza579dfdfde9d2
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.gonew file mode 100644index 00000000..53075ad6--- /dev/null+++ b/cli/beacon/cmd/memory.go@@ -0,0 +1,297 @@+package cmd++import (+ "context"+ "encoding/json"+ "fmt"+ "os"+ "strings"+ "time"++ "github.com/spf13/cobra"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/dashboard"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/lifecycle"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/learning"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++var memoryOpts struct {+ userMode bool+ systemMode bool+ logPath string+ jsonOutput bool+ projectPath string+ limit int+ page int+ query string+ harness string+ traceID string+ since string+ until string+ dryRun bool+ jevEndpoint string+ jevAPIKey string+ jevCost float64+ timeout time.Duration+}++var memoryCmd = &cobra.Command{+ Use: "memory",+ Short: "Evaluate and review cross-harness agent memory",+}++var memoryEvaluationsCmd = &cobra.Command{+ Use: "evaluations",+ Short: "Run and inspect local trace evaluations",+}++var memoryEvaluationsRunCmd = &cobra.Command{+ Use: "run",+ Short: "Evaluate local traces with Jev",+ SilenceUsage: true,+ RunE: runMemoryEvaluationsRun,+}++var memoryEvaluationsListCmd = &cobra.Command{+ Use: "list",+ Short: "List persisted trace evaluations",+ SilenceUsage: true,+ RunE: runMemoryEvaluationsList,+}++var memoryEvaluationsShowCmd = &cobra.Command{+ Use: "show <evaluation-id>",+ Short: "Show one persisted trace evaluation",+ Args: cobra.ExactArgs(1),+ SilenceUsage: true,+ RunE: runMemoryEvaluationsShow,+}++type evaluationRunResult struct {+ DryRun bool `json:"dry_run"`+ Project asymptoteobserve.LearningProjectV1 `json:"project"`+ StorePath string `json:"store_path"`+ TraceCount int `json:"trace_count"`+ EstimatedCalls int `json:"estimated_calls"`+ EstimatedCostUSD float64 `json:"estimated_cost_usd"`+ Previews []learning.EvaluationPreview `json:"previews,omitempty"`+ Evaluations []asymptoteobserve.LearningEvaluationV1 `json:"evaluations,omitempty"`+}++func init() {+ rootCmd.AddCommand(memoryCmd)+ memoryCmd.AddCommand(memoryEvaluationsCmd)+ memoryEvaluationsCmd.AddCommand(memoryEvaluationsRunCmd)+ memoryEvaluationsCmd.AddCommand(memoryEvaluationsListCmd)+ memoryEvaluationsCmd.AddCommand(memoryEvaluationsShowCmd)++ for _, c := range []*cobra.Command{memoryEvaluationsRunCmd, memoryEvaluationsListCmd, memoryEvaluationsShowCmd} {+ c.Flags().BoolVar(&memoryOpts.userMode, "user", true, "Use per-user endpoint paths")+ c.Flags().BoolVar(&memoryOpts.systemMode, "system", false, "Use system endpoint paths")+ c.Flags().StringVar(&memoryOpts.logPath, "log-path", "", "Runtime JSONL log path")+ c.Flags().BoolVar(&memoryOpts.jsonOutput, "json", false, "Print machine-readable JSON")+ c.Flags().StringVar(&memoryOpts.projectPath, "project", "", "Project path for memory scoping (defaults to current directory)")+ }+ for _, c := range []*cobra.Command{memoryEvaluationsRunCmd, memoryEvaluationsListCmd} {+ c.Flags().IntVar(&memoryOpts.limit, "limit", 25, "Limit returned traces or evaluations")+ c.Flags().IntVar(&memoryOpts.page, "page", 1, "Page number for collection output")+ c.Flags().StringVarP(&memoryOpts.query, "query", "q", "", "Free-text query")+ }+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.harness, "harness", "", "Filter traces by harness")+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.traceID, "trace", "", "Evaluate a single trace ID")+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.since, "since", "", "RFC3339 lower time bound, inclusive")+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.until, "until", "", "RFC3339 upper time bound, inclusive")+ memoryEvaluationsRunCmd.Flags().BoolVar(&memoryOpts.dryRun, "dry-run", false, "Preview selected traces, Jev calls, and estimated cost without writing evaluations")+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevEndpoint, "jev-endpoint", "", "Jev evaluator endpoint (defaults to BEACON_JEV_ENDPOINT)")+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevAPIKey, "jev-api-key", "", "Jev API key (defaults to BEACON_JEV_API_KEY)")+ memoryEvaluationsRunCmd.Flags().Float64Var(&memoryOpts.jevCost, "jev-cost-per-trace", learning.DefaultCostPerTrace, "Estimated Jev cost per trace in USD")+ memoryEvaluationsRunCmd.Flags().DurationVar(&memoryOpts.timeout, "timeout", 10*time.Second, "Jev request timeout")+}++func runMemoryEvaluationsRun(cmd *cobra.Command, args []string) error {+ project, err := learning.ResolveProject(memoryOpts.projectPath)+ if err != nil {+ return err+ }+ store := memoryStore()+ inputs, err := selectedEvaluationInputs(project)+ if err != nil {+ return err+ }+ evaluatorOpts := learning.EvaluatorOptions{+ Endpoint: firstNonEmpty(memoryOpts.jevEndpoint, os.Getenv("BEACON_JEV_ENDPOINT")),+ APIKey: firstNonEmpty(memoryOpts.jevAPIKey, os.Getenv("BEACON_JEV_API_KEY")),+ CostPerTrace: memoryOpts.jevCost,+ Timeout: memoryOpts.timeout,+ }+ result := evaluationRunResult{+ DryRun: memoryOpts.dryRun,+ Project: project,+ StorePath: store.Path(),+ TraceCount: len(inputs),+ EstimatedCalls: len(inputs),+ EstimatedCostUSD: float64(len(inputs)) * memoryOpts.jevCost,+ }+ for _, input := range inputs {+ input.DryRun = memoryOpts.dryRun+ if memoryOpts.dryRun {+ result.Previews = append(result.Previews, learning.Preview(input, evaluatorOpts))+ continue+ }+ ctx := cmd.Context()+ if ctx == nil {+ ctx = context.Background()+ }+ eval, err := learning.Evaluate(ctx, evaluatorOpts, input)+ if err != nil {+ return err+ }+ if err := store.PutEvaluation(eval); err != nil {+ return err+ }+ result.Evaluations = append(result.Evaluations, eval)+ }+ if memoryOpts.jsonOutput {+ return json.NewEncoder(cmd.OutOrStdout()).Encode(result)+ }+ if result.DryRun {+ fmt.Fprintf(cmd.OutOrStdout(), "Selected %d trace(s); estimated Jev calls: %d; estimated cost: $%.5f\n", result.TraceCount, result.EstimatedCalls, result.EstimatedCostUSD)+ for _, preview := range result.Previews {+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t$%.5f\n", preview.TraceID, preview.Title, preview.EstimatedCostUSD)+ }+ return nil+ }+ fmt.Fprintf(cmd.OutOrStdout(), "Evaluated %d trace(s); wrote results to %s\n", len(result.Evaluations), store.Path())+ for _, eval := range result.Evaluations {+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%.2f\t%s\n", eval.ID, eval.Trace.ID, eval.Score, eval.Status)+ }+ return nil+}++func runMemoryEvaluationsList(cmd *cobra.Command, args []string) error {+ query, err := memoryQuery()+ if err != nil {+ return err+ }+ evals, err := memoryStore().ListEvaluations(query)+ if err != nil {+ return err+ }+ if memoryOpts.jsonOutput {+ return json.NewEncoder(cmd.OutOrStdout()).Encode(evals)+ }+ for _, eval := range evals {+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%s\t%.2f\t%s\n", eval.ID, eval.Trace.ID, eval.Status, eval.Score, eval.Trace.Title)+ }+ return nil+}++func runMemoryEvaluationsShow(cmd *cobra.Command, args []string) error {+ eval, ok, err := memoryStore().GetEvaluation(args[0])+ if err != nil {+ return err+ }+ if !ok {+ return fmt.Errorf("evaluation not found: %s", args[0])+ }+ if memoryOpts.jsonOutput {+ return json.NewEncoder(cmd.OutOrStdout()).Encode(eval)+ }+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%s\t%.2f\n", eval.ID, eval.Trace.ID, eval.Status, eval.Score)+ for _, question := range eval.Questions {+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%.2f\t%.2f\t%s\n", question.ID, question.Probability, question.Confidence, question.Reason)+ }+ return nil+}++func selectedEvaluationInputs(project asymptoteobserve.LearningProjectV1) ([]learning.EvaluationInput, error) {+ logPath := memoryLogPath()+ if memoryOpts.traceID != "" {+ show, ok, err := dashboard.ShowTrace(logPath, memoryOpts.traceID, dashboard.TraceQuery{Limit: 2000})+ if err != nil {+ return nil, err+ }+ if !ok {+ return nil, fmt.Errorf("trace not found: %s", memoryOpts.traceID)+ }+ return []learning.EvaluationInput{{Project: project, Trace: show}}, nil+ }+ query := dashboard.TraceQuery{+ EventQuery: dashboard.EventQuery{+ Q: memoryOpts.query,+ Harness: memoryOpts.harness,+ },+ Limit: memoryOpts.limit,+ Page: memoryOpts.page,+ }+ var err error+ if query.Since, err = parseMemoryTime("since", memoryOpts.since); err != nil {+ return nil, err+ }+ if query.Until, err = parseMemoryTime("until", memoryOpts.until); err != nil {+ return nil, err+ }+ traces, err := dashboard.ReadTraceList(logPath, query)+ if err != nil {+ return nil, err+ }+ inputs := make([]learning.EvaluationInput, 0, len(traces.Traces))+ for _, summary := range traces.Traces {+ show, ok, err := dashboard.ShowTrace(logPath, summary.ID, dashboard.TraceQuery{Limit: 2000})+ if err != nil {+ return nil, err+ }+ if !ok {+ continue+ }+ inputs = append(inputs, learning.EvaluationInput{Project: project, Trace: show})+ }+ return inputs, nil+}++func memoryQuery() (learning.Query, error) {+ project, err := learning.ResolveProject(memoryOpts.projectPath)+ if err != nil {+ return learning.Query{}, err+ }+ return learning.Query{+ ProjectID: project.ID,+ Q: memoryOpts.query,+ Limit: memoryOpts.limit,+ Page: memoryOpts.page,+ }, nil+}++func memoryStore() *learning.Store {+ return learning.Open(learning.PathForRuntimeLog(memoryLogPath()))+}++func memoryLogPath() string {+ runtimeLog := lifecycle.ResolveRuntimeLog(memoryUserMode(), memoryOpts.logPath)+ return runtimeLog.EffectiveLogPath+}++func memoryUserMode() bool {+ return memoryOpts.userMode && !memoryOpts.systemMode+}++func parseMemoryTime(name, value string) (time.Time, error) {+ if strings.TrimSpace(value) == "" {+ return time.Time{}, nil+ }+ parsed, err := time.Parse(time.RFC3339, value)+ if err != nil {+ return time.Time{}, fmt.Errorf("%s must be RFC3339: %w", name, err)+ }+ return parsed, nil+}++func firstNonEmpty(values ...string) string {+ for _, value := range values {+ if strings.TrimSpace(value) != "" {+ return strings.TrimSpace(value)+ }+ }+ return ""+}diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.gonew file mode 100644index 00000000..b5e9a0b1--- /dev/null+++ b/cli/beacon/cmd/memory_test.go@@ -0,0 +1,167 @@+package cmd++import (+ "bytes"+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"++ "github.com/spf13/cobra"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/learning"+)++func TestMemoryEvaluationCommandsRegistered(t *testing.T) {+ for _, path := range [][]string{+ {"memory", "evaluations", "run"},+ {"memory", "evaluations", "list"},+ {"memory", "evaluations", "show"},+ } {+ cmd, _, err := rootCmd.Find(path)+ if err != nil || cmd == nil {+ t.Fatalf("command %v not registered: %v", path, err)+ }+ }+}++func TestMemoryEvaluationsRunDryRunPreviewsCost(t *testing.T) {+ logPath, project := writeMemoryCommandFixture(t)+ resetMemoryOpts(t)+ memoryOpts.userMode = true+ memoryOpts.logPath = logPath+ memoryOpts.projectPath = project+ memoryOpts.traceID = "session:cursor:s1"+ memoryOpts.dryRun = true+ memoryOpts.jsonOutput = true+ memoryOpts.jevCost = 0.01++ var out bytes.Buffer+ cmd := &cobra.Command{}+ cmd.SetOut(&out)+ if err := runMemoryEvaluationsRun(cmd, nil); err != nil {+ t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)+ }+ var result evaluationRunResult+ if err := json.Unmarshal(out.Bytes(), &result); err != nil {+ t.Fatalf("decode result: %v\n%s", err, out.String())+ }+ if !result.DryRun || result.TraceCount != 1 || result.EstimatedCostUSD != 0.01 {+ t.Fatalf("result = %#v", result)+ }+ if _, err := os.Stat(learning.PathForRuntimeLog(logPath)); !os.IsNotExist(err) {+ t.Fatalf("dry-run should not create memory store, stat err=%v", err)+ }+}++func TestMemoryEvaluationsRunPersistsMockedJevResult(t *testing.T) {+ logPath, project := writeMemoryCommandFixture(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Authorization") != "Bearer test-key" {+ t.Fatalf("missing auth header: %s", r.Header.Get("Authorization"))+ }+ _, _ = w.Write([]byte(`{"questions":[{"id":"task_success","probability":0.9},{"id":"reusable_correction","probability":0.8},{"id":"evidence_supported","probability":0.7}]}`))+ }))+ defer server.Close()++ resetMemoryOpts(t)+ memoryOpts.userMode = true+ memoryOpts.logPath = logPath+ memoryOpts.projectPath = project+ memoryOpts.traceID = "session:cursor:s1"+ memoryOpts.jsonOutput = true+ memoryOpts.jevEndpoint = server.URL+ memoryOpts.jevAPIKey = "test-key"+ memoryOpts.jevCost = learning.DefaultCostPerTrace++ var out bytes.Buffer+ cmd := &cobra.Command{}+ cmd.SetOut(&out)+ if err := runMemoryEvaluationsRun(cmd, nil); err != nil {+ t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)+ }+ var result evaluationRunResult+ if err := json.Unmarshal(out.Bytes(), &result); err != nil {+ t.Fatalf("decode result: %v\n%s", err, out.String())+ }+ if len(result.Evaluations) != 1 || result.Evaluations[0].Score < 0.79 || result.Evaluations[0].Score > 0.81 {+ t.Fatalf("evaluations = %#v", result.Evaluations)+ }++ memoryOpts.query = "session:cursor:s1"+ list, err := memoryStore().ListEvaluations(learning.Query{ProjectID: result.Project.ID, Q: "cursor"})+ if err != nil {+ t.Fatalf("ListEvaluations: %v", err)+ }+ if len(list) != 1 || !strings.HasPrefix(list[0].ID, "eval_") {+ t.Fatalf("persisted evaluations = %#v", list)+ }+}++func writeMemoryCommandFixture(t *testing.T) (string, string) {+ t.Helper()+ dir := t.TempDir()+ project := filepath.Join(dir, "project")+ if err := os.MkdirAll(filepath.Join(project, ".git"), 0755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(filepath.Join(project, ".git", "HEAD"), []byte("ref: refs/heads/main\n"), 0644); err != nil {+ t.Fatal(err)+ }+ logPath := filepath.Join(dir, "endpoint", "logs", "runtime.jsonl")+ if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil {+ t.Fatal(err)+ }+ event := schema.Event{+ Timestamp: "2026-09-21T01:00:00Z",+ Vendor: schema.Vendor,+ Product: schema.Product,+ SchemaVersion: schema.SchemaVersion,+ Event: schema.EventInfo{ID: "event-1", Kind: "agent_runtime", Action: "prompt.submitted", Category: "prompt"},+ Severity: schema.SeverityInfo,+ Endpoint: schema.EndpointInfo{OS: "darwin"},+ Harness: schema.HarnessInfo{Name: "cursor"},+ Session: &schema.SessionInfo{ID: "s1", WorkingDirectory: project},+ Prompt: &schema.PromptInfo{Text: "Fix flaky package smoke"},+ Message: "Fix flaky package smoke",+ Repository: project,+ }+ line, err := json.Marshal(event)+ if err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(logPath, append(line, '\n'), 0644); err != nil {+ t.Fatal(err)+ }+ return logPath, project+}++func resetMemoryOpts(t *testing.T) {+ t.Helper()+ previous := memoryOpts+ memoryOpts = struct {+ userMode bool+ systemMode bool+ logPath string+ jsonOutput bool+ projectPath string+ limit int+ page int+ query string+ harness string+ traceID string+ since string+ until string+ dryRun bool+ jevEndpoint string+ jevAPIKey string+ jevCost float64+ timeout time.Duration+ }{userMode: true, limit: 25, page: 1, jevCost: learning.DefaultCostPerTrace}+ t.Cleanup(func() { memoryOpts = previous })+}diff --git a/cli/beacon/internal/learning/evaluator.go b/cli/beacon/internal/learning/evaluator.gonew file mode 100644index 00000000..703b0e65--- /dev/null+++ b/cli/beacon/internal/learning/evaluator.go@@ -0,0 +1,317 @@+package learning++import (+ "bytes"+ "context"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "strings"+ "time"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++const (+ RubricVersion = "beacon.learning.rubric.v1"+ DefaultCostPerTrace = 0.00035+ maxProjectionEvents = 80+ maxProjectionText = 1200+)++var RubricQuestions = []asymptoteobserve.LearningEvaluationQuestionV1{+ {ID: "task_success", Prompt: "Did the trace complete the user's engineering task successfully?"},+ {ID: "reusable_correction", Prompt: "Does the trace contain a correction or debugging pattern that future agents should reuse?"},+ {ID: "evidence_supported", Prompt: "Is the reusable lesson supported by concrete events in the trace?"},+}++type EvaluatorOptions struct {+ Endpoint string+ APIKey string+ HTTPClient *http.Client+ CostPerTrace float64+ Timeout time.Duration+}++type EvaluationInput struct {+ Project asymptoteobserve.LearningProjectV1+ Trace asymptoteobserve.TraceShowResultV1+ DryRun bool+}++type EvaluationPreview struct {+ TraceID string `json:"trace_id"`+ Title string `json:"title,omitempty"`+ EventCount int `json:"event_count"`+ EstimatedCalls int `json:"estimated_calls"`+ EstimatedCostUSD float64 `json:"estimated_cost_usd"`+ Reason string `json:"reason,omitempty"`+}++type Projection struct {+ TraceID string `json:"trace_id"`+ Title string `json:"title,omitempty"`+ Harness string `json:"harness,omitempty"`+ Repository string `json:"repository,omitempty"`+ Events []ProjectedEvent `json:"events"`+ Questions []ProjectedQuestion `json:"questions"`+}++type ProjectedEvent struct {+ Number int `json:"number"`+ Type string `json:"type"`+ Action string `json:"action"`+ Title string `json:"title,omitempty"`+ Summary string `json:"summary,omitempty"`+ Content string `json:"content,omitempty"`+}++type ProjectedQuestion struct {+ ID string `json:"id"`+ Prompt string `json:"prompt"`+}++type jevRequest struct {+ RubricVersion string `json:"rubric_version"`+ RubricHash string `json:"rubric_hash"`+ Trace Projection `json:"trace"`+}++type jevResponse struct {+ Questions []asymptoteobserve.LearningEvaluationQuestionV1 `json:"questions"`+ Results []asymptoteobserve.LearningEvaluationQuestionV1 `json:"results"`+ Score *float64 `json:"score,omitempty"`+ CostEstimateUSD *float64 `json:"cost_estimate_usd,omitempty"`+}++func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput) (asymptoteobserve.LearningEvaluationV1, error) {+ projection := BuildProjection(input.Trace)+ evaluation := baseEvaluation(opts, input, projection)+ if input.DryRun {+ evaluation.Status = asymptoteobserve.LearningEvaluationStatusDryRun+ evaluation.CostEstimateUSD = costPerTrace(opts)+ evaluation.ID = EvaluationID(evaluation)+ return evaluation, nil+ }+ if strings.TrimSpace(opts.Endpoint) == "" {+ return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("Jev endpoint is required; pass --jev-endpoint or set BEACON_JEV_ENDPOINT")+ }+ resp, err := callEvaluator(ctx, opts, projection)+ if err != nil {+ evaluation.Status = asymptoteobserve.LearningEvaluationStatusFailed+ evaluation.Error = err.Error()+ evaluation.ID = EvaluationID(evaluation)+ return evaluation, err+ }+ questions := resp.Questions+ if len(questions) == 0 {+ questions = resp.Results+ }+ if len(questions) == 0 {+ return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("Jev response did not include questions or results")+ }+ evaluation.Questions = normalizeQuestionResults(questions)+ evaluation.Score = evaluationScore(evaluation.Questions)+ if resp.Score != nil {+ evaluation.Score = *resp.Score+ }+ if resp.CostEstimateUSD != nil {+ evaluation.CostEstimateUSD = *resp.CostEstimateUSD+ } else {+ evaluation.CostEstimateUSD = costPerTrace(opts)+ }+ evaluation.Status = asymptoteobserve.LearningEvaluationStatusCompleted+ evaluation.ID = EvaluationID(evaluation)+ return evaluation, nil+}++func Preview(input EvaluationInput, opts EvaluatorOptions) EvaluationPreview {+ trace := input.Trace.Trace+ return EvaluationPreview{+ TraceID: trace.ID,+ Title: trace.Title,+ EventCount: input.Trace.Range.TotalEvents,+ EstimatedCalls: 1,+ EstimatedCostUSD: costPerTrace(opts),+ Reason: "explicit evaluation command; hooks and collectors do not call Jev",+ }+}++func BuildProjection(show asymptoteobserve.TraceShowResultV1) Projection {+ projection := Projection{+ TraceID: show.Trace.ID,+ Title: show.Trace.Title,+ Harness: show.Trace.Harness.Name,+ }+ if show.Trace.Repository != nil {+ projection.Repository = firstNonEmpty(show.Trace.Repository.RemoteURL, show.Trace.Repository.Path)+ }+ for _, question := range RubricQuestions {+ projection.Questions = append(projection.Questions, ProjectedQuestion{ID: question.ID, Prompt: question.Prompt})+ }+ events := show.Events+ if len(events) > maxProjectionEvents {+ events = events[:maxProjectionEvents]+ }+ for _, event := range events {+ projected := ProjectedEvent{+ Number: event.Number,+ Type: event.Type,+ Action: event.Action,+ Title: cleanText(event.Title),+ Summary: cleanText(event.Summary),+ }+ if event.Content != nil {+ projected.Content = cleanText(event.Content.Text)+ }+ if event.Command != nil && projected.Content == "" {+ projected.Content = cleanText(event.Command.Command)+ }+ projection.Events = append(projection.Events, projected)+ }+ return projection+}++func RubricHash() string {+ data, _ := json.Marshal(RubricQuestions)+ sum := sha256.Sum256(data)+ return "sha256:" + hex.EncodeToString(sum[:])+}++func baseEvaluation(opts EvaluatorOptions, input EvaluationInput, projection Projection) asymptoteobserve.LearningEvaluationV1 {+ trace := input.Trace.Trace+ ref := asymptoteobserve.LearningTraceRefV1{+ ID: trace.ID,+ Title: trace.Title,+ Harness: trace.Harness,+ Session: trace.Session,+ Repo: trace.Repository,+ }+ for _, event := range input.Trace.Events {+ ref.EventIDs = append(ref.EventIDs, event.ID)+ }+ return asymptoteobserve.LearningEvaluationV1{+ SchemaVersion: asymptoteobserve.LearningSchemaVersion,+ Status: asymptoteobserve.LearningEvaluationStatusCompleted,+ Project: input.Project,+ RubricVersion: RubricVersion,+ RubricHash: RubricHash(),+ Evaluator: evaluatorName(opts),+ DryRun: input.DryRun,+ Trace: ref,+ CostEstimateUSD: costPerTrace(opts),+ Questions: projectionQuestionDefaults(projection),+ }+}++func callEvaluator(ctx context.Context, opts EvaluatorOptions, projection Projection) (jevResponse, error) {+ client := opts.HTTPClient+ if client == nil {+ client = http.DefaultClient+ }+ timeout := opts.Timeout+ if timeout <= 0 {+ timeout = 10 * time.Second+ }+ ctx, cancel := context.WithTimeout(ctx, timeout)+ defer cancel()+ body, err := json.Marshal(jevRequest{RubricVersion: RubricVersion, RubricHash: RubricHash(), Trace: projection})+ if err != nil {+ return jevResponse{}, err+ }+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, opts.Endpoint, bytes.NewReader(body))+ if err != nil {+ return jevResponse{}, err+ }+ req.Header.Set("Content-Type", "application/json")+ if opts.APIKey != "" {+ req.Header.Set("Authorization", "Bearer "+opts.APIKey)+ }+ resp, err := client.Do(req)+ if err != nil {+ return jevResponse{}, err+ }+ defer resp.Body.Close()+ data, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20))+ if err != nil {+ return jevResponse{}, err+ }+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {+ return jevResponse{}, fmt.Errorf("Jev endpoint returned %s: %s", resp.Status, strings.TrimSpace(string(data)))+ }+ var parsed jevResponse+ if err := json.Unmarshal(data, &parsed); err != nil {+ return jevResponse{}, err+ }+ return parsed, nil+}++func normalizeQuestionResults(results []asymptoteobserve.LearningEvaluationQuestionV1) []asymptoteobserve.LearningEvaluationQuestionV1 {+ byID := map[string]asymptoteobserve.LearningEvaluationQuestionV1{}+ for _, result := range results {+ byID[result.ID] = result+ }+ out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(RubricQuestions))+ for _, question := range RubricQuestions {+ result := byID[question.ID]+ if result.ID == "" {+ result.ID = question.ID+ }+ if result.Prompt == "" {+ result.Prompt = question.Prompt+ }+ if result.Probability < 0 {+ result.Probability = 0+ }+ if result.Probability > 1 {+ result.Probability = 1+ }+ out = append(out, result)+ }+ return out+}++func projectionQuestionDefaults(projection Projection) []asymptoteobserve.LearningEvaluationQuestionV1 {+ out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(projection.Questions))+ for _, question := range projection.Questions {+ out = append(out, asymptoteobserve.LearningEvaluationQuestionV1{ID: question.ID, Prompt: question.Prompt})+ }+ return out+}++func evaluationScore(results []asymptoteobserve.LearningEvaluationQuestionV1) float64 {+ if len(results) == 0 {+ return 0+ }+ var sum float64+ for _, result := range results {+ sum += result.Probability+ }+ return sum / float64(len(results))+}++func evaluatorName(opts EvaluatorOptions) string {+ if opts.Endpoint == "" {+ return "jev"+ }+ return "jev:" + opts.Endpoint+}++func costPerTrace(opts EvaluatorOptions) float64 {+ if opts.CostPerTrace > 0 {+ return opts.CostPerTrace+ }+ return DefaultCostPerTrace+}++func cleanText(value string) string {+ value = strings.TrimSpace(value)+ if value == "" {+ return ""+ }+ value = asymptoteobserve.CleanString(value, maxProjectionText, true)+ return value+}diff --git a/cli/beacon/internal/learning/evaluator_test.go b/cli/beacon/internal/learning/evaluator_test.gonew file mode 100644index 00000000..1a553fbf--- /dev/null+++ b/cli/beacon/internal/learning/evaluator_test.go@@ -0,0 +1,111 @@+package learning++import (+ "encoding/json"+ "net/http"+ "net/http/httptest"+ "strings"+ "testing"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++func TestEvaluateDryRunDoesNotNeedEndpoint(t *testing.T) {+ show := testTraceShow()+ project := asymptoteobserve.LearningProjectV1{ID: "project-1"}+ eval, err := Evaluate(t.Context(), EvaluatorOptions{}, EvaluationInput{Project: project, Trace: show, DryRun: true})+ if err != nil {+ t.Fatalf("Evaluate dry-run returned error: %v", err)+ }+ if eval.Status != asymptoteobserve.LearningEvaluationStatusDryRun {+ t.Fatalf("status = %s", eval.Status)+ }+ if eval.ID == "" || eval.RubricHash == "" || eval.CostEstimateUSD == 0 {+ t.Fatalf("dry-run missing identifiers/cost: %#v", eval)+ }+}++func TestEvaluatePostsProjectionToMockableEndpoint(t *testing.T) {+ var sawAuth bool+ var sawTrace bool+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.Header.Get("Authorization") == "Bearer secret" {+ sawAuth = true+ }+ var req jevRequest+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {+ t.Fatalf("decode request: %v", err)+ }+ sawTrace = req.Trace.TraceID == "trace-1" && len(req.Trace.Questions) == len(RubricQuestions)+ _ = json.NewEncoder(w).Encode(jevResponse{+ Questions: []asymptoteobserve.LearningEvaluationQuestionV1{+ {ID: "task_success", Probability: 0.9, Confidence: 0.8},+ {ID: "reusable_correction", Probability: 0.7, Confidence: 0.6},+ {ID: "evidence_supported", Probability: 0.8, Confidence: 0.9},+ },+ })+ }))+ defer server.Close()++ eval, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL, APIKey: "secret"}, EvaluationInput{+ Project: asymptoteobserve.LearningProjectV1{ID: "project-1"},+ Trace: testTraceShow(),+ })+ if err != nil {+ t.Fatalf("Evaluate returned error: %v", err)+ }+ if !sawAuth || !sawTrace {+ t.Fatalf("endpoint did not see expected request auth=%v trace=%v", sawAuth, sawTrace)+ }+ if eval.Status != asymptoteobserve.LearningEvaluationStatusCompleted {+ t.Fatalf("status = %s", eval.Status)+ }+ if eval.Score <= 0.79 || eval.Score >= 0.81 {+ t.Fatalf("score = %f", eval.Score)+ }+}++func TestProjectionRedactsAndBoundsContent(t *testing.T) {+ show := testTraceShow()+ show.Events[0].Content = &asymptoteobserve.TraceContentV1{Text: "token=supersecretvalue " + strings.Repeat("a", maxProjectionText*2)}+ projection := BuildProjection(show)+ if len(projection.Events) != 1 {+ t.Fatalf("events = %d", len(projection.Events))+ }+ content := projection.Events[0].Content+ if strings.Contains(content, "supersecretvalue") {+ t.Fatalf("projection leaked secret: %s", content)+ }+ if len(content) > maxProjectionText+32 {+ t.Fatalf("projection content not bounded: %d", len(content))+ }+}++func TestEvaluateRequiresEndpointOutsideDryRun(t *testing.T) {+ _, err := Evaluate(t.Context(), EvaluatorOptions{}, EvaluationInput{Project: asymptoteobserve.LearningProjectV1{ID: "p"}, Trace: testTraceShow()})+ if err == nil || !strings.Contains(err.Error(), "Jev endpoint is required") {+ t.Fatalf("error = %v", err)+ }+}++func testTraceShow() asymptoteobserve.TraceShowResultV1 {+ return asymptoteobserve.TraceShowResultV1{+ Trace: asymptoteobserve.TraceSummaryV1{+ ID: "trace-1",+ Title: "Fix flaky package smoke",+ Harness: asymptoteobserve.TraceHarnessV1{Name: "claude_code"},+ },+ Events: []asymptoteobserve.TraceEventV1{+ {+ ID: "event-1",+ Number: 1,+ Type: "command",+ Action: "command.executed",+ Title: "Run package smoke",+ Summary: "Package smoke failed once, then passed after retry.",+ Content: &asymptoteobserve.TraceContentV1{Text: "npm test passed"},+ },+ },+ Range: asymptoteobserve.TraceRangeV1{TotalEvents: 1, ReturnedEvents: 1},+ }+}