Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

32.0 KB
Raw Blame
package cmd

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"github.com/spf13/cobra"
	"github.com/spf13/pflag"

	"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
	"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/learning"
	"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)

func TestMemoryEvaluationsRunDryRunPreviewsCost(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	resetMemoryOpts(t)
	memoryOpts.userMode = true
	memoryOpts.logPath = logPath
	memoryOpts.projectPath = project
	memoryOpts.traceID = "session:cursor:s1"
	memoryOpts.dryRun = true
	memoryOpts.jsonOutput = true
	memoryOpts.costPerTrace = 0.01

	var out bytes.Buffer
	cmd := &cobra.Command{}
	cmd.SetOut(&out)
	if err := runMemoryEvaluationsRun(cmd, nil); err != nil {
		t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)
	}
	var result evaluationRunResult
	if err := json.Unmarshal(out.Bytes(), &result); err != nil {
		t.Fatalf("decode result: %v\n%s", err, out.String())
	}
	if !result.DryRun || result.TraceCount != 1 || result.EstimatedCostUSD != 0.01 {
		t.Fatalf("result = %#v", result)
	}
	if _, err := os.Stat(learning.PathForRuntimeLog(logPath)); !os.IsNotExist(err) {
		t.Fatalf("dry-run should not create memory store, stat err=%v", err)
	}
}

func TestMemoryEvaluationsRunPersistsMockedJevResult(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Authorization") != "Bearer test-key" {
			t.Fatalf("missing auth header: %s", r.Header.Get("Authorization"))
		}
		_, _ = w.Write([]byte(`{"questions":[{"id":"task_success","probability":0.9},{"id":"reusable_correction","probability":0.8},{"id":"evidence_supported","probability":0.7}]}`))
	}))
	defer server.Close()

	resetMemoryOpts(t)
	memoryOpts.userMode = true
	memoryOpts.logPath = logPath
	memoryOpts.projectPath = project
	memoryOpts.traceID = "session:cursor:s1"
	memoryOpts.jsonOutput = true
	memoryOpts.jevEndpoint = server.URL
	memoryOpts.jevAPIKey = "test-key"
	memoryOpts.jevModel = "jev-test"
	memoryOpts.costPerTrace = learning.DefaultCostPerTrace

	var out bytes.Buffer
	cmd := &cobra.Command{}
	cmd.SetOut(&out)
	if err := runMemoryEvaluationsRun(cmd, nil); err != nil {
		t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)
	}
	var result evaluationRunResult
	if err := json.Unmarshal(out.Bytes(), &result); err != nil {
		t.Fatalf("decode result: %v\n%s", err, out.String())
	}
	if len(result.Evaluations) != 1 || result.Evaluations[0].Score < 0.79 || result.Evaluations[0].Score > 0.81 {
		t.Fatalf("evaluations = %#v", result.Evaluations)
	}
	if len(result.Candidates) != 1 || result.Candidates[0].State != "candidate" {
		t.Fatalf("candidates = %#v", result.Candidates)
	}

	memoryOpts.query = "session:cursor:s1"
	list, err := memoryStore().ListEvaluations(learning.Query{ProjectID: result.Project.ID, Q: "cursor"})
	if err != nil {
		t.Fatalf("ListEvaluations: %v", err)
	}
	if len(list) != 1 || !strings.HasPrefix(list[0].ID, "eval_") {
		t.Fatalf("persisted evaluations = %#v", list)
	}
}

// Issue #649: a trace the judge scored as failed must not become a candidate even
// when the mean clears the score threshold. The stored and printed score is the
// unchanged mean; only the promotion decision applies the task_success gate.
func TestMemoryEvaluationsRunDoesNotPromoteFailedTask(t *testing.T) {
	for _, tc := range []struct {
		name           string
		answers        string
		wantScore      string
		wantTextScore  string
		wantCandidates int
		wantNote       string
	}{
		{
			name:          "failed task clears the mean",
			answers:       `{"answers":{"task_success":{"type":"noul","noul":0.27},"reusable_correction":{"type":"noul","noul":0.86},"evidence_supported":{"type":"noul","noul":0.69}}}`,
			wantScore:     "0.6067",
			wantTextScore: "0.61",
			wantNote:      "(task_success 0.27 is below 0.50)",
		},
		{
			name:           "successful task still promotes",
			answers:        `{"answers":{"task_success":{"type":"noul","noul":0.9},"reusable_correction":{"type":"noul","noul":0.8},"evidence_supported":{"type":"noul","noul":0.7}}}`,
			wantScore:      "0.8000",
			wantTextScore:  "0.80",
			wantCandidates: 1,
		},
	} {
		t.Run(tc.name, func(t *testing.T) {
			for _, jsonOutput := range []bool{true, false} {
				logPath, project := writeMemoryCommandFixture(t)
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					_, _ = w.Write([]byte(tc.answers))
				}))
				resetMemoryOpts(t)
				memoryOpts.userMode = true
				memoryOpts.logPath = logPath
				memoryOpts.projectPath = project
				memoryOpts.traceID = "session:cursor:s1"
				memoryOpts.jsonOutput = jsonOutput
				memoryOpts.jevEndpoint = server.URL
				memoryOpts.jevAPIKey = "test-key"

				var out bytes.Buffer
				cmd := &cobra.Command{}
				cmd.SetOut(&out)
				err := runMemoryEvaluationsRun(cmd, nil)
				server.Close()
				if err != nil {
					t.Fatalf("runMemoryEvaluationsRun(json=%v) returned error: %v", jsonOutput, err)
				}
				if jsonOutput {
					var result evaluationRunResult
					if err := json.Unmarshal(out.Bytes(), &result); err != nil {
						t.Fatalf("decode result: %v\n%s", err, out.String())
					}
					if len(result.Evaluations) != 1 || fmt.Sprintf("%.4f", result.Evaluations[0].Score) != tc.wantScore {
						t.Fatalf("evaluations = %#v, want one with score %s", result.Evaluations, tc.wantScore)
					}
					if len(result.Candidates) != tc.wantCandidates {
						t.Fatalf("candidates = %#v, want %d", result.Candidates, tc.wantCandidates)
					}
				} else {
					text := out.String()
					if !strings.Contains(text, "\t"+tc.wantTextScore+"\t") {
						t.Fatalf("text output does not print the unchanged score %s:\n%s", tc.wantTextScore, text)
					}
					switch {
					case tc.wantNote != "" && !strings.Contains(text, "Not promoted: eval_"):
						t.Fatalf("text output does not say the evaluation was not promoted:\n%s", text)
					case tc.wantNote != "" && !strings.Contains(text, tc.wantNote):
						t.Fatalf("text output missing the gate reason %q:\n%s", tc.wantNote, text)
					case tc.wantNote == "" && strings.Contains(text, "Not promoted:"):
						t.Fatalf("promoted evaluation reported as not promoted:\n%s", text)
					}
				}
				stored, err := memoryStore().ListCandidates(learning.Query{ProjectPath: project})
				if err != nil {
					t.Fatalf("ListCandidates: %v", err)
				}
				if len(stored) != tc.wantCandidates {
					t.Fatalf("stored candidates = %#v, want %d", stored, tc.wantCandidates)
				}
			}
		})
	}
}

func TestMemoryApproveWithEditsThenListAndShow(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	resetMemoryOpts(t)
	memoryOpts.userMode = true
	memoryOpts.logPath = logPath
	memoryOpts.projectPath = project
	candidate := testCommandCandidate(t)
	resolved, err := learning.ResolveProject(project)
	if err != nil {
		t.Fatal(err)
	}
	candidate.Project = resolved
	if err := memoryStore().PutCandidate(candidate); err != nil {
		t.Fatal(err)
	}
	bodyFile := filepath.Join(t.TempDir(), "lesson.md")
	if err := os.WriteFile(bodyFile, []byte("Rerun package smoke once when the failure is a launchctl timeout.\n"), 0644); err != nil {
		t.Fatal(err)
	}
	memoryOpts.title = "Package smoke launchctl timeouts are transient"
	memoryOpts.bodyFile = bodyFile
	memoryOpts.kind = asymptoteobserve.LearningMemoryKindGotcha
	memoryOpts.tags = []string{"packaging"}
	memoryOpts.jsonOutput = true
	var approveOut bytes.Buffer
	cmd := &cobra.Command{}
	cmd.SetOut(&approveOut)
	if err := runMemoryCandidatesApprove(cmd, []string{candidate.ID}); err != nil {
		t.Fatalf("runMemoryCandidatesApprove returned error: %v", err)
	}
	var approved struct {
		Memory asymptoteobserve.LearningMemoryV1 `json:"memory"`
	}
	if err := json.Unmarshal(approveOut.Bytes(), &approved); err != nil {
		t.Fatal(err)
	}
	if approved.Memory.Body != "Rerun package smoke once when the failure is a launchctl timeout." || approved.Memory.Kind != asymptoteobserve.LearningMemoryKindGotcha {
		t.Fatalf("approved memory = %#v", approved.Memory)
	}

	memoryOpts.kind = ""
	memoryOpts.query = "launchctl"
	var listOut bytes.Buffer
	cmd.SetOut(&listOut)
	if err := runMemoryList(cmd, nil); err != nil {
		t.Fatalf("runMemoryList returned error: %v", err)
	}
	var listed []asymptoteobserve.LearningMemoryV1
	if err := json.Unmarshal(listOut.Bytes(), &listed); err != nil {
		t.Fatal(err)
	}
	if len(listed) != 1 || listed[0].ID != approved.Memory.ID {
		t.Fatalf("listed = %#v", listed)
	}

	memoryOpts.jsonOutput = false
	var showOut bytes.Buffer
	cmd.SetOut(&showOut)
	if err := runMemoryShow(cmd, []string{approved.Memory.ID}); err != nil {
		t.Fatalf("runMemoryShow returned error: %v", err)
	}
	if !strings.Contains(showOut.String(), "launchctl timeout") || !strings.Contains(showOut.String(), "Trace trace-command") {
		t.Fatalf("show output = %s", showOut.String())
	}
	if err := runMemoryShow(cmd, []string{"memory_missing"}); err == nil {
		t.Fatal("showing an unknown memory should fail")
	}
}

func TestMemoryApproveRefusesBodyAndBodyFile(t *testing.T) {
	resetMemoryOpts(t)
	memoryOpts.body = "inline"
	memoryOpts.bodyFile = "-"
	if _, err := memoryApprovalEdits(&cobra.Command{}); err == nil {
		t.Fatal("--body with --body-file should be refused")
	}
}

func TestMemorySkillsPreviewAndInstallCommands(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	resetMemoryOpts(t)
	memoryOpts.userMode = true
	memoryOpts.logPath = logPath
	memoryOpts.projectPath = project
	candidate, memory := testApprovedCommandMemory()
	store := memoryStore()
	if err := store.PutCandidate(candidate); err != nil {
		t.Fatal(err)
	}
	if err := store.PutMemory(memory); err != nil {
		t.Fatal(err)
	}

	var preview bytes.Buffer
	cmd := &cobra.Command{}
	cmd.SetOut(&preview)
	if err := runMemorySkillsPreview(cmd, []string{candidate.ID}); err != nil {
		t.Fatalf("runMemorySkillsPreview returned error: %v", err)
	}
	if !strings.Contains(preview.String(), "beacon_memory_id") || !strings.Contains(preview.String(), "Retry package smoke") {
		t.Fatalf("preview output = %s", preview.String())
	}

	var install bytes.Buffer
	cmd.SetOut(&install)
	if err := runMemorySkillsInstall(cmd, []string{candidate.ID}); err != nil {
		t.Fatalf("runMemorySkillsInstall returned error: %v", err)
	}
	if !strings.Contains(install.String(), ".agents") {
		t.Fatalf("install output = %s", install.String())
	}
	if _, err := os.Stat(filepath.Join(project, ".agents", "skills", "beacon-debugging-pattern-retry-package-smoke", "SKILL.md")); err != nil {
		t.Fatalf("installed skill missing: %v", err)
	}
}

func writeMemoryCommandFixture(t *testing.T) (string, string) {
	t.Helper()
	dir := t.TempDir()
	project := filepath.Join(dir, "project")
	if err := os.MkdirAll(filepath.Join(project, ".git"), 0755); err != nil {
		t.Fatal(err)
	}
	if err := os.WriteFile(filepath.Join(project, ".git", "HEAD"), []byte("ref: refs/heads/main\n"), 0644); err != nil {
		t.Fatal(err)
	}
	logPath := filepath.Join(dir, "endpoint", "logs", "runtime.jsonl")
	if err := os.MkdirAll(filepath.Dir(logPath), 0755); err != nil {
		t.Fatal(err)
	}
	event := schema.Event{
		Timestamp:     "2026-09-21T01:00:00Z",
		Vendor:        schema.Vendor,
		Product:       schema.Product,
		SchemaVersion: schema.SchemaVersion,
		Event:         schema.EventInfo{ID: "event-1", Kind: "agent_runtime", Action: "prompt.submitted", Category: "prompt"},
		Severity:      schema.SeverityInfo,
		Endpoint:      schema.EndpointInfo{OS: "darwin"},
		Harness:       schema.HarnessInfo{Name: "cursor"},
		Session:       &schema.SessionInfo{ID: "s1", WorkingDirectory: project},
		Prompt:        &schema.PromptInfo{Text: "Fix flaky package smoke"},
		Message:       "Fix flaky package smoke",
		Repository:    project,
	}
	// A later OTLP metric sample with no session: it forms a single-event trace that
	// sorts above the session and must not be selected for evaluation.
	metric := schema.Event{
		Timestamp:     "2026-09-21T02:00:00Z",
		Vendor:        schema.Vendor,
		Product:       schema.Product,
		SchemaVersion: schema.SchemaVersion,
		Event:         schema.EventInfo{ID: "metric-1", Kind: "agent_runtime", Action: "metric.observed", Category: "metric"},
		Severity:      schema.SeverityInfo,
		Endpoint:      schema.EndpointInfo{OS: "darwin"},
		Harness:       schema.HarnessInfo{Name: "cursor"},
		Message:       "cursor.active_time.total",
	}
	var lines []byte
	for _, item := range []schema.Event{event, metric} {
		line, err := json.Marshal(item)
		if err != nil {
			t.Fatal(err)
		}
		lines = append(append(lines, line...), '\n')
	}
	if err := os.WriteFile(logPath, lines, 0644); err != nil {
		t.Fatal(err)
	}
	return logPath, project
}

func TestMemoryEvaluationsRunSkipsSessionlessTraces(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	resetMemoryOpts(t)
	memoryOpts.logPath = logPath
	memoryOpts.projectPath = project
	memoryOpts.dryRun = true
	memoryOpts.jsonOutput = true

	var out bytes.Buffer
	cmd := &cobra.Command{}
	cmd.SetOut(&out)
	if err := runMemoryEvaluationsRun(cmd, nil); err != nil {
		t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)
	}
	var result evaluationRunResult
	if err := json.Unmarshal(out.Bytes(), &result); err != nil {
		t.Fatalf("decode result: %v\n%s", err, out.String())
	}
	if result.TraceCount != 1 || len(result.Previews) != 1 || result.Previews[0].TraceID != "session:cursor:s1" {
		t.Fatalf("expected only the session trace to be selected, got %#v", result)
	}
	if result.Project == nil || result.Project.Path != project || result.Previews[0].Project.ID != result.Project.ID {
		t.Fatalf("with --project the result and every preview should name it, got %#v", result)
	}
}

// Without --project each trace is scoped to its own repository, so the result must
// not claim a single project that the stored records may not belong to.
func TestMemoryEvaluationsRunReportsProjectOnlyWhenPinned(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	resetMemoryOpts(t)
	memoryOpts.logPath = logPath
	memoryOpts.dryRun = true
	memoryOpts.jsonOutput = true

	var out bytes.Buffer
	cmd := &cobra.Command{}
	cmd.SetOut(&out)
	if err := runMemoryEvaluationsRun(cmd, nil); err != nil {
		t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)
	}
	var result evaluationRunResult
	if err := json.Unmarshal(out.Bytes(), &result); err != nil {
		t.Fatalf("decode result: %v\n%s", err, out.String())
	}
	if result.Project != nil {
		t.Fatalf("result should not name a project without --project, got %#v", result.Project)
	}
	want, err := learning.ResolveProject(project)
	if err != nil {
		t.Fatal(err)
	}
	if len(result.Previews) != 1 || result.Previews[0].Project.ID != want.ID {
		t.Fatalf("preview should carry the trace's repository %s, got %#v", want.ID, result.Previews)
	}
}

func TestMemoryCandidatesCreateScopesToTraceRepository(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	resetMemoryOpts(t)
	memoryOpts.logPath = logPath
	// No --project: the trace recorded its repository, so memory must land there and
	// not in the reviewer's working directory.
	memoryOpts.traceID = "session:cursor:s1"
	memoryOpts.kind = asymptoteobserve.LearningMemoryKindDebuggingPattern
	memoryOpts.title = "Retry package smoke once"
	memoryOpts.bodyFile = "-"
	memoryOpts.tags = []string{"smoke", "smoke", " "}
	memoryOpts.jsonOutput = true

	var out bytes.Buffer
	cmd := &cobra.Command{}
	cmd.SetOut(&out)
	cmd.SetIn(strings.NewReader("Rerun package smoke once after confirming the failure is transient.\n"))
	if err := runMemoryCandidatesCreate(cmd, nil); err != nil {
		t.Fatalf("runMemoryCandidatesCreate returned error: %v", err)
	}
	var candidate asymptoteobserve.LearningCandidateV1
	if err := json.Unmarshal(out.Bytes(), &candidate); err != nil {
		t.Fatalf("decode candidate: %v\n%s", err, out.String())
	}
	want, err := learning.ResolveProject(project)
	if err != nil {
		t.Fatal(err)
	}
	if candidate.Project.ID != want.ID {
		t.Fatalf("candidate project = %#v, want %#v", candidate.Project, want)
	}
	if candidate.State != asymptoteobserve.LearningCandidateStateCandidate || candidate.SourceEvaluationID != "" {
		t.Fatalf("candidate = %#v", candidate)
	}
	if candidate.Body != "Rerun package smoke once after confirming the failure is transient." {
		t.Fatalf("body = %q", candidate.Body)
	}
	if len(candidate.Evidence) != 1 || candidate.Evidence[0].TraceID != "session:cursor:s1" || len(candidate.Evidence[0].EventIDs) != 1 {
		t.Fatalf("evidence = %#v", candidate.Evidence)
	}
	if strings.Join(candidate.Tags, ",") != "smoke" {
		t.Fatalf("tags = %#v", candidate.Tags)
	}

	stored, err := memoryStore().ListCandidates(learning.Query{ProjectID: want.ID})
	if err != nil {
		t.Fatal(err)
	}
	if len(stored) != 1 || stored[0].ID != candidate.ID {
		t.Fatalf("stored candidates = %#v", stored)
	}

	// The authored candidate approves like any other, and the memory carries its text.
	memoryOpts.kind, memoryOpts.title, memoryOpts.bodyFile, memoryOpts.tags = "", "", "", nil
	memoryOpts.reason = "reviewed"
	var approved bytes.Buffer
	cmd.SetOut(&approved)
	if err := runMemoryCandidatesApprove(cmd, []string{candidate.ID}); err != nil {
		t.Fatalf("runMemoryCandidatesApprove returned error: %v", err)
	}
	if !strings.Contains(approved.String(), `"approved"`) || !strings.Contains(approved.String(), "Rerun package smoke once") {
		t.Fatalf("approve output = %s", approved.String())
	}
}

func TestMemoryCandidatesCreateRejectsUnknownKind(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	resetMemoryOpts(t)
	memoryOpts.logPath = logPath
	memoryOpts.projectPath = project
	memoryOpts.traceID = "session:cursor:s1"
	memoryOpts.kind = "anecdote"
	memoryOpts.title = "t"
	memoryOpts.body = "b"
	cmd := &cobra.Command{}
	cmd.SetOut(&bytes.Buffer{})
	err := runMemoryCandidatesCreate(cmd, nil)
	if err == nil || !strings.Contains(err.Error(), "unknown memory kind") {
		t.Fatalf("expected unknown kind error, got %v", err)
	}
	if _, statErr := os.Stat(learning.PathForRuntimeLog(logPath)); !os.IsNotExist(statErr) {
		t.Fatalf("a rejected candidate must not create the store, stat err=%v", statErr)
	}
}

func resetMemoryOpts(t *testing.T) {
	t.Helper()
	previous := memoryOpts
	memoryOpts = struct {
		userMode     bool
		systemMode   bool
		logPath      string
		jsonOutput   bool
		projectPath  string
		limit        int
		page         int
		query        string
		harness      string
		traceID      string
		since        string
		until        string
		dryRun       bool
		evaluator    string
		jevEndpoint  string
		jevAPIKey    string
		jevModel     string
		costPerTrace float64
		subqEndpoint string
		subqAPIKey   string
		subqModel    string
		timeout      time.Duration
		state        string
		kind         string
		reason       string
		replacement  string
		force        bool
		title        string
		body         string
		bodyFile     string
		applies      string
		tags         []string
	}{userMode: true, limit: 25, page: 1, costPerTrace: learning.DefaultCostPerTrace}
	t.Cleanup(func() { memoryOpts = previous })
}

func testCommandCandidate(t *testing.T) asymptoteobserve.LearningCandidateV1 {
	t.Helper()
	candidate, ok := learning.CandidateFromEvaluation(asymptoteobserve.LearningEvaluationV1{
		SchemaVersion: asymptoteobserve.LearningSchemaVersion,
		ID:            "eval-command",
		Status:        asymptoteobserve.LearningEvaluationStatusCompleted,
		Score:         0.9,
		Project:       asymptoteobserve.LearningProjectV1{ID: "project-1"},
		Trace: asymptoteobserve.LearningTraceRefV1{
			ID:       "trace-command",
			Title:    "Fix flaky package smoke",
			Harness:  asymptoteobserve.TraceHarnessV1{Name: "cursor"},
			EventIDs: []string{"event-1"},
		},
		Questions: []asymptoteobserve.LearningEvaluationQuestionV1{
			{ID: "task_success", Probability: 0.9},
			{ID: "reusable_correction", Probability: 0.9},
			{ID: "evidence_supported", Probability: 0.9},
		},
	})
	if !ok {
		t.Fatal("candidate not created")
	}
	return candidate
}

func testApprovedCommandMemory() (asymptoteobserve.LearningCandidateV1, asymptoteobserve.LearningMemoryV1) {
	project := asymptoteobserve.LearningProjectV1{ID: "project-1"}
	evidence := []asymptoteobserve.LearningEvidenceV1{{TraceID: "trace-1", Summary: "Retry fixed a transient failure"}}
	candidate := asymptoteobserve.LearningCandidateV1{
		ID:       "candidate-approved",
		MemoryID: "memory-approved",
		State:    asymptoteobserve.LearningCandidateStateApproved,
		Kind:     asymptoteobserve.LearningMemoryKindDebuggingPattern,
		Title:    "Debugging pattern: Retry package smoke",
		Body:     "Rerun package smoke once after confirming the error is transient.",
		Project:  project,
		Evidence: evidence,
	}
	memory := asymptoteobserve.LearningMemoryV1{
		ID:            candidate.MemoryID,
		CandidateID:   candidate.ID,
		Kind:          candidate.Kind,
		Title:         candidate.Title,
		Body:          candidate.Body,
		Applicability: "when package smoke fails transiently",
		Tags:          []string{"beacon", "debugging_pattern"},
		Project:       project,
		Evidence:      evidence,
	}
	return candidate, memory
}

// Issue #620: evaluations stored by earlier releases carry reason "noul", the Jev
// question type, not a rationale. The text view must not present it as one.
func TestMemoryEvaluationsShowHidesPlaceholderReason(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	resetMemoryOpts(t)
	memoryOpts.userMode = true
	memoryOpts.logPath = logPath
	memoryOpts.projectPath = project
	eval := asymptoteobserve.LearningEvaluationV1{
		SchemaVersion: asymptoteobserve.LearningSchemaVersion,
		ID:            "eval-placeholder",
		Status:        asymptoteobserve.LearningEvaluationStatusCompleted,
		Score:         0.7,
		Project:       asymptoteobserve.LearningProjectV1{ID: "project-1"},
		Trace:         asymptoteobserve.LearningTraceRefV1{ID: "trace-placeholder"},
		Questions: []asymptoteobserve.LearningEvaluationQuestionV1{
			{ID: "task_success", Probability: 0.43, Reason: "noul"},
			{ID: "reusable_correction", Probability: 0.78, Reason: "Retry isolates the flaky step."},
		},
	}
	if err := memoryStore().PutEvaluation(eval); err != nil {
		t.Fatal(err)
	}
	var out bytes.Buffer
	cmd := &cobra.Command{}
	cmd.SetOut(&out)
	if err := runMemoryEvaluationsShow(cmd, []string{eval.ID}); err != nil {
		t.Fatalf("runMemoryEvaluationsShow returned error: %v", err)
	}
	if strings.Contains(out.String(), "noul") {
		t.Fatalf("show printed the placeholder reason:\n%s", out.String())
	}
	if !strings.Contains(out.String(), "Retry isolates the flaky step.") {
		t.Fatalf("show dropped a real reason:\n%s", out.String())
	}
}

// The --jev-endpoint and --jev-model flags used to default to the hosted TypeSafe
// values, so BEACON_JEV_ENDPOINT and BEACON_JEV_MODEL were never consulted even
// though the docs and the --jev-api-key help text promise them. Leaving the flags
// unset must fall through to the environment, then to the built-in defaults.
func TestMemoryEvaluationsRunHonoursJevEnvironment(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	var gotModel string
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Authorization") != "Bearer env-key" {
			t.Errorf("Authorization = %q, want Bearer env-key", r.Header.Get("Authorization"))
		}
		var req struct {
			Model string `json:"model"`
		}
		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
			t.Errorf("decode request: %v", err)
		}
		gotModel = req.Model
		_, _ = w.Write([]byte(`{"model":"typesafe-ai/jev","answers":{"task_success":{"type":"noul","noul":0.9},"reusable_correction":{"type":"noul","noul":0.8},"evidence_supported":{"type":"noul","noul":0.7}}}`))
	}))
	defer server.Close()

	t.Setenv("BEACON_JEV_ENDPOINT", server.URL)
	t.Setenv("BEACON_JEV_MODEL", "typesafe-ai/jev")
	t.Setenv("TYPESAFE_API_KEY", "env-key")
	t.Setenv("BEACON_JEV_API_KEY", "")

	resetMemoryOpts(t)
	memoryOpts.userMode = true
	memoryOpts.logPath = logPath
	memoryOpts.projectPath = project
	memoryOpts.traceID = "session:cursor:s1"
	memoryOpts.jsonOutput = true

	var out bytes.Buffer
	cmd := &cobra.Command{}
	cmd.SetOut(&out)
	if err := runMemoryEvaluationsRun(cmd, nil); err != nil {
		t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)
	}
	if gotModel != "typesafe-ai/jev" {
		t.Fatalf("request model = %q, want typesafe-ai/jev", gotModel)
	}
	var result evaluationRunResult
	if err := json.Unmarshal(out.Bytes(), &result); err != nil {
		t.Fatalf("decode result: %v\n%s", err, out.String())
	}
	if len(result.Evaluations) != 1 {
		t.Fatalf("evaluations = %#v", result.Evaluations)
	}
	if got := result.Evaluations[0].Evaluator; !strings.Contains(got, server.URL) {
		t.Fatalf("evaluator endpoint = %q, want it to reference %s", got, server.URL)
	}
}

func TestJevEvaluatorOptionsPrecedence(t *testing.T) {
	cases := []struct {
		name         string
		flagEndpoint string
		flagModel    string
		flagKey      string
		env          map[string]string
		wantEndpoint string
		wantModel    string
		wantKey      string
	}{
		{
			name:         "defaults when nothing is set",
			env:          map[string]string{"BEACON_JEV_ENDPOINT": "", "BEACON_JEV_MODEL": "", "TYPESAFE_API_KEY": "", "BEACON_JEV_API_KEY": ""},
			wantEndpoint: learning.DefaultJevEndpoint,
			wantModel:    learning.DefaultJevModel,
		},
		{
			name: "environment overrides defaults",
			env: map[string]string{
				"BEACON_JEV_ENDPOINT": "https://gateway.example/typesafe/v1/systemone",
				"BEACON_JEV_MODEL":    "typesafe-ai/jev",
				"TYPESAFE_API_KEY":    "",
				"BEACON_JEV_API_KEY":  "fallback-key",
			},
			wantEndpoint: "https://gateway.example/typesafe/v1/systemone",
			wantModel:    "typesafe-ai/jev",
			wantKey:      "fallback-key",
		},
		{
			name:         "flags override environment",
			flagEndpoint: "https://flag.example/v1/systemone",
			flagModel:    "jev-1.13.0",
			flagKey:      "flag-key",
			env: map[string]string{
				"BEACON_JEV_ENDPOINT": "https://gateway.example/typesafe/v1/systemone",
				"BEACON_JEV_MODEL":    "typesafe-ai/jev",
				"TYPESAFE_API_KEY":    "env-key",
				"BEACON_JEV_API_KEY":  "fallback-key",
			},
			wantEndpoint: "https://flag.example/v1/systemone",
			wantModel:    "jev-1.13.0",
			wantKey:      "flag-key",
		},
	}
	for _, tc := range cases {
		t.Run(tc.name, func(t *testing.T) {
			for k, v := range tc.env {
				t.Setenv(k, v)
			}
			resetMemoryOpts(t)
			memoryOpts.jevEndpoint = tc.flagEndpoint
			memoryOpts.jevModel = tc.flagModel
			memoryOpts.jevAPIKey = tc.flagKey

			got := jevEvaluatorOptions()
			if got.Endpoint != tc.wantEndpoint || got.Model != tc.wantModel || got.APIKey != tc.wantKey {
				t.Fatalf("jevEvaluatorOptions() = endpoint %q model %q key %q, want %q %q %q",
					got.Endpoint, got.Model, got.APIKey, tc.wantEndpoint, tc.wantModel, tc.wantKey)
			}
		})
	}
}

// SubQ's Decisions API is the same System One contract as Jev, so selecting it
// must send the same request to the SubQ endpoint with SUBQ_API_KEY and record
// the evaluation under the subq provider.
func TestMemoryEvaluationsRunWithSubQ(t *testing.T) {
	logPath, project := writeMemoryCommandFixture(t)
	var gotModel string
	var gotTypes []string
	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Authorization") != "Bearer subq-key" {
			t.Errorf("Authorization = %q, want Bearer subq-key", r.Header.Get("Authorization"))
		}
		var req struct {
			Model     string                           `json:"model"`
			Questions map[string]struct{ Type string } `json:"questions"`
		}
		if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
			t.Errorf("decode request: %v", err)
		}
		gotModel = req.Model
		for _, q := range req.Questions {
			gotTypes = append(gotTypes, q.Type)
		}
		_, _ = w.Write([]byte(`{"model":"subq-snap-preview","answers":{"task_success":{"type":"noul","noul":0.9},"reusable_correction":{"type":"noul","noul":0.8},"evidence_supported":{"type":"noul","noul":0.7}},"usage":{"input_tokens":120,"output_tokens":0}}`))
	}))
	defer server.Close()

	t.Setenv("BEACON_EVALUATOR", "subq")
	t.Setenv("BEACON_SUBQ_ENDPOINT", server.URL)
	t.Setenv("BEACON_SUBQ_MODEL", "")
	t.Setenv("SUBQ_API_KEY", "subq-key")
	t.Setenv("TYPESAFE_API_KEY", "jev-key-must-not-be-sent")

	resetMemoryOpts(t)
	memoryOpts.userMode = true
	memoryOpts.logPath = logPath
	memoryOpts.projectPath = project
	memoryOpts.traceID = "session:cursor:s1"
	memoryOpts.jsonOutput = true

	var out bytes.Buffer
	cmd := &cobra.Command{}
	cmd.SetOut(&out)
	if err := runMemoryEvaluationsRun(cmd, nil); err != nil {
		t.Fatalf("runMemoryEvaluationsRun returned error: %v", err)
	}
	if gotModel != learning.DefaultSubQModel {
		t.Fatalf("request model = %q, want %s", gotModel, learning.DefaultSubQModel)
	}
	if len(gotTypes) != len(learning.RubricQuestions) {
		t.Fatalf("question types = %v", gotTypes)
	}
	for _, typ := range gotTypes {
		if typ != "noul" {
			t.Fatalf("question type = %q, want noul", typ)
		}
	}
	var result evaluationRunResult
	if err := json.Unmarshal(out.Bytes(), &result); err != nil {
		t.Fatalf("decode result: %v\n%s", err, out.String())
	}
	if len(result.Evaluations) != 1 {
		t.Fatalf("evaluations = %#v", result.Evaluations)
	}
	if got, want := result.Evaluations[0].Evaluator, "subq:"+server.URL; got != want {
		t.Fatalf("evaluator = %q, want %q", got, want)
	}
}

func TestEvaluatorOptionsSelectsProvider(t *testing.T) {
	for k, v := range map[string]string{
		"BEACON_EVALUATOR": "", "BEACON_SUBQ_ENDPOINT": "", "BEACON_SUBQ_MODEL": "", "SUBQ_API_KEY": "", "BEACON_SUBQ_API_KEY": "fallback-subq",
		"BEACON_JEV_ENDPOINT": "", "BEACON_JEV_MODEL": "", "TYPESAFE_API_KEY": "", "BEACON_JEV_API_KEY": "",
	} {
		t.Setenv(k, v)
	}
	resetMemoryOpts(t)
	got, err := evaluatorOptions()
	if err != nil || got.Provider != learning.ProviderJev || got.Endpoint != learning.DefaultJevEndpoint {
		t.Fatalf("default evaluatorOptions() = %+v, %v; want Jev defaults", got, err)
	}

	memoryOpts.evaluator = "SubQ"
	got, err = evaluatorOptions()
	if err != nil {
		t.Fatal(err)
	}
	if got.Provider != learning.ProviderSubQ || got.Endpoint != learning.DefaultSubQEndpoint || got.Model != learning.DefaultSubQModel || got.APIKey != "fallback-subq" {
		t.Fatalf("subq evaluatorOptions() = %+v", got)
	}

	memoryOpts.subqEndpoint = "https://flag.example/v1/systemone"
	memoryOpts.subqModel = "subq-pinned"
	memoryOpts.subqAPIKey = "flag-key"
	t.Setenv("SUBQ_API_KEY", "env-key")
	got, _ = evaluatorOptions()
	if got.Endpoint != "https://flag.example/v1/systemone" || got.Model != "subq-pinned" || got.APIKey != "flag-key" {
		t.Fatalf("subq flags did not win: %+v", got)
	}

	memoryOpts.evaluator = "openai"
	if _, err := evaluatorOptions(); err == nil {
		t.Fatal("unknown evaluator was accepted")
	}
}

// --cost-per-trace replaced --jev-cost-per-trace when SubQ was added. Both spellings must set the
// estimate `memory evaluations run --dry-run` previews, driven through the real command line.
func TestMemoryEvaluationsRunCostPerTraceFlagsSetTheEstimate(t *testing.T) {
	for _, name := range []string{"cost-per-trace", "jev-cost-per-trace"} {
		t.Run(name, func(t *testing.T) {
			logPath, project := writeMemoryCommandFixture(t)
			resetMemoryOpts(t)
			flags := memoryEvaluationsRunCmd.Flags()
			t.Cleanup(func() {
				flags.VisitAll(func(f *pflag.Flag) { f.Changed = false })
				rootCmd.SetOut(nil)
				rootCmd.SetErr(nil)
				rootCmd.SetArgs(nil)
			})
			var out, stderr bytes.Buffer
			rootCmd.SetOut(&out)
			rootCmd.SetErr(&stderr)
			rootCmd.SetArgs([]string{"memory", "evaluations", "run", "--dry-run", "--json",
				"--log-path", logPath, "--project", project, "--trace", "session:cursor:s1", "--" + name, "0.03"})
			if err := rootCmd.Execute(); err != nil {
				t.Fatalf("memory evaluations run --%s: %v\n%s", name, err, stderr.String())
			}
			// Cobra prints the deprecation notice with OutOrStderr: stderr in a real run, but out here
			// because the test sets it. The JSON document follows it.
			doc := out.Bytes()[max(bytes.IndexByte(out.Bytes(), '{'), 0):]
			var result evaluationRunResult
			if err := json.Unmarshal(doc, &result); err != nil {
				t.Fatalf("decode result: %v\n%s", err, out.String())
			}
			if result.TraceCount != 1 || result.EstimatedCostUSD != 0.03 {
				t.Fatalf("--%s=0.03: trace count %d, estimate %v; want 1 trace at 0.03", name, result.TraceCount, result.EstimatedCostUSD)
			}
		})
	}
}