Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Add memory candidate review lifecycle

Justin D'Souza0bb0637c024d
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.go
index 1047351d..f52b03c8 100644
--- a/cli/beacon/cmd/memory.go
+++ b/cli/beacon/cmd/memory.go
@@ -35,6 +35,10 @@ var memoryOpts struct {
jevModel string
jevCost float64
timeout time.Duration
+ state string
+ kind string
+ reason string
+ replacement string
}
var memoryCmd = &cobra.Command{
@@ -69,6 +73,50 @@ var memoryEvaluationsShowCmd = &cobra.Command{
RunE: runMemoryEvaluationsShow,
}
+var memoryCandidatesCmd = &cobra.Command{
+ Use: "candidates",
+ Short: "Review extracted memory candidates",
+}
+
+var memoryCandidatesListCmd = &cobra.Command{
+ Use: "list",
+ Short: "List memory candidates",
+ SilenceUsage: true,
+ RunE: runMemoryCandidatesList,
+}
+
+var memoryCandidatesShowCmd = &cobra.Command{
+ Use: "show <candidate-id>",
+ Short: "Show one memory candidate",
+ Args: cobra.ExactArgs(1),
+ SilenceUsage: true,
+ RunE: runMemoryCandidatesShow,
+}
+
+var memoryCandidatesApproveCmd = &cobra.Command{
+ Use: "approve <candidate-id>",
+ Short: "Approve a candidate into project memory",
+ Args: cobra.ExactArgs(1),
+ SilenceUsage: true,
+ RunE: runMemoryCandidatesApprove,
+}
+
+var memoryCandidatesRejectCmd = &cobra.Command{
+ Use: "reject <candidate-id>",
+ Short: "Reject a memory candidate",
+ Args: cobra.ExactArgs(1),
+ SilenceUsage: true,
+ RunE: runMemoryCandidatesReject,
+}
+
+var memoryCandidatesSupersedeCmd = &cobra.Command{
+ Use: "supersede <candidate-id>",
+ Short: "Mark a candidate as superseded by approved memory",
+ Args: cobra.ExactArgs(1),
+ SilenceUsage: true,
+ RunE: runMemoryCandidatesSupersede,
+}
+
type evaluationRunResult struct {
DryRun bool `json:"dry_run"`
Project asymptoteobserve.LearningProjectV1 `json:"project"`
@@ -78,27 +126,40 @@ type evaluationRunResult struct {
EstimatedCostUSD float64 `json:"estimated_cost_usd"`
Previews []learning.EvaluationPreview `json:"previews,omitempty"`
Evaluations []asymptoteobserve.LearningEvaluationV1 `json:"evaluations,omitempty"`
+ Candidates []asymptoteobserve.LearningCandidateV1 `json:"candidates,omitempty"`
}
func init() {
rootCmd.AddCommand(memoryCmd)
memoryCmd.AddCommand(memoryEvaluationsCmd)
+ memoryCmd.AddCommand(memoryCandidatesCmd)
memoryEvaluationsCmd.AddCommand(memoryEvaluationsRunCmd)
memoryEvaluationsCmd.AddCommand(memoryEvaluationsListCmd)
memoryEvaluationsCmd.AddCommand(memoryEvaluationsShowCmd)
+ memoryCandidatesCmd.AddCommand(memoryCandidatesListCmd)
+ memoryCandidatesCmd.AddCommand(memoryCandidatesShowCmd)
+ memoryCandidatesCmd.AddCommand(memoryCandidatesApproveCmd)
+ memoryCandidatesCmd.AddCommand(memoryCandidatesRejectCmd)
+ memoryCandidatesCmd.AddCommand(memoryCandidatesSupersedeCmd)
- for _, c := range []*cobra.Command{memoryEvaluationsRunCmd, memoryEvaluationsListCmd, memoryEvaluationsShowCmd} {
+ for _, c := range []*cobra.Command{memoryEvaluationsRunCmd, memoryEvaluationsListCmd, memoryEvaluationsShowCmd, memoryCandidatesListCmd, memoryCandidatesShowCmd, memoryCandidatesApproveCmd, memoryCandidatesRejectCmd, memoryCandidatesSupersedeCmd} {
c.Flags().BoolVar(&memoryOpts.userMode, "user", true, "Use per-user endpoint paths")
c.Flags().BoolVar(&memoryOpts.systemMode, "system", false, "Use system endpoint paths")
c.Flags().StringVar(&memoryOpts.logPath, "log-path", "", "Runtime JSONL log path")
c.Flags().BoolVar(&memoryOpts.jsonOutput, "json", false, "Print machine-readable JSON")
c.Flags().StringVar(&memoryOpts.projectPath, "project", "", "Project path for memory scoping (defaults to current directory)")
}
- for _, c := range []*cobra.Command{memoryEvaluationsRunCmd, memoryEvaluationsListCmd} {
+ for _, c := range []*cobra.Command{memoryEvaluationsRunCmd, memoryEvaluationsListCmd, memoryCandidatesListCmd} {
c.Flags().IntVar(&memoryOpts.limit, "limit", 25, "Limit returned traces or evaluations")
c.Flags().IntVar(&memoryOpts.page, "page", 1, "Page number for collection output")
c.Flags().StringVarP(&memoryOpts.query, "query", "q", "", "Free-text query")
}
+ memoryCandidatesListCmd.Flags().StringVar(&memoryOpts.state, "state", "", "Filter candidates by state")
+ memoryCandidatesListCmd.Flags().StringVar(&memoryOpts.kind, "kind", "", "Filter candidates by memory kind")
+ for _, c := range []*cobra.Command{memoryCandidatesApproveCmd, memoryCandidatesRejectCmd, memoryCandidatesSupersedeCmd} {
+ c.Flags().StringVar(&memoryOpts.reason, "reason", "", "Review reason recorded with the candidate")
+ }
+ memoryCandidatesSupersedeCmd.Flags().StringVar(&memoryOpts.replacement, "replacement", "", "Approved memory ID that supersedes this candidate")
memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.harness, "harness", "", "Filter traces by harness")
memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.traceID, "trace", "", "Evaluate a single trace ID")
memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.since, "since", "", "RFC3339 lower time bound, inclusive")
@@ -154,6 +215,12 @@ func runMemoryEvaluationsRun(cmd *cobra.Command, args []string) error {
return err
}
result.Evaluations = append(result.Evaluations, eval)
+ if candidate, ok := learning.CandidateFromEvaluation(eval); ok {
+ if err := store.PutCandidate(candidate); err != nil {
+ return err
+ }
+ result.Candidates = append(result.Candidates, candidate)
+ }
}
if memoryOpts.jsonOutput {
return json.NewEncoder(cmd.OutOrStdout()).Encode(result)
@@ -169,6 +236,9 @@ func runMemoryEvaluationsRun(cmd *cobra.Command, args []string) error {
for _, eval := range result.Evaluations {
fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%.2f\t%s\n", eval.ID, eval.Trace.ID, eval.Score, eval.Status)
}
+ if len(result.Candidates) > 0 {
+ fmt.Fprintf(cmd.OutOrStdout(), "Created %d candidate(s) for review.\n", len(result.Candidates))
+ }
return nil
}
@@ -208,6 +278,75 @@ func runMemoryEvaluationsShow(cmd *cobra.Command, args []string) error {
return nil
}
+func runMemoryCandidatesList(cmd *cobra.Command, args []string) error {
+ query, err := memoryCandidateQuery()
+ if err != nil {
+ return err
+ }
+ candidates, err := memoryStore().ListCandidates(query)
+ if err != nil {
+ return err
+ }
+ if memoryOpts.jsonOutput {
+ return json.NewEncoder(cmd.OutOrStdout()).Encode(candidates)
+ }
+ for _, candidate := range candidates {
+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%s\t%s\n", candidate.ID, candidate.State, candidate.Kind, candidate.Title)
+ }
+ return nil
+}
+
+func runMemoryCandidatesShow(cmd *cobra.Command, args []string) error {
+ candidate, ok, err := memoryStore().GetCandidate(args[0])
+ if err != nil {
+ return err
+ }
+ if !ok {
+ return fmt.Errorf("candidate not found: %s", args[0])
+ }
+ if memoryOpts.jsonOutput {
+ return json.NewEncoder(cmd.OutOrStdout()).Encode(candidate)
+ }
+ printCandidate(cmd, candidate)
+ return nil
+}
+
+func runMemoryCandidatesApprove(cmd *cobra.Command, args []string) error {
+ candidate, memory, err := learning.ApproveCandidate(memoryStore(), args[0], memoryOpts.reason)
+ if err != nil {
+ return err
+ }
+ if memoryOpts.jsonOutput {
+ return json.NewEncoder(cmd.OutOrStdout()).Encode(map[string]interface{}{"candidate": candidate, "memory": memory})
+ }
+ fmt.Fprintf(cmd.OutOrStdout(), "Approved %s as memory %s\n", candidate.ID, memory.ID)
+ return nil
+}
+
+func runMemoryCandidatesReject(cmd *cobra.Command, args []string) error {
+ candidate, err := learning.RejectCandidate(memoryStore(), args[0], memoryOpts.reason)
+ if err != nil {
+ return err
+ }
+ if memoryOpts.jsonOutput {
+ return json.NewEncoder(cmd.OutOrStdout()).Encode(candidate)
+ }
+ fmt.Fprintf(cmd.OutOrStdout(), "Rejected %s\n", candidate.ID)
+ return nil
+}
+
+func runMemoryCandidatesSupersede(cmd *cobra.Command, args []string) error {
+ candidate, err := learning.SupersedeCandidate(memoryStore(), args[0], memoryOpts.replacement, memoryOpts.reason)
+ if err != nil {
+ return err
+ }
+ if memoryOpts.jsonOutput {
+ return json.NewEncoder(cmd.OutOrStdout()).Encode(candidate)
+ }
+ fmt.Fprintf(cmd.OutOrStdout(), "Superseded %s with %s\n", candidate.ID, candidate.SupersededBy)
+ return nil
+}
+
func selectedEvaluationInputs(project asymptoteobserve.LearningProjectV1) ([]learning.EvaluationInput, error) {
logPath := memoryLogPath()
if memoryOpts.traceID != "" {
@@ -266,6 +405,23 @@ func memoryQuery() (learning.Query, error) {
}, nil
}
+func memoryCandidateQuery() (learning.Query, error) {
+ query, err := memoryQuery()
+ if err != nil {
+ return learning.Query{}, err
+ }
+ query.State = memoryOpts.state
+ query.Kind = memoryOpts.kind
+ return query, nil
+}
+
+func printCandidate(cmd *cobra.Command, candidate asymptoteobserve.LearningCandidateV1) {
+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%s\n", candidate.ID, candidate.State, candidate.Kind)
+ fmt.Fprintln(cmd.OutOrStdout(), candidate.Title)
+ fmt.Fprintln(cmd.OutOrStdout())
+ fmt.Fprintln(cmd.OutOrStdout(), candidate.Body)
+}
+
func memoryStore() *learning.Store {
return learning.Open(learning.PathForRuntimeLog(memoryLogPath()))
}
diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.go
index 77a9a667..10aa7147 100644
--- a/cli/beacon/cmd/memory_test.go
+++ b/cli/beacon/cmd/memory_test.go
@@ -15,6 +15,7 @@ import (
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/learning"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
func TestMemoryEvaluationCommandsRegistered(t *testing.T) {
@@ -22,6 +23,11 @@ func TestMemoryEvaluationCommandsRegistered(t *testing.T) {
{"memory", "evaluations", "run"},
{"memory", "evaluations", "list"},
{"memory", "evaluations", "show"},
+ {"memory", "candidates", "list"},
+ {"memory", "candidates", "show"},
+ {"memory", "candidates", "approve"},
+ {"memory", "candidates", "reject"},
+ {"memory", "candidates", "supersede"},
} {
cmd, _, err := rootCmd.Find(path)
if err != nil || cmd == nil {
@@ -93,6 +99,9 @@ func TestMemoryEvaluationsRunPersistsMockedJevResult(t *testing.T) {
if len(result.Evaluations) != 1 || result.Evaluations[0].Score < 0.79 || result.Evaluations[0].Score > 0.81 {
t.Fatalf("evaluations = %#v", result.Evaluations)
}
+ if len(result.Candidates) != 1 || result.Candidates[0].State != "candidate" {
+ t.Fatalf("candidates = %#v", result.Candidates)
+ }
memoryOpts.query = "session:cursor:s1"
list, err := memoryStore().ListEvaluations(learning.Query{ProjectID: result.Project.ID, Q: "cursor"})
@@ -104,6 +113,29 @@ func TestMemoryEvaluationsRunPersistsMockedJevResult(t *testing.T) {
}
}
+func TestMemoryCandidatesApproveCommand(t *testing.T) {
+ logPath, project := writeMemoryCommandFixture(t)
+ resetMemoryOpts(t)
+ memoryOpts.userMode = true
+ memoryOpts.logPath = logPath
+ memoryOpts.projectPath = project
+ candidate := testCommandCandidate(t)
+ if err := memoryStore().PutCandidate(candidate); err != nil {
+ t.Fatal(err)
+ }
+ memoryOpts.reason = "reviewed"
+ memoryOpts.jsonOutput = true
+ var out bytes.Buffer
+ cmd := &cobra.Command{}
+ cmd.SetOut(&out)
+ if err := runMemoryCandidatesApprove(cmd, []string{candidate.ID}); err != nil {
+ t.Fatalf("runMemoryCandidatesApprove returned error: %v", err)
+ }
+ if !strings.Contains(out.String(), `"memory"`) || !strings.Contains(out.String(), `"approved"`) {
+ t.Fatalf("unexpected approve output: %s", out.String())
+ }
+}
+
func writeMemoryCommandFixture(t *testing.T) (string, string) {
t.Helper()
dir := t.TempDir()
@@ -164,6 +196,36 @@ func resetMemoryOpts(t *testing.T) {
jevModel string
jevCost float64
timeout time.Duration
+ state string
+ kind string
+ reason string
+ replacement string
}{userMode: true, limit: 25, page: 1, jevEndpoint: learning.DefaultJevEndpoint, jevModel: learning.DefaultJevModel, jevCost: learning.DefaultCostPerTrace}
t.Cleanup(func() { memoryOpts = previous })
}
+
+func testCommandCandidate(t *testing.T) asymptoteobserve.LearningCandidateV1 {
+ t.Helper()
+ candidate, ok := learning.CandidateFromEvaluation(asymptoteobserve.LearningEvaluationV1{
+ SchemaVersion: asymptoteobserve.LearningSchemaVersion,
+ ID: "eval-command",
+ Status: asymptoteobserve.LearningEvaluationStatusCompleted,
+ Score: 0.9,
+ Project: asymptoteobserve.LearningProjectV1{ID: "project-1"},
+ Trace: asymptoteobserve.LearningTraceRefV1{
+ ID: "trace-command",
+ Title: "Fix flaky package smoke",
+ Harness: asymptoteobserve.TraceHarnessV1{Name: "cursor"},
+ EventIDs: []string{"event-1"},
+ },
+ Questions: []asymptoteobserve.LearningEvaluationQuestionV1{
+ {ID: "task_success", Probability: 0.9},
+ {ID: "reusable_correction", Probability: 0.9},
+ {ID: "evidence_supported", Probability: 0.9},
+ },
+ })
+ if !ok {
+ t.Fatal("candidate not created")
+ }
+ return candidate
+}
diff --git a/cli/beacon/internal/learning/candidate.go b/cli/beacon/internal/learning/candidate.go
new file mode 100644
index 00000000..e13ca16e
--- /dev/null
+++ b/cli/beacon/internal/learning/candidate.go
@@ -0,0 +1,204 @@
+package learning
+
+import (
+ "fmt"
+ "strings"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+const CandidateScoreThreshold = 0.60
+
+func CandidateFromEvaluation(eval asymptoteobserve.LearningEvaluationV1) (asymptoteobserve.LearningCandidateV1, bool) {
+ if eval.Status != asymptoteobserve.LearningEvaluationStatusCompleted {
+ return asymptoteobserve.LearningCandidateV1{}, false
+ }
+ if eval.Score < CandidateScoreThreshold {
+ return asymptoteobserve.LearningCandidateV1{}, false
+ }
+ kind := candidateKind(eval)
+ title := candidateTitle(eval, kind)
+ body := candidateBody(eval)
+ candidate := asymptoteobserve.LearningCandidateV1{
+ SchemaVersion: asymptoteobserve.LearningSchemaVersion,
+ State: asymptoteobserve.LearningCandidateStateCandidate,
+ Kind: kind,
+ Title: title,
+ Body: body,
+ Applicability: candidateApplicability(eval),
+ Tags: candidateTags(eval, kind),
+ Project: eval.Project,
+ SourceEvaluationID: eval.ID,
+ Evidence: []asymptoteobserve.LearningEvidenceV1{
+ {
+ TraceID: eval.Trace.ID,
+ EventIDs: append([]string(nil), eval.Trace.EventIDs...),
+ Summary: strings.TrimSpace(eval.Trace.Title),
+ },
+ },
+ }
+ candidate.ID = CandidateID(candidate)
+ return candidate, true
+}
+
+func ApproveCandidate(store *Store, id, reason string) (asymptoteobserve.LearningCandidateV1, asymptoteobserve.LearningMemoryV1, error) {
+ candidate, ok, err := store.GetCandidate(id)
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
+ }
+ if !ok {
+ return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, fmt.Errorf("candidate not found: %s", id)
+ }
+ if candidate.State != asymptoteobserve.LearningCandidateStateCandidate {
+ return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, fmt.Errorf("candidate %s is %s, not candidate", id, candidate.State)
+ }
+ now := nowString()
+ memory := asymptoteobserve.LearningMemoryV1{
+ SchemaVersion: asymptoteobserve.LearningSchemaVersion,
+ CandidateID: candidate.ID,
+ Kind: candidate.Kind,
+ Title: candidate.Title,
+ Body: candidate.Body,
+ Applicability: candidate.Applicability,
+ Tags: append([]string(nil), candidate.Tags...),
+ Project: candidate.Project,
+ Evidence: append([]asymptoteobserve.LearningEvidenceV1(nil), candidate.Evidence...),
+ CreatedAt: now,
+ UpdatedAt: now,
+ }
+ memory.ID = MemoryID(memory)
+ candidate.State = asymptoteobserve.LearningCandidateStateApproved
+ candidate.MemoryID = memory.ID
+ candidate.ApprovedAt = now
+ candidate.ReviewReason = strings.TrimSpace(reason)
+ if err := store.PutMemory(memory); err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
+ }
+ if err := store.PutCandidate(candidate); err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
+ }
+ return candidate, memory, nil
+}
+
+func RejectCandidate(store *Store, id, reason string) (asymptoteobserve.LearningCandidateV1, error) {
+ candidate, ok, err := store.GetCandidate(id)
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ if !ok {
+ return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("candidate not found: %s", id)
+ }
+ if candidate.State != asymptoteobserve.LearningCandidateStateCandidate {
+ return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("candidate %s is %s, not candidate", id, candidate.State)
+ }
+ candidate.State = asymptoteobserve.LearningCandidateStateRejected
+ candidate.RejectedAt = nowString()
+ candidate.ReviewReason = strings.TrimSpace(reason)
+ if err := store.PutCandidate(candidate); err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ return candidate, nil
+}
+
+func SupersedeCandidate(store *Store, id, replacementMemoryID, reason string) (asymptoteobserve.LearningCandidateV1, error) {
+ if strings.TrimSpace(replacementMemoryID) == "" {
+ return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("replacement memory id is required")
+ }
+ replacement, ok, err := store.GetMemory(replacementMemoryID)
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ if !ok {
+ return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("replacement memory not found: %s", replacementMemoryID)
+ }
+ candidate, ok, err := store.GetCandidate(id)
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ if !ok {
+ return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("candidate not found: %s", id)
+ }
+ if candidate.Project.ID != replacement.Project.ID {
+ return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("replacement memory belongs to a different project")
+ }
+ now := nowString()
+ candidate.State = asymptoteobserve.LearningCandidateStateSuperseded
+ candidate.SupersededAt = now
+ candidate.SupersededBy = replacement.ID
+ candidate.ReviewReason = strings.TrimSpace(reason)
+ if err := store.PutCandidate(candidate); err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ if candidate.MemoryID != "" {
+ memory, ok, err := store.GetMemory(candidate.MemoryID)
+ if err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ if ok {
+ memory.SupersededBy = replacement.ID
+ memory.UpdatedAt = now
+ if err := store.PutMemory(memory); err != nil {
+ return asymptoteobserve.LearningCandidateV1{}, err
+ }
+ }
+ }
+ return candidate, nil
+}
+
+func candidateKind(eval asymptoteobserve.LearningEvaluationV1) string {
+ title := strings.ToLower(eval.Trace.Title)
+ switch {
+ case strings.Contains(title, "convention") || strings.Contains(title, "standard"):
+ return asymptoteobserve.LearningMemoryKindConvention
+ case strings.Contains(title, "gotcha") || strings.Contains(title, "pitfall"):
+ return asymptoteobserve.LearningMemoryKindGotcha
+ case strings.Contains(title, "workflow") || strings.Contains(title, "process"):
+ return asymptoteobserve.LearningMemoryKindWorkflow
+ case strings.Contains(title, "fix") || strings.Contains(title, "debug") || strings.Contains(title, "fail"):
+ return asymptoteobserve.LearningMemoryKindDebuggingPattern
+ default:
+ return asymptoteobserve.LearningMemoryKindCorrection
+ }
+}
+
+func candidateTitle(eval asymptoteobserve.LearningEvaluationV1, kind string) string {
+ title := strings.TrimSpace(eval.Trace.Title)
+ if title == "" {
+ title = eval.Trace.ID
+ }
+ return strings.TrimSpace(strings.ReplaceAll(kind, "_", " ")) + ": " + title
+}
+
+func candidateBody(eval asymptoteobserve.LearningEvaluationV1) string {
+ var lines []string
+ lines = append(lines, "Reusable lesson extracted from a reviewed Beacon trace.")
+ lines = append(lines, "")
+ lines = append(lines, "Trace: "+eval.Trace.ID)
+ if eval.Trace.Harness.Name != "" {
+ lines = append(lines, "Harness: "+eval.Trace.Harness.Name)
+ }
+ if eval.Trace.Title != "" {
+ lines = append(lines, "Observed workflow: "+eval.Trace.Title)
+ }
+ lines = append(lines, "")
+ lines = append(lines, "Evaluation signals:")
+ for _, question := range eval.Questions {
+ lines = append(lines, fmt.Sprintf("- %s: %.2f", question.ID, question.Probability))
+ }
+ return strings.Join(lines, "\n")
+}
+
+func candidateApplicability(eval asymptoteobserve.LearningEvaluationV1) string {
+ if eval.Trace.Harness.Name == "" {
+ return "When a future agent works on a similar task in this project."
+ }
+ return "When a future agent in this project hits a similar workflow, regardless of harness."
+}
+
+func candidateTags(eval asymptoteobserve.LearningEvaluationV1, kind string) []string {
+ tags := []string{"beacon", kind}
+ if eval.Trace.Harness.Name != "" {
+ tags = append(tags, eval.Trace.Harness.Name)
+ }
+ return tags
+}
diff --git a/cli/beacon/internal/learning/candidate_test.go b/cli/beacon/internal/learning/candidate_test.go
new file mode 100644
index 00000000..78215d22
--- /dev/null
+++ b/cli/beacon/internal/learning/candidate_test.go
@@ -0,0 +1,103 @@
+package learning
+
+import (
+ "path/filepath"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+func TestCandidateFromEvaluationRequiresHighSignal(t *testing.T) {
+ eval := testLearningEvaluation()
+ eval.Score = 0.3
+ if _, ok := CandidateFromEvaluation(eval); ok {
+ t.Fatal("low-signal evaluation should not form a candidate")
+ }
+ eval.Score = 0.9
+ candidate, ok := CandidateFromEvaluation(eval)
+ if !ok {
+ t.Fatal("high-signal evaluation did not form a candidate")
+ }
+ if candidate.Kind != asymptoteobserve.LearningMemoryKindDebuggingPattern {
+ t.Fatalf("kind = %s", candidate.Kind)
+ }
+ if candidate.SourceEvaluationID != eval.ID || len(candidate.Evidence) != 1 || candidate.Evidence[0].TraceID != eval.Trace.ID {
+ t.Fatalf("candidate evidence = %#v", candidate)
+ }
+}
+
+func TestApproveRejectAndSupersedeCandidate(t *testing.T) {
+ store := Open(filepath.Join(t.TempDir(), "memory.db"))
+ candidate, ok := CandidateFromEvaluation(testLearningEvaluation())
+ if !ok {
+ t.Fatal("candidate not created")
+ }
+ if err := store.PutCandidate(candidate); err != nil {
+ t.Fatal(err)
+ }
+ approved, memory, err := ApproveCandidate(store, candidate.ID, "reviewed")
+ if err != nil {
+ t.Fatalf("ApproveCandidate: %v", err)
+ }
+ if approved.State != asymptoteobserve.LearningCandidateStateApproved || approved.MemoryID != memory.ID {
+ t.Fatalf("approved candidate = %#v memory=%#v", approved, memory)
+ }
+ if _, _, err := ApproveCandidate(store, candidate.ID, "again"); err == nil {
+ t.Fatal("approving an approved candidate should fail")
+ }
+
+ rejectCandidate, ok := CandidateFromEvaluation(testLearningEvaluationWithID("eval-reject", "trace-reject"))
+ if !ok {
+ t.Fatal("reject candidate not created")
+ }
+ if err := store.PutCandidate(rejectCandidate); err != nil {
+ t.Fatal(err)
+ }
+ rejected, err := RejectCandidate(store, rejectCandidate.ID, "not reusable")
+ if err != nil {
+ t.Fatalf("RejectCandidate: %v", err)
+ }
+ if rejected.State != asymptoteobserve.LearningCandidateStateRejected || rejected.ReviewReason != "not reusable" {
+ t.Fatalf("rejected = %#v", rejected)
+ }
+
+ supersedeCandidate, ok := CandidateFromEvaluation(testLearningEvaluationWithID("eval-supersede", "trace-supersede"))
+ if !ok {
+ t.Fatal("supersede candidate not created")
+ }
+ if err := store.PutCandidate(supersedeCandidate); err != nil {
+ t.Fatal(err)
+ }
+ superseded, err := SupersedeCandidate(store, supersedeCandidate.ID, memory.ID, "covered by approved memory")
+ if err != nil {
+ t.Fatalf("SupersedeCandidate: %v", err)
+ }
+ if superseded.State != asymptoteobserve.LearningCandidateStateSuperseded || superseded.SupersededBy != memory.ID {
+ t.Fatalf("superseded = %#v", superseded)
+ }
+}
+
+func testLearningEvaluation() asymptoteobserve.LearningEvaluationV1 {
+ return testLearningEvaluationWithID("eval-1", "trace-1")
+}
+
+func testLearningEvaluationWithID(evalID, traceID string) asymptoteobserve.LearningEvaluationV1 {
+ return asymptoteobserve.LearningEvaluationV1{
+ SchemaVersion: asymptoteobserve.LearningSchemaVersion,
+ ID: evalID,
+ Status: asymptoteobserve.LearningEvaluationStatusCompleted,
+ Score: 0.9,
+ Project: asymptoteobserve.LearningProjectV1{ID: "project-1", Path: "/repo"},
+ Trace: asymptoteobserve.LearningTraceRefV1{
+ ID: traceID,
+ Title: "Fix flaky package smoke",
+ Harness: asymptoteobserve.TraceHarnessV1{Name: "cursor"},
+ EventIDs: []string{"event-1", "event-2"},
+ },
+ Questions: []asymptoteobserve.LearningEvaluationQuestionV1{
+ {ID: "task_success", Probability: 0.95},
+ {ID: "reusable_correction", Probability: 0.90},
+ {ID: "evidence_supported", Probability: 0.85},
+ },
+ }
+}