The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(memory): never promote traces the judge scored as failed (#649)
Claudedb41ff0614ac
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.goindex fb4c77ce..0cda15f2 100644--- a/cli/beacon/cmd/memory.go+++ b/cli/beacon/cmd/memory.go@@ -262,6 +262,14 @@ func runMemoryEvaluationsRun(cmd *cobra.Command, args []string) error {for _, eval := range result.Evaluations {fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%.2f\t%s\n", eval.ID, eval.Trace.ID, eval.Score, eval.Status)}+ for _, eval := range result.Evaluations {+ if eval.Status != asymptoteobserve.LearningEvaluationStatusCompleted {+ continue+ }+ if ok, reason := learning.PromotionDecision(eval); !ok {+ fmt.Fprintf(cmd.OutOrStdout(), "Not promoted: %s (%s)\n", eval.ID, reason)+ }+ }if len(result.Candidates) > 0 {fmt.Fprintf(cmd.OutOrStdout(), "Created %d candidate(s) for review.\n", len(result.Candidates))}diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.goindex f5fb9786..20602102 100644--- a/cli/beacon/cmd/memory_test.go+++ b/cli/beacon/cmd/memory_test.go@@ -3,6 +3,7 @@ package cmdimport ("bytes""encoding/json"+ "fmt""net/http""net/http/httptest""os"@@ -115,6 +116,93 @@ func TestMemoryEvaluationsRunPersistsMockedJevResult(t *testing.T) {}}+// Issue #649: a trace the judge scored as failed must not become a candidate even+// when the mean clears the score threshold. The stored and printed score is the+// unchanged mean; only the promotion decision applies the task_success gate.+func TestMemoryEvaluationsRunDoesNotPromoteFailedTask(t *testing.T) {+ for _, tc := range []struct {+ name string+ answers string+ wantScore string+ wantTextScore string+ wantCandidates int+ wantNote string+ }{+ {+ name: "failed task clears the mean",+ answers: `{"answers":{"task_success":{"type":"noul","noul":0.27},"reusable_correction":{"type":"noul","noul":0.86},"evidence_supported":{"type":"noul","noul":0.69}}}`,+ wantScore: "0.6067",+ wantTextScore: "0.61",+ wantNote: "(task_success 0.27 is below 0.50)",+ },+ {+ name: "successful task still promotes",+ answers: `{"answers":{"task_success":{"type":"noul","noul":0.9},"reusable_correction":{"type":"noul","noul":0.8},"evidence_supported":{"type":"noul","noul":0.7}}}`,+ wantScore: "0.8000",+ wantTextScore: "0.80",+ wantCandidates: 1,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ for _, jsonOutput := range []bool{true, false} {+ logPath, project := writeMemoryCommandFixture(t)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = w.Write([]byte(tc.answers))+ }))+ resetMemoryOpts(t)+ memoryOpts.userMode = true+ memoryOpts.logPath = logPath+ memoryOpts.projectPath = project+ memoryOpts.traceID = "session:cursor:s1"+ memoryOpts.jsonOutput = jsonOutput+ memoryOpts.jevEndpoint = server.URL+ memoryOpts.jevAPIKey = "test-key"++ var out bytes.Buffer+ cmd := &cobra.Command{}+ cmd.SetOut(&out)+ err := runMemoryEvaluationsRun(cmd, nil)+ server.Close()+ if err != nil {+ t.Fatalf("runMemoryEvaluationsRun(json=%v) returned error: %v", jsonOutput, err)+ }+ if jsonOutput {+ var result evaluationRunResult+ if err := json.Unmarshal(out.Bytes(), &result); err != nil {+ t.Fatalf("decode result: %v\n%s", err, out.String())+ }+ if len(result.Evaluations) != 1 || fmt.Sprintf("%.4f", result.Evaluations[0].Score) != tc.wantScore {+ t.Fatalf("evaluations = %#v, want one with score %s", result.Evaluations, tc.wantScore)+ }+ if len(result.Candidates) != tc.wantCandidates {+ t.Fatalf("candidates = %#v, want %d", result.Candidates, tc.wantCandidates)+ }+ } else {+ text := out.String()+ if !strings.Contains(text, "\t"+tc.wantTextScore+"\t") {+ t.Fatalf("text output does not print the unchanged score %s:\n%s", tc.wantTextScore, text)+ }+ switch {+ case tc.wantNote != "" && !strings.Contains(text, "Not promoted: eval_"):+ t.Fatalf("text output does not say the evaluation was not promoted:\n%s", text)+ case tc.wantNote != "" && !strings.Contains(text, tc.wantNote):+ t.Fatalf("text output missing the gate reason %q:\n%s", tc.wantNote, text)+ case tc.wantNote == "" && strings.Contains(text, "Not promoted:"):+ t.Fatalf("promoted evaluation reported as not promoted:\n%s", text)+ }+ }+ stored, err := memoryStore().ListCandidates(learning.Query{ProjectPath: project})+ if err != nil {+ t.Fatalf("ListCandidates: %v", err)+ }+ if len(stored) != tc.wantCandidates {+ t.Fatalf("stored candidates = %#v, want %d", stored, tc.wantCandidates)+ }+ }+ })+ }+}+func TestMemoryCandidatesApproveCommand(t *testing.T) {logPath, project := writeMemoryCommandFixture(t)resetMemoryOpts(t)diff --git a/cli/beacon/internal/learning/candidate.go b/cli/beacon/internal/learning/candidate.goindex a3ee0d25..62d8258f 100644--- a/cli/beacon/internal/learning/candidate.go+++ b/cli/beacon/internal/learning/candidate.go@@ -2,18 +2,74 @@ package learningimport ("fmt"+ "strconv""strings""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve")+// CandidateScoreThreshold is the minimum evaluation score (the mean of the+// rubric probabilities) a completed evaluation needs to become a candidate.const CandidateScoreThreshold = 0.60-func CandidateFromEvaluation(eval asymptoteobserve.LearningEvaluationV1) (asymptoteobserve.LearningCandidateV1, bool) {+// TaskSuccessQuestionID names the rubric question that asks whether the trace+// completed the user's task.+const TaskSuccessQuestionID = "task_success"++// CandidateTaskSuccessThreshold is a precondition that the score cannot stand in+// for. The score is a mean, so two high answers about reusability and evidence+// could outvote a judge who said the task failed: 0.27/0.86/0.69 averages 0.6067+// and cleared CandidateScoreThreshold (#649). A trace whose task_success+// probability is below this value is never promoted, whatever its score.+const CandidateTaskSuccessThreshold = 0.50++// PromotionDecision reports whether a completed evaluation should become a+// memory candidate, and when it should not, why. It applies the task_success+// precondition before the score threshold, so the stored Score keeps its meaning+// (the plain mean shown to reviewers) while a failed task can no longer be+// averaged into a candidate. An evaluation without a task_success answer fails+// the precondition, because nothing says the task succeeded.+func PromotionDecision(eval asymptoteobserve.LearningEvaluationV1) (bool, string) {if eval.Status != asymptoteobserve.LearningEvaluationStatusCompleted {- return asymptoteobserve.LearningCandidateV1{}, false+ return false, fmt.Sprintf("evaluation status is %s, not %s", eval.Status, asymptoteobserve.LearningEvaluationStatusCompleted)+ }+ taskSuccess, ok := questionProbability(eval, TaskSuccessQuestionID)+ if !ok {+ return false, TaskSuccessQuestionID + " was not answered"+ }+ if taskSuccess < CandidateTaskSuccessThreshold {+ return false, fmt.Sprintf("%s %s is below %.2f", TaskSuccessQuestionID, formatBelow(taskSuccess, CandidateTaskSuccessThreshold), CandidateTaskSuccessThreshold)}if eval.Score < CandidateScoreThreshold {+ return false, fmt.Sprintf("score %s is below %.2f", formatBelow(eval.Score, CandidateScoreThreshold), CandidateScoreThreshold)+ }+ return true, ""+}++// formatBelow renders value, known to be below limit, with the fewest decimals+// (at least two) that still read as below it, so a reason never says "0.50 is+// below 0.50" for 0.4999.+func formatBelow(value, limit float64) string {+ for precision := 2; precision <= 6; precision++ {+ text := strconv.FormatFloat(value, 'f', precision, 64)+ if parsed, err := strconv.ParseFloat(text, 64); err == nil && parsed < limit {+ return text+ }+ }+ return strconv.FormatFloat(value, 'g', -1, 64)+}++func questionProbability(eval asymptoteobserve.LearningEvaluationV1, id string) (float64, bool) {+ for _, question := range eval.Questions {+ if question.ID == id {+ return question.Probability, true+ }+ }+ return 0, false+}++func CandidateFromEvaluation(eval asymptoteobserve.LearningEvaluationV1) (asymptoteobserve.LearningCandidateV1, bool) {+ if ok, _ := PromotionDecision(eval); !ok {return asymptoteobserve.LearningCandidateV1{}, false}kind := candidateKind(eval)diff --git a/cli/beacon/internal/learning/evaluator.go b/cli/beacon/internal/learning/evaluator.goindex 0461540a..e306cc38 100644--- a/cli/beacon/internal/learning/evaluator.go+++ b/cli/beacon/internal/learning/evaluator.go@@ -29,7 +29,7 @@ const ()var RubricQuestions = []asymptoteobserve.LearningEvaluationQuestionV1{- {ID: "task_success", Prompt: "Did the trace complete the user's engineering task successfully?"},+ {ID: TaskSuccessQuestionID, Prompt: "Did the trace complete the user's engineering task successfully?"},{ID: "reusable_correction", Prompt: "Does the trace contain a correction or debugging pattern that future agents should reuse?"},{ID: "evidence_supported", Prompt: "Is the reusable lesson supported by concrete events in the trace?"},}diff --git a/cli/beacon/internal/learning/promotion_test.go b/cli/beacon/internal/learning/promotion_test.gonew file mode 100644index 00000000..f4690936--- /dev/null+++ b/cli/beacon/internal/learning/promotion_test.go@@ -0,0 +1,169 @@+package learning++import (+ "fmt"+ "math"+ "strings"+ "testing"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// rubricEvaluation builds a completed evaluation whose Score is computed exactly+// as Evaluate computes it, so the tests exercise the real score and the gate+// together rather than a hand-picked Score.+func rubricEvaluation(taskSuccess, reusable, evidence float64) asymptoteobserve.LearningEvaluationV1 {+ eval := testLearningEvaluation()+ eval.Questions = normalizeQuestionResults([]asymptoteobserve.LearningEvaluationQuestionV1{+ {ID: TaskSuccessQuestionID, Probability: taskSuccess},+ {ID: "reusable_correction", Probability: reusable},+ {ID: "evidence_supported", Probability: evidence},+ })+ eval.Score = evaluationScore(eval.Questions)+ return eval+}++// Issue #649: the three real traces that were promoted although the judge said+// the task failed. Each clears CandidateScoreThreshold on the mean alone.+func TestPromotionRejectsFailedTasksThatClearTheMean(t *testing.T) {+ for _, tc := range []struct {+ name string+ task, reusable, evidence float64+ wantScore float64+ wantReasonContains string+ }{+ {name: "task_success 0.27 promoted by +0.0067", task: 0.27, reusable: 0.86, evidence: 0.69, wantScore: 0.6067, wantReasonContains: "task_success 0.27 is below 0.50"},+ {name: "task_success 0.42 scored 0.6667", task: 0.42, reusable: 0.86, evidence: 0.72, wantScore: 0.6667, wantReasonContains: "task_success 0.42 is below 0.50"},+ {name: "task_success 0.42 scored 0.6367", task: 0.42, reusable: 0.80, evidence: 0.69, wantScore: 0.6367, wantReasonContains: "task_success 0.42 is below 0.50"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ eval := rubricEvaluation(tc.task, tc.reusable, tc.evidence)+ if math.Abs(eval.Score-tc.wantScore) > 0.00005 {+ t.Fatalf("score = %.6f, want %.4f (the reported mean must not change)", eval.Score, tc.wantScore)+ }+ if eval.Score < CandidateScoreThreshold {+ t.Fatalf("fixture score %.4f no longer clears the score threshold, so it does not test the gate", eval.Score)+ }+ ok, reason := PromotionDecision(eval)+ if ok {+ t.Fatalf("PromotionDecision promoted a failed task (score %.4f)", eval.Score)+ }+ if !strings.Contains(reason, tc.wantReasonContains) {+ t.Fatalf("reason = %q, want it to contain %q", reason, tc.wantReasonContains)+ }+ if candidate, ok := CandidateFromEvaluation(eval); ok {+ t.Fatalf("CandidateFromEvaluation promoted a failed task: %#v", candidate)+ }+ })+ }+}++func TestPromotionTaskSuccessGateBoundaries(t *testing.T) {+ for _, tc := range []struct {+ name string+ task, reusable, evidence float64+ wantPromoted bool+ wantReasonContains string+ }{+ {name: "at gate, mean over threshold", task: CandidateTaskSuccessThreshold, reusable: 0.80, evidence: 0.80, wantPromoted: true},+ {name: "at gate, mean under threshold", task: CandidateTaskSuccessThreshold, reusable: 0.60, evidence: 0.60, wantReasonContains: "score 0.57 is below 0.60"},+ {name: "just below gate, mean well over threshold", task: 0.49, reusable: 0.95, evidence: 0.95, wantReasonContains: "task_success 0.49 is below 0.50"},+ {name: "fractionally below gate", task: 0.4999, reusable: 1.0, evidence: 1.0, wantReasonContains: "task_success 0.4999 is below 0.50"},+ {name: "below gate, mean under threshold", task: 0.10, reusable: 0.60, evidence: 0.60, wantReasonContains: "task_success 0.10 is below 0.50"},+ {name: "just above gate, mean over threshold", task: 0.51, reusable: 0.80, evidence: 0.80, wantPromoted: true},+ {name: "just above gate, mean under threshold", task: 0.51, reusable: 0.60, evidence: 0.60, wantReasonContains: "score 0.57 is below 0.60"},+ } {+ t.Run(tc.name, func(t *testing.T) {+ eval := rubricEvaluation(tc.task, tc.reusable, tc.evidence)+ ok, reason := PromotionDecision(eval)+ if ok != tc.wantPromoted {+ t.Fatalf("PromotionDecision(%v/%v/%v, score %.4f) = %v (%q), want %v", tc.task, tc.reusable, tc.evidence, eval.Score, ok, reason, tc.wantPromoted)+ }+ if tc.wantPromoted && reason != "" {+ t.Fatalf("promoted evaluation carries a reason: %q", reason)+ }+ if !tc.wantPromoted && !strings.Contains(reason, tc.wantReasonContains) {+ t.Fatalf("reason = %q, want it to contain %q", reason, tc.wantReasonContains)+ }+ if _, created := CandidateFromEvaluation(eval); created != tc.wantPromoted {+ t.Fatalf("CandidateFromEvaluation created=%v, want %v (must agree with PromotionDecision)", created, tc.wantPromoted)+ }+ })+ }+}++func TestPromotionKeepsSuccessfulTraceAndItsScore(t *testing.T) {+ eval := rubricEvaluation(0.9, 0.8, 0.7)+ if want := (0.9 + 0.8 + 0.7) / 3; math.Abs(eval.Score-want) > 1e-12 {+ t.Fatalf("score = %v, want the plain mean %v", eval.Score, want)+ }+ candidate, ok := CandidateFromEvaluation(eval)+ if !ok {+ t.Fatal("a successful, reusable trace was not promoted")+ }+ if candidate.SourceEvaluationID != eval.ID || candidate.State != asymptoteobserve.LearningCandidateStateCandidate {+ t.Fatalf("candidate = %#v", candidate)+ }+ if !strings.Contains(candidate.Body, "- task_success: 0.90") {+ t.Fatalf("candidate body lost the task_success signal:\n%s", candidate.Body)+ }+}++func TestPromotionRequiresTaskSuccessAnswer(t *testing.T) {+ eval := testLearningEvaluation()+ eval.Questions = []asymptoteobserve.LearningEvaluationQuestionV1{+ {ID: "reusable_correction", Probability: 0.95},+ {ID: "evidence_supported", Probability: 0.95},+ }+ eval.Score = 0.95+ ok, reason := PromotionDecision(eval)+ if ok || reason != "task_success was not answered" {+ t.Fatalf("PromotionDecision = %v, %q; want a refusal for a missing task_success answer", ok, reason)+ }+}++func TestPromotionRequiresCompletedStatus(t *testing.T) {+ for _, status := range []string{asymptoteobserve.LearningEvaluationStatusDryRun, asymptoteobserve.LearningEvaluationStatusFailed} {+ eval := rubricEvaluation(0.9, 0.9, 0.9)+ eval.Status = status+ ok, reason := PromotionDecision(eval)+ if ok || !strings.Contains(reason, status) {+ t.Fatalf("status %s: PromotionDecision = %v, %q", status, ok, reason)+ }+ }+}++// The gate sits on the whole Evaluate path: a real Jev answer for the #649 trace+// keeps its reported score and is not promoted, and an evaluator-supplied+// top-level score cannot bypass the task_success precondition.+func TestEvaluateThenPromoteAppliesTaskSuccessGate(t *testing.T) {+ eval := evaluateWithJevBody(t, `{"answers":{+ "task_success":{"type":"noul","noul":0.27},+ "reusable_correction":{"type":"noul","noul":0.86},+ "evidence_supported":{"type":"noul","noul":0.69}}}`)+ if got := fmt.Sprintf("%.4f", eval.Score); got != "0.6067" {+ t.Fatalf("stored score = %s, want the unchanged mean 0.6067", got)+ }+ if _, ok := CandidateFromEvaluation(eval); ok {+ t.Fatal("Jev-scored failed task was promoted")+ }++ legacy := evaluateWithJevBody(t, `{"score":0.95,"questions":[+ {"id":"task_success","probability":0.2},+ {"id":"reusable_correction","probability":0.95},+ {"id":"evidence_supported","probability":0.95}]}`)+ if legacy.Score != 0.95 {+ t.Fatalf("evaluator-supplied score = %v, want 0.95 kept as reported", legacy.Score)+ }+ if ok, reason := PromotionDecision(legacy); ok || !strings.Contains(reason, "task_success 0.20 is below 0.50") {+ t.Fatalf("evaluator-supplied score bypassed the gate: %v, %q", ok, reason)+ }++ passing := evaluateWithJevBody(t, `{"answers":{+ "task_success":{"type":"noul","noul":0.9},+ "reusable_correction":{"type":"noul","noul":0.8},+ "evidence_supported":{"type":"noul","noul":0.7}}}`)+ if _, ok := CandidateFromEvaluation(passing); !ok {+ t.Fatalf("successful Jev-scored trace was not promoted (score %.4f)", passing.Score)+ }+}diff --git a/docs/cli/memory.mdx b/docs/cli/memory.mdxindex c4907c97..892ddbc6 100644--- a/docs/cli/memory.mdx+++ b/docs/cli/memory.mdx@@ -63,6 +63,17 @@ Useful flags:`--jev-endpoint` can also come from `BEACON_JEV_ENDPOINT`, and the model can comefrom `BEACON_JEV_MODEL`.+Each completed evaluation stores a `score`, the plain mean of the three rubric+probabilities. A trace becomes a review candidate only when both hold:++- `task_success` is at least `0.50`. This is a precondition, so a trace the+ evaluator judged as failed is never promoted, however high the other two+ answers are.+- `score` is at least `0.60`.++Text output lists each completed evaluation that was not promoted with the+reason, for example `Not promoted: eval_... (task_success 0.27 is below 0.50)`.+List and inspect stored evaluations:```bashdiff --git a/docs/concepts/cross-harness-memory.mdx b/docs/concepts/cross-harness-memory.mdxindex 31bdc75b..5961e352 100644--- a/docs/concepts/cross-harness-memory.mdx+++ b/docs/concepts/cross-harness-memory.mdx@@ -48,6 +48,11 @@ presenting the scores as a lesson. Review the source trace before approving it.A compatible internal evaluator using the legacy `questions` or `results`response shape can also provide a stored `reason`.+A trace becomes a candidate only when the task succeeded (`task_success` of at+least 0.50) and the mean of the three probabilities is at least 0.60. The+success answer is a precondition rather than one vote in the mean, so a failed+task cannot be promoted on the strength of the other two answers.+Use `--dry-run` first to preview selected traces and estimated calls without makinga network request.