Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(memory): never promote traces the judge scored as failed (#649)

Claudedb41ff0614ac
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.go
index fb4c77ce..0cda15f2 100644
--- a/cli/beacon/cmd/memory.go
+++ b/cli/beacon/cmd/memory.go
@@ -262,6 +262,14 @@ func runMemoryEvaluationsRun(cmd *cobra.Command, args []string) error {
for _, eval := range result.Evaluations {
fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%.2f\t%s\n", eval.ID, eval.Trace.ID, eval.Score, eval.Status)
}
+ for _, eval := range result.Evaluations {
+ if eval.Status != asymptoteobserve.LearningEvaluationStatusCompleted {
+ continue
+ }
+ if ok, reason := learning.PromotionDecision(eval); !ok {
+ fmt.Fprintf(cmd.OutOrStdout(), "Not promoted: %s (%s)\n", eval.ID, reason)
+ }
+ }
if len(result.Candidates) > 0 {
fmt.Fprintf(cmd.OutOrStdout(), "Created %d candidate(s) for review.\n", len(result.Candidates))
}
diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.go
index f5fb9786..20602102 100644
--- a/cli/beacon/cmd/memory_test.go
+++ b/cli/beacon/cmd/memory_test.go
@@ -3,6 +3,7 @@ package cmd
import (
"bytes"
"encoding/json"
+ "fmt"
"net/http"
"net/http/httptest"
"os"
@@ -115,6 +116,93 @@ func TestMemoryEvaluationsRunPersistsMockedJevResult(t *testing.T) {
}
}
+// Issue #649: a trace the judge scored as failed must not become a candidate even
+// when the mean clears the score threshold. The stored and printed score is the
+// unchanged mean; only the promotion decision applies the task_success gate.
+func TestMemoryEvaluationsRunDoesNotPromoteFailedTask(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ answers string
+ wantScore string
+ wantTextScore string
+ wantCandidates int
+ wantNote string
+ }{
+ {
+ name: "failed task clears the mean",
+ answers: `{"answers":{"task_success":{"type":"noul","noul":0.27},"reusable_correction":{"type":"noul","noul":0.86},"evidence_supported":{"type":"noul","noul":0.69}}}`,
+ wantScore: "0.6067",
+ wantTextScore: "0.61",
+ wantNote: "(task_success 0.27 is below 0.50)",
+ },
+ {
+ name: "successful task still promotes",
+ answers: `{"answers":{"task_success":{"type":"noul","noul":0.9},"reusable_correction":{"type":"noul","noul":0.8},"evidence_supported":{"type":"noul","noul":0.7}}}`,
+ wantScore: "0.8000",
+ wantTextScore: "0.80",
+ wantCandidates: 1,
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ for _, jsonOutput := range []bool{true, false} {
+ logPath, project := writeMemoryCommandFixture(t)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write([]byte(tc.answers))
+ }))
+ resetMemoryOpts(t)
+ memoryOpts.userMode = true
+ memoryOpts.logPath = logPath
+ memoryOpts.projectPath = project
+ memoryOpts.traceID = "session:cursor:s1"
+ memoryOpts.jsonOutput = jsonOutput
+ memoryOpts.jevEndpoint = server.URL
+ memoryOpts.jevAPIKey = "test-key"
+
+ var out bytes.Buffer
+ cmd := &cobra.Command{}
+ cmd.SetOut(&out)
+ err := runMemoryEvaluationsRun(cmd, nil)
+ server.Close()
+ if err != nil {
+ t.Fatalf("runMemoryEvaluationsRun(json=%v) returned error: %v", jsonOutput, err)
+ }
+ if jsonOutput {
+ var result evaluationRunResult
+ if err := json.Unmarshal(out.Bytes(), &result); err != nil {
+ t.Fatalf("decode result: %v\n%s", err, out.String())
+ }
+ if len(result.Evaluations) != 1 || fmt.Sprintf("%.4f", result.Evaluations[0].Score) != tc.wantScore {
+ t.Fatalf("evaluations = %#v, want one with score %s", result.Evaluations, tc.wantScore)
+ }
+ if len(result.Candidates) != tc.wantCandidates {
+ t.Fatalf("candidates = %#v, want %d", result.Candidates, tc.wantCandidates)
+ }
+ } else {
+ text := out.String()
+ if !strings.Contains(text, "\t"+tc.wantTextScore+"\t") {
+ t.Fatalf("text output does not print the unchanged score %s:\n%s", tc.wantTextScore, text)
+ }
+ switch {
+ case tc.wantNote != "" && !strings.Contains(text, "Not promoted: eval_"):
+ t.Fatalf("text output does not say the evaluation was not promoted:\n%s", text)
+ case tc.wantNote != "" && !strings.Contains(text, tc.wantNote):
+ t.Fatalf("text output missing the gate reason %q:\n%s", tc.wantNote, text)
+ case tc.wantNote == "" && strings.Contains(text, "Not promoted:"):
+ t.Fatalf("promoted evaluation reported as not promoted:\n%s", text)
+ }
+ }
+ stored, err := memoryStore().ListCandidates(learning.Query{ProjectPath: project})
+ if err != nil {
+ t.Fatalf("ListCandidates: %v", err)
+ }
+ if len(stored) != tc.wantCandidates {
+ t.Fatalf("stored candidates = %#v, want %d", stored, tc.wantCandidates)
+ }
+ }
+ })
+ }
+}
+
func TestMemoryCandidatesApproveCommand(t *testing.T) {
logPath, project := writeMemoryCommandFixture(t)
resetMemoryOpts(t)
diff --git a/cli/beacon/internal/learning/candidate.go b/cli/beacon/internal/learning/candidate.go
index a3ee0d25..62d8258f 100644
--- a/cli/beacon/internal/learning/candidate.go
+++ b/cli/beacon/internal/learning/candidate.go
@@ -2,18 +2,74 @@ package learning
import (
"fmt"
+ "strconv"
"strings"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
+// CandidateScoreThreshold is the minimum evaluation score (the mean of the
+// rubric probabilities) a completed evaluation needs to become a candidate.
const CandidateScoreThreshold = 0.60
-func CandidateFromEvaluation(eval asymptoteobserve.LearningEvaluationV1) (asymptoteobserve.LearningCandidateV1, bool) {
+// TaskSuccessQuestionID names the rubric question that asks whether the trace
+// completed the user's task.
+const TaskSuccessQuestionID = "task_success"
+
+// CandidateTaskSuccessThreshold is a precondition that the score cannot stand in
+// for. The score is a mean, so two high answers about reusability and evidence
+// could outvote a judge who said the task failed: 0.27/0.86/0.69 averages 0.6067
+// and cleared CandidateScoreThreshold (#649). A trace whose task_success
+// probability is below this value is never promoted, whatever its score.
+const CandidateTaskSuccessThreshold = 0.50
+
+// PromotionDecision reports whether a completed evaluation should become a
+// memory candidate, and when it should not, why. It applies the task_success
+// precondition before the score threshold, so the stored Score keeps its meaning
+// (the plain mean shown to reviewers) while a failed task can no longer be
+// averaged into a candidate. An evaluation without a task_success answer fails
+// the precondition, because nothing says the task succeeded.
+func PromotionDecision(eval asymptoteobserve.LearningEvaluationV1) (bool, string) {
if eval.Status != asymptoteobserve.LearningEvaluationStatusCompleted {
- return asymptoteobserve.LearningCandidateV1{}, false
+ return false, fmt.Sprintf("evaluation status is %s, not %s", eval.Status, asymptoteobserve.LearningEvaluationStatusCompleted)
+ }
+ taskSuccess, ok := questionProbability(eval, TaskSuccessQuestionID)
+ if !ok {
+ return false, TaskSuccessQuestionID + " was not answered"
+ }
+ if taskSuccess < CandidateTaskSuccessThreshold {
+ return false, fmt.Sprintf("%s %s is below %.2f", TaskSuccessQuestionID, formatBelow(taskSuccess, CandidateTaskSuccessThreshold), CandidateTaskSuccessThreshold)
}
if eval.Score < CandidateScoreThreshold {
+ return false, fmt.Sprintf("score %s is below %.2f", formatBelow(eval.Score, CandidateScoreThreshold), CandidateScoreThreshold)
+ }
+ return true, ""
+}
+
+// formatBelow renders value, known to be below limit, with the fewest decimals
+// (at least two) that still read as below it, so a reason never says "0.50 is
+// below 0.50" for 0.4999.
+func formatBelow(value, limit float64) string {
+ for precision := 2; precision <= 6; precision++ {
+ text := strconv.FormatFloat(value, 'f', precision, 64)
+ if parsed, err := strconv.ParseFloat(text, 64); err == nil && parsed < limit {
+ return text
+ }
+ }
+ return strconv.FormatFloat(value, 'g', -1, 64)
+}
+
+func questionProbability(eval asymptoteobserve.LearningEvaluationV1, id string) (float64, bool) {
+ for _, question := range eval.Questions {
+ if question.ID == id {
+ return question.Probability, true
+ }
+ }
+ return 0, false
+}
+
+func CandidateFromEvaluation(eval asymptoteobserve.LearningEvaluationV1) (asymptoteobserve.LearningCandidateV1, bool) {
+ if ok, _ := PromotionDecision(eval); !ok {
return asymptoteobserve.LearningCandidateV1{}, false
}
kind := candidateKind(eval)
diff --git a/cli/beacon/internal/learning/evaluator.go b/cli/beacon/internal/learning/evaluator.go
index 0461540a..e306cc38 100644
--- a/cli/beacon/internal/learning/evaluator.go
+++ b/cli/beacon/internal/learning/evaluator.go
@@ -29,7 +29,7 @@ const (
)
var RubricQuestions = []asymptoteobserve.LearningEvaluationQuestionV1{
- {ID: "task_success", Prompt: "Did the trace complete the user's engineering task successfully?"},
+ {ID: TaskSuccessQuestionID, Prompt: "Did the trace complete the user's engineering task successfully?"},
{ID: "reusable_correction", Prompt: "Does the trace contain a correction or debugging pattern that future agents should reuse?"},
{ID: "evidence_supported", Prompt: "Is the reusable lesson supported by concrete events in the trace?"},
}
diff --git a/cli/beacon/internal/learning/promotion_test.go b/cli/beacon/internal/learning/promotion_test.go
new file mode 100644
index 00000000..f4690936
--- /dev/null
+++ b/cli/beacon/internal/learning/promotion_test.go
@@ -0,0 +1,169 @@
+package learning
+
+import (
+ "fmt"
+ "math"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+// rubricEvaluation builds a completed evaluation whose Score is computed exactly
+// as Evaluate computes it, so the tests exercise the real score and the gate
+// together rather than a hand-picked Score.
+func rubricEvaluation(taskSuccess, reusable, evidence float64) asymptoteobserve.LearningEvaluationV1 {
+ eval := testLearningEvaluation()
+ eval.Questions = normalizeQuestionResults([]asymptoteobserve.LearningEvaluationQuestionV1{
+ {ID: TaskSuccessQuestionID, Probability: taskSuccess},
+ {ID: "reusable_correction", Probability: reusable},
+ {ID: "evidence_supported", Probability: evidence},
+ })
+ eval.Score = evaluationScore(eval.Questions)
+ return eval
+}
+
+// Issue #649: the three real traces that were promoted although the judge said
+// the task failed. Each clears CandidateScoreThreshold on the mean alone.
+func TestPromotionRejectsFailedTasksThatClearTheMean(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ task, reusable, evidence float64
+ wantScore float64
+ wantReasonContains string
+ }{
+ {name: "task_success 0.27 promoted by +0.0067", task: 0.27, reusable: 0.86, evidence: 0.69, wantScore: 0.6067, wantReasonContains: "task_success 0.27 is below 0.50"},
+ {name: "task_success 0.42 scored 0.6667", task: 0.42, reusable: 0.86, evidence: 0.72, wantScore: 0.6667, wantReasonContains: "task_success 0.42 is below 0.50"},
+ {name: "task_success 0.42 scored 0.6367", task: 0.42, reusable: 0.80, evidence: 0.69, wantScore: 0.6367, wantReasonContains: "task_success 0.42 is below 0.50"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ eval := rubricEvaluation(tc.task, tc.reusable, tc.evidence)
+ if math.Abs(eval.Score-tc.wantScore) > 0.00005 {
+ t.Fatalf("score = %.6f, want %.4f (the reported mean must not change)", eval.Score, tc.wantScore)
+ }
+ if eval.Score < CandidateScoreThreshold {
+ t.Fatalf("fixture score %.4f no longer clears the score threshold, so it does not test the gate", eval.Score)
+ }
+ ok, reason := PromotionDecision(eval)
+ if ok {
+ t.Fatalf("PromotionDecision promoted a failed task (score %.4f)", eval.Score)
+ }
+ if !strings.Contains(reason, tc.wantReasonContains) {
+ t.Fatalf("reason = %q, want it to contain %q", reason, tc.wantReasonContains)
+ }
+ if candidate, ok := CandidateFromEvaluation(eval); ok {
+ t.Fatalf("CandidateFromEvaluation promoted a failed task: %#v", candidate)
+ }
+ })
+ }
+}
+
+func TestPromotionTaskSuccessGateBoundaries(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ task, reusable, evidence float64
+ wantPromoted bool
+ wantReasonContains string
+ }{
+ {name: "at gate, mean over threshold", task: CandidateTaskSuccessThreshold, reusable: 0.80, evidence: 0.80, wantPromoted: true},
+ {name: "at gate, mean under threshold", task: CandidateTaskSuccessThreshold, reusable: 0.60, evidence: 0.60, wantReasonContains: "score 0.57 is below 0.60"},
+ {name: "just below gate, mean well over threshold", task: 0.49, reusable: 0.95, evidence: 0.95, wantReasonContains: "task_success 0.49 is below 0.50"},
+ {name: "fractionally below gate", task: 0.4999, reusable: 1.0, evidence: 1.0, wantReasonContains: "task_success 0.4999 is below 0.50"},
+ {name: "below gate, mean under threshold", task: 0.10, reusable: 0.60, evidence: 0.60, wantReasonContains: "task_success 0.10 is below 0.50"},
+ {name: "just above gate, mean over threshold", task: 0.51, reusable: 0.80, evidence: 0.80, wantPromoted: true},
+ {name: "just above gate, mean under threshold", task: 0.51, reusable: 0.60, evidence: 0.60, wantReasonContains: "score 0.57 is below 0.60"},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ eval := rubricEvaluation(tc.task, tc.reusable, tc.evidence)
+ ok, reason := PromotionDecision(eval)
+ if ok != tc.wantPromoted {
+ t.Fatalf("PromotionDecision(%v/%v/%v, score %.4f) = %v (%q), want %v", tc.task, tc.reusable, tc.evidence, eval.Score, ok, reason, tc.wantPromoted)
+ }
+ if tc.wantPromoted && reason != "" {
+ t.Fatalf("promoted evaluation carries a reason: %q", reason)
+ }
+ if !tc.wantPromoted && !strings.Contains(reason, tc.wantReasonContains) {
+ t.Fatalf("reason = %q, want it to contain %q", reason, tc.wantReasonContains)
+ }
+ if _, created := CandidateFromEvaluation(eval); created != tc.wantPromoted {
+ t.Fatalf("CandidateFromEvaluation created=%v, want %v (must agree with PromotionDecision)", created, tc.wantPromoted)
+ }
+ })
+ }
+}
+
+func TestPromotionKeepsSuccessfulTraceAndItsScore(t *testing.T) {
+ eval := rubricEvaluation(0.9, 0.8, 0.7)
+ if want := (0.9 + 0.8 + 0.7) / 3; math.Abs(eval.Score-want) > 1e-12 {
+ t.Fatalf("score = %v, want the plain mean %v", eval.Score, want)
+ }
+ candidate, ok := CandidateFromEvaluation(eval)
+ if !ok {
+ t.Fatal("a successful, reusable trace was not promoted")
+ }
+ if candidate.SourceEvaluationID != eval.ID || candidate.State != asymptoteobserve.LearningCandidateStateCandidate {
+ t.Fatalf("candidate = %#v", candidate)
+ }
+ if !strings.Contains(candidate.Body, "- task_success: 0.90") {
+ t.Fatalf("candidate body lost the task_success signal:\n%s", candidate.Body)
+ }
+}
+
+func TestPromotionRequiresTaskSuccessAnswer(t *testing.T) {
+ eval := testLearningEvaluation()
+ eval.Questions = []asymptoteobserve.LearningEvaluationQuestionV1{
+ {ID: "reusable_correction", Probability: 0.95},
+ {ID: "evidence_supported", Probability: 0.95},
+ }
+ eval.Score = 0.95
+ ok, reason := PromotionDecision(eval)
+ if ok || reason != "task_success was not answered" {
+ t.Fatalf("PromotionDecision = %v, %q; want a refusal for a missing task_success answer", ok, reason)
+ }
+}
+
+func TestPromotionRequiresCompletedStatus(t *testing.T) {
+ for _, status := range []string{asymptoteobserve.LearningEvaluationStatusDryRun, asymptoteobserve.LearningEvaluationStatusFailed} {
+ eval := rubricEvaluation(0.9, 0.9, 0.9)
+ eval.Status = status
+ ok, reason := PromotionDecision(eval)
+ if ok || !strings.Contains(reason, status) {
+ t.Fatalf("status %s: PromotionDecision = %v, %q", status, ok, reason)
+ }
+ }
+}
+
+// The gate sits on the whole Evaluate path: a real Jev answer for the #649 trace
+// keeps its reported score and is not promoted, and an evaluator-supplied
+// top-level score cannot bypass the task_success precondition.
+func TestEvaluateThenPromoteAppliesTaskSuccessGate(t *testing.T) {
+ eval := evaluateWithJevBody(t, `{"answers":{
+ "task_success":{"type":"noul","noul":0.27},
+ "reusable_correction":{"type":"noul","noul":0.86},
+ "evidence_supported":{"type":"noul","noul":0.69}}}`)
+ if got := fmt.Sprintf("%.4f", eval.Score); got != "0.6067" {
+ t.Fatalf("stored score = %s, want the unchanged mean 0.6067", got)
+ }
+ if _, ok := CandidateFromEvaluation(eval); ok {
+ t.Fatal("Jev-scored failed task was promoted")
+ }
+
+ legacy := evaluateWithJevBody(t, `{"score":0.95,"questions":[
+ {"id":"task_success","probability":0.2},
+ {"id":"reusable_correction","probability":0.95},
+ {"id":"evidence_supported","probability":0.95}]}`)
+ if legacy.Score != 0.95 {
+ t.Fatalf("evaluator-supplied score = %v, want 0.95 kept as reported", legacy.Score)
+ }
+ if ok, reason := PromotionDecision(legacy); ok || !strings.Contains(reason, "task_success 0.20 is below 0.50") {
+ t.Fatalf("evaluator-supplied score bypassed the gate: %v, %q", ok, reason)
+ }
+
+ passing := evaluateWithJevBody(t, `{"answers":{
+ "task_success":{"type":"noul","noul":0.9},
+ "reusable_correction":{"type":"noul","noul":0.8},
+ "evidence_supported":{"type":"noul","noul":0.7}}}`)
+ if _, ok := CandidateFromEvaluation(passing); !ok {
+ t.Fatalf("successful Jev-scored trace was not promoted (score %.4f)", passing.Score)
+ }
+}
diff --git a/docs/cli/memory.mdx b/docs/cli/memory.mdx
index c4907c97..892ddbc6 100644
--- a/docs/cli/memory.mdx
+++ b/docs/cli/memory.mdx
@@ -63,6 +63,17 @@ Useful flags:
`--jev-endpoint` can also come from `BEACON_JEV_ENDPOINT`, and the model can come
from `BEACON_JEV_MODEL`.
+Each completed evaluation stores a `score`, the plain mean of the three rubric
+probabilities. A trace becomes a review candidate only when both hold:
+
+- `task_success` is at least `0.50`. This is a precondition, so a trace the
+ evaluator judged as failed is never promoted, however high the other two
+ answers are.
+- `score` is at least `0.60`.
+
+Text output lists each completed evaluation that was not promoted with the
+reason, for example `Not promoted: eval_... (task_success 0.27 is below 0.50)`.
+
List and inspect stored evaluations:
```bash
diff --git a/docs/concepts/cross-harness-memory.mdx b/docs/concepts/cross-harness-memory.mdx
index 31bdc75b..5961e352 100644
--- a/docs/concepts/cross-harness-memory.mdx
+++ b/docs/concepts/cross-harness-memory.mdx
@@ -48,6 +48,11 @@ presenting the scores as a lesson. Review the source trace before approving it.
A compatible internal evaluator using the legacy `questions` or `results`
response shape can also provide a stored `reason`.
+A trace becomes a candidate only when the task succeeded (`task_success` of at
+least 0.50) and the mean of the three probabilities is at least 0.60. The
+success answer is a precondition rather than one vote in the mean, so a failed
+task cannot be promoted on the strength of the other two answers.
+
Use `--dry-run` first to preview selected traces and estimated calls without making
a network request.