Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(memory): stop storing the Jev question type as each rubric reason

Claude9b450d956b00
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.go
index 2e16412d..fb4c77ce 100644
--- a/cli/beacon/cmd/memory.go
+++ b/cli/beacon/cmd/memory.go
@@ -299,7 +299,7 @@ func runMemoryEvaluationsShow(cmd *cobra.Command, args []string) error {
}
fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%s\t%.2f\n", eval.ID, eval.Trace.ID, eval.Status, eval.Score)
for _, question := range eval.Questions {
- fmt.Fprintf(cmd.OutOrStdout(), "%s\t%.2f\t%.2f\t%s\n", question.ID, question.Probability, question.Confidence, question.Reason)
+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%.2f\t%.2f\t%s\n", question.ID, question.Probability, question.Confidence, learning.QuestionReason(question))
}
return nil
}
diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.go
index 253875fe..f5fb9786 100644
--- a/cli/beacon/cmd/memory_test.go
+++ b/cli/beacon/cmd/memory_test.go
@@ -297,3 +297,40 @@ func testApprovedCommandMemory() (asymptoteobserve.LearningCandidateV1, asymptot
}
return candidate, memory
}
+
+// Issue #620: evaluations stored by earlier releases carry reason "noul", the Jev
+// question type, not a rationale. The text view must not present it as one.
+func TestMemoryEvaluationsShowHidesPlaceholderReason(t *testing.T) {
+ logPath, project := writeMemoryCommandFixture(t)
+ resetMemoryOpts(t)
+ memoryOpts.userMode = true
+ memoryOpts.logPath = logPath
+ memoryOpts.projectPath = project
+ eval := asymptoteobserve.LearningEvaluationV1{
+ SchemaVersion: asymptoteobserve.LearningSchemaVersion,
+ ID: "eval-placeholder",
+ Status: asymptoteobserve.LearningEvaluationStatusCompleted,
+ Score: 0.7,
+ Project: asymptoteobserve.LearningProjectV1{ID: "project-1"},
+ Trace: asymptoteobserve.LearningTraceRefV1{ID: "trace-placeholder"},
+ Questions: []asymptoteobserve.LearningEvaluationQuestionV1{
+ {ID: "task_success", Probability: 0.43, Reason: "noul"},
+ {ID: "reusable_correction", Probability: 0.78, Reason: "Retry isolates the flaky step."},
+ },
+ }
+ if err := memoryStore().PutEvaluation(eval); err != nil {
+ t.Fatal(err)
+ }
+ var out bytes.Buffer
+ cmd := &cobra.Command{}
+ cmd.SetOut(&out)
+ if err := runMemoryEvaluationsShow(cmd, []string{eval.ID}); err != nil {
+ t.Fatalf("runMemoryEvaluationsShow returned error: %v", err)
+ }
+ if strings.Contains(out.String(), "noul") {
+ t.Fatalf("show printed the placeholder reason:\n%s", out.String())
+ }
+ if !strings.Contains(out.String(), "Retry isolates the flaky step.") {
+ t.Fatalf("show dropped a real reason:\n%s", out.String())
+ }
+}
diff --git a/cli/beacon/internal/learning/candidate.go b/cli/beacon/internal/learning/candidate.go
index e13ca16e..f1fb16f2 100644
--- a/cli/beacon/internal/learning/candidate.go
+++ b/cli/beacon/internal/learning/candidate.go
@@ -169,9 +169,27 @@ func candidateTitle(eval asymptoteobserve.LearningEvaluationV1, kind string) str
return strings.TrimSpace(strings.ReplaceAll(kind, "_", " ")) + ": " + title
}
+// candidateBody builds the candidate text from the evaluator's per-question
+// rationale. Without any rationale there is no lesson to extract, so the body says
+// so instead of presenting bare scores as guidance (#620).
func candidateBody(eval asymptoteobserve.LearningEvaluationV1) string {
+ var rationale []string
+ for _, question := range eval.Questions {
+ if reason := QuestionReason(question); reason != "" {
+ rationale = append(rationale, fmt.Sprintf("- %s: %s", question.ID, reason))
+ }
+ }
var lines []string
- lines = append(lines, "Reusable lesson extracted from a reviewed Beacon trace.")
+ if len(rationale) > 0 {
+ lines = append(lines, "Reusable lesson extracted from a reviewed Beacon trace.")
+ lines = append(lines, "")
+ lines = append(lines, "Evaluator rationale:")
+ lines = append(lines, rationale...)
+ } else {
+ lines = append(lines, "Beacon trace flagged for review by evaluation scores.")
+ lines = append(lines, "")
+ lines = append(lines, "The evaluator returned scores with no rationale, so no lesson text was extracted. Review the source trace before approving.")
+ }
lines = append(lines, "")
lines = append(lines, "Trace: "+eval.Trace.ID)
if eval.Trace.Harness.Name != "" {
diff --git a/cli/beacon/internal/learning/evaluator.go b/cli/beacon/internal/learning/evaluator.go
index dc517d0d..6a7fa003 100644
--- a/cli/beacon/internal/learning/evaluator.go
+++ b/cli/beacon/internal/learning/evaluator.go
@@ -22,6 +22,10 @@ const (
DefaultCostPerTrace = 0.00035
maxProjectionEvents = 80
maxProjectionText = 1200
+
+ // jevQuestionType is the System One question type Beacon asks for. Jev echoes
+ // it back as each answer's "type", which is not a rationale.
+ jevQuestionType = "noul"
)
var RubricQuestions = []asymptoteobserve.LearningEvaluationQuestionV1{
@@ -108,6 +112,11 @@ type jevAnswer struct {
Score *float64 `json:"score,omitempty"`
Confidence float64 `json:"confidence,omitempty"`
Probabilities map[string]float64 `json:"probabilities,omitempty"`
+ // Rationale for the probability, when the evaluator returns one. The
+ // spellings cover compatible evaluators; the first non-empty one wins.
+ Reason string `json:"reason,omitempty"`
+ Rationale string `json:"rationale,omitempty"`
+ Explanation string `json:"explanation,omitempty"`
}
func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput) (asymptoteobserve.LearningEvaluationV1, error) {
@@ -328,6 +337,9 @@ func normalizeQuestionResults(results []asymptoteobserve.LearningEvaluationQuest
if result.Probability > 1 {
result.Probability = 1
}
+ // One line, so it stays a single bullet in the candidate body and a
+ // single row in `memory evaluations show`.
+ result.Reason = cleanText(strings.Join(strings.Fields(QuestionReason(result)), " "))
out = append(out, result)
}
return out
@@ -337,7 +349,7 @@ func jevQuestions() map[string]jevQuestion {
out := make(map[string]jevQuestion, len(RubricQuestions))
for _, question := range RubricQuestions {
out[question.ID] = jevQuestion{
- Type: "noul",
+ Type: jevQuestionType,
Instructions: question.Prompt,
Criteria: map[string]string{
"true": "The trace satisfies this criterion.",
@@ -372,12 +384,23 @@ func questionsFromJevAnswers(answers map[string]jevAnswer) []asymptoteobserve.Le
Prompt: question.Prompt,
Probability: probability,
Confidence: answer.Confidence,
- Reason: answer.Type,
+ Reason: firstNonEmpty(answer.Reason, answer.Rationale, answer.Explanation),
})
}
return out
}
+// QuestionReason returns the evaluator's rationale for one rubric question, or ""
+// when it gave none. Releases before the fix for #620 stored the Jev question
+// type ("noul") as the reason, so that value is treated as absent too.
+func QuestionReason(question asymptoteobserve.LearningEvaluationQuestionV1) string {
+ reason := strings.TrimSpace(question.Reason)
+ if strings.EqualFold(reason, jevQuestionType) {
+ return ""
+ }
+ return reason
+}
+
func projectionQuestionDefaults(projection Projection) []asymptoteobserve.LearningEvaluationQuestionV1 {
out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(projection.Questions))
for _, question := range projection.Questions {
diff --git a/cli/beacon/internal/learning/reasons_test.go b/cli/beacon/internal/learning/reasons_test.go
new file mode 100644
index 00000000..cbcfeaa3
--- /dev/null
+++ b/cli/beacon/internal/learning/reasons_test.go
@@ -0,0 +1,174 @@
+package learning
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+// evaluateWithJevBody runs Evaluate against a local stub that answers with body.
+func evaluateWithJevBody(t *testing.T, body string) asymptoteobserve.LearningEvaluationV1 {
+ t.Helper()
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("Content-Type", "application/json")
+ _, _ = w.Write([]byte(body))
+ }))
+ defer server.Close()
+ eval, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL}, EvaluationInput{
+ Project: asymptoteobserve.LearningProjectV1{ID: "project-1"},
+ Trace: testTraceShow(),
+ })
+ if err != nil {
+ t.Fatalf("Evaluate returned error: %v", err)
+ }
+ return eval
+}
+
+func questionByID(t *testing.T, eval asymptoteobserve.LearningEvaluationV1, id string) asymptoteobserve.LearningEvaluationQuestionV1 {
+ t.Helper()
+ for _, question := range eval.Questions {
+ if question.ID == id {
+ return question
+ }
+ }
+ t.Fatalf("question %s missing from %#v", id, eval.Questions)
+ return asymptoteobserve.LearningEvaluationQuestionV1{}
+}
+
+// Issue #620: the answer's question type ("noul") was stored as the reason.
+func TestEvaluateDoesNotReportAnswerTypeAsReason(t *testing.T) {
+ eval := evaluateWithJevBody(t, `{"answers":{
+ "task_success":{"type":"noul","noul":0.43},
+ "reusable_correction":{"type":"noul","noul":0.78},
+ "evidence_supported":{"type":"noul","noul":0.49}}}`)
+ for _, question := range eval.Questions {
+ if question.Reason != "" {
+ t.Fatalf("question %s reason = %q, want empty when Jev returns no rationale", question.ID, question.Reason)
+ }
+ }
+ if got := questionByID(t, eval, "reusable_correction").Probability; got != 0.78 {
+ t.Fatalf("reusable_correction probability = %v, want 0.78", got)
+ }
+}
+
+func TestEvaluateKeepsJevAnswerReason(t *testing.T) {
+ eval := evaluateWithJevBody(t, `{"answers":{
+ "task_success":{"type":"noul","noul":0.9,"reason":" The final test run\n\tpassed after the retry. "},
+ "reusable_correction":{"type":"noul","noul":0.8,"rationale":"Retrying the smoke once isolates the flaky network step."},
+ "evidence_supported":{"type":"noul","noul":0.7,"explanation":"Event 1 shows the failure and the passing rerun."}}}`)
+ cases := map[string]string{
+ "task_success": "The final test run passed after the retry.",
+ "reusable_correction": "Retrying the smoke once isolates the flaky network step.",
+ "evidence_supported": "Event 1 shows the failure and the passing rerun.",
+ }
+ for id, want := range cases {
+ if got := questionByID(t, eval, id).Reason; got != want {
+ t.Fatalf("question %s reason = %q, want %q", id, got, want)
+ }
+ }
+}
+
+func TestEvaluateRedactsAndBoundsJevReason(t *testing.T) {
+ long := strings.Repeat("b", maxProjectionText*2)
+ eval := evaluateWithJevBody(t, `{"answers":{
+ "task_success":{"type":"noul","noul":0.9,"reason":"token=supersecretvalue `+long+`"}}}`)
+ reason := questionByID(t, eval, "task_success").Reason
+ if strings.Contains(reason, "supersecretvalue") {
+ t.Fatalf("reason leaked secret: %s", reason)
+ }
+ if !strings.Contains(reason, "bbbbbbbb") || len(reason) > maxProjectionText+32 {
+ t.Fatalf("reason not bounded: %d bytes", len(reason))
+ }
+}
+
+func TestEvaluateDropsPlaceholderReasonFromLegacyShape(t *testing.T) {
+ eval := evaluateWithJevBody(t, `{"questions":[
+ {"id":"task_success","probability":0.9,"reason":"noul"},
+ {"id":"reusable_correction","probability":0.8,"reason":"Pin the retry to the network step."},
+ {"id":"evidence_supported","probability":0.7}]}`)
+ if got := questionByID(t, eval, "task_success").Reason; got != "" {
+ t.Fatalf("placeholder reason kept: %q", got)
+ }
+ if got := questionByID(t, eval, "reusable_correction").Reason; got != "Pin the retry to the network step." {
+ t.Fatalf("real reason = %q", got)
+ }
+}
+
+func TestQuestionReasonHidesPlaceholder(t *testing.T) {
+ for _, value := range []string{"", " ", "noul", " NOUL "} {
+ if got := QuestionReason(asymptoteobserve.LearningEvaluationQuestionV1{Reason: value}); got != "" {
+ t.Fatalf("QuestionReason(%q) = %q, want empty", value, got)
+ }
+ }
+ if got := QuestionReason(asymptoteobserve.LearningEvaluationQuestionV1{Reason: " real reason "}); got != "real reason" {
+ t.Fatalf("QuestionReason trimmed = %q", got)
+ }
+}
+
+func TestCandidateBodyIncludesEvaluatorReasons(t *testing.T) {
+ eval := testLearningEvaluation()
+ eval.Questions[1].Reason = "Retrying the smoke once isolates the flaky network step."
+ eval.Questions[2].Reason = "Event 1 shows the failure and the passing rerun."
+ candidate, ok := CandidateFromEvaluation(eval)
+ if !ok {
+ t.Fatal("candidate not created")
+ }
+ for _, want := range []string{
+ "Reusable lesson extracted from a reviewed Beacon trace.",
+ "Evaluator rationale:",
+ "- reusable_correction: Retrying the smoke once isolates the flaky network step.",
+ "- evidence_supported: Event 1 shows the failure and the passing rerun.",
+ "- task_success: 0.95",
+ } {
+ if !strings.Contains(candidate.Body, want) {
+ t.Fatalf("candidate body missing %q:\n%s", want, candidate.Body)
+ }
+ }
+ if n := strings.Count(candidate.Body, "- task_success:"); n != 1 {
+ t.Fatalf("task_success has no rationale and should appear once (signals only), got %d:\n%s", n, candidate.Body)
+ }
+ // The rationale reaches an installed skill through approval.
+ store := Open(filepath.Join(t.TempDir(), "memory.db"))
+ if err := store.PutCandidate(candidate); err != nil {
+ t.Fatal(err)
+ }
+ approved, memory, err := ApproveCandidate(store, candidate.ID, "reviewed")
+ if err != nil {
+ t.Fatal(err)
+ }
+ skill := RenderSkill(approved, memory)
+ if !strings.Contains(skill, "Retrying the smoke once isolates the flaky network step.") {
+ t.Fatalf("skill missing rationale:\n%s", skill)
+ }
+}
+
+func TestCandidateBodyWithoutReasonsSaysNoLessonWasExtracted(t *testing.T) {
+ eval := testLearningEvaluation()
+ eval.Questions[0].Reason = "noul" // stored by earlier releases
+ candidate, ok := CandidateFromEvaluation(eval)
+ if !ok {
+ t.Fatal("candidate not created")
+ }
+ if strings.Contains(candidate.Body, "Reusable lesson extracted") {
+ t.Fatalf("body presents scores as an extracted lesson:\n%s", candidate.Body)
+ }
+ if strings.Contains(candidate.Body, "noul") || strings.Contains(candidate.Body, "Evaluator rationale:") {
+ t.Fatalf("body renders a placeholder rationale:\n%s", candidate.Body)
+ }
+ for _, want := range []string{
+ "no rationale",
+ "Review the source trace before approving.",
+ "Trace: trace-1",
+ "- task_success: 0.95",
+ "- reusable_correction: 0.90",
+ "- evidence_supported: 0.85",
+ } {
+ if !strings.Contains(candidate.Body, want) {
+ t.Fatalf("candidate body missing %q:\n%s", want, candidate.Body)
+ }
+ }
+}
diff --git a/docs/cli/memory.mdx b/docs/cli/memory.mdx
index f4645f41..ec5621e7 100644
--- a/docs/cli/memory.mdx
+++ b/docs/cli/memory.mdx
@@ -91,6 +91,10 @@ Inspect one candidate:
beacon memory candidates show <candidate-id>
```
+The candidate body lists the evaluator's rationale for each rubric question when
+Jev returns one. If it returned scores only, the body says no lesson text was
+extracted; check the source trace before you approve it.
+
Approve a candidate into project memory:
```bash
diff --git a/docs/concepts/cross-harness-memory.mdx b/docs/concepts/cross-harness-memory.mdx
index ab37930b..45d11868 100644
--- a/docs/concepts/cross-harness-memory.mdx
+++ b/docs/concepts/cross-harness-memory.mdx
@@ -40,7 +40,14 @@ bounded, redacted trace projection. Beacon sends typed yes/no questions such as:
- Is the reusable lesson supported by concrete trace evidence?
Jev returns probabilities. Beacon stores those probabilities with the trace
-provenance, evaluator model, and usage metadata when the API reports it.
+provenance, evaluator model, and usage metadata when the API reports it. When the
+evaluator also returns a rationale for a question, Beacon stores it, redacted and
+size-limited, as that question's `reason`.
+
+A candidate's body is built from those rationales. When the evaluator returns
+scores with no rationale, the candidate says that no lesson text was extracted
+instead of presenting the scores as a lesson, so review the source trace before
+approving it.
Use `--dry-run` first to preview selected traces and estimated calls without making
a network request.