Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(memory): align Jev reasons with documented Noul contract

Justin D'Souza7b48dff36f9d
diff --git a/cli/beacon/internal/learning/candidate.go b/cli/beacon/internal/learning/candidate.go
index f1fb16f2..a3ee0d25 100644
--- a/cli/beacon/internal/learning/candidate.go
+++ b/cli/beacon/internal/learning/candidate.go
@@ -169,9 +169,10 @@ func candidateTitle(eval asymptoteobserve.LearningEvaluationV1, kind string) str
return strings.TrimSpace(strings.ReplaceAll(kind, "_", " ")) + ": " + title
}
-// candidateBody builds the candidate text from the evaluator's per-question
-// rationale. Without any rationale there is no lesson to extract, so the body says
-// so instead of presenting bare scores as guidance (#620).
+// candidateBody uses per-question rationale only when a compatible evaluator
+// supplied it through the legacy questions/results response shapes. TypeSafe Noul
+// answers contain probabilities, not rationale, so the normal Jev path explicitly
+// says that no lesson text was extracted instead of presenting scores as guidance.
func candidateBody(eval asymptoteobserve.LearningEvaluationV1) string {
var rationale []string
for _, question := range eval.Questions {
diff --git a/cli/beacon/internal/learning/evaluator.go b/cli/beacon/internal/learning/evaluator.go
index 6a7fa003..0461540a 100644
--- a/cli/beacon/internal/learning/evaluator.go
+++ b/cli/beacon/internal/learning/evaluator.go
@@ -112,11 +112,6 @@ type jevAnswer struct {
Score *float64 `json:"score,omitempty"`
Confidence float64 `json:"confidence,omitempty"`
Probabilities map[string]float64 `json:"probabilities,omitempty"`
- // Rationale for the probability, when the evaluator returns one. The
- // spellings cover compatible evaluators; the first non-empty one wins.
- Reason string `json:"reason,omitempty"`
- Rationale string `json:"rationale,omitempty"`
- Explanation string `json:"explanation,omitempty"`
}
func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput) (asymptoteobserve.LearningEvaluationV1, error) {
@@ -337,8 +332,9 @@ func normalizeQuestionResults(results []asymptoteobserve.LearningEvaluationQuest
if result.Probability > 1 {
result.Probability = 1
}
- // One line, so it stays a single bullet in the candidate body and a
- // single row in `memory evaluations show`.
+ // Compatibility responses can still use the older questions/results
+ // shapes, whose schema includes reason. Keep real values while dropping
+ // the question-type placeholder written by older Beacon releases.
result.Reason = cleanText(strings.Join(strings.Fields(QuestionReason(result)), " "))
out = append(out, result)
}
@@ -384,15 +380,14 @@ func questionsFromJevAnswers(answers map[string]jevAnswer) []asymptoteobserve.Le
Prompt: question.Prompt,
Probability: probability,
Confidence: answer.Confidence,
- Reason: firstNonEmpty(answer.Reason, answer.Rationale, answer.Explanation),
})
}
return out
}
-// QuestionReason returns the evaluator's rationale for one rubric question, or ""
-// when it gave none. Releases before the fix for #620 stored the Jev question
-// type ("noul") as the reason, so that value is treated as absent too.
+// QuestionReason returns a stored reason for one rubric question, or "" when it
+// has none. TypeSafe Noul answers do not include reasons. Releases before the fix
+// for #620 copied the answer type ("noul") into this field, so hide that value.
func QuestionReason(question asymptoteobserve.LearningEvaluationQuestionV1) string {
reason := strings.TrimSpace(question.Reason)
if strings.EqualFold(reason, jevQuestionType) {
diff --git a/cli/beacon/internal/learning/reasons_test.go b/cli/beacon/internal/learning/reasons_test.go
index cbcfeaa3..10d2ea9a 100644
--- a/cli/beacon/internal/learning/reasons_test.go
+++ b/cli/beacon/internal/learning/reasons_test.go
@@ -43,11 +43,11 @@ func questionByID(t *testing.T, eval asymptoteobserve.LearningEvaluationV1, id s
func TestEvaluateDoesNotReportAnswerTypeAsReason(t *testing.T) {
eval := evaluateWithJevBody(t, `{"answers":{
"task_success":{"type":"noul","noul":0.43},
- "reusable_correction":{"type":"noul","noul":0.78},
- "evidence_supported":{"type":"noul","noul":0.49}}}`)
+ "reusable_correction":{"type":"noul","noul":0.78,"reason":"undocumented"},
+ "evidence_supported":{"type":"noul","noul":0.49,"explanation":"undocumented"}}}`)
for _, question := range eval.Questions {
if question.Reason != "" {
- t.Fatalf("question %s reason = %q, want empty when Jev returns no rationale", question.ID, question.Reason)
+ t.Fatalf("question %s reason = %q, want empty for a TypeSafe Noul answer", question.ID, question.Reason)
}
}
if got := questionByID(t, eval, "reusable_correction").Probability; got != 0.78 {
@@ -55,36 +55,6 @@ func TestEvaluateDoesNotReportAnswerTypeAsReason(t *testing.T) {
}
}
-func TestEvaluateKeepsJevAnswerReason(t *testing.T) {
- eval := evaluateWithJevBody(t, `{"answers":{
- "task_success":{"type":"noul","noul":0.9,"reason":" The final test run\n\tpassed after the retry. "},
- "reusable_correction":{"type":"noul","noul":0.8,"rationale":"Retrying the smoke once isolates the flaky network step."},
- "evidence_supported":{"type":"noul","noul":0.7,"explanation":"Event 1 shows the failure and the passing rerun."}}}`)
- cases := map[string]string{
- "task_success": "The final test run passed after the retry.",
- "reusable_correction": "Retrying the smoke once isolates the flaky network step.",
- "evidence_supported": "Event 1 shows the failure and the passing rerun.",
- }
- for id, want := range cases {
- if got := questionByID(t, eval, id).Reason; got != want {
- t.Fatalf("question %s reason = %q, want %q", id, got, want)
- }
- }
-}
-
-func TestEvaluateRedactsAndBoundsJevReason(t *testing.T) {
- long := strings.Repeat("b", maxProjectionText*2)
- eval := evaluateWithJevBody(t, `{"answers":{
- "task_success":{"type":"noul","noul":0.9,"reason":"token=supersecretvalue `+long+`"}}}`)
- reason := questionByID(t, eval, "task_success").Reason
- if strings.Contains(reason, "supersecretvalue") {
- t.Fatalf("reason leaked secret: %s", reason)
- }
- if !strings.Contains(reason, "bbbbbbbb") || len(reason) > maxProjectionText+32 {
- t.Fatalf("reason not bounded: %d bytes", len(reason))
- }
-}
-
func TestEvaluateDropsPlaceholderReasonFromLegacyShape(t *testing.T) {
eval := evaluateWithJevBody(t, `{"questions":[
{"id":"task_success","probability":0.9,"reason":"noul"},
@@ -109,7 +79,7 @@ func TestQuestionReasonHidesPlaceholder(t *testing.T) {
}
}
-func TestCandidateBodyIncludesEvaluatorReasons(t *testing.T) {
+func TestCandidateBodyIncludesCompatibilityReasons(t *testing.T) {
eval := testLearningEvaluation()
eval.Questions[1].Reason = "Retrying the smoke once isolates the flaky network step."
eval.Questions[2].Reason = "Event 1 shows the failure and the passing rerun."
diff --git a/docs/cli/memory.mdx b/docs/cli/memory.mdx
index ec5621e7..c4907c97 100644
--- a/docs/cli/memory.mdx
+++ b/docs/cli/memory.mdx
@@ -91,9 +91,10 @@ Inspect one candidate:
beacon memory candidates show <candidate-id>
```
-The candidate body lists the evaluator's rationale for each rubric question when
-Jev returns one. If it returned scores only, the body says no lesson text was
-extracted; check the source trace before you approve it.
+TypeSafe Noul answers contain probabilities, not explanatory text. The candidate
+body therefore says that no lesson text was extracted; check the source trace
+before you approve it. A compatible internal evaluator using the legacy
+`questions` or `results` response shape can provide a stored `reason`.
Approve a candidate into project memory:
diff --git a/docs/concepts/cross-harness-memory.mdx b/docs/concepts/cross-harness-memory.mdx
index 45d11868..31bdc75b 100644
--- a/docs/concepts/cross-harness-memory.mdx
+++ b/docs/concepts/cross-harness-memory.mdx
@@ -39,15 +39,14 @@ bounded, redacted trace projection. Beacon sends typed yes/no questions such as:
- Does the trace contain a reusable correction or debugging pattern?
- Is the reusable lesson supported by concrete trace evidence?
-Jev returns probabilities. Beacon stores those probabilities with the trace
-provenance, evaluator model, and usage metadata when the API reports it. When the
-evaluator also returns a rationale for a question, Beacon stores it, redacted and
-size-limited, as that question's `reason`.
-
-A candidate's body is built from those rationales. When the evaluator returns
-scores with no rationale, the candidate says that no lesson text was extracted
-instead of presenting the scores as a lesson, so review the source trace before
-approving it.
+Jev returns probabilities. TypeSafe System One models do not generate
+explanations or rationale. Beacon stores those probabilities with the trace
+provenance, evaluator model, and usage metadata when the API reports it.
+
+The candidate therefore says that no lesson text was extracted instead of
+presenting the scores as a lesson. Review the source trace before approving it.
+A compatible internal evaluator using the legacy `questions` or `results`
+response shape can also provide a stored `reason`.
Use `--dry-run` first to preview selected traces and estimated calls without making
a network request.