The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(memory): align Jev reasons with documented Noul contract
Justin D'Souza7b48dff36f9d
diff --git a/cli/beacon/internal/learning/candidate.go b/cli/beacon/internal/learning/candidate.goindex f1fb16f2..a3ee0d25 100644--- a/cli/beacon/internal/learning/candidate.go+++ b/cli/beacon/internal/learning/candidate.go@@ -169,9 +169,10 @@ func candidateTitle(eval asymptoteobserve.LearningEvaluationV1, kind string) strreturn strings.TrimSpace(strings.ReplaceAll(kind, "_", " ")) + ": " + title}-// candidateBody builds the candidate text from the evaluator's per-question-// rationale. Without any rationale there is no lesson to extract, so the body says-// so instead of presenting bare scores as guidance (#620).+// candidateBody uses per-question rationale only when a compatible evaluator+// supplied it through the legacy questions/results response shapes. TypeSafe Noul+// answers contain probabilities, not rationale, so the normal Jev path explicitly+// says that no lesson text was extracted instead of presenting scores as guidance.func candidateBody(eval asymptoteobserve.LearningEvaluationV1) string {var rationale []stringfor _, question := range eval.Questions {diff --git a/cli/beacon/internal/learning/evaluator.go b/cli/beacon/internal/learning/evaluator.goindex 6a7fa003..0461540a 100644--- a/cli/beacon/internal/learning/evaluator.go+++ b/cli/beacon/internal/learning/evaluator.go@@ -112,11 +112,6 @@ type jevAnswer struct {Score *float64 `json:"score,omitempty"`Confidence float64 `json:"confidence,omitempty"`Probabilities map[string]float64 `json:"probabilities,omitempty"`- // Rationale for the probability, when the evaluator returns one. The- // spellings cover compatible evaluators; the first non-empty one wins.- Reason string `json:"reason,omitempty"`- Rationale string `json:"rationale,omitempty"`- Explanation string `json:"explanation,omitempty"`}func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput) (asymptoteobserve.LearningEvaluationV1, error) {@@ -337,8 +332,9 @@ func normalizeQuestionResults(results []asymptoteobserve.LearningEvaluationQuestif result.Probability > 1 {result.Probability = 1}- // One line, so it stays a single bullet in the candidate body and a- // single row in `memory evaluations show`.+ // Compatibility responses can still use the older questions/results+ // shapes, whose schema includes reason. Keep real values while dropping+ // the question-type placeholder written by older Beacon releases.result.Reason = cleanText(strings.Join(strings.Fields(QuestionReason(result)), " "))out = append(out, result)}@@ -384,15 +380,14 @@ func questionsFromJevAnswers(answers map[string]jevAnswer) []asymptoteobserve.LePrompt: question.Prompt,Probability: probability,Confidence: answer.Confidence,- Reason: firstNonEmpty(answer.Reason, answer.Rationale, answer.Explanation),})}return out}-// QuestionReason returns the evaluator's rationale for one rubric question, or ""-// when it gave none. Releases before the fix for #620 stored the Jev question-// type ("noul") as the reason, so that value is treated as absent too.+// QuestionReason returns a stored reason for one rubric question, or "" when it+// has none. TypeSafe Noul answers do not include reasons. Releases before the fix+// for #620 copied the answer type ("noul") into this field, so hide that value.func QuestionReason(question asymptoteobserve.LearningEvaluationQuestionV1) string {reason := strings.TrimSpace(question.Reason)if strings.EqualFold(reason, jevQuestionType) {diff --git a/cli/beacon/internal/learning/reasons_test.go b/cli/beacon/internal/learning/reasons_test.goindex cbcfeaa3..10d2ea9a 100644--- a/cli/beacon/internal/learning/reasons_test.go+++ b/cli/beacon/internal/learning/reasons_test.go@@ -43,11 +43,11 @@ func questionByID(t *testing.T, eval asymptoteobserve.LearningEvaluationV1, id sfunc TestEvaluateDoesNotReportAnswerTypeAsReason(t *testing.T) {eval := evaluateWithJevBody(t, `{"answers":{"task_success":{"type":"noul","noul":0.43},- "reusable_correction":{"type":"noul","noul":0.78},- "evidence_supported":{"type":"noul","noul":0.49}}}`)+ "reusable_correction":{"type":"noul","noul":0.78,"reason":"undocumented"},+ "evidence_supported":{"type":"noul","noul":0.49,"explanation":"undocumented"}}}`)for _, question := range eval.Questions {if question.Reason != "" {- t.Fatalf("question %s reason = %q, want empty when Jev returns no rationale", question.ID, question.Reason)+ t.Fatalf("question %s reason = %q, want empty for a TypeSafe Noul answer", question.ID, question.Reason)}}if got := questionByID(t, eval, "reusable_correction").Probability; got != 0.78 {@@ -55,36 +55,6 @@ func TestEvaluateDoesNotReportAnswerTypeAsReason(t *testing.T) {}}-func TestEvaluateKeepsJevAnswerReason(t *testing.T) {- eval := evaluateWithJevBody(t, `{"answers":{- "task_success":{"type":"noul","noul":0.9,"reason":" The final test run\n\tpassed after the retry. "},- "reusable_correction":{"type":"noul","noul":0.8,"rationale":"Retrying the smoke once isolates the flaky network step."},- "evidence_supported":{"type":"noul","noul":0.7,"explanation":"Event 1 shows the failure and the passing rerun."}}}`)- cases := map[string]string{- "task_success": "The final test run passed after the retry.",- "reusable_correction": "Retrying the smoke once isolates the flaky network step.",- "evidence_supported": "Event 1 shows the failure and the passing rerun.",- }- for id, want := range cases {- if got := questionByID(t, eval, id).Reason; got != want {- t.Fatalf("question %s reason = %q, want %q", id, got, want)- }- }-}--func TestEvaluateRedactsAndBoundsJevReason(t *testing.T) {- long := strings.Repeat("b", maxProjectionText*2)- eval := evaluateWithJevBody(t, `{"answers":{- "task_success":{"type":"noul","noul":0.9,"reason":"token=supersecretvalue `+long+`"}}}`)- reason := questionByID(t, eval, "task_success").Reason- if strings.Contains(reason, "supersecretvalue") {- t.Fatalf("reason leaked secret: %s", reason)- }- if !strings.Contains(reason, "bbbbbbbb") || len(reason) > maxProjectionText+32 {- t.Fatalf("reason not bounded: %d bytes", len(reason))- }-}-func TestEvaluateDropsPlaceholderReasonFromLegacyShape(t *testing.T) {eval := evaluateWithJevBody(t, `{"questions":[{"id":"task_success","probability":0.9,"reason":"noul"},@@ -109,7 +79,7 @@ func TestQuestionReasonHidesPlaceholder(t *testing.T) {}}-func TestCandidateBodyIncludesEvaluatorReasons(t *testing.T) {+func TestCandidateBodyIncludesCompatibilityReasons(t *testing.T) {eval := testLearningEvaluation()eval.Questions[1].Reason = "Retrying the smoke once isolates the flaky network step."eval.Questions[2].Reason = "Event 1 shows the failure and the passing rerun."diff --git a/docs/cli/memory.mdx b/docs/cli/memory.mdxindex ec5621e7..c4907c97 100644--- a/docs/cli/memory.mdx+++ b/docs/cli/memory.mdx@@ -91,9 +91,10 @@ Inspect one candidate:beacon memory candidates show <candidate-id>```-The candidate body lists the evaluator's rationale for each rubric question when-Jev returns one. If it returned scores only, the body says no lesson text was-extracted; check the source trace before you approve it.+TypeSafe Noul answers contain probabilities, not explanatory text. The candidate+body therefore says that no lesson text was extracted; check the source trace+before you approve it. A compatible internal evaluator using the legacy+`questions` or `results` response shape can provide a stored `reason`.Approve a candidate into project memory:diff --git a/docs/concepts/cross-harness-memory.mdx b/docs/concepts/cross-harness-memory.mdxindex 45d11868..31bdc75b 100644--- a/docs/concepts/cross-harness-memory.mdx+++ b/docs/concepts/cross-harness-memory.mdx@@ -39,15 +39,14 @@ bounded, redacted trace projection. Beacon sends typed yes/no questions such as:- Does the trace contain a reusable correction or debugging pattern?- Is the reusable lesson supported by concrete trace evidence?-Jev returns probabilities. Beacon stores those probabilities with the trace-provenance, evaluator model, and usage metadata when the API reports it. When the-evaluator also returns a rationale for a question, Beacon stores it, redacted and-size-limited, as that question's `reason`.--A candidate's body is built from those rationales. When the evaluator returns-scores with no rationale, the candidate says that no lesson text was extracted-instead of presenting the scores as a lesson, so review the source trace before-approving it.+Jev returns probabilities. TypeSafe System One models do not generate+explanations or rationale. Beacon stores those probabilities with the trace+provenance, evaluator model, and usage metadata when the API reports it.++The candidate therefore says that no lesson text was extracted instead of+presenting the scores as a lesson. Review the source trace before approving it.+A compatible internal evaluator using the legacy `questions` or `results`+response shape can also provide a stored `reason`.Use `--dry-run` first to preview selected traces and estimated calls without makinga network request.