The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(memory): stop storing the Jev question type as each rubric reason
Claude9b450d956b00
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.goindex 2e16412d..fb4c77ce 100644--- a/cli/beacon/cmd/memory.go+++ b/cli/beacon/cmd/memory.go@@ -299,7 +299,7 @@ func runMemoryEvaluationsShow(cmd *cobra.Command, args []string) error {}fmt.Fprintf(cmd.OutOrStdout(), "%s\t%s\t%s\t%.2f\n", eval.ID, eval.Trace.ID, eval.Status, eval.Score)for _, question := range eval.Questions {- fmt.Fprintf(cmd.OutOrStdout(), "%s\t%.2f\t%.2f\t%s\n", question.ID, question.Probability, question.Confidence, question.Reason)+ fmt.Fprintf(cmd.OutOrStdout(), "%s\t%.2f\t%.2f\t%s\n", question.ID, question.Probability, question.Confidence, learning.QuestionReason(question))}return nil}diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.goindex 253875fe..f5fb9786 100644--- a/cli/beacon/cmd/memory_test.go+++ b/cli/beacon/cmd/memory_test.go@@ -297,3 +297,40 @@ func testApprovedCommandMemory() (asymptoteobserve.LearningCandidateV1, asymptot}return candidate, memory}++// Issue #620: evaluations stored by earlier releases carry reason "noul", the Jev+// question type, not a rationale. The text view must not present it as one.+func TestMemoryEvaluationsShowHidesPlaceholderReason(t *testing.T) {+ logPath, project := writeMemoryCommandFixture(t)+ resetMemoryOpts(t)+ memoryOpts.userMode = true+ memoryOpts.logPath = logPath+ memoryOpts.projectPath = project+ eval := asymptoteobserve.LearningEvaluationV1{+ SchemaVersion: asymptoteobserve.LearningSchemaVersion,+ ID: "eval-placeholder",+ Status: asymptoteobserve.LearningEvaluationStatusCompleted,+ Score: 0.7,+ Project: asymptoteobserve.LearningProjectV1{ID: "project-1"},+ Trace: asymptoteobserve.LearningTraceRefV1{ID: "trace-placeholder"},+ Questions: []asymptoteobserve.LearningEvaluationQuestionV1{+ {ID: "task_success", Probability: 0.43, Reason: "noul"},+ {ID: "reusable_correction", Probability: 0.78, Reason: "Retry isolates the flaky step."},+ },+ }+ if err := memoryStore().PutEvaluation(eval); err != nil {+ t.Fatal(err)+ }+ var out bytes.Buffer+ cmd := &cobra.Command{}+ cmd.SetOut(&out)+ if err := runMemoryEvaluationsShow(cmd, []string{eval.ID}); err != nil {+ t.Fatalf("runMemoryEvaluationsShow returned error: %v", err)+ }+ if strings.Contains(out.String(), "noul") {+ t.Fatalf("show printed the placeholder reason:\n%s", out.String())+ }+ if !strings.Contains(out.String(), "Retry isolates the flaky step.") {+ t.Fatalf("show dropped a real reason:\n%s", out.String())+ }+}diff --git a/cli/beacon/internal/learning/candidate.go b/cli/beacon/internal/learning/candidate.goindex e13ca16e..f1fb16f2 100644--- a/cli/beacon/internal/learning/candidate.go+++ b/cli/beacon/internal/learning/candidate.go@@ -169,9 +169,27 @@ func candidateTitle(eval asymptoteobserve.LearningEvaluationV1, kind string) strreturn strings.TrimSpace(strings.ReplaceAll(kind, "_", " ")) + ": " + title}+// candidateBody builds the candidate text from the evaluator's per-question+// rationale. Without any rationale there is no lesson to extract, so the body says+// so instead of presenting bare scores as guidance (#620).func candidateBody(eval asymptoteobserve.LearningEvaluationV1) string {+ var rationale []string+ for _, question := range eval.Questions {+ if reason := QuestionReason(question); reason != "" {+ rationale = append(rationale, fmt.Sprintf("- %s: %s", question.ID, reason))+ }+ }var lines []string- lines = append(lines, "Reusable lesson extracted from a reviewed Beacon trace.")+ if len(rationale) > 0 {+ lines = append(lines, "Reusable lesson extracted from a reviewed Beacon trace.")+ lines = append(lines, "")+ lines = append(lines, "Evaluator rationale:")+ lines = append(lines, rationale...)+ } else {+ lines = append(lines, "Beacon trace flagged for review by evaluation scores.")+ lines = append(lines, "")+ lines = append(lines, "The evaluator returned scores with no rationale, so no lesson text was extracted. Review the source trace before approving.")+ }lines = append(lines, "")lines = append(lines, "Trace: "+eval.Trace.ID)if eval.Trace.Harness.Name != "" {diff --git a/cli/beacon/internal/learning/evaluator.go b/cli/beacon/internal/learning/evaluator.goindex dc517d0d..6a7fa003 100644--- a/cli/beacon/internal/learning/evaluator.go+++ b/cli/beacon/internal/learning/evaluator.go@@ -22,6 +22,10 @@ const (DefaultCostPerTrace = 0.00035maxProjectionEvents = 80maxProjectionText = 1200++ // jevQuestionType is the System One question type Beacon asks for. Jev echoes+ // it back as each answer's "type", which is not a rationale.+ jevQuestionType = "noul")var RubricQuestions = []asymptoteobserve.LearningEvaluationQuestionV1{@@ -108,6 +112,11 @@ type jevAnswer struct {Score *float64 `json:"score,omitempty"`Confidence float64 `json:"confidence,omitempty"`Probabilities map[string]float64 `json:"probabilities,omitempty"`+ // Rationale for the probability, when the evaluator returns one. The+ // spellings cover compatible evaluators; the first non-empty one wins.+ Reason string `json:"reason,omitempty"`+ Rationale string `json:"rationale,omitempty"`+ Explanation string `json:"explanation,omitempty"`}func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput) (asymptoteobserve.LearningEvaluationV1, error) {@@ -328,6 +337,9 @@ func normalizeQuestionResults(results []asymptoteobserve.LearningEvaluationQuestif result.Probability > 1 {result.Probability = 1}+ // One line, so it stays a single bullet in the candidate body and a+ // single row in `memory evaluations show`.+ result.Reason = cleanText(strings.Join(strings.Fields(QuestionReason(result)), " "))out = append(out, result)}return out@@ -337,7 +349,7 @@ func jevQuestions() map[string]jevQuestion {out := make(map[string]jevQuestion, len(RubricQuestions))for _, question := range RubricQuestions {out[question.ID] = jevQuestion{- Type: "noul",+ Type: jevQuestionType,Instructions: question.Prompt,Criteria: map[string]string{"true": "The trace satisfies this criterion.",@@ -372,12 +384,23 @@ func questionsFromJevAnswers(answers map[string]jevAnswer) []asymptoteobserve.LePrompt: question.Prompt,Probability: probability,Confidence: answer.Confidence,- Reason: answer.Type,+ Reason: firstNonEmpty(answer.Reason, answer.Rationale, answer.Explanation),})}return out}+// QuestionReason returns the evaluator's rationale for one rubric question, or ""+// when it gave none. Releases before the fix for #620 stored the Jev question+// type ("noul") as the reason, so that value is treated as absent too.+func QuestionReason(question asymptoteobserve.LearningEvaluationQuestionV1) string {+ reason := strings.TrimSpace(question.Reason)+ if strings.EqualFold(reason, jevQuestionType) {+ return ""+ }+ return reason+}+func projectionQuestionDefaults(projection Projection) []asymptoteobserve.LearningEvaluationQuestionV1 {out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(projection.Questions))for _, question := range projection.Questions {diff --git a/cli/beacon/internal/learning/reasons_test.go b/cli/beacon/internal/learning/reasons_test.gonew file mode 100644index 00000000..cbcfeaa3--- /dev/null+++ b/cli/beacon/internal/learning/reasons_test.go@@ -0,0 +1,174 @@+package learning++import (+ "net/http"+ "net/http/httptest"+ "path/filepath"+ "strings"+ "testing"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// evaluateWithJevBody runs Evaluate against a local stub that answers with body.+func evaluateWithJevBody(t *testing.T, body string) asymptoteobserve.LearningEvaluationV1 {+ t.Helper()+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.Header().Set("Content-Type", "application/json")+ _, _ = w.Write([]byte(body))+ }))+ defer server.Close()+ eval, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL}, EvaluationInput{+ Project: asymptoteobserve.LearningProjectV1{ID: "project-1"},+ Trace: testTraceShow(),+ })+ if err != nil {+ t.Fatalf("Evaluate returned error: %v", err)+ }+ return eval+}++func questionByID(t *testing.T, eval asymptoteobserve.LearningEvaluationV1, id string) asymptoteobserve.LearningEvaluationQuestionV1 {+ t.Helper()+ for _, question := range eval.Questions {+ if question.ID == id {+ return question+ }+ }+ t.Fatalf("question %s missing from %#v", id, eval.Questions)+ return asymptoteobserve.LearningEvaluationQuestionV1{}+}++// Issue #620: the answer's question type ("noul") was stored as the reason.+func TestEvaluateDoesNotReportAnswerTypeAsReason(t *testing.T) {+ eval := evaluateWithJevBody(t, `{"answers":{+ "task_success":{"type":"noul","noul":0.43},+ "reusable_correction":{"type":"noul","noul":0.78},+ "evidence_supported":{"type":"noul","noul":0.49}}}`)+ for _, question := range eval.Questions {+ if question.Reason != "" {+ t.Fatalf("question %s reason = %q, want empty when Jev returns no rationale", question.ID, question.Reason)+ }+ }+ if got := questionByID(t, eval, "reusable_correction").Probability; got != 0.78 {+ t.Fatalf("reusable_correction probability = %v, want 0.78", got)+ }+}++func TestEvaluateKeepsJevAnswerReason(t *testing.T) {+ eval := evaluateWithJevBody(t, `{"answers":{+ "task_success":{"type":"noul","noul":0.9,"reason":" The final test run\n\tpassed after the retry. "},+ "reusable_correction":{"type":"noul","noul":0.8,"rationale":"Retrying the smoke once isolates the flaky network step."},+ "evidence_supported":{"type":"noul","noul":0.7,"explanation":"Event 1 shows the failure and the passing rerun."}}}`)+ cases := map[string]string{+ "task_success": "The final test run passed after the retry.",+ "reusable_correction": "Retrying the smoke once isolates the flaky network step.",+ "evidence_supported": "Event 1 shows the failure and the passing rerun.",+ }+ for id, want := range cases {+ if got := questionByID(t, eval, id).Reason; got != want {+ t.Fatalf("question %s reason = %q, want %q", id, got, want)+ }+ }+}++func TestEvaluateRedactsAndBoundsJevReason(t *testing.T) {+ long := strings.Repeat("b", maxProjectionText*2)+ eval := evaluateWithJevBody(t, `{"answers":{+ "task_success":{"type":"noul","noul":0.9,"reason":"token=supersecretvalue `+long+`"}}}`)+ reason := questionByID(t, eval, "task_success").Reason+ if strings.Contains(reason, "supersecretvalue") {+ t.Fatalf("reason leaked secret: %s", reason)+ }+ if !strings.Contains(reason, "bbbbbbbb") || len(reason) > maxProjectionText+32 {+ t.Fatalf("reason not bounded: %d bytes", len(reason))+ }+}++func TestEvaluateDropsPlaceholderReasonFromLegacyShape(t *testing.T) {+ eval := evaluateWithJevBody(t, `{"questions":[+ {"id":"task_success","probability":0.9,"reason":"noul"},+ {"id":"reusable_correction","probability":0.8,"reason":"Pin the retry to the network step."},+ {"id":"evidence_supported","probability":0.7}]}`)+ if got := questionByID(t, eval, "task_success").Reason; got != "" {+ t.Fatalf("placeholder reason kept: %q", got)+ }+ if got := questionByID(t, eval, "reusable_correction").Reason; got != "Pin the retry to the network step." {+ t.Fatalf("real reason = %q", got)+ }+}++func TestQuestionReasonHidesPlaceholder(t *testing.T) {+ for _, value := range []string{"", " ", "noul", " NOUL "} {+ if got := QuestionReason(asymptoteobserve.LearningEvaluationQuestionV1{Reason: value}); got != "" {+ t.Fatalf("QuestionReason(%q) = %q, want empty", value, got)+ }+ }+ if got := QuestionReason(asymptoteobserve.LearningEvaluationQuestionV1{Reason: " real reason "}); got != "real reason" {+ t.Fatalf("QuestionReason trimmed = %q", got)+ }+}++func TestCandidateBodyIncludesEvaluatorReasons(t *testing.T) {+ eval := testLearningEvaluation()+ eval.Questions[1].Reason = "Retrying the smoke once isolates the flaky network step."+ eval.Questions[2].Reason = "Event 1 shows the failure and the passing rerun."+ candidate, ok := CandidateFromEvaluation(eval)+ if !ok {+ t.Fatal("candidate not created")+ }+ for _, want := range []string{+ "Reusable lesson extracted from a reviewed Beacon trace.",+ "Evaluator rationale:",+ "- reusable_correction: Retrying the smoke once isolates the flaky network step.",+ "- evidence_supported: Event 1 shows the failure and the passing rerun.",+ "- task_success: 0.95",+ } {+ if !strings.Contains(candidate.Body, want) {+ t.Fatalf("candidate body missing %q:\n%s", want, candidate.Body)+ }+ }+ if n := strings.Count(candidate.Body, "- task_success:"); n != 1 {+ t.Fatalf("task_success has no rationale and should appear once (signals only), got %d:\n%s", n, candidate.Body)+ }+ // The rationale reaches an installed skill through approval.+ store := Open(filepath.Join(t.TempDir(), "memory.db"))+ if err := store.PutCandidate(candidate); err != nil {+ t.Fatal(err)+ }+ approved, memory, err := ApproveCandidate(store, candidate.ID, "reviewed")+ if err != nil {+ t.Fatal(err)+ }+ skill := RenderSkill(approved, memory)+ if !strings.Contains(skill, "Retrying the smoke once isolates the flaky network step.") {+ t.Fatalf("skill missing rationale:\n%s", skill)+ }+}++func TestCandidateBodyWithoutReasonsSaysNoLessonWasExtracted(t *testing.T) {+ eval := testLearningEvaluation()+ eval.Questions[0].Reason = "noul" // stored by earlier releases+ candidate, ok := CandidateFromEvaluation(eval)+ if !ok {+ t.Fatal("candidate not created")+ }+ if strings.Contains(candidate.Body, "Reusable lesson extracted") {+ t.Fatalf("body presents scores as an extracted lesson:\n%s", candidate.Body)+ }+ if strings.Contains(candidate.Body, "noul") || strings.Contains(candidate.Body, "Evaluator rationale:") {+ t.Fatalf("body renders a placeholder rationale:\n%s", candidate.Body)+ }+ for _, want := range []string{+ "no rationale",+ "Review the source trace before approving.",+ "Trace: trace-1",+ "- task_success: 0.95",+ "- reusable_correction: 0.90",+ "- evidence_supported: 0.85",+ } {+ if !strings.Contains(candidate.Body, want) {+ t.Fatalf("candidate body missing %q:\n%s", want, candidate.Body)+ }+ }+}diff --git a/docs/cli/memory.mdx b/docs/cli/memory.mdxindex f4645f41..ec5621e7 100644--- a/docs/cli/memory.mdx+++ b/docs/cli/memory.mdx@@ -91,6 +91,10 @@ Inspect one candidate:beacon memory candidates show <candidate-id>```+The candidate body lists the evaluator's rationale for each rubric question when+Jev returns one. If it returned scores only, the body says no lesson text was+extracted; check the source trace before you approve it.+Approve a candidate into project memory:```bashdiff --git a/docs/concepts/cross-harness-memory.mdx b/docs/concepts/cross-harness-memory.mdxindex ab37930b..45d11868 100644--- a/docs/concepts/cross-harness-memory.mdx+++ b/docs/concepts/cross-harness-memory.mdx@@ -40,7 +40,14 @@ bounded, redacted trace projection. Beacon sends typed yes/no questions such as:- Is the reusable lesson supported by concrete trace evidence?Jev returns probabilities. Beacon stores those probabilities with the trace-provenance, evaluator model, and usage metadata when the API reports it.+provenance, evaluator model, and usage metadata when the API reports it. When the+evaluator also returns a rationale for a question, Beacon stores it, redacted and+size-limited, as that question's `reason`.++A candidate's body is built from those rationales. When the evaluator returns+scores with no rationale, the candidate says that no lesson text was extracted+instead of presenting the scores as a lesson, so review the source trace before+approving it.Use `--dry-run` first to preview selected traces and estimated calls without makinga network request.