The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
package learning
import (
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
// evaluateWithJevBody runs Evaluate against a local stub that answers with body.
func evaluateWithJevBody(t *testing.T, body string) asymptoteobserve.LearningEvaluationV1 {
t.Helper()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(body))
}))
defer server.Close()
eval, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL}, EvaluationInput{
Project: asymptoteobserve.LearningProjectV1{ID: "project-1"},
Trace: testTraceShow(),
})
if err != nil {
t.Fatalf("Evaluate returned error: %v", err)
}
return eval
}
func questionByID(t *testing.T, eval asymptoteobserve.LearningEvaluationV1, id string) asymptoteobserve.LearningEvaluationQuestionV1 {
t.Helper()
for _, question := range eval.Questions {
if question.ID == id {
return question
}
}
t.Fatalf("question %s missing from %#v", id, eval.Questions)
return asymptoteobserve.LearningEvaluationQuestionV1{}
}
// Issue #620: the answer's question type ("noul") was stored as the reason.
func TestEvaluateDoesNotReportAnswerTypeAsReason(t *testing.T) {
eval := evaluateWithJevBody(t, `{"answers":{
"task_success":{"type":"noul","noul":0.43},
"reusable_correction":{"type":"noul","noul":0.78,"reason":"undocumented"},
"evidence_supported":{"type":"noul","noul":0.49,"explanation":"undocumented"}}}`)
for _, question := range eval.Questions {
if question.Reason != "" {
t.Fatalf("question %s reason = %q, want empty for a TypeSafe Noul answer", question.ID, question.Reason)
}
}
if got := questionByID(t, eval, "reusable_correction").Probability; got != 0.78 {
t.Fatalf("reusable_correction probability = %v, want 0.78", got)
}
}
func TestEvaluateDropsPlaceholderReasonFromLegacyShape(t *testing.T) {
eval := evaluateWithJevBody(t, `{"questions":[
{"id":"task_success","probability":0.9,"reason":"noul"},
{"id":"reusable_correction","probability":0.8,"reason":"Pin the retry to the network step."},
{"id":"evidence_supported","probability":0.7}]}`)
if got := questionByID(t, eval, "task_success").Reason; got != "" {
t.Fatalf("placeholder reason kept: %q", got)
}
if got := questionByID(t, eval, "reusable_correction").Reason; got != "Pin the retry to the network step." {
t.Fatalf("real reason = %q", got)
}
}
func TestQuestionReasonHidesPlaceholder(t *testing.T) {
for _, value := range []string{"", " ", "noul", " NOUL "} {
if got := QuestionReason(asymptoteobserve.LearningEvaluationQuestionV1{Reason: value}); got != "" {
t.Fatalf("QuestionReason(%q) = %q, want empty", value, got)
}
}
if got := QuestionReason(asymptoteobserve.LearningEvaluationQuestionV1{Reason: " real reason "}); got != "real reason" {
t.Fatalf("QuestionReason trimmed = %q", got)
}
}
func TestCandidateBodyIncludesCompatibilityReasons(t *testing.T) {
eval := testLearningEvaluation()
eval.Questions[1].Reason = "Retrying the smoke once isolates the flaky network step."
eval.Questions[2].Reason = "Event 1 shows the failure and the passing rerun."
candidate, ok := CandidateFromEvaluation(eval)
if !ok {
t.Fatal("candidate not created")
}
for _, want := range []string{
"Reusable lesson extracted from a reviewed Beacon trace.",
"Evaluator rationale:",
"- reusable_correction: Retrying the smoke once isolates the flaky network step.",
"- evidence_supported: Event 1 shows the failure and the passing rerun.",
"- task_success: 0.95",
} {
if !strings.Contains(candidate.Body, want) {
t.Fatalf("candidate body missing %q:\n%s", want, candidate.Body)
}
}
if n := strings.Count(candidate.Body, "- task_success:"); n != 1 {
t.Fatalf("task_success has no rationale and should appear once (signals only), got %d:\n%s", n, candidate.Body)
}
// The rationale reaches an installed skill through approval.
store := Open(filepath.Join(t.TempDir(), "memory.db"))
if err := store.PutCandidate(candidate); err != nil {
t.Fatal(err)
}
approved, memory, err := ApproveCandidate(store, candidate.ID, "reviewed")
if err != nil {
t.Fatal(err)
}
skill := RenderSkill(approved, memory)
if !strings.Contains(skill, "Retrying the smoke once isolates the flaky network step.") {
t.Fatalf("skill missing rationale:\n%s", skill)
}
}
func TestCandidateBodyWithoutReasonsSaysNoLessonWasExtracted(t *testing.T) {
eval := testLearningEvaluation()
eval.Questions[0].Reason = "noul" // stored by earlier releases
candidate, ok := CandidateFromEvaluation(eval)
if !ok {
t.Fatal("candidate not created")
}
if strings.Contains(candidate.Body, "Reusable lesson extracted") {
t.Fatalf("body presents scores as an extracted lesson:\n%s", candidate.Body)
}
if strings.Contains(candidate.Body, "noul") || strings.Contains(candidate.Body, "Evaluator rationale:") {
t.Fatalf("body renders a placeholder rationale:\n%s", candidate.Body)
}
for _, want := range []string{
"no rationale",
"Review the source trace before approving.",
"Trace: trace-1",
"- task_success: 0.95",
"- reusable_correction: 0.90",
"- evidence_supported: 0.85",
} {
if !strings.Contains(candidate.Body, want) {
t.Fatalf("candidate body missing %q:\n%s", want, candidate.Body)
}
}
}