Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

5.0 KB
Raw Blame
package learning

import (
	"fmt"
	"strings"
	"testing"

	"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)

// rubricEvaluation builds a completed evaluation whose Score is computed exactly
// as Evaluate computes it, so the tests exercise the real score and the gate
// together rather than a hand-picked Score.
func rubricEvaluation(taskSuccess, reusable, evidence float64) asymptoteobserve.LearningEvaluationV1 {
	eval := testLearningEvaluation()
	eval.Questions = normalizeQuestionResults([]asymptoteobserve.LearningEvaluationQuestionV1{
		{ID: TaskSuccessQuestionID, Probability: taskSuccess},
		{ID: "reusable_correction", Probability: reusable},
		{ID: "evidence_supported", Probability: evidence},
	})
	eval.Score = evaluationScore(eval.Questions)
	return eval
}

func TestPromotionTaskSuccessGateBoundaries(t *testing.T) {
	for _, tc := range []struct {
		name                     string
		task, reusable, evidence float64
		wantPromoted             bool
		wantReasonContains       string
	}{
		{name: "at gate, mean over threshold", task: CandidateTaskSuccessThreshold, reusable: 0.80, evidence: 0.80, wantPromoted: true},
		{name: "at gate, mean under threshold", task: CandidateTaskSuccessThreshold, reusable: 0.60, evidence: 0.60, wantReasonContains: "score 0.57 is below 0.60"},
		{name: "just below gate, mean well over threshold", task: 0.49, reusable: 0.95, evidence: 0.95, wantReasonContains: "task_success 0.49 is below 0.50"},
		{name: "fractionally below gate", task: 0.4999, reusable: 1.0, evidence: 1.0, wantReasonContains: "task_success 0.4999 is below 0.50"},
		{name: "below gate, mean under threshold", task: 0.10, reusable: 0.60, evidence: 0.60, wantReasonContains: "task_success 0.10 is below 0.50"},
	} {
		t.Run(tc.name, func(t *testing.T) {
			eval := rubricEvaluation(tc.task, tc.reusable, tc.evidence)
			ok, reason := PromotionDecision(eval)
			if ok != tc.wantPromoted {
				t.Fatalf("PromotionDecision(%v/%v/%v, score %.4f) = %v (%q), want %v", tc.task, tc.reusable, tc.evidence, eval.Score, ok, reason, tc.wantPromoted)
			}
			if tc.wantPromoted && reason != "" {
				t.Fatalf("promoted evaluation carries a reason: %q", reason)
			}
			if !tc.wantPromoted && !strings.Contains(reason, tc.wantReasonContains) {
				t.Fatalf("reason = %q, want it to contain %q", reason, tc.wantReasonContains)
			}
			if _, created := CandidateFromEvaluation(eval); created != tc.wantPromoted {
				t.Fatalf("CandidateFromEvaluation created=%v, want %v (must agree with PromotionDecision)", created, tc.wantPromoted)
			}
		})
	}
}

func TestPromotionRequiresTaskSuccessAnswer(t *testing.T) {
	eval := testLearningEvaluation()
	eval.Questions = []asymptoteobserve.LearningEvaluationQuestionV1{
		{ID: "reusable_correction", Probability: 0.95},
		{ID: "evidence_supported", Probability: 0.95},
	}
	eval.Score = 0.95
	ok, reason := PromotionDecision(eval)
	if ok || reason != "task_success was not answered" {
		t.Fatalf("PromotionDecision = %v, %q; want a refusal for a missing task_success answer", ok, reason)
	}
}

func TestPromotionRequiresCompletedStatus(t *testing.T) {
	for _, status := range []string{asymptoteobserve.LearningEvaluationStatusDryRun, asymptoteobserve.LearningEvaluationStatusFailed} {
		eval := rubricEvaluation(0.9, 0.9, 0.9)
		eval.Status = status
		ok, reason := PromotionDecision(eval)
		if ok || !strings.Contains(reason, status) {
			t.Fatalf("status %s: PromotionDecision = %v, %q", status, ok, reason)
		}
	}
}

// The gate sits on the whole Evaluate path: a real Jev answer for the #649 trace
// keeps its reported score and is not promoted, and an evaluator-supplied
// top-level score cannot bypass the task_success precondition.
func TestEvaluateThenPromoteAppliesTaskSuccessGate(t *testing.T) {
	eval := evaluateWithJevBody(t, `{"answers":{
		"task_success":{"type":"noul","noul":0.27},
		"reusable_correction":{"type":"noul","noul":0.86},
		"evidence_supported":{"type":"noul","noul":0.69}}}`)
	if got := fmt.Sprintf("%.4f", eval.Score); got != "0.6067" {
		t.Fatalf("stored score = %s, want the unchanged mean 0.6067", got)
	}
	if _, ok := CandidateFromEvaluation(eval); ok {
		t.Fatal("Jev-scored failed task was promoted")
	}

	legacy := evaluateWithJevBody(t, `{"score":0.95,"questions":[
		{"id":"task_success","probability":0.2},
		{"id":"reusable_correction","probability":0.95},
		{"id":"evidence_supported","probability":0.95}]}`)
	if legacy.Score != 0.95 {
		t.Fatalf("evaluator-supplied score = %v, want 0.95 kept as reported", legacy.Score)
	}
	if ok, reason := PromotionDecision(legacy); ok || !strings.Contains(reason, "task_success 0.20 is below 0.50") {
		t.Fatalf("evaluator-supplied score bypassed the gate: %v, %q", ok, reason)
	}

	passing := evaluateWithJevBody(t, `{"answers":{
		"task_success":{"type":"noul","noul":0.9},
		"reusable_correction":{"type":"noul","noul":0.8},
		"evidence_supported":{"type":"noul","noul":0.7}}}`)
	if _, ok := CandidateFromEvaluation(passing); !ok {
		t.Fatalf("successful Jev-scored trace was not promoted (score %.4f)", passing.Score)
	}
}