The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
package learning
import (
"fmt"
"strconv"
"strings"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
// CandidateScoreThreshold is the minimum evaluation score (the mean of the
// rubric probabilities) a completed evaluation needs to become a candidate.
const CandidateScoreThreshold = 0.60
// TaskSuccessQuestionID names the rubric question that asks whether the trace
// completed the user's task.
const TaskSuccessQuestionID = "task_success"
// CandidateTaskSuccessThreshold is a precondition that the score cannot stand in
// for. The score is a mean, so two high answers about reusability and evidence
// could outvote a judge who said the task failed: 0.27/0.86/0.69 averages 0.6067
// and cleared CandidateScoreThreshold (#649). A trace whose task_success
// probability is below this value is never promoted, whatever its score.
const CandidateTaskSuccessThreshold = 0.50
// PromotionDecision reports whether a completed evaluation should become a
// memory candidate, and when it should not, why. It applies the task_success
// precondition before the score threshold, so the stored Score keeps its meaning
// (the plain mean shown to reviewers) while a failed task can no longer be
// averaged into a candidate. An evaluation without a task_success answer fails
// the precondition, because nothing says the task succeeded.
func PromotionDecision(eval asymptoteobserve.LearningEvaluationV1) (bool, string) {
if eval.Status != asymptoteobserve.LearningEvaluationStatusCompleted {
return false, fmt.Sprintf("evaluation status is %s, not %s", eval.Status, asymptoteobserve.LearningEvaluationStatusCompleted)
}
taskSuccess, ok := questionProbability(eval, TaskSuccessQuestionID)
if !ok {
return false, TaskSuccessQuestionID + " was not answered"
}
if taskSuccess < CandidateTaskSuccessThreshold {
return false, fmt.Sprintf("%s %s is below %.2f", TaskSuccessQuestionID, formatBelow(taskSuccess, CandidateTaskSuccessThreshold), CandidateTaskSuccessThreshold)
}
if eval.Score < CandidateScoreThreshold {
return false, fmt.Sprintf("score %s is below %.2f", formatBelow(eval.Score, CandidateScoreThreshold), CandidateScoreThreshold)
}
return true, ""
}
// formatBelow renders value, known to be below limit, with the fewest decimals
// (at least two) that still read as below it, so a reason never says "0.50 is
// below 0.50" for 0.4999.
func formatBelow(value, limit float64) string {
for precision := 2; precision <= 6; precision++ {
text := strconv.FormatFloat(value, 'f', precision, 64)
if parsed, err := strconv.ParseFloat(text, 64); err == nil && parsed < limit {
return text
}
}
return strconv.FormatFloat(value, 'g', -1, 64)
}
func questionProbability(eval asymptoteobserve.LearningEvaluationV1, id string) (float64, bool) {
for _, question := range eval.Questions {
if question.ID == id {
return question.Probability, true
}
}
return 0, false
}
func CandidateFromEvaluation(eval asymptoteobserve.LearningEvaluationV1) (asymptoteobserve.LearningCandidateV1, bool) {
if ok, _ := PromotionDecision(eval); !ok {
return asymptoteobserve.LearningCandidateV1{}, false
}
kind := candidateKind(eval)
title := candidateTitle(eval, kind)
body := candidateBody(eval)
candidate := asymptoteobserve.LearningCandidateV1{
SchemaVersion: asymptoteobserve.LearningSchemaVersion,
State: asymptoteobserve.LearningCandidateStateCandidate,
Kind: kind,
Title: title,
Body: body,
Applicability: candidateApplicability(eval),
Tags: candidateTags(eval, kind),
Project: eval.Project,
SourceEvaluationID: eval.ID,
Evidence: []asymptoteobserve.LearningEvidenceV1{
{
TraceID: eval.Trace.ID,
EventIDs: append([]string(nil), eval.Trace.EventIDs...),
Summary: strings.TrimSpace(eval.Trace.Title),
},
},
}
candidate.ID = CandidateID(candidate)
return candidate, true
}
// ApprovalEdits carries reviewer-authored text that replaces the candidate's
// fields in the approved memory. The evaluator scores traces but does not write
// lessons, so a reviewer (or an agent the reviewer is working with) supplies the
// title, body, and applicability after reading the source trace. Empty fields
// keep the candidate's value. The candidate record itself is left as the
// evaluator produced it, so the review history shows what was changed.
type ApprovalEdits struct {
Title string
Body string
Applicability string
Kind string
Tags []string
}
// ValidMemoryKind reports whether kind is one of the memory kinds Beacon defines.
func ValidMemoryKind(kind string) bool {
switch kind {
case asymptoteobserve.LearningMemoryKindWorkflow,
asymptoteobserve.LearningMemoryKindCorrection,
asymptoteobserve.LearningMemoryKindDebuggingPattern,
asymptoteobserve.LearningMemoryKindGotcha,
asymptoteobserve.LearningMemoryKindConvention:
return true
}
return false
}
func ApproveCandidate(store *Store, id, reason string) (asymptoteobserve.LearningCandidateV1, asymptoteobserve.LearningMemoryV1, error) {
return ApproveCandidateWithEdits(store, id, reason, ApprovalEdits{})
}
func ApproveCandidateWithEdits(store *Store, id, reason string, edits ApprovalEdits) (asymptoteobserve.LearningCandidateV1, asymptoteobserve.LearningMemoryV1, error) {
kind := strings.TrimSpace(edits.Kind)
if kind != "" && !ValidMemoryKind(kind) {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, fmt.Errorf("unknown memory kind %q (want workflow, correction, debugging_pattern, gotcha, or convention)", kind)
}
db, err := store.db()
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
}
defer db.Close()
tx, err := db.Begin()
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
}
defer tx.Rollback()
candidate, ok, err := getCandidate(tx, id)
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
}
if !ok {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, fmt.Errorf("candidate not found: %s", id)
}
if candidate.State != asymptoteobserve.LearningCandidateStateCandidate {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, fmt.Errorf("candidate %s is %s, not candidate", id, candidate.State)
}
now := nowString()
memory := asymptoteobserve.LearningMemoryV1{
SchemaVersion: asymptoteobserve.LearningSchemaVersion,
CandidateID: candidate.ID,
Kind: firstNonEmpty(kind, candidate.Kind),
Title: firstNonEmpty(edits.Title, candidate.Title),
Body: firstNonEmpty(edits.Body, candidate.Body),
Applicability: firstNonEmpty(edits.Applicability, candidate.Applicability),
Tags: approvedTags(edits.Tags, candidate.Tags),
Project: candidate.Project,
Evidence: append([]asymptoteobserve.LearningEvidenceV1(nil), candidate.Evidence...),
CreatedAt: now,
UpdatedAt: now,
}
memory.ID = MemoryID(memory)
candidate.State = asymptoteobserve.LearningCandidateStateApproved
candidate.MemoryID = memory.ID
candidate.ApprovedAt = now
candidate.ReviewReason = strings.TrimSpace(reason)
if err := putMemory(tx, memory); err != nil {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
}
if err := putCandidate(tx, candidate); err != nil {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
}
if err := tx.Commit(); err != nil {
return asymptoteobserve.LearningCandidateV1{}, asymptoteobserve.LearningMemoryV1{}, err
}
return candidate, memory, nil
}
// approvedTags uses the reviewer's tags when any were given, trimmed and without
// empties or duplicates, and otherwise the candidate's.
func approvedTags(edited, original []string) []string {
var out []string
seen := map[string]bool{}
for _, tag := range edited {
tag = strings.TrimSpace(tag)
if tag == "" || seen[tag] {
continue
}
seen[tag] = true
out = append(out, tag)
}
if len(out) > 0 {
return out
}
return append([]string(nil), original...)
}
func RejectCandidate(store *Store, id, reason string) (asymptoteobserve.LearningCandidateV1, error) {
db, err := store.db()
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
defer db.Close()
tx, err := db.Begin()
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
defer tx.Rollback()
candidate, ok, err := getCandidate(tx, id)
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
if !ok {
return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("candidate not found: %s", id)
}
if candidate.State != asymptoteobserve.LearningCandidateStateCandidate {
return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("candidate %s is %s, not candidate", id, candidate.State)
}
candidate.State = asymptoteobserve.LearningCandidateStateRejected
candidate.RejectedAt = nowString()
candidate.ReviewReason = strings.TrimSpace(reason)
if err := putCandidate(tx, candidate); err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
if err := tx.Commit(); err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
return candidate, nil
}
func SupersedeCandidate(store *Store, id, replacementMemoryID, reason string) (asymptoteobserve.LearningCandidateV1, error) {
if strings.TrimSpace(replacementMemoryID) == "" {
return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("replacement memory id is required")
}
db, err := store.db()
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
defer db.Close()
tx, err := db.Begin()
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
defer tx.Rollback()
replacement, ok, err := getMemory(tx, replacementMemoryID)
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
if !ok {
return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("replacement memory not found: %s", replacementMemoryID)
}
candidate, ok, err := getCandidate(tx, id)
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
if !ok {
return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("candidate not found: %s", id)
}
if candidate.Project.ID != replacement.Project.ID {
return asymptoteobserve.LearningCandidateV1{}, fmt.Errorf("replacement memory belongs to a different project")
}
now := nowString()
candidate.State = asymptoteobserve.LearningCandidateStateSuperseded
candidate.SupersededAt = now
candidate.SupersededBy = replacement.ID
candidate.ReviewReason = strings.TrimSpace(reason)
if err := putCandidate(tx, candidate); err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
if candidate.MemoryID != "" {
memory, ok, err := getMemory(tx, candidate.MemoryID)
if err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
if ok {
memory.SupersededBy = replacement.ID
memory.UpdatedAt = now
if err := putMemory(tx, memory); err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
}
}
if err := tx.Commit(); err != nil {
return asymptoteobserve.LearningCandidateV1{}, err
}
return candidate, nil
}
func candidateKind(eval asymptoteobserve.LearningEvaluationV1) string {
title := strings.ToLower(eval.Trace.Title)
switch {
case strings.Contains(title, "convention") || strings.Contains(title, "standard"):
return asymptoteobserve.LearningMemoryKindConvention
case strings.Contains(title, "gotcha") || strings.Contains(title, "pitfall"):
return asymptoteobserve.LearningMemoryKindGotcha
case strings.Contains(title, "workflow") || strings.Contains(title, "process"):
return asymptoteobserve.LearningMemoryKindWorkflow
case strings.Contains(title, "fix") || strings.Contains(title, "debug") || strings.Contains(title, "fail"):
return asymptoteobserve.LearningMemoryKindDebuggingPattern
default:
return asymptoteobserve.LearningMemoryKindCorrection
}
}
func candidateTitle(eval asymptoteobserve.LearningEvaluationV1, kind string) string {
title := strings.TrimSpace(eval.Trace.Title)
if title == "" {
title = eval.Trace.ID
}
return strings.TrimSpace(strings.ReplaceAll(kind, "_", " ")) + ": " + title
}
// candidateBody uses per-question rationale only when a compatible evaluator
// supplied it through the legacy questions/results response shapes. TypeSafe Noul
// answers contain probabilities, not rationale, so the normal Jev path explicitly
// says that no lesson text was extracted instead of presenting scores as guidance.
func candidateBody(eval asymptoteobserve.LearningEvaluationV1) string {
var rationale []string
for _, question := range eval.Questions {
if reason := QuestionReason(question); reason != "" {
rationale = append(rationale, fmt.Sprintf("- %s: %s", question.ID, reason))
}
}
var lines []string
if len(rationale) > 0 {
lines = append(lines, "Reusable lesson extracted from a reviewed Beacon trace.")
lines = append(lines, "")
lines = append(lines, "Evaluator rationale:")
lines = append(lines, rationale...)
} else {
lines = append(lines, "Beacon trace flagged for review by evaluation scores.")
lines = append(lines, "")
lines = append(lines, "The evaluator returned scores with no rationale, so no lesson text was extracted. Review the source trace before approving.")
}
lines = append(lines, "")
lines = append(lines, "Trace: "+eval.Trace.ID)
if eval.Trace.Harness.Name != "" {
lines = append(lines, "Harness: "+eval.Trace.Harness.Name)
}
if eval.Trace.Title != "" {
lines = append(lines, "Observed workflow: "+eval.Trace.Title)
}
lines = append(lines, "")
lines = append(lines, "Evaluation signals:")
for _, question := range eval.Questions {
lines = append(lines, fmt.Sprintf("- %s: %.2f", question.ID, question.Probability))
}
return strings.Join(lines, "\n")
}
func candidateApplicability(eval asymptoteobserve.LearningEvaluationV1) string {
if eval.Trace.Harness.Name == "" {
return "When a future agent works on a similar task in this project."
}
return "When a future agent in this project hits a similar workflow, regardless of harness."
}
func candidateTags(eval asymptoteobserve.LearningEvaluationV1, kind string) []string {
tags := []string{"beacon", kind}
if eval.Trace.Harness.Name != "" {
tags = append(tags, eval.Trace.Harness.Name)
}
return tags
}