The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
package learning
import (
"bytes"
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"strings"
"time"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
const (
RubricVersion = "beacon.learning.rubric.v1"
DefaultJevEndpoint = "https://api.typesafe.ai/v1/systemone"
DefaultJevModel = "jev-latest"
DefaultSubQEndpoint = "https://api.subq.ai/v1/systemone"
DefaultSubQModel = "subq-snap-preview"
DefaultCostPerTrace = 0.00035
maxProjectionEvents = 80
maxProjectionText = 1200
// jevQuestionType is the System One question type Beacon asks for. Jev and
// SubQ echo it back as each answer's "type", which is not a rationale.
jevQuestionType = "noul"
)
// Evaluator providers. Both speak the System One decisions API (a POST of
// model, state and typed questions answered with probabilities), so they share
// one request and response shape and differ only in their hosted defaults.
const (
ProviderJev = "jev"
ProviderSubQ = "subq"
)
// Providers lists the accepted --evaluator values.
var Providers = []string{ProviderJev, ProviderSubQ}
// ProviderDefaults returns the hosted endpoint and model for a provider.
func ProviderDefaults(provider string) (endpoint, model string, err error) {
switch NormalizeProvider(provider) {
case ProviderJev:
return DefaultJevEndpoint, DefaultJevModel, nil
case ProviderSubQ:
return DefaultSubQEndpoint, DefaultSubQModel, nil
}
return "", "", fmt.Errorf("unknown evaluator %q (want one of %s)", provider, strings.Join(Providers, ", "))
}
// NormalizeProvider lowercases a provider name; empty means Jev, the original
// and default evaluator.
func NormalizeProvider(provider string) string {
provider = strings.ToLower(strings.TrimSpace(provider))
if provider == "" {
return ProviderJev
}
return provider
}
func providerLabel(opts EvaluatorOptions) string {
if NormalizeProvider(opts.Provider) == ProviderSubQ {
return "SubQ"
}
return "Jev"
}
var RubricQuestions = []asymptoteobserve.LearningEvaluationQuestionV1{
{ID: TaskSuccessQuestionID, Prompt: "Did the trace complete the user's engineering task successfully?"},
{ID: "reusable_correction", Prompt: "Does the trace contain a correction or debugging pattern that future agents should reuse?"},
{ID: "evidence_supported", Prompt: "Is the reusable lesson supported by concrete events in the trace?"},
}
type EvaluatorOptions struct {
// Provider is ProviderJev (the default when empty) or ProviderSubQ.
Provider string
Endpoint string
APIKey string
Model string
HTTPClient *http.Client
CostPerTrace float64
Timeout time.Duration
}
type EvaluationInput struct {
Project asymptoteobserve.LearningProjectV1
Trace asymptoteobserve.TraceShowResultV1
DryRun bool
}
type EvaluationPreview struct {
TraceID string `json:"trace_id"`
Title string `json:"title,omitempty"`
EventCount int `json:"event_count"`
EstimatedCalls int `json:"estimated_calls"`
EstimatedCostUSD float64 `json:"estimated_cost_usd"`
Reason string `json:"reason,omitempty"`
Project asymptoteobserve.LearningProjectV1 `json:"project"`
}
type Projection struct {
TraceID string `json:"trace_id"`
Title string `json:"title,omitempty"`
Harness string `json:"harness,omitempty"`
Repository string `json:"repository,omitempty"`
Events []ProjectedEvent `json:"events"`
Questions []ProjectedQuestion `json:"-"`
}
type ProjectedEvent struct {
Number int `json:"number"`
Type string `json:"type"`
Action string `json:"action"`
Title string `json:"title,omitempty"`
Summary string `json:"summary,omitempty"`
Content string `json:"content,omitempty"`
}
type ProjectedQuestion struct {
ID string `json:"id"`
Prompt string `json:"prompt"`
}
type jevRequest struct {
Model string `json:"model,omitempty"`
State map[string]interface{} `json:"state"`
Questions map[string]jevQuestion `json:"questions"`
}
type jevResponse struct {
Model string `json:"model,omitempty"`
Answers map[string]jevAnswer `json:"answers"`
Usage *asymptoteobserve.LearningEvaluationUsageV1 `json:"usage,omitempty"`
// Compatibility with tests or local shims that still return the earlier internal shape.
Questions []asymptoteobserve.LearningEvaluationQuestionV1 `json:"questions,omitempty"`
Results []asymptoteobserve.LearningEvaluationQuestionV1 `json:"results,omitempty"`
Score *float64 `json:"score,omitempty"`
CostEstimateUSD *float64 `json:"cost_estimate_usd,omitempty"`
}
type jevQuestion struct {
Type string `json:"type"`
Instructions string `json:"instructions"`
Criteria map[string]string `json:"criteria,omitempty"`
}
type jevAnswer struct {
Type string `json:"type,omitempty"`
Noul *float64 `json:"noul,omitempty"`
Probability *float64 `json:"probability,omitempty"`
Choice string `json:"choice,omitempty"`
Score *float64 `json:"score,omitempty"`
Confidence float64 `json:"confidence,omitempty"`
Probabilities map[string]float64 `json:"probabilities,omitempty"`
}
func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput) (asymptoteobserve.LearningEvaluationV1, error) {
projection := BuildProjection(input.Trace)
evaluation := baseEvaluation(opts, input, projection)
if input.DryRun {
evaluation.Status = asymptoteobserve.LearningEvaluationStatusDryRun
evaluation.CostEstimateUSD = costPerTrace(opts)
evaluation.ID = EvaluationID(evaluation)
return evaluation, nil
}
if strings.TrimSpace(opts.Endpoint) == "" {
opts.Endpoint = defaultEndpoint(opts)
}
resp, err := callEvaluator(ctx, opts, projection)
if err != nil {
evaluation.Status = asymptoteobserve.LearningEvaluationStatusFailed
evaluation.Error = err.Error()
evaluation.ID = EvaluationID(evaluation)
return evaluation, err
}
questions := questionsFromJevAnswers(resp.Answers)
if len(questions) == 0 {
questions = resp.Questions
}
if len(questions) == 0 {
questions = resp.Results
}
if len(questions) == 0 {
return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("%s response did not include answers, questions, or results", providerLabel(opts))
}
evaluation.Questions = normalizeQuestionResults(questions)
evaluation.Score = evaluationScore(evaluation.Questions)
if resp.Score != nil {
evaluation.Score = *resp.Score
}
if resp.Model != "" {
evaluation.EvaluatorModel = resp.Model
}
if resp.Usage != nil {
evaluation.Usage = resp.Usage
if resp.Usage.CostUSD > 0 {
evaluation.CostEstimateUSD = resp.Usage.CostUSD
}
}
if resp.CostEstimateUSD != nil {
evaluation.CostEstimateUSD = *resp.CostEstimateUSD
} else if evaluation.CostEstimateUSD == 0 {
evaluation.CostEstimateUSD = costPerTrace(opts)
}
evaluation.Status = asymptoteobserve.LearningEvaluationStatusCompleted
evaluation.ID = EvaluationID(evaluation)
return evaluation, nil
}
func Preview(input EvaluationInput, opts EvaluatorOptions) EvaluationPreview {
trace := input.Trace.Trace
return EvaluationPreview{
TraceID: trace.ID,
Title: trace.Title,
EventCount: input.Trace.Range.TotalEvents,
EstimatedCalls: 1,
EstimatedCostUSD: costPerTrace(opts),
Reason: "explicit evaluation command; hooks and collectors do not call " + providerLabel(opts),
Project: input.Project,
}
}
func BuildProjection(show asymptoteobserve.TraceShowResultV1) Projection {
projection := Projection{
TraceID: show.Trace.ID,
Title: show.Trace.Title,
Harness: show.Trace.Harness.Name,
}
if show.Trace.Repository != nil {
projection.Repository = firstNonEmpty(show.Trace.Repository.RemoteURL, show.Trace.Repository.Path)
}
for _, question := range RubricQuestions {
projection.Questions = append(projection.Questions, ProjectedQuestion{ID: question.ID, Prompt: question.Prompt})
}
events := show.Events
if len(events) > maxProjectionEvents {
// Long sessions resolve at the end: keep the opening context and the tail
// instead of only the first maxProjectionEvents events, otherwise the
// evaluator never sees the fix or outcome a reusable lesson depends on.
head := maxProjectionEvents / 2
tail := maxProjectionEvents - head
combined := make([]asymptoteobserve.TraceEventV1, 0, maxProjectionEvents+1)
combined = append(combined, events[:head]...)
combined = append(combined, asymptoteobserve.TraceEventV1{
Type: "omitted",
Summary: fmt.Sprintf("%d events omitted from the middle of this trace", len(events)-maxProjectionEvents),
})
combined = append(combined, events[len(events)-tail:]...)
events = combined
}
for _, event := range events {
projected := ProjectedEvent{
Number: event.Number,
Type: event.Type,
Action: event.Action,
Title: cleanText(event.Title),
Summary: cleanText(event.Summary),
}
if event.Content != nil {
projected.Content = cleanText(event.Content.Text)
}
if event.Command != nil && projected.Content == "" {
projected.Content = cleanText(event.Command.Command)
}
projection.Events = append(projection.Events, projected)
}
return projection
}
func RubricHash() string {
data, _ := json.Marshal(RubricQuestions)
sum := sha256.Sum256(data)
return "sha256:" + hex.EncodeToString(sum[:])
}
func baseEvaluation(opts EvaluatorOptions, input EvaluationInput, projection Projection) asymptoteobserve.LearningEvaluationV1 {
trace := input.Trace.Trace
now := time.Now().UTC().Format(time.RFC3339Nano)
ref := asymptoteobserve.LearningTraceRefV1{
ID: trace.ID,
Title: trace.Title,
Harness: trace.Harness,
Session: trace.Session,
Repo: trace.Repository,
}
for _, event := range input.Trace.Events {
ref.EventIDs = append(ref.EventIDs, event.ID)
}
return asymptoteobserve.LearningEvaluationV1{
SchemaVersion: asymptoteobserve.LearningSchemaVersion,
Status: asymptoteobserve.LearningEvaluationStatusCompleted,
CreatedAt: now,
UpdatedAt: now,
Project: input.Project,
RubricVersion: RubricVersion,
RubricHash: RubricHash(),
Evaluator: evaluatorName(opts),
EvaluatorModel: modelName(opts),
DryRun: input.DryRun,
Trace: ref,
CostEstimateUSD: costPerTrace(opts),
Questions: projectionQuestionDefaults(projection),
}
}
func callEvaluator(ctx context.Context, opts EvaluatorOptions, projection Projection) (jevResponse, error) {
client := opts.HTTPClient
if client == nil {
client = http.DefaultClient
}
timeout := opts.Timeout
if timeout <= 0 {
timeout = 10 * time.Second
}
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
body, err := json.Marshal(jevRequest{
Model: modelName(opts),
State: map[string]interface{}{
"trace": projection,
"rubric_version": RubricVersion,
"rubric_hash": RubricHash(),
},
Questions: jevQuestions(),
})
if err != nil {
return jevResponse{}, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, opts.Endpoint, bytes.NewReader(body))
if err != nil {
return jevResponse{}, err
}
req.Header.Set("Content-Type", "application/json")
if opts.APIKey != "" {
req.Header.Set("Authorization", "Bearer "+opts.APIKey)
}
resp, err := client.Do(req)
if err != nil {
return jevResponse{}, err
}
defer resp.Body.Close()
data, err := io.ReadAll(io.LimitReader(resp.Body, 2<<20))
if err != nil {
return jevResponse{}, err
}
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
return jevResponse{}, fmt.Errorf("%s endpoint returned %s: %s", providerLabel(opts), resp.Status, strings.TrimSpace(string(data)))
}
var parsed jevResponse
if err := json.Unmarshal(data, &parsed); err != nil {
return jevResponse{}, err
}
return parsed, nil
}
func normalizeQuestionResults(results []asymptoteobserve.LearningEvaluationQuestionV1) []asymptoteobserve.LearningEvaluationQuestionV1 {
byID := map[string]asymptoteobserve.LearningEvaluationQuestionV1{}
for _, result := range results {
byID[result.ID] = result
}
out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(RubricQuestions))
for _, question := range RubricQuestions {
result := byID[question.ID]
if result.ID == "" {
result.ID = question.ID
}
if result.Prompt == "" {
result.Prompt = question.Prompt
}
if result.Probability < 0 {
result.Probability = 0
}
if result.Probability > 1 {
result.Probability = 1
}
// Compatibility responses can still use the older questions/results
// shapes, whose schema includes reason. Keep real values while dropping
// the question-type placeholder written by older Beacon releases.
result.Reason = cleanText(strings.Join(strings.Fields(QuestionReason(result)), " "))
out = append(out, result)
}
return out
}
func jevQuestions() map[string]jevQuestion {
out := make(map[string]jevQuestion, len(RubricQuestions))
for _, question := range RubricQuestions {
out[question.ID] = jevQuestion{
Type: jevQuestionType,
Instructions: question.Prompt,
Criteria: map[string]string{
"true": "The trace satisfies this criterion.",
"false": "The trace does not satisfy this criterion.",
},
}
}
return out
}
func questionsFromJevAnswers(answers map[string]jevAnswer) []asymptoteobserve.LearningEvaluationQuestionV1 {
if len(answers) == 0 {
return nil
}
out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(RubricQuestions))
for _, question := range RubricQuestions {
answer, ok := answers[question.ID]
if !ok {
continue
}
probability := 0.0
switch {
case answer.Noul != nil:
probability = *answer.Noul
case answer.Probability != nil:
probability = *answer.Probability
case answer.Score != nil:
probability = *answer.Score
}
out = append(out, asymptoteobserve.LearningEvaluationQuestionV1{
ID: question.ID,
Prompt: question.Prompt,
Probability: probability,
Confidence: answer.Confidence,
})
}
return out
}
// QuestionReason returns a stored reason for one rubric question, or "" when it
// has none. TypeSafe Noul answers do not include reasons. Releases before the fix
// for #620 copied the answer type ("noul") into this field, so hide that value.
func QuestionReason(question asymptoteobserve.LearningEvaluationQuestionV1) string {
reason := strings.TrimSpace(question.Reason)
if strings.EqualFold(reason, jevQuestionType) {
return ""
}
return reason
}
func projectionQuestionDefaults(projection Projection) []asymptoteobserve.LearningEvaluationQuestionV1 {
out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(projection.Questions))
for _, question := range projection.Questions {
out = append(out, asymptoteobserve.LearningEvaluationQuestionV1{ID: question.ID, Prompt: question.Prompt})
}
return out
}
func evaluationScore(results []asymptoteobserve.LearningEvaluationQuestionV1) float64 {
if len(results) == 0 {
return 0
}
var sum float64
for _, result := range results {
sum += result.Probability
}
return sum / float64(len(results))
}
func evaluatorName(opts EvaluatorOptions) string {
provider := NormalizeProvider(opts.Provider)
if opts.Endpoint == "" {
return provider + ":" + defaultEndpoint(opts)
}
return provider + ":" + opts.Endpoint
}
func defaultEndpoint(opts EvaluatorOptions) string {
endpoint, _ := providerDefaultsOrJev(opts.Provider)
return endpoint
}
func providerDefaultsOrJev(provider string) (endpoint, model string) {
endpoint, model, err := ProviderDefaults(provider)
if err != nil {
return DefaultJevEndpoint, DefaultJevModel
}
return endpoint, model
}
func modelName(opts EvaluatorOptions) string {
if strings.TrimSpace(opts.Model) != "" {
return strings.TrimSpace(opts.Model)
}
_, model := providerDefaultsOrJev(opts.Provider)
return model
}
func costPerTrace(opts EvaluatorOptions) float64 {
if opts.CostPerTrace > 0 {
return opts.CostPerTrace
}
return DefaultCostPerTrace
}
func cleanText(value string) string {
value = strings.TrimSpace(value)
if value == "" {
return ""
}
value = asymptoteobserve.CleanString(value, maxProjectionText, true)
return value
}