Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Use TypeSafe Jev API

Justin D'Souza6e590d599d28
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.go
index 53075ad6..1047351d 100644
--- a/cli/beacon/cmd/memory.go
+++ b/cli/beacon/cmd/memory.go
@@ -32,6 +32,7 @@ var memoryOpts struct {
dryRun bool
jevEndpoint string
jevAPIKey string
+ jevModel string
jevCost float64
timeout time.Duration
}
@@ -103,8 +104,9 @@ func init() {
memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.since, "since", "", "RFC3339 lower time bound, inclusive")
memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.until, "until", "", "RFC3339 upper time bound, inclusive")
memoryEvaluationsRunCmd.Flags().BoolVar(&memoryOpts.dryRun, "dry-run", false, "Preview selected traces, Jev calls, and estimated cost without writing evaluations")
- memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevEndpoint, "jev-endpoint", "", "Jev evaluator endpoint (defaults to BEACON_JEV_ENDPOINT)")
- memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevAPIKey, "jev-api-key", "", "Jev API key (defaults to BEACON_JEV_API_KEY)")
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevEndpoint, "jev-endpoint", learning.DefaultJevEndpoint, "Jev System One endpoint")
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevAPIKey, "jev-api-key", "", "Jev API key (defaults to TYPESAFE_API_KEY, then BEACON_JEV_API_KEY)")
+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevModel, "jev-model", learning.DefaultJevModel, "Jev model name, such as jev-latest or a pinned Jev version")
memoryEvaluationsRunCmd.Flags().Float64Var(&memoryOpts.jevCost, "jev-cost-per-trace", learning.DefaultCostPerTrace, "Estimated Jev cost per trace in USD")
memoryEvaluationsRunCmd.Flags().DurationVar(&memoryOpts.timeout, "timeout", 10*time.Second, "Jev request timeout")
}
@@ -120,8 +122,9 @@ func runMemoryEvaluationsRun(cmd *cobra.Command, args []string) error {
return err
}
evaluatorOpts := learning.EvaluatorOptions{
- Endpoint: firstNonEmpty(memoryOpts.jevEndpoint, os.Getenv("BEACON_JEV_ENDPOINT")),
- APIKey: firstNonEmpty(memoryOpts.jevAPIKey, os.Getenv("BEACON_JEV_API_KEY")),
+ Endpoint: firstNonEmpty(memoryOpts.jevEndpoint, os.Getenv("BEACON_JEV_ENDPOINT"), learning.DefaultJevEndpoint),
+ APIKey: firstNonEmpty(memoryOpts.jevAPIKey, os.Getenv("TYPESAFE_API_KEY"), os.Getenv("BEACON_JEV_API_KEY")),
+ Model: firstNonEmpty(memoryOpts.jevModel, os.Getenv("BEACON_JEV_MODEL"), learning.DefaultJevModel),
CostPerTrace: memoryOpts.jevCost,
Timeout: memoryOpts.timeout,
}
diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.go
index b5e9a0b1..77a9a667 100644
--- a/cli/beacon/cmd/memory_test.go
+++ b/cli/beacon/cmd/memory_test.go
@@ -77,6 +77,7 @@ func TestMemoryEvaluationsRunPersistsMockedJevResult(t *testing.T) {
memoryOpts.jsonOutput = true
memoryOpts.jevEndpoint = server.URL
memoryOpts.jevAPIKey = "test-key"
+ memoryOpts.jevModel = "jev-test"
memoryOpts.jevCost = learning.DefaultCostPerTrace
var out bytes.Buffer
@@ -160,8 +161,9 @@ func resetMemoryOpts(t *testing.T) {
dryRun bool
jevEndpoint string
jevAPIKey string
+ jevModel string
jevCost float64
timeout time.Duration
- }{userMode: true, limit: 25, page: 1, jevCost: learning.DefaultCostPerTrace}
+ }{userMode: true, limit: 25, page: 1, jevEndpoint: learning.DefaultJevEndpoint, jevModel: learning.DefaultJevModel, jevCost: learning.DefaultCostPerTrace}
t.Cleanup(func() { memoryOpts = previous })
}
diff --git a/cli/beacon/internal/learning/evaluator.go b/cli/beacon/internal/learning/evaluator.go
index 703b0e65..1241e3c0 100644
--- a/cli/beacon/internal/learning/evaluator.go
+++ b/cli/beacon/internal/learning/evaluator.go
@@ -17,6 +17,8 @@ import (
const (
RubricVersion = "beacon.learning.rubric.v1"
+ DefaultJevEndpoint = "https://api.typesafe.ai/v1/systemone"
+ DefaultJevModel = "jev-latest"
DefaultCostPerTrace = 0.00035
maxProjectionEvents = 80
maxProjectionText = 1200
@@ -31,6 +33,7 @@ var RubricQuestions = []asymptoteobserve.LearningEvaluationQuestionV1{
type EvaluatorOptions struct {
Endpoint string
APIKey string
+ Model string
HTTPClient *http.Client
CostPerTrace float64
Timeout time.Duration
@@ -57,7 +60,7 @@ type Projection struct {
Harness string `json:"harness,omitempty"`
Repository string `json:"repository,omitempty"`
Events []ProjectedEvent `json:"events"`
- Questions []ProjectedQuestion `json:"questions"`
+ Questions []ProjectedQuestion `json:"-"`
}
type ProjectedEvent struct {
@@ -75,18 +78,38 @@ type ProjectedQuestion struct {
}
type jevRequest struct {
- RubricVersion string `json:"rubric_version"`
- RubricHash string `json:"rubric_hash"`
- Trace Projection `json:"trace"`
+ Model string `json:"model,omitempty"`
+ State map[string]interface{} `json:"state"`
+ Questions map[string]jevQuestion `json:"questions"`
}
type jevResponse struct {
- Questions []asymptoteobserve.LearningEvaluationQuestionV1 `json:"questions"`
- Results []asymptoteobserve.LearningEvaluationQuestionV1 `json:"results"`
+ Model string `json:"model,omitempty"`
+ Answers map[string]jevAnswer `json:"answers"`
+ Usage *asymptoteobserve.LearningEvaluationUsageV1 `json:"usage,omitempty"`
+ // Compatibility with tests or local shims that still return the earlier internal shape.
+ Questions []asymptoteobserve.LearningEvaluationQuestionV1 `json:"questions,omitempty"`
+ Results []asymptoteobserve.LearningEvaluationQuestionV1 `json:"results,omitempty"`
Score *float64 `json:"score,omitempty"`
CostEstimateUSD *float64 `json:"cost_estimate_usd,omitempty"`
}
+type jevQuestion struct {
+ Type string `json:"type"`
+ Instructions string `json:"instructions"`
+ Criteria map[string]string `json:"criteria,omitempty"`
+}
+
+type jevAnswer struct {
+ Type string `json:"type,omitempty"`
+ Noul *float64 `json:"noul,omitempty"`
+ Probability *float64 `json:"probability,omitempty"`
+ Choice string `json:"choice,omitempty"`
+ Score *float64 `json:"score,omitempty"`
+ Confidence float64 `json:"confidence,omitempty"`
+ Probabilities map[string]float64 `json:"probabilities,omitempty"`
+}
+
func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput) (asymptoteobserve.LearningEvaluationV1, error) {
projection := BuildProjection(input.Trace)
evaluation := baseEvaluation(opts, input, projection)
@@ -97,7 +120,7 @@ func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput)
return evaluation, nil
}
if strings.TrimSpace(opts.Endpoint) == "" {
- return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("Jev endpoint is required; pass --jev-endpoint or set BEACON_JEV_ENDPOINT")
+ opts.Endpoint = DefaultJevEndpoint
}
resp, err := callEvaluator(ctx, opts, projection)
if err != nil {
@@ -106,21 +129,33 @@ func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput)
evaluation.ID = EvaluationID(evaluation)
return evaluation, err
}
- questions := resp.Questions
+ questions := questionsFromJevAnswers(resp.Answers)
+ if len(questions) == 0 {
+ questions = resp.Questions
+ }
if len(questions) == 0 {
questions = resp.Results
}
if len(questions) == 0 {
- return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("Jev response did not include questions or results")
+ return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("Jev response did not include answers, questions, or results")
}
evaluation.Questions = normalizeQuestionResults(questions)
evaluation.Score = evaluationScore(evaluation.Questions)
if resp.Score != nil {
evaluation.Score = *resp.Score
}
+ if resp.Model != "" {
+ evaluation.EvaluatorModel = resp.Model
+ }
+ if resp.Usage != nil {
+ evaluation.Usage = resp.Usage
+ if resp.Usage.CostUSD > 0 {
+ evaluation.CostEstimateUSD = resp.Usage.CostUSD
+ }
+ }
if resp.CostEstimateUSD != nil {
evaluation.CostEstimateUSD = *resp.CostEstimateUSD
- } else {
+ } else if evaluation.CostEstimateUSD == 0 {
evaluation.CostEstimateUSD = costPerTrace(opts)
}
evaluation.Status = asymptoteobserve.LearningEvaluationStatusCompleted
@@ -183,6 +218,7 @@ func RubricHash() string {
func baseEvaluation(opts EvaluatorOptions, input EvaluationInput, projection Projection) asymptoteobserve.LearningEvaluationV1 {
trace := input.Trace.Trace
+ now := time.Now().UTC().Format(time.RFC3339Nano)
ref := asymptoteobserve.LearningTraceRefV1{
ID: trace.ID,
Title: trace.Title,
@@ -196,10 +232,13 @@ func baseEvaluation(opts EvaluatorOptions, input EvaluationInput, projection Pro
return asymptoteobserve.LearningEvaluationV1{
SchemaVersion: asymptoteobserve.LearningSchemaVersion,
Status: asymptoteobserve.LearningEvaluationStatusCompleted,
+ CreatedAt: now,
+ UpdatedAt: now,
Project: input.Project,
RubricVersion: RubricVersion,
RubricHash: RubricHash(),
Evaluator: evaluatorName(opts),
+ EvaluatorModel: modelName(opts),
DryRun: input.DryRun,
Trace: ref,
CostEstimateUSD: costPerTrace(opts),
@@ -218,7 +257,15 @@ func callEvaluator(ctx context.Context, opts EvaluatorOptions, projection Projec
}
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
- body, err := json.Marshal(jevRequest{RubricVersion: RubricVersion, RubricHash: RubricHash(), Trace: projection})
+ body, err := json.Marshal(jevRequest{
+ Model: modelName(opts),
+ State: map[string]interface{}{
+ "trace": projection,
+ "rubric_version": RubricVersion,
+ "rubric_hash": RubricHash(),
+ },
+ Questions: jevQuestions(),
+ })
if err != nil {
return jevResponse{}, err
}
@@ -274,6 +321,51 @@ func normalizeQuestionResults(results []asymptoteobserve.LearningEvaluationQuest
return out
}
+func jevQuestions() map[string]jevQuestion {
+ out := make(map[string]jevQuestion, len(RubricQuestions))
+ for _, question := range RubricQuestions {
+ out[question.ID] = jevQuestion{
+ Type: "noul",
+ Instructions: question.Prompt,
+ Criteria: map[string]string{
+ "true": "The trace satisfies this criterion.",
+ "false": "The trace does not satisfy this criterion.",
+ },
+ }
+ }
+ return out
+}
+
+func questionsFromJevAnswers(answers map[string]jevAnswer) []asymptoteobserve.LearningEvaluationQuestionV1 {
+ if len(answers) == 0 {
+ return nil
+ }
+ out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(RubricQuestions))
+ for _, question := range RubricQuestions {
+ answer, ok := answers[question.ID]
+ if !ok {
+ continue
+ }
+ probability := 0.0
+ switch {
+ case answer.Noul != nil:
+ probability = *answer.Noul
+ case answer.Probability != nil:
+ probability = *answer.Probability
+ case answer.Score != nil:
+ probability = *answer.Score
+ }
+ out = append(out, asymptoteobserve.LearningEvaluationQuestionV1{
+ ID: question.ID,
+ Prompt: question.Prompt,
+ Probability: probability,
+ Confidence: answer.Confidence,
+ Reason: answer.Type,
+ })
+ }
+ return out
+}
+
func projectionQuestionDefaults(projection Projection) []asymptoteobserve.LearningEvaluationQuestionV1 {
out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(projection.Questions))
for _, question := range projection.Questions {
@@ -295,11 +387,18 @@ func evaluationScore(results []asymptoteobserve.LearningEvaluationQuestionV1) fl
func evaluatorName(opts EvaluatorOptions) string {
if opts.Endpoint == "" {
- return "jev"
+ return "jev:" + DefaultJevEndpoint
}
return "jev:" + opts.Endpoint
}
+func modelName(opts EvaluatorOptions) string {
+ if strings.TrimSpace(opts.Model) != "" {
+ return strings.TrimSpace(opts.Model)
+ }
+ return DefaultJevModel
+}
+
func costPerTrace(opts EvaluatorOptions) float64 {
if opts.CostPerTrace > 0 {
return opts.CostPerTrace
diff --git a/cli/beacon/internal/learning/evaluator_test.go b/cli/beacon/internal/learning/evaluator_test.go
index 1a553fbf..43c7b663 100644
--- a/cli/beacon/internal/learning/evaluator_test.go
+++ b/cli/beacon/internal/learning/evaluator_test.go
@@ -23,9 +23,12 @@ func TestEvaluateDryRunDoesNotNeedEndpoint(t *testing.T) {
if eval.ID == "" || eval.RubricHash == "" || eval.CostEstimateUSD == 0 {
t.Fatalf("dry-run missing identifiers/cost: %#v", eval)
}
+ if eval.EvaluatorModel != DefaultJevModel {
+ t.Fatalf("model = %q", eval.EvaluatorModel)
+ }
}
-func TestEvaluatePostsProjectionToMockableEndpoint(t *testing.T) {
+func TestEvaluatePostsSystemOneRequestToMockableEndpoint(t *testing.T) {
var sawAuth bool
var sawTrace bool
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -36,18 +39,24 @@ func TestEvaluatePostsProjectionToMockableEndpoint(t *testing.T) {
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
t.Fatalf("decode request: %v", err)
}
- sawTrace = req.Trace.TraceID == "trace-1" && len(req.Trace.Questions) == len(RubricQuestions)
+ trace, _ := req.State["trace"].(map[string]interface{})
+ sawTrace = req.Model == "jev-test" && trace["trace_id"] == "trace-1" && len(req.Questions) == len(RubricQuestions)
+ nine := 0.9
+ seven := 0.7
+ eight := 0.8
_ = json.NewEncoder(w).Encode(jevResponse{
- Questions: []asymptoteobserve.LearningEvaluationQuestionV1{
- {ID: "task_success", Probability: 0.9, Confidence: 0.8},
- {ID: "reusable_correction", Probability: 0.7, Confidence: 0.6},
- {ID: "evidence_supported", Probability: 0.8, Confidence: 0.9},
+ Model: "jev-test-resolved",
+ Answers: map[string]jevAnswer{
+ "task_success": {Type: "noul", Noul: &nine},
+ "reusable_correction": {Type: "noul", Noul: &seven},
+ "evidence_supported": {Type: "noul", Noul: &eight},
},
+ Usage: &asymptoteobserve.LearningEvaluationUsageV1{InputTokens: 100, CostUSD: 0.0001},
})
}))
defer server.Close()
- eval, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL, APIKey: "secret"}, EvaluationInput{
+ eval, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL, APIKey: "secret", Model: "jev-test"}, EvaluationInput{
Project: asymptoteobserve.LearningProjectV1{ID: "project-1"},
Trace: testTraceShow(),
})
@@ -63,6 +72,9 @@ func TestEvaluatePostsProjectionToMockableEndpoint(t *testing.T) {
if eval.Score <= 0.79 || eval.Score >= 0.81 {
t.Fatalf("score = %f", eval.Score)
}
+ if eval.EvaluatorModel != "jev-test-resolved" || eval.Usage == nil || eval.CostEstimateUSD != 0.0001 {
+ t.Fatalf("model/usage not retained: %#v", eval)
+ }
}
func TestProjectionRedactsAndBoundsContent(t *testing.T) {
@@ -81,9 +93,13 @@ func TestProjectionRedactsAndBoundsContent(t *testing.T) {
}
}
-func TestEvaluateRequiresEndpointOutsideDryRun(t *testing.T) {
- _, err := Evaluate(t.Context(), EvaluatorOptions{}, EvaluationInput{Project: asymptoteobserve.LearningProjectV1{ID: "p"}, Trace: testTraceShow()})
- if err == nil || !strings.Contains(err.Error(), "Jev endpoint is required") {
+func TestEvaluateRequiresAnswersOutsideDryRun(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, _ = w.Write([]byte(`{"answers":{}}`))
+ }))
+ defer server.Close()
+ _, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL}, EvaluationInput{Project: asymptoteobserve.LearningProjectV1{ID: "p"}, Trace: testTraceShow()})
+ if err == nil || !strings.Contains(err.Error(), "answers") {
t.Fatalf("error = %v", err)
}
}
diff --git a/pkg/asymptoteobserve/learning.go b/pkg/asymptoteobserve/learning.go
index ed03573e..b95dd08b 100644
--- a/pkg/asymptoteobserve/learning.go
+++ b/pkg/asymptoteobserve/learning.go
@@ -57,14 +57,23 @@ type LearningEvaluationV1 struct {
RubricVersion string `json:"rubric_version"`
RubricHash string `json:"rubric_hash"`
Evaluator string `json:"evaluator"`
+ EvaluatorModel string `json:"evaluator_model,omitempty"`
DryRun bool `json:"dry_run,omitempty"`
Trace LearningTraceRefV1 `json:"trace"`
Questions []LearningEvaluationQuestionV1 `json:"questions,omitempty"`
Score float64 `json:"score,omitempty"`
CostEstimateUSD float64 `json:"cost_estimate_usd,omitempty"`
+ Usage *LearningEvaluationUsageV1 `json:"usage,omitempty"`
Error string `json:"error,omitempty"`
}
+type LearningEvaluationUsageV1 struct {
+ InputTokens int64 `json:"input_tokens,omitempty"`
+ OutputTokens int64 `json:"output_tokens,omitempty"`
+ CostUSD float64 `json:"cost_usd,omitempty"`
+ CreditsRemainingUSD float64 `json:"credits_remaining_usd,omitempty"`
+}
+
type LearningEvidenceV1 struct {
TraceID string `json:"trace_id"`
EventIDs []string `json:"event_ids,omitempty"`