The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
Use TypeSafe Jev API
Justin D'Souza6e590d599d28
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.goindex 53075ad6..1047351d 100644--- a/cli/beacon/cmd/memory.go+++ b/cli/beacon/cmd/memory.go@@ -32,6 +32,7 @@ var memoryOpts struct {dryRun booljevEndpoint stringjevAPIKey string+ jevModel stringjevCost float64timeout time.Duration}@@ -103,8 +104,9 @@ func init() {memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.since, "since", "", "RFC3339 lower time bound, inclusive")memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.until, "until", "", "RFC3339 upper time bound, inclusive")memoryEvaluationsRunCmd.Flags().BoolVar(&memoryOpts.dryRun, "dry-run", false, "Preview selected traces, Jev calls, and estimated cost without writing evaluations")- memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevEndpoint, "jev-endpoint", "", "Jev evaluator endpoint (defaults to BEACON_JEV_ENDPOINT)")- memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevAPIKey, "jev-api-key", "", "Jev API key (defaults to BEACON_JEV_API_KEY)")+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevEndpoint, "jev-endpoint", learning.DefaultJevEndpoint, "Jev System One endpoint")+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevAPIKey, "jev-api-key", "", "Jev API key (defaults to TYPESAFE_API_KEY, then BEACON_JEV_API_KEY)")+ memoryEvaluationsRunCmd.Flags().StringVar(&memoryOpts.jevModel, "jev-model", learning.DefaultJevModel, "Jev model name, such as jev-latest or a pinned Jev version")memoryEvaluationsRunCmd.Flags().Float64Var(&memoryOpts.jevCost, "jev-cost-per-trace", learning.DefaultCostPerTrace, "Estimated Jev cost per trace in USD")memoryEvaluationsRunCmd.Flags().DurationVar(&memoryOpts.timeout, "timeout", 10*time.Second, "Jev request timeout")}@@ -120,8 +122,9 @@ func runMemoryEvaluationsRun(cmd *cobra.Command, args []string) error {return err}evaluatorOpts := learning.EvaluatorOptions{- Endpoint: firstNonEmpty(memoryOpts.jevEndpoint, os.Getenv("BEACON_JEV_ENDPOINT")),- APIKey: firstNonEmpty(memoryOpts.jevAPIKey, os.Getenv("BEACON_JEV_API_KEY")),+ Endpoint: firstNonEmpty(memoryOpts.jevEndpoint, os.Getenv("BEACON_JEV_ENDPOINT"), learning.DefaultJevEndpoint),+ APIKey: firstNonEmpty(memoryOpts.jevAPIKey, os.Getenv("TYPESAFE_API_KEY"), os.Getenv("BEACON_JEV_API_KEY")),+ Model: firstNonEmpty(memoryOpts.jevModel, os.Getenv("BEACON_JEV_MODEL"), learning.DefaultJevModel),CostPerTrace: memoryOpts.jevCost,Timeout: memoryOpts.timeout,}diff --git a/cli/beacon/cmd/memory_test.go b/cli/beacon/cmd/memory_test.goindex b5e9a0b1..77a9a667 100644--- a/cli/beacon/cmd/memory_test.go+++ b/cli/beacon/cmd/memory_test.go@@ -77,6 +77,7 @@ func TestMemoryEvaluationsRunPersistsMockedJevResult(t *testing.T) {memoryOpts.jsonOutput = truememoryOpts.jevEndpoint = server.URLmemoryOpts.jevAPIKey = "test-key"+ memoryOpts.jevModel = "jev-test"memoryOpts.jevCost = learning.DefaultCostPerTracevar out bytes.Buffer@@ -160,8 +161,9 @@ func resetMemoryOpts(t *testing.T) {dryRun booljevEndpoint stringjevAPIKey string+ jevModel stringjevCost float64timeout time.Duration- }{userMode: true, limit: 25, page: 1, jevCost: learning.DefaultCostPerTrace}+ }{userMode: true, limit: 25, page: 1, jevEndpoint: learning.DefaultJevEndpoint, jevModel: learning.DefaultJevModel, jevCost: learning.DefaultCostPerTrace}t.Cleanup(func() { memoryOpts = previous })}diff --git a/cli/beacon/internal/learning/evaluator.go b/cli/beacon/internal/learning/evaluator.goindex 703b0e65..1241e3c0 100644--- a/cli/beacon/internal/learning/evaluator.go+++ b/cli/beacon/internal/learning/evaluator.go@@ -17,6 +17,8 @@ import (const (RubricVersion = "beacon.learning.rubric.v1"+ DefaultJevEndpoint = "https://api.typesafe.ai/v1/systemone"+ DefaultJevModel = "jev-latest"DefaultCostPerTrace = 0.00035maxProjectionEvents = 80maxProjectionText = 1200@@ -31,6 +33,7 @@ var RubricQuestions = []asymptoteobserve.LearningEvaluationQuestionV1{type EvaluatorOptions struct {Endpoint stringAPIKey string+ Model stringHTTPClient *http.ClientCostPerTrace float64Timeout time.Duration@@ -57,7 +60,7 @@ type Projection struct {Harness string `json:"harness,omitempty"`Repository string `json:"repository,omitempty"`Events []ProjectedEvent `json:"events"`- Questions []ProjectedQuestion `json:"questions"`+ Questions []ProjectedQuestion `json:"-"`}type ProjectedEvent struct {@@ -75,18 +78,38 @@ type ProjectedQuestion struct {}type jevRequest struct {- RubricVersion string `json:"rubric_version"`- RubricHash string `json:"rubric_hash"`- Trace Projection `json:"trace"`+ Model string `json:"model,omitempty"`+ State map[string]interface{} `json:"state"`+ Questions map[string]jevQuestion `json:"questions"`}type jevResponse struct {- Questions []asymptoteobserve.LearningEvaluationQuestionV1 `json:"questions"`- Results []asymptoteobserve.LearningEvaluationQuestionV1 `json:"results"`+ Model string `json:"model,omitempty"`+ Answers map[string]jevAnswer `json:"answers"`+ Usage *asymptoteobserve.LearningEvaluationUsageV1 `json:"usage,omitempty"`+ // Compatibility with tests or local shims that still return the earlier internal shape.+ Questions []asymptoteobserve.LearningEvaluationQuestionV1 `json:"questions,omitempty"`+ Results []asymptoteobserve.LearningEvaluationQuestionV1 `json:"results,omitempty"`Score *float64 `json:"score,omitempty"`CostEstimateUSD *float64 `json:"cost_estimate_usd,omitempty"`}+type jevQuestion struct {+ Type string `json:"type"`+ Instructions string `json:"instructions"`+ Criteria map[string]string `json:"criteria,omitempty"`+}++type jevAnswer struct {+ Type string `json:"type,omitempty"`+ Noul *float64 `json:"noul,omitempty"`+ Probability *float64 `json:"probability,omitempty"`+ Choice string `json:"choice,omitempty"`+ Score *float64 `json:"score,omitempty"`+ Confidence float64 `json:"confidence,omitempty"`+ Probabilities map[string]float64 `json:"probabilities,omitempty"`+}+func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput) (asymptoteobserve.LearningEvaluationV1, error) {projection := BuildProjection(input.Trace)evaluation := baseEvaluation(opts, input, projection)@@ -97,7 +120,7 @@ func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput)return evaluation, nil}if strings.TrimSpace(opts.Endpoint) == "" {- return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("Jev endpoint is required; pass --jev-endpoint or set BEACON_JEV_ENDPOINT")+ opts.Endpoint = DefaultJevEndpoint}resp, err := callEvaluator(ctx, opts, projection)if err != nil {@@ -106,21 +129,33 @@ func Evaluate(ctx context.Context, opts EvaluatorOptions, input EvaluationInput)evaluation.ID = EvaluationID(evaluation)return evaluation, err}- questions := resp.Questions+ questions := questionsFromJevAnswers(resp.Answers)+ if len(questions) == 0 {+ questions = resp.Questions+ }if len(questions) == 0 {questions = resp.Results}if len(questions) == 0 {- return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("Jev response did not include questions or results")+ return asymptoteobserve.LearningEvaluationV1{}, fmt.Errorf("Jev response did not include answers, questions, or results")}evaluation.Questions = normalizeQuestionResults(questions)evaluation.Score = evaluationScore(evaluation.Questions)if resp.Score != nil {evaluation.Score = *resp.Score}+ if resp.Model != "" {+ evaluation.EvaluatorModel = resp.Model+ }+ if resp.Usage != nil {+ evaluation.Usage = resp.Usage+ if resp.Usage.CostUSD > 0 {+ evaluation.CostEstimateUSD = resp.Usage.CostUSD+ }+ }if resp.CostEstimateUSD != nil {evaluation.CostEstimateUSD = *resp.CostEstimateUSD- } else {+ } else if evaluation.CostEstimateUSD == 0 {evaluation.CostEstimateUSD = costPerTrace(opts)}evaluation.Status = asymptoteobserve.LearningEvaluationStatusCompleted@@ -183,6 +218,7 @@ func RubricHash() string {func baseEvaluation(opts EvaluatorOptions, input EvaluationInput, projection Projection) asymptoteobserve.LearningEvaluationV1 {trace := input.Trace.Trace+ now := time.Now().UTC().Format(time.RFC3339Nano)ref := asymptoteobserve.LearningTraceRefV1{ID: trace.ID,Title: trace.Title,@@ -196,10 +232,13 @@ func baseEvaluation(opts EvaluatorOptions, input EvaluationInput, projection Proreturn asymptoteobserve.LearningEvaluationV1{SchemaVersion: asymptoteobserve.LearningSchemaVersion,Status: asymptoteobserve.LearningEvaluationStatusCompleted,+ CreatedAt: now,+ UpdatedAt: now,Project: input.Project,RubricVersion: RubricVersion,RubricHash: RubricHash(),Evaluator: evaluatorName(opts),+ EvaluatorModel: modelName(opts),DryRun: input.DryRun,Trace: ref,CostEstimateUSD: costPerTrace(opts),@@ -218,7 +257,15 @@ func callEvaluator(ctx context.Context, opts EvaluatorOptions, projection Projec}ctx, cancel := context.WithTimeout(ctx, timeout)defer cancel()- body, err := json.Marshal(jevRequest{RubricVersion: RubricVersion, RubricHash: RubricHash(), Trace: projection})+ body, err := json.Marshal(jevRequest{+ Model: modelName(opts),+ State: map[string]interface{}{+ "trace": projection,+ "rubric_version": RubricVersion,+ "rubric_hash": RubricHash(),+ },+ Questions: jevQuestions(),+ })if err != nil {return jevResponse{}, err}@@ -274,6 +321,51 @@ func normalizeQuestionResults(results []asymptoteobserve.LearningEvaluationQuestreturn out}+func jevQuestions() map[string]jevQuestion {+ out := make(map[string]jevQuestion, len(RubricQuestions))+ for _, question := range RubricQuestions {+ out[question.ID] = jevQuestion{+ Type: "noul",+ Instructions: question.Prompt,+ Criteria: map[string]string{+ "true": "The trace satisfies this criterion.",+ "false": "The trace does not satisfy this criterion.",+ },+ }+ }+ return out+}++func questionsFromJevAnswers(answers map[string]jevAnswer) []asymptoteobserve.LearningEvaluationQuestionV1 {+ if len(answers) == 0 {+ return nil+ }+ out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(RubricQuestions))+ for _, question := range RubricQuestions {+ answer, ok := answers[question.ID]+ if !ok {+ continue+ }+ probability := 0.0+ switch {+ case answer.Noul != nil:+ probability = *answer.Noul+ case answer.Probability != nil:+ probability = *answer.Probability+ case answer.Score != nil:+ probability = *answer.Score+ }+ out = append(out, asymptoteobserve.LearningEvaluationQuestionV1{+ ID: question.ID,+ Prompt: question.Prompt,+ Probability: probability,+ Confidence: answer.Confidence,+ Reason: answer.Type,+ })+ }+ return out+}+func projectionQuestionDefaults(projection Projection) []asymptoteobserve.LearningEvaluationQuestionV1 {out := make([]asymptoteobserve.LearningEvaluationQuestionV1, 0, len(projection.Questions))for _, question := range projection.Questions {@@ -295,11 +387,18 @@ func evaluationScore(results []asymptoteobserve.LearningEvaluationQuestionV1) flfunc evaluatorName(opts EvaluatorOptions) string {if opts.Endpoint == "" {- return "jev"+ return "jev:" + DefaultJevEndpoint}return "jev:" + opts.Endpoint}+func modelName(opts EvaluatorOptions) string {+ if strings.TrimSpace(opts.Model) != "" {+ return strings.TrimSpace(opts.Model)+ }+ return DefaultJevModel+}+func costPerTrace(opts EvaluatorOptions) float64 {if opts.CostPerTrace > 0 {return opts.CostPerTracediff --git a/cli/beacon/internal/learning/evaluator_test.go b/cli/beacon/internal/learning/evaluator_test.goindex 1a553fbf..43c7b663 100644--- a/cli/beacon/internal/learning/evaluator_test.go+++ b/cli/beacon/internal/learning/evaluator_test.go@@ -23,9 +23,12 @@ func TestEvaluateDryRunDoesNotNeedEndpoint(t *testing.T) {if eval.ID == "" || eval.RubricHash == "" || eval.CostEstimateUSD == 0 {t.Fatalf("dry-run missing identifiers/cost: %#v", eval)}+ if eval.EvaluatorModel != DefaultJevModel {+ t.Fatalf("model = %q", eval.EvaluatorModel)+ }}-func TestEvaluatePostsProjectionToMockableEndpoint(t *testing.T) {+func TestEvaluatePostsSystemOneRequestToMockableEndpoint(t *testing.T) {var sawAuth boolvar sawTrace boolserver := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {@@ -36,18 +39,24 @@ func TestEvaluatePostsProjectionToMockableEndpoint(t *testing.T) {if err := json.NewDecoder(r.Body).Decode(&req); err != nil {t.Fatalf("decode request: %v", err)}- sawTrace = req.Trace.TraceID == "trace-1" && len(req.Trace.Questions) == len(RubricQuestions)+ trace, _ := req.State["trace"].(map[string]interface{})+ sawTrace = req.Model == "jev-test" && trace["trace_id"] == "trace-1" && len(req.Questions) == len(RubricQuestions)+ nine := 0.9+ seven := 0.7+ eight := 0.8_ = json.NewEncoder(w).Encode(jevResponse{- Questions: []asymptoteobserve.LearningEvaluationQuestionV1{- {ID: "task_success", Probability: 0.9, Confidence: 0.8},- {ID: "reusable_correction", Probability: 0.7, Confidence: 0.6},- {ID: "evidence_supported", Probability: 0.8, Confidence: 0.9},+ Model: "jev-test-resolved",+ Answers: map[string]jevAnswer{+ "task_success": {Type: "noul", Noul: &nine},+ "reusable_correction": {Type: "noul", Noul: &seven},+ "evidence_supported": {Type: "noul", Noul: &eight},},+ Usage: &asymptoteobserve.LearningEvaluationUsageV1{InputTokens: 100, CostUSD: 0.0001},})}))defer server.Close()- eval, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL, APIKey: "secret"}, EvaluationInput{+ eval, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL, APIKey: "secret", Model: "jev-test"}, EvaluationInput{Project: asymptoteobserve.LearningProjectV1{ID: "project-1"},Trace: testTraceShow(),})@@ -63,6 +72,9 @@ func TestEvaluatePostsProjectionToMockableEndpoint(t *testing.T) {if eval.Score <= 0.79 || eval.Score >= 0.81 {t.Fatalf("score = %f", eval.Score)}+ if eval.EvaluatorModel != "jev-test-resolved" || eval.Usage == nil || eval.CostEstimateUSD != 0.0001 {+ t.Fatalf("model/usage not retained: %#v", eval)+ }}func TestProjectionRedactsAndBoundsContent(t *testing.T) {@@ -81,9 +93,13 @@ func TestProjectionRedactsAndBoundsContent(t *testing.T) {}}-func TestEvaluateRequiresEndpointOutsideDryRun(t *testing.T) {- _, err := Evaluate(t.Context(), EvaluatorOptions{}, EvaluationInput{Project: asymptoteobserve.LearningProjectV1{ID: "p"}, Trace: testTraceShow()})- if err == nil || !strings.Contains(err.Error(), "Jev endpoint is required") {+func TestEvaluateRequiresAnswersOutsideDryRun(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, _ = w.Write([]byte(`{"answers":{}}`))+ }))+ defer server.Close()+ _, err := Evaluate(t.Context(), EvaluatorOptions{Endpoint: server.URL}, EvaluationInput{Project: asymptoteobserve.LearningProjectV1{ID: "p"}, Trace: testTraceShow()})+ if err == nil || !strings.Contains(err.Error(), "answers") {t.Fatalf("error = %v", err)}}diff --git a/pkg/asymptoteobserve/learning.go b/pkg/asymptoteobserve/learning.goindex ed03573e..b95dd08b 100644--- a/pkg/asymptoteobserve/learning.go+++ b/pkg/asymptoteobserve/learning.go@@ -57,14 +57,23 @@ type LearningEvaluationV1 struct {RubricVersion string `json:"rubric_version"`RubricHash string `json:"rubric_hash"`Evaluator string `json:"evaluator"`+ EvaluatorModel string `json:"evaluator_model,omitempty"`DryRun bool `json:"dry_run,omitempty"`Trace LearningTraceRefV1 `json:"trace"`Questions []LearningEvaluationQuestionV1 `json:"questions,omitempty"`Score float64 `json:"score,omitempty"`CostEstimateUSD float64 `json:"cost_estimate_usd,omitempty"`+ Usage *LearningEvaluationUsageV1 `json:"usage,omitempty"`Error string `json:"error,omitempty"`}+type LearningEvaluationUsageV1 struct {+ InputTokens int64 `json:"input_tokens,omitempty"`+ OutputTokens int64 `json:"output_tokens,omitempty"`+ CostUSD float64 `json:"cost_usd,omitempty"`+ CreditsRemainingUSD float64 `json:"credits_remaining_usd,omitempty"`+}+type LearningEvidenceV1 struct {TraceID string `json:"trace_id"`EventIDs []string `json:"event_ids,omitempty"`