Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(memory): preserve the true tail of long evaluation traces (#980)

Divyam Talwar9f9dad81fafe
diff --git a/cli/beacon/cmd/memory.go b/cli/beacon/cmd/memory.go
index df333f4f..b561840d 100644
--- a/cli/beacon/cmd/memory.go
+++ b/cli/beacon/cmd/memory.go
@@ -471,7 +471,7 @@ func runMemoryCandidatesCreate(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
- show, ok, err := dashboard.ShowTrace(memoryLogPath(), memoryOpts.traceID, dashboard.TraceQuery{Limit: 2000})
+ show, ok, err := dashboard.ShowTrace(memoryLogPath(), memoryOpts.traceID, dashboard.TraceQuery{EventQuery: dashboard.EventQuery{NoLimit: true}})
if err != nil {
return err
}
@@ -608,8 +608,9 @@ func runMemorySkillsInstall(cmd *cobra.Command, args []string) error {
// because each fresh sample sorts to the top.
func selectedEvaluationInputs() ([]learning.EvaluationInput, error) {
logPath := memoryLogPath()
+ // The evaluator bounds its projection after sampling the actual head and tail.
if memoryOpts.traceID != "" {
- show, ok, err := dashboard.ShowTrace(logPath, memoryOpts.traceID, dashboard.TraceQuery{Limit: 2000})
+ show, ok, err := dashboard.ShowTrace(logPath, memoryOpts.traceID, dashboard.TraceQuery{EventQuery: dashboard.EventQuery{NoLimit: true}})
if err != nil {
return nil, err
}
@@ -646,7 +647,7 @@ func selectedEvaluationInputs() ([]learning.EvaluationInput, error) {
if strings.HasPrefix(summary.ID, "event:") {
continue
}
- show, ok, err := dashboard.ShowTrace(logPath, summary.ID, dashboard.TraceQuery{Limit: 2000})
+ show, ok, err := dashboard.ShowTrace(logPath, summary.ID, dashboard.TraceQuery{EventQuery: dashboard.EventQuery{NoLimit: true}})
if err != nil {
return nil, err
}
diff --git a/cli/beacon/cmd/memory_evaluation_trace_test.go b/cli/beacon/cmd/memory_evaluation_trace_test.go
new file mode 100644
index 00000000..81fd3c14
--- /dev/null
+++ b/cli/beacon/cmd/memory_evaluation_trace_test.go
@@ -0,0 +1,101 @@
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/learning"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+// A trace longer than 2,000 events must reach memory with its real final event.
+func TestMemoryReadsTheEndOfLongTraces(t *testing.T) {
+ const count = 2001
+ const finalEventID = "evaluation-session-event-2000"
+ for _, route := range []string{"evaluate --trace", "evaluate list", "candidates create"} {
+ t.Run(route, func(t *testing.T) {
+ logPath, project := writeEvaluationTraceFixture(t, count)
+ resetMemoryOpts(t)
+ memoryOpts.logPath, memoryOpts.projectPath = logPath, project
+ if route != "evaluate list" {
+ memoryOpts.traceID = "session:cursor:evaluation-session"
+ }
+ if route == "candidates create" {
+ memoryOpts.kind = asymptoteobserve.LearningMemoryKindDebuggingPattern
+ memoryOpts.title = "Long trace lesson"
+ memoryOpts.bodyFile = "-"
+ memoryOpts.jsonOutput = true
+ var out bytes.Buffer
+ cmd := &cobra.Command{}
+ cmd.SetOut(&out)
+ cmd.SetIn(strings.NewReader("Lesson body.\n"))
+ if err := runMemoryCandidatesCreate(cmd, nil); err != nil {
+ t.Fatal(err)
+ }
+ var candidate asymptoteobserve.LearningCandidateV1
+ if err := json.Unmarshal(out.Bytes(), &candidate); err != nil {
+ t.Fatal(err)
+ }
+ ids := candidate.Evidence[0].EventIDs
+ if len(ids) != count || ids[len(ids)-1] != finalEventID {
+ t.Fatalf("evidence has %d events ending %q, want %d ending %q", len(ids), ids[len(ids)-1], count, finalEventID)
+ }
+ return
+ }
+ inputs, err := selectedEvaluationInputs()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(inputs) != 1 {
+ t.Fatalf("selected %d traces, want one", len(inputs))
+ }
+ projection := learning.BuildProjection(inputs[0].Trace)
+ last := projection.Events[len(projection.Events)-1]
+ if last.Number != count || last.Content != "Final outcome: all tests passed." {
+ t.Fatalf("last projected event = %+v, want event %d with the final answer", last, count)
+ }
+ })
+ }
+}
+
+func writeEvaluationTraceFixture(t *testing.T, count int) (string, string) {
+ t.Helper()
+ dir := t.TempDir()
+ logPath := filepath.Join(dir, "runtime.jsonl")
+ var lines bytes.Buffer
+ for i := range count {
+ event := schema.Event{
+ Timestamp: time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC).Add(time.Duration(i) * time.Second).Format(time.RFC3339),
+ Vendor: schema.Vendor, Product: schema.Product, SchemaVersion: schema.SchemaVersion,
+ Event: schema.EventInfo{ID: fmt.Sprintf("evaluation-session-event-%d", i), Kind: "agent_runtime", Action: "prompt.submitted", Category: "prompt"},
+ Severity: schema.SeverityInfo,
+ Harness: schema.HarnessInfo{Name: "cursor"},
+ Session: &schema.SessionInfo{ID: "evaluation-session", WorkingDirectory: dir},
+ Repository: dir,
+ Prompt: &schema.PromptInfo{Text: "Evaluation target task"},
+ Message: "Prompt submitted",
+ }
+ if i == count-1 {
+ event.Event.Action, event.Event.Category = "agent.message", "agent"
+ event.Prompt = nil
+ event.GenAI = &schema.GenAIInfo{Output: &schema.GenAIOutputInfo{Messages: asymptoteobserve.TextOutputMessages("Final outcome: all tests passed.")}}
+ event.Message = "Agent response"
+ }
+ if err := json.NewEncoder(&lines).Encode(event); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := os.WriteFile(logPath, lines.Bytes(), 0600); err != nil {
+ t.Fatal(err)
+ }
+ return logPath, dir
+}