Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(traces): retain completed assistant answers and refresh history (#971)

Divyam Talwar09296aaa7944
diff --git a/cli/beacon/internal/endpoint/dashboard/history_store.go b/cli/beacon/internal/endpoint/dashboard/history_store.go
index ce11d90c..da25a2ce 100644
--- a/cli/beacon/internal/endpoint/dashboard/history_store.go
+++ b/cli/beacon/internal/endpoint/dashboard/history_store.go
@@ -43,7 +43,7 @@ const (
// historyDerivationVersion covers everything computed from a stored line: trace keys, event
// numbers, types, summaries and search text. Bump it when that projection changes; the next
// catch-up recomputes it from the stored lines, so history survives the change.
- historyDerivationVersion = 14 // 14: trace labels preserve UTF-8 at byte limits; 2: trace events carry the event's policy block; 3: and its error; 4: usage carries the 1h cache-write split; 5: agent.response is an agent message; 6: the summary names its project; 7-11 were the sharing-staleness steps on a parallel branch; 12: sharing goes stale on activity after what a share covered (shared_through), ignoring a share's own events and events the share projection skips, and carries a refresh command; 13: redaction recognizes more credential shapes, so trace text and search text change
+ historyDerivationVersion = 15 // 15: completed assistant responses and plans retain their answer; 14: trace labels preserve UTF-8 at byte limits; 2: trace events carry the event's policy block; 3: and its error; 4: usage carries the 1h cache-write split; 5: agent.response is an agent message; 6: the summary names its project; 7-11 were the sharing-staleness steps on a parallel branch; 12: sharing goes stale on activity after what a share covered (shared_through), ignoring a share's own events and events the share projection skips, and carries a refresh command; 13: redaction recognizes more credential shapes, so trace text and search text change
defaultHistoryRetentionDays = 90
// maxHistoryRetentionDays is 100 years: a retention this long or longer means nothing expires.
diff --git a/cli/beacon/internal/endpoint/dashboard/history_test.go b/cli/beacon/internal/endpoint/dashboard/history_test.go
index 8c1159dd..7799069f 100644
--- a/cli/beacon/internal/endpoint/dashboard/history_test.go
+++ b/cli/beacon/internal/endpoint/dashboard/history_test.go
@@ -18,6 +18,7 @@ import (
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/writer"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/testenv"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
// Every test in this package runs with the local history pointed at a path that does not exist,
@@ -742,6 +743,89 @@ func TestHistoryRebuildsDerivedDataForANewProjection(t *testing.T) {
}
}
+func TestHistoryRebuildsVersion14AssistantCompletion(t *testing.T) {
+ line := []byte(`{"timestamp":"2026-10-01T10:00:02Z","event":{"id":"completed-answer","action":"agent.response.completed","category":"agent"},"harness":{"name":"chatgpt_web","collection_method":"otlp"},"session":{"id":"browser-session"},"message":"chatgpt_web assistant response completed","model":"gpt-5","gen_ai":{"output":{"messages":[{"role":"assistant","parts":[{"type":"text","content":"A uniquely retained browser answer."}]}]},"usage":{"input_tokens":10,"output_tokens":4}}}`)
+ path := traceStoreBlockedLog(t, [][]byte{line})
+ const traceID = "session:chatgpt_web:browser-session"
+ want, ok, err := ShowTrace(path, traceID, TraceQuery{Limit: 100})
+ if err != nil || !ok {
+ t.Fatalf("fresh ShowTrace ok=%v err=%v", ok, err)
+ }
+ withHistory(t)
+ optIn(t, path)
+ store, err := openHistoryStore(false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var aggregate, storedLine string
+ if err := store.db.QueryRow(`SELECT aggregate FROM traces WHERE trace_key = ?`, traceID).Scan(&aggregate); err != nil {
+ t.Fatal(err)
+ }
+ if err := store.db.QueryRow(`SELECT hex(line) FROM events`).Scan(&storedLine); err != nil {
+ t.Fatal(err)
+ }
+ var prior traceAggregateState
+ if err := json.Unmarshal([]byte(aggregate), &prior); err != nil {
+ t.Fatal(err)
+ }
+ prior.Summary.Title = "agent.response.completed"
+ prior.Summary.Preview = "chatgpt_web assistant response completed"
+ prior.Summary.LocalMessageCount = 0
+ prior.TitlePriority = 0
+ if _, err := store.db.Exec(`UPDATE traces SET aggregate = ?, hay = 'stale completion'`, mustJSON(t, prior)); err != nil {
+ t.Fatal(err)
+ }
+ for _, stmt := range []string{
+ `UPDATE events SET event_type = 'other', hay = 'stale completion'`,
+ `INSERT INTO event_fts (event_fts) VALUES ('rebuild')`,
+ `INSERT INTO trace_fts (trace_fts) VALUES ('rebuild')`,
+ `UPDATE meta SET value = '14' WHERE key = 'derivation_version'`,
+ } {
+ if _, err := store.db.Exec(stmt); err != nil {
+ t.Fatal(err)
+ }
+ }
+ store.close()
+
+ got, ok, err := ShowTrace(path, traceID, TraceQuery{Limit: 100})
+ if err != nil || !ok {
+ t.Fatalf("upgraded ShowTrace ok=%v err=%v", ok, err)
+ }
+ if mustJSON(t, got) != mustJSON(t, want) {
+ t.Fatalf("version 14 projection was not rebuilt\n got %s\nwant %s", mustJSON(t, got), mustJSON(t, want))
+ }
+ if len(got.Events) != 1 || got.Events[0].Type != "agent_message" || got.Events[0].Content == nil || got.Events[0].Content.Text != "A uniquely retained browser answer." || got.Trace.LocalMessageCount != 1 {
+ t.Fatalf("completed answer was not projected: %+v", got)
+ }
+ messages := asymptoteobserve.TraceMessages(TraceBundleV1{Events: got.Events})
+ if len(messages) != 1 || messages[0].Role != "assistant" || messages[0].Text != "A uniquely retained browser answer." {
+ t.Fatalf("completed answer missing from chat: %+v", messages)
+ }
+ search, err := SearchTraces(path, TraceQuery{EventQuery: EventQuery{Q: "uniquely retained browser answer"}, ResultLevel: "event", Limit: 10})
+ if err != nil || search.TotalMatched != 1 {
+ t.Fatalf("completed answer search = %+v err=%v", search, err)
+ }
+ store, err = openHistoryStore(false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer store.close()
+ if got := store.metaInt("derivation_version", 0); got != historyDerivationVersion || got <= 14 {
+ t.Fatalf("derivation version = %d, want current version above 14", got)
+ }
+ var count int
+ var eventType, unchangedLine string
+ if err := store.db.QueryRow(`SELECT count(*), event_type, hex(line) FROM events`).Scan(&count, &eventType, &unchangedLine); err != nil {
+ t.Fatal(err)
+ }
+ if count != 1 || eventType != "agent_message" || unchangedLine != storedLine {
+ t.Fatalf("stored events changed: count=%d type=%q same-line=%v", count, eventType, unchangedLine == storedLine)
+ }
+ if again, ok, err := ShowTrace(path, traceID, TraceQuery{Limit: 100}); err != nil || !ok || mustJSON(t, again) != mustJSON(t, got) {
+ t.Fatalf("unchanged second read differs: ok=%v err=%v", ok, err)
+ }
+}
+
func TestHistoryResetRemovesTheStore(t *testing.T) {
path := traceStoreIndexedLog(t, parityFixture())
if !HistoryStoreEnabled() {
diff --git a/docs/changelog/cli.mdx b/docs/changelog/cli.mdx
index 8eedbf3d..5dc631b0 100644
--- a/docs/changelog/cli.mdx
+++ b/docs/changelog/cli.mdx
@@ -12,6 +12,7 @@ Stay up to date with the latest changes to the Agent Beacon CLI.
## Unreleased
- **Devin CLI session history sync** · `beacon endpoint devin sync` reads Devin CLI's committed SQLite session store locally and read-only into the runtime log, adding what the live hooks cannot see: assistant replies, reasoning, per-message token usage, the per-turn model with model-change events, compaction summaries, and `approval.denied` for rejected tool calls. Sessions the hooks already cover get only those additions, so nothing is counted twice, and the install backfill includes Devin CLI. Verified on macOS; the Linux and Windows paths are best effort. See [Devin CLI](/runtimes/devin#session-database-sync).
+- **Claude.ai and ChatGPT answers show in traces** · Answers the browser extension records as `agent.response.completed` were classified as `other` in trace views, so `beacon endpoint traces show`, the dashboard and shared traces showed the prompt without the answer. They are now assistant messages carrying the answer, as are plans recorded as `agent.plan`. OpenCode's separate completion event, which carries only usage, stays out of the conversation. Local history is rebuilt on its next read, so earlier sessions show their answers too.
- **Dashboard output tokens count reasoning once** · The local dashboard's token view added reasoning tokens to output, but `gen_ai.usage.output_tokens` already includes them, so models that report reasoning had their output and total tokens overstated. Output and the total now count reasoning once, the split bar no longer shows reasoning as a separate segment, and a note below it gives the reasoning count included in output. `beacon token-usage` already counted it this way.
---
diff --git a/pkg/asymptoteobserve/tracebuild.go b/pkg/asymptoteobserve/tracebuild.go
index 5e2cb9bb..b60fca67 100644
--- a/pkg/asymptoteobserve/tracebuild.go
+++ b/pkg/asymptoteobserve/tracebuild.go
@@ -194,8 +194,11 @@ func TraceEventType(event Event) string {
return "user_message"
case action == "agent.reasoning" || strings.Contains(action, "reasoning"):
return "agent_reasoning"
- case action == "agent.message" || action == "assistant.message" || action == "agent.response":
- // OpenCode records the assistant's text as agent.response.
+ case action == "agent.response.completed" && category == "session" && RetainedOutputText(event.GenAI, GenAIPartTypeText) == "":
+ // OpenCode reports the answer separately from its session completion's usage.
+ return "session"
+ case action == "agent.message" || action == "assistant.message" || action == "agent.response" ||
+ action == "agent.response.completed" || action == "agent.plan":
return "agent_message"
case strings.HasPrefix(action, "tool.invoked") || action == "tool.started":
return "tool_call"
@@ -270,6 +273,10 @@ func traceEventContent(event Event) *TraceContentV1 {
if text := RetainedOutputText(event.GenAI, partType); text != "" {
return retainedOutputContent(text, event.Content)
}
+ if strings.EqualFold(event.Event.Action, "agent.response.completed") {
+ // The completion's message is a lifecycle label, not a retained answer.
+ return contentFromContentInfo(event.Content)
+ }
if event.Message != "" {
return contentFromText(event.Message, event.Content)
}
diff --git a/pkg/asymptoteobserve/tracebuild_test.go b/pkg/asymptoteobserve/tracebuild_test.go
index ef559e48..e75a40f0 100644
--- a/pkg/asymptoteobserve/tracebuild_test.go
+++ b/pkg/asymptoteobserve/tracebuild_test.go
@@ -49,6 +49,139 @@ func TestTraceEventFromEvent(t *testing.T) {
}
}
+func TestAssistantActionsRetainTheirAnswerInTraceMessages(t *testing.T) {
+ for _, tc := range []struct{ action, category string }{
+ {"agent.message", "agent"},
+ {"assistant.message", "agent"},
+ {"agent.response", "agent"},
+ {"agent.response.completed", "agent"},
+ {"agent.plan", "agent"},
+ {"agent.response.completed", "session"},
+ } {
+ t.Run(tc.action+"/"+tc.category, func(t *testing.T) {
+ event := shareEventAt("2026-10-01T10:00:02Z", "chatgpt_web", CollectionMethodOTLP, tc.action, tc.category)
+ event.Event.ID = "answer-event"
+ event.Message = "chatgpt_web assistant response completed"
+ event.Model = "gpt-5"
+ event.GenAI = &GenAIInfo{
+ Output: &GenAIOutputInfo{Messages: []interface{}{
+ map[string]interface{}{"role": RoleAssistant, "parts": []interface{}{
+ map[string]interface{}{"type": GenAIPartTypeText, "content": "The retained answer."},
+ map[string]interface{}{"type": GenAIPartTypeReasoning, "content": "Private reasoning."},
+ }},
+ }},
+ Usage: &GenAIUsageInfo{InputTokens: int64Ptr(10), OutputTokens: int64Ptr(4)},
+ }
+ event.Content = RetainedContent("The retained answer.", DefaultStringLimit)
+ builder := NewTraceBuilder("browser-answer", event)
+ projected := builder.Add(event, TraceEventFromEvent(event, "", 1))
+ if projected.Type != "agent_message" || projected.Actor != "assistant" || projected.Content == nil || projected.Content.Text != "The retained answer." {
+ t.Fatalf("assistant projection = %+v", projected)
+ }
+ if projected.ID != event.Event.ID || projected.Action != tc.action || projected.Summary != event.Message || projected.Model != event.Model ||
+ projected.Usage == nil || projected.Usage.InputTokens != 10 || projected.Usage.OutputTokens != 4 {
+ t.Fatalf("assistant metadata changed = %+v", projected)
+ }
+ bundle := builder.Bundle(TraceProvenanceV1{})
+ messages := TraceMessages(bundle)
+ if len(messages) != 1 || messages[0].Role != TraceRoleAssistant || messages[0].Kind != TraceMessageKindAssistant || messages[0].Text != "The retained answer." {
+ t.Fatalf("assistant messages = %+v", messages)
+ }
+ if messages[0].ID != event.Event.ID || messages[0].Model != event.Model || messages[0].TokenUsage == nil || messages[0].TokenUsage.OutputTokens != 4 {
+ t.Fatalf("message metadata changed = %+v", messages[0])
+ }
+ if bundle.Summary.LocalMessageCount != 1 {
+ t.Fatalf("message count = %d, want 1", bundle.Summary.LocalMessageCount)
+ }
+ })
+ }
+}
+
+func TestTextlessAssistantCompletionDoesNotInventAnAnswer(t *testing.T) {
+ for _, category := range []string{"agent", "session"} {
+ t.Run(category, func(t *testing.T) {
+ event := shareEventAt("2026-10-01T10:00:02Z", "opencode", CollectionMethodPlugin, "agent.response.completed", category)
+ event.Event.ID = "textless-completion"
+ event.Message = "opencode assistant response completed"
+ projected := TraceEventFromEvent(event, "", 1)
+ messages := TraceMessages(TraceBundleV1{Events: []TraceEventV1{projected}})
+ if len(messages) != 0 {
+ t.Fatalf("textless completion invented assistant text: %+v", messages)
+ }
+ })
+ }
+}
+
+func TestOpenCodeCompletionKeepsOneAssistantAnswer(t *testing.T) {
+ prompt := shareEventAt("2026-10-01T10:00:00Z", "opencode", CollectionMethodPlugin, "prompt.submitted", "prompt")
+ prompt.Event.ID = "opencode-prompt"
+ prompt.Prompt = &PromptInfo{Text: "Explain the lookup cache."}
+ response := shareEventAt("2026-10-01T10:00:01Z", "opencode", CollectionMethodPlugin, "agent.response", "session")
+ response.Event.ID = "opencode-answer"
+ response.Message = "opencode assistant response captured"
+ response.GenAI = &GenAIInfo{Output: &GenAIOutputInfo{Messages: TextOutputMessages("It memoizes lookups.")}}
+ completion := shareEventAt("2026-10-01T10:00:02Z", "opencode", CollectionMethodPlugin, "agent.response.completed", "session")
+ completion.Event.ID = "opencode-completion"
+ completion.Message = "opencode assistant response completed"
+ completion.GenAI = &GenAIInfo{Response: &GenAIResponseInfo{ID: "assistant-turn"}, Usage: &GenAIUsageInfo{
+ InputTokens: int64Ptr(10), OutputTokens: int64Ptr(4),
+ }}
+ builder := NewTraceBuilder("opencode-turn", prompt)
+ for i, event := range []Event{prompt, response, completion} {
+ builder.Add(event, TraceEventFromEvent(event, "", i+1))
+ }
+ bundle := builder.Bundle(TraceProvenanceV1{})
+ if bundle.Summary.LocalMessageCount != 2 || bundle.Summary.EventCount != 3 || bundle.Events[2].Type != "session" {
+ t.Fatalf("completion changed message count or lifecycle type: summary=%+v events=%+v", bundle.Summary, bundle.Events)
+ }
+ if bundle.Summary.Title != prompt.Prompt.Text || bundle.Summary.Preview != prompt.Prompt.Text ||
+ bundle.Summary.TokenUsage == nil || bundle.Summary.TokenUsage.InputTokens != 10 || bundle.Summary.TokenUsage.OutputTokens != 4 {
+ t.Fatalf("completion changed title, preview or usage: %+v", bundle.Summary)
+ }
+ messages := TraceMessages(bundle)
+ if len(messages) != 2 || messages[0].Role != TraceRoleUser || messages[1].Role != TraceRoleAssistant || messages[1].Text != "It memoizes lookups." {
+ t.Fatalf("completion changed the conversation: %+v", messages)
+ }
+}
+
+func TestMetadataOnlyBrowserCompletionStillCountsAsAnAnswer(t *testing.T) {
+ event := shareEventAt("2026-10-01T10:00:02Z", "chatgpt_web", CollectionMethodOTLP, "agent.response.completed", "agent")
+ event.Event.ID = "browser-completion"
+ event.Message = "chatgpt_web agent.response.completed"
+ event.Content = &ContentInfo{Retention: ContentRetentionMetadata}
+ builder := NewTraceBuilder("metadata-browser-turn", event)
+ projected := builder.Add(event, TraceEventFromEvent(event, "", 1))
+ bundle := builder.Bundle(TraceProvenanceV1{})
+ if projected.Type != "agent_message" || projected.Actor != "assistant" || bundle.Summary.LocalMessageCount != 1 {
+ t.Fatalf("browser completion lost its assistant identity: event=%+v summary=%+v", projected, bundle.Summary)
+ }
+ if messages := TraceMessages(bundle); len(messages) != 0 {
+ t.Fatalf("metadata-only browser completion invented text: %+v", messages)
+ }
+}
+
+func TestTraceEventTypeKeepsUnrelatedClassifications(t *testing.T) {
+ for _, tc := range []struct{ action, category, want string }{
+ {"prompt.submitted", "prompt", "user_message"},
+ {"agent.reasoning", "agent", "agent_reasoning"},
+ {"tool.invoked", "tool", "tool_call"},
+ {"tool.completed", "tool", "tool_result"},
+ {"command.executed", "command", "command"},
+ {"file.read", "file", "file"},
+ {"metric.observed", "metric", "other"},
+ {"session.ended", "session", "session"},
+ {"agent.unknown", "agent", "other"},
+ {"AGENT.RESPONSE", "agent", "agent_message"},
+ {" agent.response", "agent", "other"},
+ } {
+ t.Run(tc.action, func(t *testing.T) {
+ if got := TraceEventType(Event{Event: EventInfo{Action: tc.action, Category: tc.category}}); got != tc.want {
+ t.Fatalf("type = %q, want %q", got, tc.want)
+ }
+ })
+ }
+}
+
func TestNormalizeTraceEvent(t *testing.T) {
metric := Event{Raw: map[string]interface{}{"metric_name": "claude_code.cost.usage"}}
NormalizeTraceEvent(&metric)