Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(claude): record the final assistant reply from the Stop hook

Claude9ea5dd242fc5
diff --git a/cli/beacon-hooks/cmd/assistant_response.go b/cli/beacon-hooks/cmd/assistant_response.go
new file mode 100644
index 00000000..4f38be49
--- /dev/null
+++ b/cli/beacon-hooks/cmd/assistant_response.go
@@ -0,0 +1,65 @@
+package cmd
+
+import (
+ "strings"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/logging"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+// lastAssistantMessagePlatforms names the runtimes whose Stop payload documents the agent's final
+// reply as `last_assistant_message`.
+//
+// Claude Code added the field so a hook would not have to read the transcript, which is written
+// asynchronously and can be missing the turn's last message when Stop fires; reading it here is
+// race-free where the transcript is not. A list rather than every Claude-shaped runtime: Devin CLI
+// and Factory Droid send Claude-shaped Stop payloads but document no such field, and a key read
+// for a runtime that never wrote it is a guess about what it would mean there.
+var lastAssistantMessagePlatforms = map[string]bool{
+ "claude": true,
+}
+
+// stopAssistantResponse is the agent's final reply as this platform's Stop payload reports it, or
+// "" when the payload carries none: a turn that ended on a tool call, an interrupt, or a runtime
+// that reports no reply at all.
+func stopAssistantResponse(platform string, input map[string]interface{}) string {
+ switch {
+ case platform == kiroPlatform:
+ return kiroAssistantResponse(input)
+ case lastAssistantMessagePlatforms[platform]:
+ return getFirstStr(input, "last_assistant_message")
+ default:
+ return ""
+ }
+}
+
+// emitStopAssistantResponse records the reply a Stop payload carries, if any.
+//
+// The shared stop command records that a turn finished. Without this, what the agent said at the
+// end of it -- the half of a session prompt.submitted does not cover -- reached the log only when
+// someone later ran the runtime's session sync, and never on a runtime without one.
+func emitStopAssistantResponse(logger *logging.Logger, input map[string]interface{}, sessionID string) {
+ emitAssistantMessage(logger, input, sessionFields(sessionID, input), stopAssistantResponse(platformFlag, input))
+}
+
+// emitAssistantMessage writes one `agent.message` event carrying text as the assistant's reply.
+//
+// Every hook-side capture of a reply goes through here so they share one shape: the text in the
+// OTel GenAI output-messages form with the retention marker beside it, exactly as agent-thought
+// writes reasoning, under the action the session-store mappers already use for the same text. A
+// reader that understands one capture path understands all of them. Empty or blank text writes
+// nothing: an empty agent.message would assert the agent answered with nothing, which is different
+// from the runtime not reporting an answer.
+func emitAssistantMessage(logger *logging.Logger, input, fields map[string]interface{}, text string) {
+ if strings.TrimSpace(text) == "" {
+ return
+ }
+ if fields == nil {
+ fields = map[string]interface{}{}
+ }
+ fields["gen_ai"] = mergeNested(fields["gen_ai"], map[string]interface{}{
+ "output": map[string]interface{}{"messages": asymptoteobserve.TextOutputMessages(text)},
+ })
+ fields["content"] = retainedContentFields(text)
+ emitHookEvent(logger, "agent.message", "session", "info", "Agent response captured", input, fields)
+}
diff --git a/cli/beacon-hooks/cmd/assistant_response_test.go b/cli/beacon-hooks/cmd/assistant_response_test.go
new file mode 100644
index 00000000..9fad2b26
--- /dev/null
+++ b/cli/beacon-hooks/cmd/assistant_response_test.go
@@ -0,0 +1,267 @@
+package cmd
+
+import (
+ "bytes"
+ "encoding/json"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+// hookProcessArgsEnv carries the command line TestHookProcess runs, as a JSON array.
+const hookProcessArgsEnv = "BEACON_HOOKS_TEST_PROCESS_ARGS"
+
+// TestHookProcess is not a test. runHookProcess re-executes the test binary into it to run one hook
+// command end to end, because the hook commands end with outputJSONAndExit and cannot be called
+// in-process without ending the test run.
+func TestHookProcess(t *testing.T) {
+ raw := os.Getenv(hookProcessArgsEnv)
+ if raw == "" {
+ t.Skip("helper process for runHookProcess")
+ }
+ var args []string
+ if err := json.Unmarshal([]byte(raw), &args); err != nil {
+ os.Exit(3)
+ }
+ rootCmd.SetArgs(args)
+ if err := rootCmd.Execute(); err != nil {
+ os.Exit(4)
+ }
+ os.Exit(0)
+}
+
+// runHookProcess runs `beacon-hooks <args...>` in a child process with input on stdin, the way a
+// runtime invokes it, and returns the endpoint log it wrote and what it printed. The child gets a
+// home directory of its own, so no hook state or session record reaches the real one.
+func runHookProcess(t *testing.T, input map[string]interface{}, args ...string) (logPath string, stdout []byte) {
+ t.Helper()
+ dir := t.TempDir()
+ logPath = filepath.Join(dir, "runtime.jsonl")
+ home := filepath.Join(dir, "home")
+ if err := os.MkdirAll(home, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ encodedArgs, err := json.Marshal(args)
+ if err != nil {
+ t.Fatal(err)
+ }
+ stdin, err := json.Marshal(input)
+ if err != nil {
+ t.Fatal(err)
+ }
+ cmd := exec.Command(os.Args[0], "-test.run=^TestHookProcess$")
+ cmd.Env = append(os.Environ(),
+ hookProcessArgsEnv+"="+string(encodedArgs),
+ "HOME="+home,
+ "USERPROFILE="+home,
+ "BEACON_ENDPOINT_MODE=1",
+ "BEACON_ENDPOINT_LOG="+logPath,
+ "BEACON_DISABLE_GIT_METADATA=1",
+ )
+ cmd.Stdin = bytes.NewReader(stdin)
+ var out, errOut bytes.Buffer
+ cmd.Stdout = &out
+ cmd.Stderr = &errOut
+ if err := cmd.Run(); err != nil {
+ t.Fatalf("beacon-hooks %v: %v\nstderr: %s", args, err, errOut.String())
+ }
+ return logPath, out.Bytes()
+}
+
+// assistantMessages returns the agent.message events in a log, in order.
+func assistantMessages(t *testing.T, logPath string) []map[string]interface{} {
+ t.Helper()
+ var out []map[string]interface{}
+ for _, event := range endpointEvents(t, logPath) {
+ if leaf(event, "event", "action") == "agent.message" {
+ out = append(out, event)
+ }
+ }
+ return out
+}
+
+// outputText is the assistant text an event carries in gen_ai.output.messages, read with the same
+// reader the trace projection uses.
+func outputText(t *testing.T, event map[string]interface{}) string {
+ t.Helper()
+ genAI, _ := event["gen_ai"].(map[string]interface{})
+ output, _ := genAI["output"].(map[string]interface{})
+ return strings.Join(asymptoteobserve.GenAIText(output["messages"], asymptoteobserve.RoleAssistant, asymptoteobserve.GenAIPartTypeText), "\n\n")
+}
+
+// chatView projects a log the way the dashboard and beacon share do, and returns its messages.
+func chatView(t *testing.T, logPath string) []asymptoteobserve.TraceMessageV1 {
+ t.Helper()
+ data, err := os.ReadFile(logPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var bundle asymptoteobserve.TraceBundleV1
+ for i, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
+ var event asymptoteobserve.Event
+ if err := json.Unmarshal([]byte(line), &event); err != nil {
+ t.Fatalf("decode log line %d: %v", i+1, err)
+ }
+ bundle.Events = append(bundle.Events, asymptoteobserve.TraceEventFromEvent(event, "", i+1))
+ }
+ return asymptoteobserve.TraceMessages(bundle)
+}
+
+// claudeStopPayload is Claude Code's documented Stop input.
+func claudeStopPayload(lastMessage interface{}) map[string]interface{} {
+ payload := map[string]interface{}{
+ "session_id": "claude-stop-1",
+ "prompt_id": "prompt-7",
+ "transcript_path": "/nonexistent/claude-stop-1.jsonl",
+ "cwd": "/repo",
+ "permission_mode": "default",
+ "hook_event_name": "Stop",
+ "stop_hook_active": false,
+ }
+ if lastMessage != nil {
+ payload["last_assistant_message"] = lastMessage
+ }
+ return payload
+}
+
+// Claude Code's Stop payload carries the turn's final reply. Before this, the hook recorded that the
+// turn ended and dropped what was said, so a live Claude Code session had prompts and tool calls
+// but no answers until someone ran `beacon endpoint claude sync`.
+func TestClaudeStopRecordsTheFinalAssistantMessage(t *testing.T) {
+ reply := "I've completed the refactoring.\n\nThe tests pass."
+ logPath, stdout := runHookProcess(t, claudeStopPayload(reply), "--platform", "claude", "stop")
+
+ if strings.TrimSpace(string(stdout)) != "{}" {
+ t.Fatalf("stop printed %q, want the {} no-op reply", stdout)
+ }
+ events := endpointEvents(t, logPath)
+ messages := assistantMessages(t, logPath)
+ if len(messages) != 1 {
+ t.Fatalf("got %d agent.message events, want 1: %#v", len(messages), events)
+ }
+ message := messages[0]
+ if got := outputText(t, message); got != reply {
+ t.Fatalf("gen_ai.output.messages text = %q, want the reply verbatim", got)
+ }
+ for _, check := range []struct {
+ path []string
+ want string
+ }{
+ {[]string{"event", "category"}, "session"},
+ {[]string{"event", "fidelity"}, asymptoteobserve.FidelityObserved},
+ {[]string{"harness", "name"}, "claude_code"},
+ {[]string{"harness", "collection_method"}, asymptoteobserve.CollectionMethodHook},
+ {[]string{"session", "id"}, "claude-stop-1"},
+ {[]string{"prompt", "id"}, "prompt-7"},
+ } {
+ if got := leaf(message, check.path...); got != check.want {
+ t.Errorf("%s = %q, want %q", strings.Join(check.path, "."), got, check.want)
+ }
+ }
+ content, ok := message["content"].(map[string]interface{})
+ if !ok {
+ t.Fatalf("no content marker: %#v", message)
+ }
+ if content["bytes"] != float64(len(reply)) || content["included"] != true || content["hash"] == "" {
+ t.Fatalf("content marker = %#v, want the reply's size and hash", content)
+ }
+
+ // The reply precedes the turn's completion, because it happened first.
+ if last := events[len(events)-1]; leaf(last, "event", "action") != "tool.completed" {
+ t.Fatalf("last event = %q, want the turn completion after the reply", leaf(last, "event", "action"))
+ }
+
+ // What the dashboard and beacon share show: the reply is an assistant message, not a label.
+ var texts []string
+ for _, m := range chatView(t, logPath) {
+ if m.Role == asymptoteobserve.TraceRoleAssistant && m.Kind == asymptoteobserve.TraceMessageKindAssistant {
+ texts = append(texts, m.Text)
+ }
+ }
+ if len(texts) != 1 || texts[0] != reply {
+ t.Fatalf("chat view assistant text = %q, want exactly the reply", texts)
+ }
+}
+
+// A turn can end without a reply: on a tool call, an interrupt, or a field the runtime left null.
+// That is not an empty answer, so no message event is written -- but the turn's completion still is.
+func TestClaudeStopWithoutAReplyWritesNoMessage(t *testing.T) {
+ for name, value := range map[string]interface{}{
+ "absent": nil,
+ "null": json.RawMessage("null"),
+ "empty": "",
+ "blank": " \n\t",
+ } {
+ t.Run(name, func(t *testing.T) {
+ logPath, _ := runHookProcess(t, claudeStopPayload(value), "--platform", "claude", "stop")
+ if messages := assistantMessages(t, logPath); len(messages) != 0 {
+ t.Fatalf("wrote agent.message for a Stop with no reply: %#v", messages)
+ }
+ if got := leaf(lastEndpointEvent(t, logPath), "event", "action"); got != "tool.completed" {
+ t.Fatalf("last event = %q, want the turn completion", got)
+ }
+ })
+ }
+}
+
+// The reply goes through the same redaction and size limit as every other retained string, and the
+// marker still describes the original, so a reader can tell the stored copy was altered.
+func TestClaudeStopReplyIsRedactedAndBounded(t *testing.T) {
+ secret := "AKIA" + "IOSFODNN7EXAMPLE"
+ reply := "Use key " + secret + " then " + strings.Repeat("x", asymptoteobserve.DefaultStringLimit)
+ logPath, _ := runHookProcess(t, claudeStopPayload(reply), "--platform", "claude", "stop")
+
+ messages := assistantMessages(t, logPath)
+ if len(messages) != 1 {
+ t.Fatalf("got %d agent.message events, want 1", len(messages))
+ }
+ stored := outputText(t, messages[0])
+ if strings.Contains(stored, secret) {
+ t.Fatalf("stored reply kept the secret: %q", stored[:80])
+ }
+ if len(stored) >= len(reply) {
+ t.Fatalf("stored reply is %d bytes, want it bounded below the original %d", len(stored), len(reply))
+ }
+ content, _ := messages[0]["content"].(map[string]interface{})
+ if content["truncated"] != true || content["redacted"] != true || content["bytes"] != float64(len(reply)) {
+ t.Fatalf("content marker = %#v, want truncated, redacted, and the original size", content)
+ }
+}
+
+// last_assistant_message is read only where a runtime documents it. Devin CLI, Factory Droid and
+// Cursor send Stop payloads through the same command, and a key read where the runtime never wrote
+// it would be a guess about what it means there.
+func TestLastAssistantMessageIsReadOnlyWhereDocumented(t *testing.T) {
+ for _, platform := range []string{"devin-cli", "factory", "cursor", "qwen"} {
+ t.Run(platform, func(t *testing.T) {
+ logPath, _ := runHookProcess(t, claudeStopPayload("not this runtime's field"), "--platform", platform, "stop")
+ if messages := assistantMessages(t, logPath); len(messages) != 0 {
+ t.Fatalf("%s recorded agent.message from last_assistant_message: %#v", platform, messages)
+ }
+ })
+ }
+}
+
+// The Kiro reply, which now goes through the shared emitter, still arrives end to end and is still
+// read only from Kiro's own key.
+func TestKiroStopReplyEndToEnd(t *testing.T) {
+ payload := map[string]interface{}{
+ "hook_event_name": "stop",
+ "session_id": "k-e2e",
+ "cwd": "/repo",
+ "assistant_response": "Kiro's reply.",
+ "last_assistant_message": "not Kiro's field",
+ }
+ logPath, stdout := runHookProcess(t, payload, "--platform", kiroPlatform, "stop")
+ if len(bytes.TrimSpace(stdout)) != 0 {
+ t.Fatalf("kiro stop printed %q, want nothing", stdout)
+ }
+ messages := assistantMessages(t, logPath)
+ if len(messages) != 1 || outputText(t, messages[0]) != "Kiro's reply." {
+ t.Fatalf("kiro agent.message = %#v, want one carrying assistant_response", messages)
+ }
+}
diff --git a/cli/beacon-hooks/cmd/kiro.go b/cli/beacon-hooks/cmd/kiro.go
index 2492d18d..bd7c83c5 100644
--- a/cli/beacon-hooks/cmd/kiro.go
+++ b/cli/beacon-hooks/cmd/kiro.go
@@ -6,7 +6,6 @@ import (
"github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/config"
hookdiff "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/diff"
"github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/logging"
- "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
)
// Kiro payload readers and tool taxonomy.
@@ -477,26 +476,11 @@ func kiroAssistantResponse(input map[string]interface{}) string {
// emitKiroAssistantResponse records the agent's final reply as its own event.
//
-// The shared stop command records that a turn finished; it does not record what the agent said,
-// because most runtimes do not tell it. Kiro does, on the same payload, and dropping it would
-// throw away the one piece of model output this runtime puts on a hook -- the half of a session
-// that prompt.submitted only covers the user's side of.
-//
-// Written in the OTel GenAI output-messages shape with the retention marker beside it, exactly as
-// the agent-thought command writes reasoning, so a reader that already understands one understands
-// the other. `agent.message` rather than `agent.reasoning`: this is the reply, not the thinking,
-// and it is the action the fx and Devin Cloud mappers already use for the same thing.
+// Kiro puts the reply on the Stop payload and nowhere else, so dropping it would throw away the one
+// piece of model output this runtime puts on a hook. The event's shape is emitAssistantMessage's,
+// shared with every other hook-side reply capture.
func emitKiroAssistantResponse(logger *logging.Logger, input map[string]interface{}, sessionID string) {
- text := kiroAssistantResponse(input)
- if text == "" {
- return
- }
- fields := sessionFields(sessionID, input)
- fields["gen_ai"] = mergeNested(fields["gen_ai"], map[string]interface{}{
- "output": map[string]interface{}{"messages": asymptoteobserve.TextOutputMessages(text)},
- })
- fields["content"] = retainedContentFields(text)
- emitHookEvent(logger, "agent.message", "session", "info", "Agent response captured", input, fields)
+ emitAssistantMessage(logger, input, sessionFields(sessionID, input), kiroAssistantResponse(input))
}
// applyKiroToolResult attaches what the shared toolFieldsWithResponse cannot reach: the operation a
diff --git a/cli/beacon-hooks/cmd/stop.go b/cli/beacon-hooks/cmd/stop.go
index 55b4c1fc..b319ebc3 100644
--- a/cli/beacon-hooks/cmd/stop.go
+++ b/cli/beacon-hooks/cmd/stop.go
@@ -54,13 +54,11 @@ func runStop(cmd *cobra.Command, args []string) {
logger := newHookLogger("stop", platformFlag, sessionID)
logger.Debug("Stop hook called", "session_id", sessionID, "has_transcript", transcriptPath != "", "platform", platformFlag)
- // Kiro puts the agent's final reply on this payload and nowhere else. Recorded before the
+ // Kiro and Claude Code put the agent's final reply on this payload. Recorded before the
// completion event below and before the sessionID guard, because the reply happened before the
// turn ended and because a payload with no session id still carries real model output --
// dropping it there would lose the message to protect a field it does not need.
- if platformFlag == kiroPlatform {
- emitKiroAssistantResponse(logger, input, sessionID)
- }
+ emitStopAssistantResponse(logger, input, sessionID)
if sessionID == "" {
if isDevinLikePlatform(platformFlag) {
logger.Info("stop completed")
diff --git a/docs/runtimes/claude-code.mdx b/docs/runtimes/claude-code.mdx
index 9218dcef..e6973a89 100644
--- a/docs/runtimes/claude-code.mdx
+++ b/docs/runtimes/claude-code.mdx
@@ -122,6 +122,7 @@ Managed-ingest privacy is a separate forwarding policy and does not switch off l
| Area | Support |
|------|---------|
| Prompt telemetry | Supported when emitted through Claude Code OTLP or hooks |
+| Agent responses | Supported live. The `Stop` hook records the turn's final reply from the payload's `last_assistant_message` as `agent.message`, with the text in `gen_ai.output.messages`; nothing is written for a turn that ended without one. `beacon endpoint claude sync` also recovers every committed assistant message from transcripts |
| Turn identity | Supported. Claude Code's `prompt.id` is kept as `prompt.id` on OTLP events; hook events after the prompt read the same id from the session transcript, and `beacon endpoint claude sync` reads it from `promptId`. The hook-recorded prompt itself takes its id from the OTLP `user_prompt` event, because the hook runs before Claude Code records the prompt |
| Command, tool, and file telemetry | Supported through Claude Code OTLP with detailed tool inputs enabled. Command output requires hooks |
| Token usage and cost | Supported through OTLP token and cost metrics, with per-model and per-session attribution |