The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(claude): record the final assistant reply from the Stop hook
Claude9ea5dd242fc5
diff --git a/cli/beacon-hooks/cmd/assistant_response.go b/cli/beacon-hooks/cmd/assistant_response.gonew file mode 100644index 00000000..4f38be49--- /dev/null+++ b/cli/beacon-hooks/cmd/assistant_response.go@@ -0,0 +1,65 @@+package cmd++import (+ "strings"++ "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/logging"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// lastAssistantMessagePlatforms names the runtimes whose Stop payload documents the agent's final+// reply as `last_assistant_message`.+//+// Claude Code added the field so a hook would not have to read the transcript, which is written+// asynchronously and can be missing the turn's last message when Stop fires; reading it here is+// race-free where the transcript is not. A list rather than every Claude-shaped runtime: Devin CLI+// and Factory Droid send Claude-shaped Stop payloads but document no such field, and a key read+// for a runtime that never wrote it is a guess about what it would mean there.+var lastAssistantMessagePlatforms = map[string]bool{+ "claude": true,+}++// stopAssistantResponse is the agent's final reply as this platform's Stop payload reports it, or+// "" when the payload carries none: a turn that ended on a tool call, an interrupt, or a runtime+// that reports no reply at all.+func stopAssistantResponse(platform string, input map[string]interface{}) string {+ switch {+ case platform == kiroPlatform:+ return kiroAssistantResponse(input)+ case lastAssistantMessagePlatforms[platform]:+ return getFirstStr(input, "last_assistant_message")+ default:+ return ""+ }+}++// emitStopAssistantResponse records the reply a Stop payload carries, if any.+//+// The shared stop command records that a turn finished. Without this, what the agent said at the+// end of it -- the half of a session prompt.submitted does not cover -- reached the log only when+// someone later ran the runtime's session sync, and never on a runtime without one.+func emitStopAssistantResponse(logger *logging.Logger, input map[string]interface{}, sessionID string) {+ emitAssistantMessage(logger, input, sessionFields(sessionID, input), stopAssistantResponse(platformFlag, input))+}++// emitAssistantMessage writes one `agent.message` event carrying text as the assistant's reply.+//+// Every hook-side capture of a reply goes through here so they share one shape: the text in the+// OTel GenAI output-messages form with the retention marker beside it, exactly as agent-thought+// writes reasoning, under the action the session-store mappers already use for the same text. A+// reader that understands one capture path understands all of them. Empty or blank text writes+// nothing: an empty agent.message would assert the agent answered with nothing, which is different+// from the runtime not reporting an answer.+func emitAssistantMessage(logger *logging.Logger, input, fields map[string]interface{}, text string) {+ if strings.TrimSpace(text) == "" {+ return+ }+ if fields == nil {+ fields = map[string]interface{}{}+ }+ fields["gen_ai"] = mergeNested(fields["gen_ai"], map[string]interface{}{+ "output": map[string]interface{}{"messages": asymptoteobserve.TextOutputMessages(text)},+ })+ fields["content"] = retainedContentFields(text)+ emitHookEvent(logger, "agent.message", "session", "info", "Agent response captured", input, fields)+}diff --git a/cli/beacon-hooks/cmd/assistant_response_test.go b/cli/beacon-hooks/cmd/assistant_response_test.gonew file mode 100644index 00000000..9fad2b26--- /dev/null+++ b/cli/beacon-hooks/cmd/assistant_response_test.go@@ -0,0 +1,267 @@+package cmd++import (+ "bytes"+ "encoding/json"+ "os"+ "os/exec"+ "path/filepath"+ "strings"+ "testing"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// hookProcessArgsEnv carries the command line TestHookProcess runs, as a JSON array.+const hookProcessArgsEnv = "BEACON_HOOKS_TEST_PROCESS_ARGS"++// TestHookProcess is not a test. runHookProcess re-executes the test binary into it to run one hook+// command end to end, because the hook commands end with outputJSONAndExit and cannot be called+// in-process without ending the test run.+func TestHookProcess(t *testing.T) {+ raw := os.Getenv(hookProcessArgsEnv)+ if raw == "" {+ t.Skip("helper process for runHookProcess")+ }+ var args []string+ if err := json.Unmarshal([]byte(raw), &args); err != nil {+ os.Exit(3)+ }+ rootCmd.SetArgs(args)+ if err := rootCmd.Execute(); err != nil {+ os.Exit(4)+ }+ os.Exit(0)+}++// runHookProcess runs `beacon-hooks <args...>` in a child process with input on stdin, the way a+// runtime invokes it, and returns the endpoint log it wrote and what it printed. The child gets a+// home directory of its own, so no hook state or session record reaches the real one.+func runHookProcess(t *testing.T, input map[string]interface{}, args ...string) (logPath string, stdout []byte) {+ t.Helper()+ dir := t.TempDir()+ logPath = filepath.Join(dir, "runtime.jsonl")+ home := filepath.Join(dir, "home")+ if err := os.MkdirAll(home, 0o700); err != nil {+ t.Fatal(err)+ }+ encodedArgs, err := json.Marshal(args)+ if err != nil {+ t.Fatal(err)+ }+ stdin, err := json.Marshal(input)+ if err != nil {+ t.Fatal(err)+ }+ cmd := exec.Command(os.Args[0], "-test.run=^TestHookProcess$")+ cmd.Env = append(os.Environ(),+ hookProcessArgsEnv+"="+string(encodedArgs),+ "HOME="+home,+ "USERPROFILE="+home,+ "BEACON_ENDPOINT_MODE=1",+ "BEACON_ENDPOINT_LOG="+logPath,+ "BEACON_DISABLE_GIT_METADATA=1",+ )+ cmd.Stdin = bytes.NewReader(stdin)+ var out, errOut bytes.Buffer+ cmd.Stdout = &out+ cmd.Stderr = &errOut+ if err := cmd.Run(); err != nil {+ t.Fatalf("beacon-hooks %v: %v\nstderr: %s", args, err, errOut.String())+ }+ return logPath, out.Bytes()+}++// assistantMessages returns the agent.message events in a log, in order.+func assistantMessages(t *testing.T, logPath string) []map[string]interface{} {+ t.Helper()+ var out []map[string]interface{}+ for _, event := range endpointEvents(t, logPath) {+ if leaf(event, "event", "action") == "agent.message" {+ out = append(out, event)+ }+ }+ return out+}++// outputText is the assistant text an event carries in gen_ai.output.messages, read with the same+// reader the trace projection uses.+func outputText(t *testing.T, event map[string]interface{}) string {+ t.Helper()+ genAI, _ := event["gen_ai"].(map[string]interface{})+ output, _ := genAI["output"].(map[string]interface{})+ return strings.Join(asymptoteobserve.GenAIText(output["messages"], asymptoteobserve.RoleAssistant, asymptoteobserve.GenAIPartTypeText), "\n\n")+}++// chatView projects a log the way the dashboard and beacon share do, and returns its messages.+func chatView(t *testing.T, logPath string) []asymptoteobserve.TraceMessageV1 {+ t.Helper()+ data, err := os.ReadFile(logPath)+ if err != nil {+ t.Fatal(err)+ }+ var bundle asymptoteobserve.TraceBundleV1+ for i, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {+ var event asymptoteobserve.Event+ if err := json.Unmarshal([]byte(line), &event); err != nil {+ t.Fatalf("decode log line %d: %v", i+1, err)+ }+ bundle.Events = append(bundle.Events, asymptoteobserve.TraceEventFromEvent(event, "", i+1))+ }+ return asymptoteobserve.TraceMessages(bundle)+}++// claudeStopPayload is Claude Code's documented Stop input.+func claudeStopPayload(lastMessage interface{}) map[string]interface{} {+ payload := map[string]interface{}{+ "session_id": "claude-stop-1",+ "prompt_id": "prompt-7",+ "transcript_path": "/nonexistent/claude-stop-1.jsonl",+ "cwd": "/repo",+ "permission_mode": "default",+ "hook_event_name": "Stop",+ "stop_hook_active": false,+ }+ if lastMessage != nil {+ payload["last_assistant_message"] = lastMessage+ }+ return payload+}++// Claude Code's Stop payload carries the turn's final reply. Before this, the hook recorded that the+// turn ended and dropped what was said, so a live Claude Code session had prompts and tool calls+// but no answers until someone ran `beacon endpoint claude sync`.+func TestClaudeStopRecordsTheFinalAssistantMessage(t *testing.T) {+ reply := "I've completed the refactoring.\n\nThe tests pass."+ logPath, stdout := runHookProcess(t, claudeStopPayload(reply), "--platform", "claude", "stop")++ if strings.TrimSpace(string(stdout)) != "{}" {+ t.Fatalf("stop printed %q, want the {} no-op reply", stdout)+ }+ events := endpointEvents(t, logPath)+ messages := assistantMessages(t, logPath)+ if len(messages) != 1 {+ t.Fatalf("got %d agent.message events, want 1: %#v", len(messages), events)+ }+ message := messages[0]+ if got := outputText(t, message); got != reply {+ t.Fatalf("gen_ai.output.messages text = %q, want the reply verbatim", got)+ }+ for _, check := range []struct {+ path []string+ want string+ }{+ {[]string{"event", "category"}, "session"},+ {[]string{"event", "fidelity"}, asymptoteobserve.FidelityObserved},+ {[]string{"harness", "name"}, "claude_code"},+ {[]string{"harness", "collection_method"}, asymptoteobserve.CollectionMethodHook},+ {[]string{"session", "id"}, "claude-stop-1"},+ {[]string{"prompt", "id"}, "prompt-7"},+ } {+ if got := leaf(message, check.path...); got != check.want {+ t.Errorf("%s = %q, want %q", strings.Join(check.path, "."), got, check.want)+ }+ }+ content, ok := message["content"].(map[string]interface{})+ if !ok {+ t.Fatalf("no content marker: %#v", message)+ }+ if content["bytes"] != float64(len(reply)) || content["included"] != true || content["hash"] == "" {+ t.Fatalf("content marker = %#v, want the reply's size and hash", content)+ }++ // The reply precedes the turn's completion, because it happened first.+ if last := events[len(events)-1]; leaf(last, "event", "action") != "tool.completed" {+ t.Fatalf("last event = %q, want the turn completion after the reply", leaf(last, "event", "action"))+ }++ // What the dashboard and beacon share show: the reply is an assistant message, not a label.+ var texts []string+ for _, m := range chatView(t, logPath) {+ if m.Role == asymptoteobserve.TraceRoleAssistant && m.Kind == asymptoteobserve.TraceMessageKindAssistant {+ texts = append(texts, m.Text)+ }+ }+ if len(texts) != 1 || texts[0] != reply {+ t.Fatalf("chat view assistant text = %q, want exactly the reply", texts)+ }+}++// A turn can end without a reply: on a tool call, an interrupt, or a field the runtime left null.+// That is not an empty answer, so no message event is written -- but the turn's completion still is.+func TestClaudeStopWithoutAReplyWritesNoMessage(t *testing.T) {+ for name, value := range map[string]interface{}{+ "absent": nil,+ "null": json.RawMessage("null"),+ "empty": "",+ "blank": " \n\t",+ } {+ t.Run(name, func(t *testing.T) {+ logPath, _ := runHookProcess(t, claudeStopPayload(value), "--platform", "claude", "stop")+ if messages := assistantMessages(t, logPath); len(messages) != 0 {+ t.Fatalf("wrote agent.message for a Stop with no reply: %#v", messages)+ }+ if got := leaf(lastEndpointEvent(t, logPath), "event", "action"); got != "tool.completed" {+ t.Fatalf("last event = %q, want the turn completion", got)+ }+ })+ }+}++// The reply goes through the same redaction and size limit as every other retained string, and the+// marker still describes the original, so a reader can tell the stored copy was altered.+func TestClaudeStopReplyIsRedactedAndBounded(t *testing.T) {+ secret := "AKIA" + "IOSFODNN7EXAMPLE"+ reply := "Use key " + secret + " then " + strings.Repeat("x", asymptoteobserve.DefaultStringLimit)+ logPath, _ := runHookProcess(t, claudeStopPayload(reply), "--platform", "claude", "stop")++ messages := assistantMessages(t, logPath)+ if len(messages) != 1 {+ t.Fatalf("got %d agent.message events, want 1", len(messages))+ }+ stored := outputText(t, messages[0])+ if strings.Contains(stored, secret) {+ t.Fatalf("stored reply kept the secret: %q", stored[:80])+ }+ if len(stored) >= len(reply) {+ t.Fatalf("stored reply is %d bytes, want it bounded below the original %d", len(stored), len(reply))+ }+ content, _ := messages[0]["content"].(map[string]interface{})+ if content["truncated"] != true || content["redacted"] != true || content["bytes"] != float64(len(reply)) {+ t.Fatalf("content marker = %#v, want truncated, redacted, and the original size", content)+ }+}++// last_assistant_message is read only where a runtime documents it. Devin CLI, Factory Droid and+// Cursor send Stop payloads through the same command, and a key read where the runtime never wrote+// it would be a guess about what it means there.+func TestLastAssistantMessageIsReadOnlyWhereDocumented(t *testing.T) {+ for _, platform := range []string{"devin-cli", "factory", "cursor", "qwen"} {+ t.Run(platform, func(t *testing.T) {+ logPath, _ := runHookProcess(t, claudeStopPayload("not this runtime's field"), "--platform", platform, "stop")+ if messages := assistantMessages(t, logPath); len(messages) != 0 {+ t.Fatalf("%s recorded agent.message from last_assistant_message: %#v", platform, messages)+ }+ })+ }+}++// The Kiro reply, which now goes through the shared emitter, still arrives end to end and is still+// read only from Kiro's own key.+func TestKiroStopReplyEndToEnd(t *testing.T) {+ payload := map[string]interface{}{+ "hook_event_name": "stop",+ "session_id": "k-e2e",+ "cwd": "/repo",+ "assistant_response": "Kiro's reply.",+ "last_assistant_message": "not Kiro's field",+ }+ logPath, stdout := runHookProcess(t, payload, "--platform", kiroPlatform, "stop")+ if len(bytes.TrimSpace(stdout)) != 0 {+ t.Fatalf("kiro stop printed %q, want nothing", stdout)+ }+ messages := assistantMessages(t, logPath)+ if len(messages) != 1 || outputText(t, messages[0]) != "Kiro's reply." {+ t.Fatalf("kiro agent.message = %#v, want one carrying assistant_response", messages)+ }+}diff --git a/cli/beacon-hooks/cmd/kiro.go b/cli/beacon-hooks/cmd/kiro.goindex 2492d18d..bd7c83c5 100644--- a/cli/beacon-hooks/cmd/kiro.go+++ b/cli/beacon-hooks/cmd/kiro.go@@ -6,7 +6,6 @@ import ("github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/config"hookdiff "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/diff""github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/logging"- "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve")// Kiro payload readers and tool taxonomy.@@ -477,26 +476,11 @@ func kiroAssistantResponse(input map[string]interface{}) string {// emitKiroAssistantResponse records the agent's final reply as its own event.//-// The shared stop command records that a turn finished; it does not record what the agent said,-// because most runtimes do not tell it. Kiro does, on the same payload, and dropping it would-// throw away the one piece of model output this runtime puts on a hook -- the half of a session-// that prompt.submitted only covers the user's side of.-//-// Written in the OTel GenAI output-messages shape with the retention marker beside it, exactly as-// the agent-thought command writes reasoning, so a reader that already understands one understands-// the other. `agent.message` rather than `agent.reasoning`: this is the reply, not the thinking,-// and it is the action the fx and Devin Cloud mappers already use for the same thing.+// Kiro puts the reply on the Stop payload and nowhere else, so dropping it would throw away the one+// piece of model output this runtime puts on a hook. The event's shape is emitAssistantMessage's,+// shared with every other hook-side reply capture.func emitKiroAssistantResponse(logger *logging.Logger, input map[string]interface{}, sessionID string) {- text := kiroAssistantResponse(input)- if text == "" {- return- }- fields := sessionFields(sessionID, input)- fields["gen_ai"] = mergeNested(fields["gen_ai"], map[string]interface{}{- "output": map[string]interface{}{"messages": asymptoteobserve.TextOutputMessages(text)},- })- fields["content"] = retainedContentFields(text)- emitHookEvent(logger, "agent.message", "session", "info", "Agent response captured", input, fields)+ emitAssistantMessage(logger, input, sessionFields(sessionID, input), kiroAssistantResponse(input))}// applyKiroToolResult attaches what the shared toolFieldsWithResponse cannot reach: the operation adiff --git a/cli/beacon-hooks/cmd/stop.go b/cli/beacon-hooks/cmd/stop.goindex 55b4c1fc..b319ebc3 100644--- a/cli/beacon-hooks/cmd/stop.go+++ b/cli/beacon-hooks/cmd/stop.go@@ -54,13 +54,11 @@ func runStop(cmd *cobra.Command, args []string) {logger := newHookLogger("stop", platformFlag, sessionID)logger.Debug("Stop hook called", "session_id", sessionID, "has_transcript", transcriptPath != "", "platform", platformFlag)- // Kiro puts the agent's final reply on this payload and nowhere else. Recorded before the+ // Kiro and Claude Code put the agent's final reply on this payload. Recorded before the// completion event below and before the sessionID guard, because the reply happened before the// turn ended and because a payload with no session id still carries real model output --// dropping it there would lose the message to protect a field it does not need.- if platformFlag == kiroPlatform {- emitKiroAssistantResponse(logger, input, sessionID)- }+ emitStopAssistantResponse(logger, input, sessionID)if sessionID == "" {if isDevinLikePlatform(platformFlag) {logger.Info("stop completed")diff --git a/docs/runtimes/claude-code.mdx b/docs/runtimes/claude-code.mdxindex 9218dcef..e6973a89 100644--- a/docs/runtimes/claude-code.mdx+++ b/docs/runtimes/claude-code.mdx@@ -122,6 +122,7 @@ Managed-ingest privacy is a separate forwarding policy and does not switch off l| Area | Support ||------|---------|| Prompt telemetry | Supported when emitted through Claude Code OTLP or hooks |+| Agent responses | Supported live. The `Stop` hook records the turn's final reply from the payload's `last_assistant_message` as `agent.message`, with the text in `gen_ai.output.messages`; nothing is written for a turn that ended without one. `beacon endpoint claude sync` also recovers every committed assistant message from transcripts || Turn identity | Supported. Claude Code's `prompt.id` is kept as `prompt.id` on OTLP events; hook events after the prompt read the same id from the session transcript, and `beacon endpoint claude sync` reads it from `promptId`. The hook-recorded prompt itself takes its id from the OTLP `user_prompt` event, because the hook runs before Claude Code records the prompt || Command, tool, and file telemetry | Supported through Claude Code OTLP with detailed tool inputs enabled. Command output requires hooks || Token usage and cost | Supported through OTLP token and cost metrics, with per-model and per-session attribution |