The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(factory): record the final assistant reply from the session transcript
Claude1446200a3e34
diff --git a/cli/beacon-hooks/cmd/assistant_response_test.go b/cli/beacon-hooks/cmd/assistant_response_test.goindex 52d396d2..6f875eb4 100644--- a/cli/beacon-hooks/cmd/assistant_response_test.go+++ b/cli/beacon-hooks/cmd/assistant_response_test.go@@ -39,7 +39,13 @@ func TestHookProcess(t *testing.T) {// home directory of its own, so no hook state or session record reaches the real one.func runHookProcess(t *testing.T, input map[string]interface{}, args ...string) (logPath string, stdout []byte) {t.Helper()- dir := t.TempDir()+ return runHookProcessIn(t, t.TempDir(), input, args...)+}++// runHookProcessIn is runHookProcess with the log and home directory under dir, so consecutive+// hooks of one session share their log and hook state.+func runHookProcessIn(t *testing.T, dir string, input map[string]interface{}, args ...string) (logPath string, stdout []byte) {+ t.Helper()logPath = filepath.Join(dir, "runtime.jsonl")home := filepath.Join(dir, "home")if err := os.MkdirAll(home, 0o700); err != nil {diff --git a/cli/beacon-hooks/cmd/factory_reply.go b/cli/beacon-hooks/cmd/factory_reply.gonew file mode 100644index 00000000..d2971397--- /dev/null+++ b/cli/beacon-hooks/cmd/factory_reply.go@@ -0,0 +1,157 @@+package cmd++import (+ "bytes"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "io"+ "os"+ "path/filepath"+ "strings"++ "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/logging"+ "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/state"+)++// factoryTranscriptTailBytes bounds how much of a Droid transcript the Stop hook reads. Only the+// last message record matters, and one record is far smaller than this.+const factoryTranscriptTailBytes = 1 << 20++// factoryReply is the final assistant message of a Droid turn as its transcript records it.+type factoryReply struct {+ id string+ text string+}++// factoryTranscriptReply reads the turn's final reply from a Factory Droid session transcript.+//+// Droid's Stop payload carries no reply text, only transcript_path, so this is the one place a live+// hook can learn what the agent said. A transcript can lag the hook that names it, so a reply is+// returned only when the transcript's newest message record is an assistant message that has text+// and no tool call. A turn's final answer never ends on a tool call, so an assistant message that+// does is one the turn has not finished with -- the transcript is behind -- and reporting it as the+// answer would be wrong. Anything else returns false and the session sync reads the reply later.+//+// Only a regular .jsonl file is read, from its last factoryTranscriptTailBytes, and any failure+// returns nothing: a missed reply costs one event, a stalled hook costs the agent's turn.+func factoryTranscriptReply(path string) (factoryReply, bool) {+ tail, partial := readTranscriptTail(path, factoryTranscriptTailBytes)+ if len(tail) == 0 {+ return factoryReply{}, false+ }+ lines := bytes.Split(tail, []byte("\n"))+ if partial {+ lines = lines[1:]+ }+ for i := len(lines) - 1; i >= 0; i-- {+ line := bytes.TrimSpace(lines[i])+ if len(line) == 0 {+ continue+ }+ var record struct {+ Type string `json:"type"`+ ID string `json:"id"`+ Message *struct {+ Role string `json:"role"`+ Content json.RawMessage `json:"content"`+ } `json:"message"`+ }+ if json.Unmarshal(line, &record) != nil {+ // A record still being written is the newest line and is not yet a message.+ continue+ }+ if record.Type != "message" || record.Message == nil {+ continue+ }+ if record.Message.Role != "assistant" {+ return factoryReply{}, false+ }+ text, hasToolUse := factoryAssistantText(record.Message.Content)+ if hasToolUse || strings.TrimSpace(text) == "" {+ return factoryReply{}, false+ }+ id := record.ID+ if id == "" {+ sum := sha256.Sum256([]byte(text))+ id = "sha256:" + hex.EncodeToString(sum[:])+ }+ return factoryReply{id: id, text: text}, true+ }+ return factoryReply{}, false+}++// factoryAssistantText joins the text blocks of a Droid message body, which is either a string or+// a list of content blocks, and reports whether any block is a tool call.+func factoryAssistantText(content json.RawMessage) (string, bool) {+ var plain string+ if json.Unmarshal(content, &plain) == nil {+ return plain, false+ }+ var blocks []struct {+ Type string `json:"type"`+ Text string `json:"text"`+ }+ if json.Unmarshal(content, &blocks) != nil {+ return "", false+ }+ var parts []string+ hasToolUse := false+ for _, block := range blocks {+ switch block.Type {+ case "text":+ if strings.TrimSpace(block.Text) != "" {+ parts = append(parts, block.Text)+ }+ case "tool_use":+ hasToolUse = true+ }+ }+ return strings.Join(parts, "\n\n"), hasToolUse+}++// readTranscriptTail returns up to the last max bytes of a regular .jsonl file, and whether they+// start mid-file (so the first line is a fragment).+func readTranscriptTail(path string, max int64) ([]byte, bool) {+ path = strings.TrimSpace(path)+ if path == "" || !strings.EqualFold(filepath.Ext(path), ".jsonl") {+ return nil, false+ }+ file, err := os.Open(path)+ if err != nil {+ return nil, false+ }+ defer file.Close()+ info, err := file.Stat()+ if err != nil || !info.Mode().IsRegular() {+ return nil, false+ }+ size := info.Size()+ window := size+ if window > max {+ window = max+ }+ buf := make([]byte, window)+ if _, err := file.ReadAt(buf, size-window); err != nil && err != io.EOF {+ return nil, false+ }+ return buf, window < size+}++// emitFactoryTranscriptReply records the final reply of a Droid turn read from its transcript, at+// most once per reply: a transcript that has not moved since the previous Stop names the same+// record, and the session state remembers which one was written last.+func emitFactoryTranscriptReply(logger *logging.Logger, input map[string]interface{}, sessionID, transcriptPath string, st *state.SessionState) {+ reply, ok := factoryTranscriptReply(transcriptPath)+ if !ok {+ return+ }+ // A state file that cannot be written costs at most a duplicate, which is better than a lost+ // reply; only a reply the state says was already written is skipped.+ if fresh, err := st.MarkReplyRecorded(reply.id); err == nil && !fresh {+ return+ }+ fields := sessionFields(sessionID, input)+ fields["raw"] = mergeNested(fields["raw"], map[string]interface{}{"factory": map[string]interface{}{"reply_message_id": reply.id, "source": "transcript"}})+ emitAssistantMessage(logger, input, fields, reply.text)+}diff --git a/cli/beacon-hooks/cmd/factory_reply_test.go b/cli/beacon-hooks/cmd/factory_reply_test.gonew file mode 100644index 00000000..e904246f--- /dev/null+++ b/cli/beacon-hooks/cmd/factory_reply_test.go@@ -0,0 +1,233 @@+package cmd++import (+ "encoding/json"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++// factoryRecord is one line of a Droid session transcript, in the shape factorysession reads.+func factoryRecord(id, role string, content interface{}) string {+ data, err := json.Marshal(map[string]interface{}{+ "type": "message",+ "id": id,+ "timestamp": "2026-10-15T10:00:00Z",+ "message": map[string]interface{}{"role": role, "content": content},+ })+ if err != nil {+ panic(err)+ }+ return string(data)+}++func factoryText(text string) map[string]interface{} {+ return map[string]interface{}{"type": "text", "text": text}+}++func factoryToolUse(id string) map[string]interface{} {+ return map[string]interface{}{"type": "tool_use", "id": id, "name": "Read", "input": map[string]interface{}{"file_path": "/repo/a.go"}}+}++func factoryToolResult(id string) map[string]interface{} {+ return map[string]interface{}{"type": "tool_result", "tool_use_id": id, "content": "package a"}+}++// factoryTurn is a whole Droid turn: the prompt, a tool call with intermediate text, its result,+// and the final answer in two text blocks.+func factoryTurn(prefix, final string) []string {+ return []string{+ factoryRecord(prefix+"-u", "user", []interface{}{factoryText("explain a.go")}),+ factoryRecord(prefix+"-a1", "assistant", []interface{}{factoryText("Let me read it."), factoryToolUse(prefix + "-t")}),+ factoryRecord(prefix+"-r", "user", []interface{}{factoryToolResult(prefix + "-t")}),+ factoryRecord(prefix+"-a2", "assistant", []interface{}{factoryText(final), factoryText("Anything else?")}),+ }+}++func writeTranscript(t *testing.T, path string, lines ...string) {+ t.Helper()+ if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {+ t.Fatal(err)+ }+}++// factoryStopPayload is Droid's documented Stop input: the common fields plus stop_hook_active,+// tool_execution_count and elapsed_time. No field carries the reply.+func factoryStopPayload(transcriptPath string) map[string]interface{} {+ return map[string]interface{}{+ "session_id": "droid-1",+ "transcript_path": transcriptPath,+ "cwd": "/repo",+ "permission_mode": "auto-low",+ "hook_event_name": "Stop",+ "stop_hook_active": false,+ "tool_execution_count": 1,+ "elapsed_time": 4200,+ }+}++func TestFactoryTranscriptReplyReadsTheFinalAssistantMessage(t *testing.T) {+ path := filepath.Join(t.TempDir(), "droid-1.jsonl")+ lines := append([]string{`{"type":"session_start","id":"droid-1","cwd":"/repo"}`}, factoryTurn("t1", "a.go declares package a.")...)+ lines = append(lines, `{"type":"todo_state","id":"todo-1"}`)+ writeTranscript(t, path, lines...)++ reply, ok := factoryTranscriptReply(path)+ if !ok {+ t.Fatal("no reply read from a finished turn")+ }+ if reply.id != "t1-a2" || reply.text != "a.go declares package a.\n\nAnything else?" {+ t.Fatalf("reply = %#v, want t1-a2 with both text blocks", reply)+ }+}++// The transcript can lag the Stop hook. Every case below is a turn the transcript has not finished+// recording, and none of them may be reported as the answer.+func TestFactoryTranscriptReplyRefusesAnUnfinishedTurn(t *testing.T) {+ turn := factoryTurn("t1", "done")+ cases := map[string][]string{+ "ends on a tool call": turn[:2],+ "ends on a tool result": turn[:3],+ "ends on the prompt": turn[:1],+ "text-only block list is blank": {turn[0], factoryRecord("x", "assistant", []interface{}{factoryText(" ")})},+ "assistant message with no body": {turn[0], `{"type":"message","id":"x","message":{"role":"assistant"}}`},+ }+ for name, lines := range cases {+ t.Run(name, func(t *testing.T) {+ path := filepath.Join(t.TempDir(), "s.jsonl")+ writeTranscript(t, path, lines...)+ if reply, ok := factoryTranscriptReply(path); ok {+ t.Fatalf("reported %#v as the answer of an unfinished turn", reply)+ }+ })+ }+}++// A record still being written is skipped, and the newest complete record decides.+func TestFactoryTranscriptReplySkipsAPartialLastLine(t *testing.T) {+ path := filepath.Join(t.TempDir(), "s.jsonl")+ turn := factoryTurn("t1", "done")+ data := strings.Join(turn[:3], "\n") + "\n" + `{"type":"message","id":"t1-a2","message":{"role":"assis`+ if err := os.WriteFile(path, []byte(data), 0o600); err != nil {+ t.Fatal(err)+ }+ if reply, ok := factoryTranscriptReply(path); ok {+ t.Fatalf("reported %#v while the final record was still being written", reply)+ }+}++func TestFactoryTranscriptReplyAcceptsAStringBody(t *testing.T) {+ path := filepath.Join(t.TempDir(), "s.jsonl")+ writeTranscript(t, path, factoryRecord("u", "user", "hi"), factoryRecord("a", "assistant", "Hello."))+ if reply, ok := factoryTranscriptReply(path); !ok || reply.text != "Hello." {+ t.Fatalf("reply = %#v, %v; want the string body", reply, ok)+ }+}++// Only the tail of a long session is read, and the reply at its end is still found.+func TestFactoryTranscriptReplyReadsOnlyTheTail(t *testing.T) {+ path := filepath.Join(t.TempDir(), "s.jsonl")+ filler := factoryRecord("old", "assistant", []interface{}{factoryText(strings.Repeat("y", 4096))})+ var lines []string+ for i := 0; i < (factoryTranscriptTailBytes/len(filler))*2; i++ {+ lines = append(lines, filler)+ }+ lines = append(lines, factoryTurn("t9", "the newest answer")...)+ writeTranscript(t, path, lines...)+ if reply, ok := factoryTranscriptReply(path); !ok || reply.id != "t9-a2" {+ t.Fatalf("reply = %#v, %v; want t9-a2", reply, ok)+ }+}++func TestFactoryTranscriptReplyRefusesWhatIsNotATranscript(t *testing.T) {+ dir := t.TempDir()+ notJSONL := filepath.Join(dir, "s.json")+ writeTranscript(t, notJSONL, factoryTurn("t1", "x")...)+ for name, path := range map[string]string{"missing": filepath.Join(dir, "none.jsonl"), "wrong extension": notJSONL, "directory": dir + ".jsonl", "empty": ""} {+ if name == "directory" {+ _ = os.Mkdir(path, 0o700)+ }+ if reply, ok := factoryTranscriptReply(path); ok {+ t.Fatalf("%s: read %#v", name, reply)+ }+ }+}++// End to end through the real stop command: the reply is recorded once, ahead of the turn's+// completion, and a second Stop over the same transcript does not record it again. The next turn's+// reply is recorded.+func TestFactoryStopRecordsTheTranscriptReplyOnce(t *testing.T) {+ dir := t.TempDir()+ path := filepath.Join(dir, "droid-1.jsonl")+ turn1 := factoryTurn("t1", "a.go declares package a.")+ writeTranscript(t, path, turn1...)++ logPath, stdout := runHookProcessIn(t, dir, factoryStopPayload(path), "--platform", "factory", "stop")+ if strings.TrimSpace(string(stdout)) != "{}" {+ t.Fatalf("stdout = %q, want {}", stdout)+ }+ messages := assistantMessages(t, logPath)+ if len(messages) != 1 {+ t.Fatalf("got %d agent.message events, want 1", len(messages))+ }+ if got := outputText(t, messages[0]); got != "a.go declares package a.\n\nAnything else?" {+ t.Fatalf("reply = %q", got)+ }+ for _, check := range []struct {+ path []string+ want string+ }{+ {[]string{"harness", "name"}, "factory"},+ {[]string{"harness", "collection_method"}, "hook"},+ {[]string{"session", "id"}, "droid-1"},+ {[]string{"raw", "factory", "reply_message_id"}, "t1-a2"},+ } {+ if got := leaf(messages[0], check.path...); got != check.want {+ t.Errorf("%s = %q, want %q", strings.Join(check.path, "."), got, check.want)+ }+ }+ if last := lastEndpointEvent(t, logPath); leaf(last, "event", "action") != "tool.completed" {+ t.Fatalf("last event = %q, want the turn completion after the reply", leaf(last, "event", "action"))+ }++ runHookProcessIn(t, dir, factoryStopPayload(path), "--platform", "factory", "stop")+ if n := len(assistantMessages(t, logPath)); n != 1 {+ t.Fatalf("an unchanged transcript recorded the reply %d times, want once", n)+ }++ writeTranscript(t, path, append(turn1, factoryTurn("t2", "Second answer.")...)...)+ runHookProcessIn(t, dir, factoryStopPayload(path), "--platform", "factory", "stop")+ messages = assistantMessages(t, logPath)+ if len(messages) != 2 || leaf(messages[1], "raw", "factory", "reply_message_id") != "t2-a2" {+ t.Fatalf("second turn's reply not recorded: %d messages", len(messages))+ }+}++// A lagging transcript at Stop records no reply, and the turn still completes.+func TestFactoryStopWithALaggingTranscriptRecordsNoReply(t *testing.T) {+ dir := t.TempDir()+ path := filepath.Join(dir, "droid-1.jsonl")+ writeTranscript(t, path, factoryTurn("t1", "x")[:2]...)+ logPath, _ := runHookProcessIn(t, dir, factoryStopPayload(path), "--platform", "factory", "stop")+ if messages := assistantMessages(t, logPath); len(messages) != 0 {+ t.Fatalf("recorded %d replies from a transcript that ended on a tool call", len(messages))+ }+ if last := lastEndpointEvent(t, logPath); leaf(last, "event", "action") != "tool.completed" {+ t.Fatalf("last event = %q, want the turn completion", leaf(last, "event", "action"))+ }+}++// The transcript read is Droid's alone: a Claude-shaped runtime whose Stop names a transcript in+// the same format records nothing from it.+func TestTranscriptReplyIsReadOnlyForFactory(t *testing.T) {+ dir := t.TempDir()+ path := filepath.Join(dir, "s.jsonl")+ writeTranscript(t, path, factoryTurn("t1", "x")...)+ for _, platform := range []string{"devin-cli", "qwen"} {+ logPath, _ := runHookProcessIn(t, t.TempDir(), factoryStopPayload(path), "--platform", platform, "stop")+ if messages := assistantMessages(t, logPath); len(messages) != 0 {+ t.Fatalf("%s read a reply from the transcript: %#v", platform, messages)+ }+ }+}diff --git a/cli/beacon-hooks/cmd/stop.go b/cli/beacon-hooks/cmd/stop.goindex b319ebc3..de6ee11a 100644--- a/cli/beacon-hooks/cmd/stop.go+++ b/cli/beacon-hooks/cmd/stop.go@@ -74,6 +74,10 @@ func runStop(cmd *cobra.Command, args []string) {config.RotateLogIfNeededForPlatform(platformFlag)st := state.NewSessionState(sessionID, platformFlag)+ // Factory Droid's Stop payload carries no reply; its transcript does.+ if platformFlag == "factory" {+ emitFactoryTranscriptReply(logger, input, sessionID, transcriptPath, st)+ }// The public Beacon build is local-only. Clear any pending remote evaluations// left over from older hook versions instead of polling a hosted service.diff --git a/cli/beacon-hooks/internal/state/state.go b/cli/beacon-hooks/internal/state/state.goindex b44bd84f..81104f4f 100644--- a/cli/beacon-hooks/internal/state/state.go+++ b/cli/beacon-hooks/internal/state/state.go@@ -85,7 +85,10 @@ type SessionData struct {RepromptCount int `json:"reprompt_count"`Model string `json:"model,omitempty"`PromptEmitted bool `json:"prompt_emitted,omitempty"`- CreatedAt string `json:"created_at,omitempty"`+ // LastReplyID names the last assistant reply a hook recorded for this session, so a reply read+ // from a transcript that has not moved since the previous turn is not recorded twice.+ LastReplyID string `json:"last_reply_id,omitempty"`+ CreatedAt string `json:"created_at,omitempty"`}// SessionState manages session state@@ -196,6 +199,24 @@ func (s *SessionState) MarkPromptEmittedIfNeeded() (bool, error) {return true, nil}+// MarkReplyRecorded records id as the session's last recorded assistant reply. It reports false,+// and changes nothing, when id is already the last one recorded, so a caller writes a reply at+// most once even when two Stop hooks of one session race.+func (s *SessionState) MarkReplyRecorded(id string) (bool, error) {+ defer lockState(s.platform)()++ state := s.loadStateFile()+ s.ensureSession(state)+ if state[s.sessionID].LastReplyID == id {+ return false, nil+ }+ state[s.sessionID].LastReplyID = id+ if err := s.saveStateFile(state); err != nil {+ return false, err+ }+ return true, nil+}+// AddEvaluation adds a new pending evaluationfunc (s *SessionState) AddEvaluation(evaluationID, filePath string) error {defer lockState(s.platform)()diff --git a/cli/beacon-hooks/internal/state/state_test.go b/cli/beacon-hooks/internal/state/state_test.goindex c099e7fe..72363df1 100644--- a/cli/beacon-hooks/internal/state/state_test.go+++ b/cli/beacon-hooks/internal/state/state_test.go@@ -156,3 +156,30 @@ func TestStateIsSafeAcrossProcesses(t *testing.T) {}}}++func TestMarkReplyRecordedOncePerReplyAndSession(t *testing.T) {+ _, cleanup := setupTestStateDir(t)+ defer cleanup()++ a := NewSessionState("session-a", "cursor")+ for i, want := range []bool{true, false} {+ fresh, err := a.MarkReplyRecorded("msg-1")+ if err != nil || fresh != want {+ t.Fatalf("mark %d = %v, %v; want %v", i+1, fresh, err, want)+ }+ }+ // A later reply is new, and a reply id is per session.+ if fresh, _ := NewSessionState("session-a", "cursor").MarkReplyRecorded("msg-2"); !fresh {+ t.Fatal("a new reply id was treated as already recorded")+ }+ if fresh, _ := NewSessionState("session-b", "cursor").MarkReplyRecorded("msg-2"); !fresh {+ t.Fatal("another session's reply id suppressed this session's")+ }+ // The marker sits beside the other session data without disturbing it.+ if err := a.SetModel("m"); err != nil {+ t.Fatal(err)+ }+ if fresh, _ := a.MarkReplyRecorded("msg-2"); fresh {+ t.Fatal("setting the model reset the recorded reply")+ }+}diff --git a/docs/runtimes/factory-droid.mdx b/docs/runtimes/factory-droid.mdxindex f283c246..906c0a1e 100644--- a/docs/runtimes/factory-droid.mdx+++ b/docs/runtimes/factory-droid.mdx@@ -55,6 +55,7 @@ For scheduled collection, run `beacon endpoint factory sync --watch`, or run one| Area | Support ||------|---------|| Prompt telemetry | Supported through `UserPromptSubmit` hooks |+| Agent responses | Supported live through the `Stop` hook. Droid's Stop payload carries no reply, so the hook reads the end of the session transcript it names and records the final assistant message as `agent.message` (`raw.factory.reply_message_id` names the record). It records nothing when the transcript has not caught up, ending on a tool call, a tool result or the prompt, and records each reply once. `beacon endpoint factory sync` recovers every committed assistant message || Command, tool, and file telemetry | Supported for session, write/edit/create hook payloads, committed session tool calls/results, stop, session-end, and available OTLP telemetry || Historical sessions | Supported through `beacon endpoint factory sync`, which reads `~/.factory/sessions` locally and offline || Token usage | `sync` reads the running totals Factory keeps in each session's `.settings.json` and emits a `token.usage` event with only what they grew by since the last sync, so `beacon token-usage` sums to the session's final total. It counts the session's own `tokenUsage`; `inclusiveTokenUsage` is used only when `tokenUsage` is absent and no child session is listed, because it adds child sessions, which are synced on their own. A total that goes down is treated as a rewritten file and counts nothing. Factory credits are not USD, so they stay in `raw` (`factory_credits_delta`, `factory_credits_total`) rather than `cost_usd` |