Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(factory): record the final assistant reply from the session transcript

Claude1446200a3e34
diff --git a/cli/beacon-hooks/cmd/assistant_response_test.go b/cli/beacon-hooks/cmd/assistant_response_test.go
index 52d396d2..6f875eb4 100644
--- a/cli/beacon-hooks/cmd/assistant_response_test.go
+++ b/cli/beacon-hooks/cmd/assistant_response_test.go
@@ -39,7 +39,13 @@ func TestHookProcess(t *testing.T) {
// home directory of its own, so no hook state or session record reaches the real one.
func runHookProcess(t *testing.T, input map[string]interface{}, args ...string) (logPath string, stdout []byte) {
t.Helper()
- dir := t.TempDir()
+ return runHookProcessIn(t, t.TempDir(), input, args...)
+}
+
+// runHookProcessIn is runHookProcess with the log and home directory under dir, so consecutive
+// hooks of one session share their log and hook state.
+func runHookProcessIn(t *testing.T, dir string, input map[string]interface{}, args ...string) (logPath string, stdout []byte) {
+ t.Helper()
logPath = filepath.Join(dir, "runtime.jsonl")
home := filepath.Join(dir, "home")
if err := os.MkdirAll(home, 0o700); err != nil {
diff --git a/cli/beacon-hooks/cmd/factory_reply.go b/cli/beacon-hooks/cmd/factory_reply.go
new file mode 100644
index 00000000..d2971397
--- /dev/null
+++ b/cli/beacon-hooks/cmd/factory_reply.go
@@ -0,0 +1,157 @@
+package cmd
+
+import (
+ "bytes"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "io"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/logging"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/state"
+)
+
+// factoryTranscriptTailBytes bounds how much of a Droid transcript the Stop hook reads. Only the
+// last message record matters, and one record is far smaller than this.
+const factoryTranscriptTailBytes = 1 << 20
+
+// factoryReply is the final assistant message of a Droid turn as its transcript records it.
+type factoryReply struct {
+ id string
+ text string
+}
+
+// factoryTranscriptReply reads the turn's final reply from a Factory Droid session transcript.
+//
+// Droid's Stop payload carries no reply text, only transcript_path, so this is the one place a live
+// hook can learn what the agent said. A transcript can lag the hook that names it, so a reply is
+// returned only when the transcript's newest message record is an assistant message that has text
+// and no tool call. A turn's final answer never ends on a tool call, so an assistant message that
+// does is one the turn has not finished with -- the transcript is behind -- and reporting it as the
+// answer would be wrong. Anything else returns false and the session sync reads the reply later.
+//
+// Only a regular .jsonl file is read, from its last factoryTranscriptTailBytes, and any failure
+// returns nothing: a missed reply costs one event, a stalled hook costs the agent's turn.
+func factoryTranscriptReply(path string) (factoryReply, bool) {
+ tail, partial := readTranscriptTail(path, factoryTranscriptTailBytes)
+ if len(tail) == 0 {
+ return factoryReply{}, false
+ }
+ lines := bytes.Split(tail, []byte("\n"))
+ if partial {
+ lines = lines[1:]
+ }
+ for i := len(lines) - 1; i >= 0; i-- {
+ line := bytes.TrimSpace(lines[i])
+ if len(line) == 0 {
+ continue
+ }
+ var record struct {
+ Type string `json:"type"`
+ ID string `json:"id"`
+ Message *struct {
+ Role string `json:"role"`
+ Content json.RawMessage `json:"content"`
+ } `json:"message"`
+ }
+ if json.Unmarshal(line, &record) != nil {
+ // A record still being written is the newest line and is not yet a message.
+ continue
+ }
+ if record.Type != "message" || record.Message == nil {
+ continue
+ }
+ if record.Message.Role != "assistant" {
+ return factoryReply{}, false
+ }
+ text, hasToolUse := factoryAssistantText(record.Message.Content)
+ if hasToolUse || strings.TrimSpace(text) == "" {
+ return factoryReply{}, false
+ }
+ id := record.ID
+ if id == "" {
+ sum := sha256.Sum256([]byte(text))
+ id = "sha256:" + hex.EncodeToString(sum[:])
+ }
+ return factoryReply{id: id, text: text}, true
+ }
+ return factoryReply{}, false
+}
+
+// factoryAssistantText joins the text blocks of a Droid message body, which is either a string or
+// a list of content blocks, and reports whether any block is a tool call.
+func factoryAssistantText(content json.RawMessage) (string, bool) {
+ var plain string
+ if json.Unmarshal(content, &plain) == nil {
+ return plain, false
+ }
+ var blocks []struct {
+ Type string `json:"type"`
+ Text string `json:"text"`
+ }
+ if json.Unmarshal(content, &blocks) != nil {
+ return "", false
+ }
+ var parts []string
+ hasToolUse := false
+ for _, block := range blocks {
+ switch block.Type {
+ case "text":
+ if strings.TrimSpace(block.Text) != "" {
+ parts = append(parts, block.Text)
+ }
+ case "tool_use":
+ hasToolUse = true
+ }
+ }
+ return strings.Join(parts, "\n\n"), hasToolUse
+}
+
+// readTranscriptTail returns up to the last max bytes of a regular .jsonl file, and whether they
+// start mid-file (so the first line is a fragment).
+func readTranscriptTail(path string, max int64) ([]byte, bool) {
+ path = strings.TrimSpace(path)
+ if path == "" || !strings.EqualFold(filepath.Ext(path), ".jsonl") {
+ return nil, false
+ }
+ file, err := os.Open(path)
+ if err != nil {
+ return nil, false
+ }
+ defer file.Close()
+ info, err := file.Stat()
+ if err != nil || !info.Mode().IsRegular() {
+ return nil, false
+ }
+ size := info.Size()
+ window := size
+ if window > max {
+ window = max
+ }
+ buf := make([]byte, window)
+ if _, err := file.ReadAt(buf, size-window); err != nil && err != io.EOF {
+ return nil, false
+ }
+ return buf, window < size
+}
+
+// emitFactoryTranscriptReply records the final reply of a Droid turn read from its transcript, at
+// most once per reply: a transcript that has not moved since the previous Stop names the same
+// record, and the session state remembers which one was written last.
+func emitFactoryTranscriptReply(logger *logging.Logger, input map[string]interface{}, sessionID, transcriptPath string, st *state.SessionState) {
+ reply, ok := factoryTranscriptReply(transcriptPath)
+ if !ok {
+ return
+ }
+ // A state file that cannot be written costs at most a duplicate, which is better than a lost
+ // reply; only a reply the state says was already written is skipped.
+ if fresh, err := st.MarkReplyRecorded(reply.id); err == nil && !fresh {
+ return
+ }
+ fields := sessionFields(sessionID, input)
+ fields["raw"] = mergeNested(fields["raw"], map[string]interface{}{"factory": map[string]interface{}{"reply_message_id": reply.id, "source": "transcript"}})
+ emitAssistantMessage(logger, input, fields, reply.text)
+}
diff --git a/cli/beacon-hooks/cmd/factory_reply_test.go b/cli/beacon-hooks/cmd/factory_reply_test.go
new file mode 100644
index 00000000..e904246f
--- /dev/null
+++ b/cli/beacon-hooks/cmd/factory_reply_test.go
@@ -0,0 +1,233 @@
+package cmd
+
+import (
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// factoryRecord is one line of a Droid session transcript, in the shape factorysession reads.
+func factoryRecord(id, role string, content interface{}) string {
+ data, err := json.Marshal(map[string]interface{}{
+ "type": "message",
+ "id": id,
+ "timestamp": "2026-10-15T10:00:00Z",
+ "message": map[string]interface{}{"role": role, "content": content},
+ })
+ if err != nil {
+ panic(err)
+ }
+ return string(data)
+}
+
+func factoryText(text string) map[string]interface{} {
+ return map[string]interface{}{"type": "text", "text": text}
+}
+
+func factoryToolUse(id string) map[string]interface{} {
+ return map[string]interface{}{"type": "tool_use", "id": id, "name": "Read", "input": map[string]interface{}{"file_path": "/repo/a.go"}}
+}
+
+func factoryToolResult(id string) map[string]interface{} {
+ return map[string]interface{}{"type": "tool_result", "tool_use_id": id, "content": "package a"}
+}
+
+// factoryTurn is a whole Droid turn: the prompt, a tool call with intermediate text, its result,
+// and the final answer in two text blocks.
+func factoryTurn(prefix, final string) []string {
+ return []string{
+ factoryRecord(prefix+"-u", "user", []interface{}{factoryText("explain a.go")}),
+ factoryRecord(prefix+"-a1", "assistant", []interface{}{factoryText("Let me read it."), factoryToolUse(prefix + "-t")}),
+ factoryRecord(prefix+"-r", "user", []interface{}{factoryToolResult(prefix + "-t")}),
+ factoryRecord(prefix+"-a2", "assistant", []interface{}{factoryText(final), factoryText("Anything else?")}),
+ }
+}
+
+func writeTranscript(t *testing.T, path string, lines ...string) {
+ t.Helper()
+ if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// factoryStopPayload is Droid's documented Stop input: the common fields plus stop_hook_active,
+// tool_execution_count and elapsed_time. No field carries the reply.
+func factoryStopPayload(transcriptPath string) map[string]interface{} {
+ return map[string]interface{}{
+ "session_id": "droid-1",
+ "transcript_path": transcriptPath,
+ "cwd": "/repo",
+ "permission_mode": "auto-low",
+ "hook_event_name": "Stop",
+ "stop_hook_active": false,
+ "tool_execution_count": 1,
+ "elapsed_time": 4200,
+ }
+}
+
+func TestFactoryTranscriptReplyReadsTheFinalAssistantMessage(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "droid-1.jsonl")
+ lines := append([]string{`{"type":"session_start","id":"droid-1","cwd":"/repo"}`}, factoryTurn("t1", "a.go declares package a.")...)
+ lines = append(lines, `{"type":"todo_state","id":"todo-1"}`)
+ writeTranscript(t, path, lines...)
+
+ reply, ok := factoryTranscriptReply(path)
+ if !ok {
+ t.Fatal("no reply read from a finished turn")
+ }
+ if reply.id != "t1-a2" || reply.text != "a.go declares package a.\n\nAnything else?" {
+ t.Fatalf("reply = %#v, want t1-a2 with both text blocks", reply)
+ }
+}
+
+// The transcript can lag the Stop hook. Every case below is a turn the transcript has not finished
+// recording, and none of them may be reported as the answer.
+func TestFactoryTranscriptReplyRefusesAnUnfinishedTurn(t *testing.T) {
+ turn := factoryTurn("t1", "done")
+ cases := map[string][]string{
+ "ends on a tool call": turn[:2],
+ "ends on a tool result": turn[:3],
+ "ends on the prompt": turn[:1],
+ "text-only block list is blank": {turn[0], factoryRecord("x", "assistant", []interface{}{factoryText(" ")})},
+ "assistant message with no body": {turn[0], `{"type":"message","id":"x","message":{"role":"assistant"}}`},
+ }
+ for name, lines := range cases {
+ t.Run(name, func(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "s.jsonl")
+ writeTranscript(t, path, lines...)
+ if reply, ok := factoryTranscriptReply(path); ok {
+ t.Fatalf("reported %#v as the answer of an unfinished turn", reply)
+ }
+ })
+ }
+}
+
+// A record still being written is skipped, and the newest complete record decides.
+func TestFactoryTranscriptReplySkipsAPartialLastLine(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "s.jsonl")
+ turn := factoryTurn("t1", "done")
+ data := strings.Join(turn[:3], "\n") + "\n" + `{"type":"message","id":"t1-a2","message":{"role":"assis`
+ if err := os.WriteFile(path, []byte(data), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ if reply, ok := factoryTranscriptReply(path); ok {
+ t.Fatalf("reported %#v while the final record was still being written", reply)
+ }
+}
+
+func TestFactoryTranscriptReplyAcceptsAStringBody(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "s.jsonl")
+ writeTranscript(t, path, factoryRecord("u", "user", "hi"), factoryRecord("a", "assistant", "Hello."))
+ if reply, ok := factoryTranscriptReply(path); !ok || reply.text != "Hello." {
+ t.Fatalf("reply = %#v, %v; want the string body", reply, ok)
+ }
+}
+
+// Only the tail of a long session is read, and the reply at its end is still found.
+func TestFactoryTranscriptReplyReadsOnlyTheTail(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "s.jsonl")
+ filler := factoryRecord("old", "assistant", []interface{}{factoryText(strings.Repeat("y", 4096))})
+ var lines []string
+ for i := 0; i < (factoryTranscriptTailBytes/len(filler))*2; i++ {
+ lines = append(lines, filler)
+ }
+ lines = append(lines, factoryTurn("t9", "the newest answer")...)
+ writeTranscript(t, path, lines...)
+ if reply, ok := factoryTranscriptReply(path); !ok || reply.id != "t9-a2" {
+ t.Fatalf("reply = %#v, %v; want t9-a2", reply, ok)
+ }
+}
+
+func TestFactoryTranscriptReplyRefusesWhatIsNotATranscript(t *testing.T) {
+ dir := t.TempDir()
+ notJSONL := filepath.Join(dir, "s.json")
+ writeTranscript(t, notJSONL, factoryTurn("t1", "x")...)
+ for name, path := range map[string]string{"missing": filepath.Join(dir, "none.jsonl"), "wrong extension": notJSONL, "directory": dir + ".jsonl", "empty": ""} {
+ if name == "directory" {
+ _ = os.Mkdir(path, 0o700)
+ }
+ if reply, ok := factoryTranscriptReply(path); ok {
+ t.Fatalf("%s: read %#v", name, reply)
+ }
+ }
+}
+
+// End to end through the real stop command: the reply is recorded once, ahead of the turn's
+// completion, and a second Stop over the same transcript does not record it again. The next turn's
+// reply is recorded.
+func TestFactoryStopRecordsTheTranscriptReplyOnce(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "droid-1.jsonl")
+ turn1 := factoryTurn("t1", "a.go declares package a.")
+ writeTranscript(t, path, turn1...)
+
+ logPath, stdout := runHookProcessIn(t, dir, factoryStopPayload(path), "--platform", "factory", "stop")
+ if strings.TrimSpace(string(stdout)) != "{}" {
+ t.Fatalf("stdout = %q, want {}", stdout)
+ }
+ messages := assistantMessages(t, logPath)
+ if len(messages) != 1 {
+ t.Fatalf("got %d agent.message events, want 1", len(messages))
+ }
+ if got := outputText(t, messages[0]); got != "a.go declares package a.\n\nAnything else?" {
+ t.Fatalf("reply = %q", got)
+ }
+ for _, check := range []struct {
+ path []string
+ want string
+ }{
+ {[]string{"harness", "name"}, "factory"},
+ {[]string{"harness", "collection_method"}, "hook"},
+ {[]string{"session", "id"}, "droid-1"},
+ {[]string{"raw", "factory", "reply_message_id"}, "t1-a2"},
+ } {
+ if got := leaf(messages[0], check.path...); got != check.want {
+ t.Errorf("%s = %q, want %q", strings.Join(check.path, "."), got, check.want)
+ }
+ }
+ if last := lastEndpointEvent(t, logPath); leaf(last, "event", "action") != "tool.completed" {
+ t.Fatalf("last event = %q, want the turn completion after the reply", leaf(last, "event", "action"))
+ }
+
+ runHookProcessIn(t, dir, factoryStopPayload(path), "--platform", "factory", "stop")
+ if n := len(assistantMessages(t, logPath)); n != 1 {
+ t.Fatalf("an unchanged transcript recorded the reply %d times, want once", n)
+ }
+
+ writeTranscript(t, path, append(turn1, factoryTurn("t2", "Second answer.")...)...)
+ runHookProcessIn(t, dir, factoryStopPayload(path), "--platform", "factory", "stop")
+ messages = assistantMessages(t, logPath)
+ if len(messages) != 2 || leaf(messages[1], "raw", "factory", "reply_message_id") != "t2-a2" {
+ t.Fatalf("second turn's reply not recorded: %d messages", len(messages))
+ }
+}
+
+// A lagging transcript at Stop records no reply, and the turn still completes.
+func TestFactoryStopWithALaggingTranscriptRecordsNoReply(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "droid-1.jsonl")
+ writeTranscript(t, path, factoryTurn("t1", "x")[:2]...)
+ logPath, _ := runHookProcessIn(t, dir, factoryStopPayload(path), "--platform", "factory", "stop")
+ if messages := assistantMessages(t, logPath); len(messages) != 0 {
+ t.Fatalf("recorded %d replies from a transcript that ended on a tool call", len(messages))
+ }
+ if last := lastEndpointEvent(t, logPath); leaf(last, "event", "action") != "tool.completed" {
+ t.Fatalf("last event = %q, want the turn completion", leaf(last, "event", "action"))
+ }
+}
+
+// The transcript read is Droid's alone: a Claude-shaped runtime whose Stop names a transcript in
+// the same format records nothing from it.
+func TestTranscriptReplyIsReadOnlyForFactory(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "s.jsonl")
+ writeTranscript(t, path, factoryTurn("t1", "x")...)
+ for _, platform := range []string{"devin-cli", "qwen"} {
+ logPath, _ := runHookProcessIn(t, t.TempDir(), factoryStopPayload(path), "--platform", platform, "stop")
+ if messages := assistantMessages(t, logPath); len(messages) != 0 {
+ t.Fatalf("%s read a reply from the transcript: %#v", platform, messages)
+ }
+ }
+}
diff --git a/cli/beacon-hooks/cmd/stop.go b/cli/beacon-hooks/cmd/stop.go
index b319ebc3..de6ee11a 100644
--- a/cli/beacon-hooks/cmd/stop.go
+++ b/cli/beacon-hooks/cmd/stop.go
@@ -74,6 +74,10 @@ func runStop(cmd *cobra.Command, args []string) {
config.RotateLogIfNeededForPlatform(platformFlag)
st := state.NewSessionState(sessionID, platformFlag)
+ // Factory Droid's Stop payload carries no reply; its transcript does.
+ if platformFlag == "factory" {
+ emitFactoryTranscriptReply(logger, input, sessionID, transcriptPath, st)
+ }
// The public Beacon build is local-only. Clear any pending remote evaluations
// left over from older hook versions instead of polling a hosted service.
diff --git a/cli/beacon-hooks/internal/state/state.go b/cli/beacon-hooks/internal/state/state.go
index b44bd84f..81104f4f 100644
--- a/cli/beacon-hooks/internal/state/state.go
+++ b/cli/beacon-hooks/internal/state/state.go
@@ -85,7 +85,10 @@ type SessionData struct {
RepromptCount int `json:"reprompt_count"`
Model string `json:"model,omitempty"`
PromptEmitted bool `json:"prompt_emitted,omitempty"`
- CreatedAt string `json:"created_at,omitempty"`
+ // LastReplyID names the last assistant reply a hook recorded for this session, so a reply read
+ // from a transcript that has not moved since the previous turn is not recorded twice.
+ LastReplyID string `json:"last_reply_id,omitempty"`
+ CreatedAt string `json:"created_at,omitempty"`
}
// SessionState manages session state
@@ -196,6 +199,24 @@ func (s *SessionState) MarkPromptEmittedIfNeeded() (bool, error) {
return true, nil
}
+// MarkReplyRecorded records id as the session's last recorded assistant reply. It reports false,
+// and changes nothing, when id is already the last one recorded, so a caller writes a reply at
+// most once even when two Stop hooks of one session race.
+func (s *SessionState) MarkReplyRecorded(id string) (bool, error) {
+ defer lockState(s.platform)()
+
+ state := s.loadStateFile()
+ s.ensureSession(state)
+ if state[s.sessionID].LastReplyID == id {
+ return false, nil
+ }
+ state[s.sessionID].LastReplyID = id
+ if err := s.saveStateFile(state); err != nil {
+ return false, err
+ }
+ return true, nil
+}
+
// AddEvaluation adds a new pending evaluation
func (s *SessionState) AddEvaluation(evaluationID, filePath string) error {
defer lockState(s.platform)()
diff --git a/cli/beacon-hooks/internal/state/state_test.go b/cli/beacon-hooks/internal/state/state_test.go
index c099e7fe..72363df1 100644
--- a/cli/beacon-hooks/internal/state/state_test.go
+++ b/cli/beacon-hooks/internal/state/state_test.go
@@ -156,3 +156,30 @@ func TestStateIsSafeAcrossProcesses(t *testing.T) {
}
}
}
+
+func TestMarkReplyRecordedOncePerReplyAndSession(t *testing.T) {
+ _, cleanup := setupTestStateDir(t)
+ defer cleanup()
+
+ a := NewSessionState("session-a", "cursor")
+ for i, want := range []bool{true, false} {
+ fresh, err := a.MarkReplyRecorded("msg-1")
+ if err != nil || fresh != want {
+ t.Fatalf("mark %d = %v, %v; want %v", i+1, fresh, err, want)
+ }
+ }
+ // A later reply is new, and a reply id is per session.
+ if fresh, _ := NewSessionState("session-a", "cursor").MarkReplyRecorded("msg-2"); !fresh {
+ t.Fatal("a new reply id was treated as already recorded")
+ }
+ if fresh, _ := NewSessionState("session-b", "cursor").MarkReplyRecorded("msg-2"); !fresh {
+ t.Fatal("another session's reply id suppressed this session's")
+ }
+ // The marker sits beside the other session data without disturbing it.
+ if err := a.SetModel("m"); err != nil {
+ t.Fatal(err)
+ }
+ if fresh, _ := a.MarkReplyRecorded("msg-2"); fresh {
+ t.Fatal("setting the model reset the recorded reply")
+ }
+}
diff --git a/docs/runtimes/factory-droid.mdx b/docs/runtimes/factory-droid.mdx
index f283c246..906c0a1e 100644
--- a/docs/runtimes/factory-droid.mdx
+++ b/docs/runtimes/factory-droid.mdx
@@ -55,6 +55,7 @@ For scheduled collection, run `beacon endpoint factory sync --watch`, or run one
| Area | Support |
|------|---------|
| Prompt telemetry | Supported through `UserPromptSubmit` hooks |
+| Agent responses | Supported live through the `Stop` hook. Droid's Stop payload carries no reply, so the hook reads the end of the session transcript it names and records the final assistant message as `agent.message` (`raw.factory.reply_message_id` names the record). It records nothing when the transcript has not caught up, ending on a tool call, a tool result or the prompt, and records each reply once. `beacon endpoint factory sync` recovers every committed assistant message |
| Command, tool, and file telemetry | Supported for session, write/edit/create hook payloads, committed session tool calls/results, stop, session-end, and available OTLP telemetry |
| Historical sessions | Supported through `beacon endpoint factory sync`, which reads `~/.factory/sessions` locally and offline |
| Token usage | `sync` reads the running totals Factory keeps in each session's `.settings.json` and emits a `token.usage` event with only what they grew by since the last sync, so `beacon token-usage` sums to the session's final total. It counts the session's own `tokenUsage`; `inclusiveTokenUsage` is used only when `tokenUsage` is absent and no child session is listed, because it adds child sessions, which are synced on their own. A total that goes down is treated as a rewritten file and counts nothing. Factory credits are not USD, so they stay in `raw` (`factory_credits_delta`, `factory_credits_total`) rather than `cost_usd` |