Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Fix four Codex session syncing bugs: partial tail, native tool calls, token counts, oversized lines

Cursor Agenta934d705defa
diff --git a/cli/beacon/internal/codexsession/mapper.go b/cli/beacon/internal/codexsession/mapper.go
index d71d9c83..e7797840 100644
--- a/cli/beacon/internal/codexsession/mapper.go
+++ b/cli/beacon/internal/codexsession/mapper.go
@@ -121,6 +121,19 @@ func (m *mapper) consumeResponseItem(record Record, emit bool) {
if emit {
m.emitToolResult(record, item)
}
+ case "function_call":
+ input := parseFunctionArguments(item.Arguments, item.Input)
+ call := codexToolCall{ID: item.CallID, Name: item.Name, Input: input}
+ if call.ID != "" {
+ m.tools[call.ID] = call
+ }
+ if emit {
+ m.emitToolInvoked(record, call)
+ }
+ case "function_call_output":
+ if emit {
+ m.emitToolResult(record, item)
+ }
}
}
@@ -145,21 +158,51 @@ func (m *mapper) consumeEventMessage(record Record, emit bool) {
if msg.TurnID != "" {
m.turnID = msg.TurnID
}
- if !emit || msg.Type != "task_complete" {
+ if !emit {
+ return
+ }
+ switch msg.Type {
+ case "task_complete":
+ ev := m.base(record, "session.status", "session", schema.SeverityInfo, schema.FidelityObserved, "Codex task completed")
+ ev.Raw = map[string]interface{}{
+ "codex_session": map[string]interface{}{
+ "source_path": recordSourcePath(m.ref),
+ "line": record.Line,
+ "payload_type": msg.Type,
+ "turn_id": msg.TurnID,
+ "duration_ms": msg.DurationMS,
+ "time_to_first_token_ms": msg.TimeToFirstTokenMS,
+ },
+ }
+ m.append(record, "task.complete", ev)
+ case "token_count":
+ m.consumeTokenCount(record, msg)
+ }
+}
+
+func (m *mapper) consumeTokenCount(record Record, msg *EventMessage) {
+ if msg.Info == nil {
return
}
- ev := m.base(record, "session.status", "session", schema.SeverityInfo, schema.FidelityObserved, "Codex task completed")
+ usage := usageFromCodex(firstUsage(msg.Info.LastTokenUsage, msg.Info.TotalTokenUsage))
+ if usage == nil {
+ return
+ }
+ ev := m.base(record, "token.usage", "metric", schema.SeverityInfo, schema.FidelityObserved, "Codex token usage observed")
+ ev.GenAI = mergeGenAI(ev.GenAI, &schema.GenAIInfo{Usage: usage})
+ if ev.GenAI == nil {
+ ev.GenAI = &schema.GenAIInfo{}
+ }
+ ev.GenAI.Conversation = &schema.GenAIConversationInfo{ID: m.sessionID}
ev.Raw = map[string]interface{}{
"codex_session": map[string]interface{}{
- "source_path": recordSourcePath(m.ref),
- "line": record.Line,
- "payload_type": msg.Type,
- "turn_id": msg.TurnID,
- "duration_ms": msg.DurationMS,
- "time_to_first_token_ms": msg.TimeToFirstTokenMS,
+ "source_path": recordSourcePath(m.ref),
+ "line": record.Line,
+ "turn_id": msg.TurnID,
+ "source": "codex_session_token_count",
},
}
- m.append(record, "task.complete", ev)
+ m.append(record, "usage.token_count."+firstNonEmpty(msg.TurnID, itoa(record.Line)), ev)
}
func (m *mapper) consumeTokenUsageRecord(record Record, emit bool) {
@@ -373,6 +416,18 @@ func toolOutputText(value interface{}) string {
}
}
+func parseFunctionArguments(args string, fallback interface{}) interface{} {
+ args = strings.TrimSpace(args)
+ if args == "" {
+ return fallback
+ }
+ var parsed interface{}
+ if err := json.Unmarshal([]byte(args), &parsed); err != nil {
+ return args
+ }
+ return parsed
+}
+
func commandFromTool(call codexToolCall) string {
switch v := call.Input.(type) {
case string:
diff --git a/cli/beacon/internal/codexsession/store.go b/cli/beacon/internal/codexsession/store.go
index f73bc9a3..e557c57a 100644
--- a/cli/beacon/internal/codexsession/store.go
+++ b/cli/beacon/internal/codexsession/store.go
@@ -2,6 +2,7 @@ package codexsession
import (
"bufio"
+ "bytes"
"encoding/json"
"errors"
"fmt"
@@ -134,15 +135,30 @@ func (s *Store) Read(ref SessionRef) ([]Record, Stats, error) {
}
func decodeRecords(r io.Reader) ([]Record, Stats, error) {
- scanner := bufio.NewScanner(r)
- scanner.Buffer(make([]byte, 0, 64*1024), MaxLineBytes)
+ br := bufio.NewReaderSize(r, 64*1024)
var records []Record
var stats Stats
lineNo := 0
- for scanner.Scan() {
+ for {
+ line, partial, oversized, err := readLine(br)
+ if err != nil {
+ if errors.Is(err, io.EOF) {
+ if partial {
+ stats.PartialTail = true
+ }
+ break
+ }
+ return records, stats, err
+ }
lineNo++
stats.Lines = lineNo
- line := scanner.Bytes()
+ if oversized {
+ stats.Malformed++
+ if stats.FirstError == nil {
+ stats.FirstError = fmt.Errorf("codex session line exceeds %d bytes", MaxLineBytes)
+ }
+ continue
+ }
if len(strings.TrimSpace(string(line))) == 0 {
continue
}
@@ -157,17 +173,48 @@ func decodeRecords(r io.Reader) ([]Record, Stats, error) {
stats.Decoded++
records = append(records, Record{Line: lineNo, Entry: *entry})
}
- if err := scanner.Err(); err != nil {
- if errors.Is(err, bufio.ErrTooLong) {
- stats.Malformed++
- if stats.FirstError == nil {
- stats.FirstError = fmt.Errorf("codex session line exceeds %d bytes", MaxLineBytes)
+ return records, stats, nil
+}
+
+// readLine returns one complete line without its trailing newline.
+//
+// Three endings are distinguished. A complete line (terminated by \n) is returned with partial
+// and oversized both false. A trailing fragment with no newline is a record Codex is still
+// writing: it is reported as partial so the caller can set PartialTail and avoid advancing the
+// cursor past an incomplete record. A line longer than MaxLineBytes is consumed and discarded
+// as oversized so a corrupt file cannot dictate memory use.
+func readLine(br *bufio.Reader) (line []byte, partial, oversized bool, err error) {
+ var buf []byte
+ for {
+ chunk, readErr := br.ReadSlice('\n')
+ if oversized || len(buf)+len(chunk) > MaxLineBytes {
+ oversized = true
+ buf = nil
+ } else {
+ buf = append(buf, chunk...)
+ }
+ if errors.Is(readErr, bufio.ErrBufferFull) {
+ continue
+ }
+ if errors.Is(readErr, io.EOF) {
+ if oversized {
+ return nil, false, true, nil
+ }
+ if len(buf) > 0 {
+ return buf, true, false, io.EOF
}
- return records, stats, nil
+ return nil, false, false, io.EOF
}
- return records, stats, err
+ if readErr != nil {
+ return nil, false, false, readErr
+ }
+ break
}
- return records, stats, nil
+ if oversized {
+ return nil, false, true, nil
+ }
+ line = bytes.TrimSuffix(bytes.TrimSuffix(buf, []byte("\n")), []byte("\r"))
+ return line, false, false, nil
}
func (s *Store) readIndex() map[string]*IndexEntry {
diff --git a/cli/beacon/internal/codexsession/types.go b/cli/beacon/internal/codexsession/types.go
index 3bd94c17..d99f89b9 100644
--- a/cli/beacon/internal/codexsession/types.go
+++ b/cli/beacon/internal/codexsession/types.go
@@ -56,15 +56,16 @@ type TurnContext struct {
}
type ResponseItem struct {
- Type string `json:"type"`
- ID string `json:"id,omitempty"`
- Role string `json:"role,omitempty"`
- Content interface{} `json:"content,omitempty"`
- Status string `json:"status,omitempty"`
- CallID string `json:"call_id,omitempty"`
- Name string `json:"name,omitempty"`
- Input interface{} `json:"input,omitempty"`
- Output interface{} `json:"output,omitempty"`
+ Type string `json:"type"`
+ ID string `json:"id,omitempty"`
+ Role string `json:"role,omitempty"`
+ Content interface{} `json:"content,omitempty"`
+ Status string `json:"status,omitempty"`
+ CallID string `json:"call_id,omitempty"`
+ Name string `json:"name,omitempty"`
+ Input interface{} `json:"input,omitempty"`
+ Output interface{} `json:"output,omitempty"`
+ Arguments string `json:"arguments,omitempty"`
}
type EventMessage struct {