Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Count every Codex token_count snapshot once instead of keeping one per turn

Claude80eefb99bc8d
diff --git a/cli/beacon/internal/codexsession/codexsession_test.go b/cli/beacon/internal/codexsession/codexsession_test.go
index cbd3d467..4b8b8b88 100644
--- a/cli/beacon/internal/codexsession/codexsession_test.go
+++ b/cli/beacon/internal/codexsession/codexsession_test.go
@@ -84,32 +84,31 @@ func TestMapCodexTranscriptProducesEndpointEvents(t *testing.T) {
}
}
-func TestTokenCountLastWinsForMultipleSnapshotsPerTurn(t *testing.T) {
+func TestTokenCountCoversEveryCompletionInAMultiCallTurn(t *testing.T) {
ref := SessionRef{ID: "sess-1", Path: "/tmp/codex.jsonl", Workspace: "/tmp/repo"}
records := decodeFixture(t, []string{
sessionMetaLine("sess-1", "/tmp/repo"),
turnContextLine("turn-1", "gpt-6-astra"),
messageLine("user", "msg-user", "build it"),
- tokenCountLine("turn-1", 0, 0, 0, 0),
- tokenCountLine("turn-1", 50, 20, 50, 20),
- tokenCountLine("turn-1", 80, 30, 130, 50),
+ tokenCountLine("turn-1", &TokenUsage{}, &TokenUsage{}),
+ tokenCountLine("turn-1", &TokenUsage{InputTokens: 50, OutputTokens: 20}, &TokenUsage{InputTokens: 50, OutputTokens: 20}),
+ tokenCountLine("turn-1", &TokenUsage{InputTokens: 80, OutputTokens: 30}, &TokenUsage{InputTokens: 130, OutputTokens: 50}),
})
mapped := MapSession(ref, records, MapOptions{})
- var usageEvents []schema.Event
+ var reported int64
+ var count int
for _, item := range mapped {
- if item.Event.Event.Action == "token.usage" {
- usageEvents = append(usageEvents, item.Event)
+ if item.Event.Event.Action != "token.usage" {
+ continue
}
+ count++
+ reported += *item.Event.GenAI.Usage.OutputTokens
}
- if len(usageEvents) != 1 {
- t.Fatalf("expected 1 token.usage event, got %d", len(usageEvents))
- }
- ev := usageEvents[0]
- if ev.GenAI == nil || ev.GenAI.Usage == nil || ev.GenAI.Usage.OutputTokens == nil {
- t.Fatal("missing usage on token.usage event")
- }
- if *ev.GenAI.Usage.OutputTokens != 30 {
- t.Fatalf("output tokens = %d, want 30 (last snapshot)", *ev.GenAI.Usage.OutputTokens)
+ // Each model completion in the turn is reported once, so the reported
+ // output tokens add up to the turn's cumulative total_token_usage rather
+ // than to whichever single snapshot was kept.
+ if count != 2 || reported != 50 {
+ t.Fatalf("token.usage events = %d totalling %d output tokens, want 2 totalling 50", count, reported)
}
}
@@ -260,10 +259,109 @@ func quote(s string) string {
return string(data)
}
-func tokenCountLine(turnID string, lastInput, lastOutput, totalInput, totalOutput int64) string {
- return `{"timestamp":"2026-09-19T22:00:04.000Z","type":"event_msg","payload":{"type":"token_count","turn_id":` + quote(turnID) + `,"info":{"last_token_usage":{"input_tokens":` + i64(lastInput) + `,"output_tokens":` + i64(lastOutput) + `},"total_token_usage":{"input_tokens":` + i64(totalInput) + `,"output_tokens":` + i64(totalOutput) + `}}}}`
-}
-
func i64(v int64) string {
return strconv.FormatInt(v, 10)
}
+
+func TestTokenCountEmitsEveryModelCompletionOnce(t *testing.T) {
+ ref := SessionRef{ID: "sess-1", Path: "/tmp/codex.jsonl", Workspace: "/tmp/repo"}
+ first := &TokenUsage{InputTokens: 100, CachedInputTokens: 90, OutputTokens: 8, TotalTokens: 108}
+ second := &TokenUsage{InputTokens: 40, CachedInputTokens: 30, OutputTokens: 12, TotalTokens: 52}
+ records := decodeFixture(t, []string{
+ sessionMetaLine("sess-1", "/tmp/repo"),
+ turnContextLine("turn-1", "gpt-6-astra"),
+ // An empty snapshot before the first completion carries no usage.
+ tokenCountLine("turn-1", &TokenUsage{}, &TokenUsage{}),
+ tokenCountLine("turn-1", first, first),
+ // Codex re-emits the same snapshot without a new completion.
+ tokenCountLine("turn-1", first, first),
+ tokenCountLine("turn-1", second, &TokenUsage{InputTokens: 140, CachedInputTokens: 120, OutputTokens: 20, TotalTokens: 160}),
+ })
+
+ mapped := MapSession(ref, records, MapOptions{})
+ var outputs []int64
+ for _, item := range mapped {
+ if item.Event.Event.Action != "token.usage" {
+ continue
+ }
+ usage := item.Event.GenAI.Usage
+ if usage == nil || usage.OutputTokens == nil {
+ t.Fatalf("token.usage missing output tokens: %+v", usage)
+ }
+ outputs = append(outputs, *usage.OutputTokens)
+ if err := item.Event.Validate(); err != nil {
+ t.Fatalf("token.usage failed validation: %v", err)
+ }
+ }
+ if len(outputs) != 2 || outputs[0] != 8 || outputs[1] != 12 {
+ t.Fatalf("token.usage output tokens = %v, want [8 12]", outputs)
+ }
+}
+
+func TestTokenCountYieldsToTokenUsageRecord(t *testing.T) {
+ ref := SessionRef{ID: "sess-1", Path: "/tmp/codex.jsonl", Workspace: "/tmp/repo"}
+ snapshot := &TokenUsage{InputTokens: 120, CachedInputTokens: 90, OutputTokens: 8, TotalTokens: 128}
+ records := decodeFixture(t, []string{
+ sessionMetaLine("sess-1", "/tmp/repo"),
+ turnContextLine("turn-1", "gpt-6-astra"),
+ tokenCountLine("turn-1", snapshot, snapshot),
+ tokenUsageLine("sess-1", "turn-1", 120, 90, 10, 8, 2),
+ })
+
+ mapped := MapSession(ref, records, MapOptions{})
+ var sources []string
+ for _, item := range mapped {
+ if item.Event.Event.Action != "token.usage" {
+ continue
+ }
+ raw, _ := item.Event.Raw["codex_session"].(map[string]interface{})
+ source, _ := raw["source"].(string)
+ sources = append(sources, source)
+ }
+ if len(sources) != 1 || sources[0] != "codex_session_token_usage_record" {
+ t.Fatalf("token.usage sources = %v, want the token_usage_record only", sources)
+ }
+}
+
+func TestTokenCountWithoutLastUsageReportsDelta(t *testing.T) {
+ ref := SessionRef{ID: "sess-1", Path: "/tmp/codex.jsonl", Workspace: "/tmp/repo"}
+ records := decodeFixture(t, []string{
+ sessionMetaLine("sess-1", "/tmp/repo"),
+ turnContextLine("turn-1", "gpt-6-astra"),
+ tokenCountLine("turn-1", nil, &TokenUsage{InputTokens: 100, CachedInputTokens: 90, OutputTokens: 8, TotalTokens: 108}),
+ tokenCountLine("turn-1", nil, &TokenUsage{InputTokens: 140, CachedInputTokens: 120, OutputTokens: 20, TotalTokens: 160}),
+ })
+
+ mapped := MapSession(ref, records, MapOptions{})
+ var outputs []int64
+ for _, item := range mapped {
+ if item.Event.Event.Action != "token.usage" {
+ continue
+ }
+ outputs = append(outputs, *item.Event.GenAI.Usage.OutputTokens)
+ }
+ if len(outputs) != 2 || outputs[0] != 8 || outputs[1] != 12 {
+ t.Fatalf("token.usage output tokens = %v, want [8 12] from the cumulative delta", outputs)
+ }
+}
+
+func tokenCountLine(turnID string, last, total *TokenUsage) string {
+ info := map[string]interface{}{"model_context_window": 272000}
+ if last != nil {
+ info["last_token_usage"] = last
+ }
+ if total != nil {
+ info["total_token_usage"] = total
+ }
+ entry := map[string]interface{}{
+ "timestamp": "2026-09-19T22:00:04.500Z",
+ "type": "event_msg",
+ "payload": map[string]interface{}{
+ "type": "token_count",
+ "turn_id": turnID,
+ "info": info,
+ },
+ }
+ data, _ := json.Marshal(entry)
+ return string(data)
+}
diff --git a/cli/beacon/internal/codexsession/mapper.go b/cli/beacon/internal/codexsession/mapper.go
index 6dbc8f86..99c9452d 100644
--- a/cli/beacon/internal/codexsession/mapper.go
+++ b/cli/beacon/internal/codexsession/mapper.go
@@ -42,8 +42,11 @@ type mapper struct {
turnID string
started bool
- tokenRecordTurns map[string]bool
- emittedTokenCountIdx map[string]int
+ tokenRecordTurns map[string]bool
+ // tokenCountTotals remembers the last cumulative total_token_usage seen per
+ // turn so repeated token_count snapshots are skipped while each new model
+ // completion inside a turn is still counted once.
+ tokenCountTotals map[string]TokenUsage
}
func MapSession(ref SessionRef, records []Record, opts MapOptions) []MappedEvent {
@@ -54,13 +57,13 @@ func MapSession(ref SessionRef, records []Record, opts MapOptions) []MappedEvent
}
}
m := &mapper{
- ref: ref,
- opts: opts,
- tools: map[string]codexToolCall{},
- sessionID: ref.ID,
- workspace: ref.Workspace,
- tokenRecordTurns: tokenRecordTurns,
- emittedTokenCountIdx: map[string]int{},
+ ref: ref,
+ opts: opts,
+ tools: map[string]codexToolCall{},
+ sessionID: ref.ID,
+ workspace: ref.Workspace,
+ tokenRecordTurns: tokenRecordTurns,
+ tokenCountTotals: map[string]TokenUsage{},
}
for i := range records {
m.consume(records[i])
@@ -169,6 +172,12 @@ func (m *mapper) consumeEventMessage(record Record, emit bool) {
if msg.TurnID != "" {
m.turnID = msg.TurnID
}
+ if msg.Type == "token_count" {
+ // Handled before the emit guard so the cumulative totals of records
+ // already collected still inform the next sweep's deduplication.
+ m.consumeTokenCount(record, msg, emit)
+ return
+ }
if !emit {
return
}
@@ -186,20 +195,39 @@ func (m *mapper) consumeEventMessage(record Record, emit bool) {
},
}
m.append(record, "task.complete", ev)
- case "token_count":
- m.consumeTokenCount(record, msg)
}
}
-func (m *mapper) consumeTokenCount(record Record, msg *EventMessage) {
+func (m *mapper) consumeTokenCount(record Record, msg *EventMessage, emit bool) {
if msg.Info == nil {
return
}
turnID := firstNonEmpty(msg.TurnID, m.turnID)
+ total := msg.Info.TotalTokenUsage
+ previous, hadPrevious := m.tokenCountTotals[turnID]
+ if total != nil {
+ if hadPrevious && previous == *total {
+ // Codex re-emits token_count without a new model completion (a
+ // rate-limit refresh, for example); nothing was spent since the
+ // last snapshot.
+ return
+ }
+ m.tokenCountTotals[turnID] = *total
+ }
if turnID != "" && m.tokenRecordTurns[turnID] {
+ // token_usage_record carries the authoritative per-turn total.
return
}
- usage := usageFromCodex(firstUsage(msg.Info.LastTokenUsage, msg.Info.TotalTokenUsage))
+ if !emit {
+ return
+ }
+ snapshot := msg.Info.LastTokenUsage
+ if snapshot == nil {
+ // Without the per-completion snapshot, report what the cumulative
+ // total grew by rather than the running total itself.
+ snapshot = deltaTokenUsage(total, previous, hadPrevious)
+ }
+ usage := usageFromCodex(snapshot)
if usage == nil {
return
}
@@ -217,17 +245,7 @@ func (m *mapper) consumeTokenCount(record Record, msg *EventMessage) {
"source": "codex_session_token_count",
},
}
- suffix := "usage.token_count." + firstNonEmpty(msg.TurnID, itoa(record.Line))
- if turnID != "" {
- if idx, ok := m.emittedTokenCountIdx[turnID]; ok {
- coordinate := firstNonEmpty(m.sessionID, m.ref.ID, m.ref.Path) + ":" + itoa(record.Line) + ":" + suffix
- ev.Event.ID = codexEventID(coordinate)
- m.out[idx] = MappedEvent{DedupID: ev.Event.ID, SourceLine: record.Line, Event: ev}
- return
- }
- m.emittedTokenCountIdx[turnID] = len(m.out)
- }
- m.append(record, suffix, ev)
+ m.append(record, "usage.token_count."+firstNonEmpty(msg.TurnID, itoa(record.Line)), ev)
}
func (m *mapper) consumeTokenUsageRecord(record Record, emit bool) {
@@ -512,7 +530,7 @@ func toolResultGenAI(call codexToolCall, result string) *schema.GenAIInfo {
}
func usageFromCodex(raw *TokenUsage) *schema.GenAIUsageInfo {
- if raw == nil {
+ if raw == nil || *raw == (TokenUsage{}) {
return nil
}
usage := &schema.GenAIUsageInfo{}
@@ -532,6 +550,27 @@ func usageFromCodex(raw *TokenUsage) *schema.GenAIUsageInfo {
return usage
}
+// deltaTokenUsage reports how much the cumulative total grew since the previous
+// snapshot, so a session without per-completion usage is not counted as the
+// running total every time it is observed.
+func deltaTokenUsage(total *TokenUsage, previous TokenUsage, hadPrevious bool) *TokenUsage {
+ if total == nil {
+ return nil
+ }
+ if !hadPrevious {
+ return total
+ }
+ delta := TokenUsage{
+ InputTokens: max64(total.InputTokens-previous.InputTokens, 0),
+ CachedInputTokens: max64(total.CachedInputTokens-previous.CachedInputTokens, 0),
+ CacheWriteInputTokens: max64(total.CacheWriteInputTokens-previous.CacheWriteInputTokens, 0),
+ OutputTokens: max64(total.OutputTokens-previous.OutputTokens, 0),
+ ReasoningOutputTokens: max64(total.ReasoningOutputTokens-previous.ReasoningOutputTokens, 0),
+ TotalTokens: max64(total.TotalTokens-previous.TotalTokens, 0),
+ }
+ return &delta
+}
+
func firstUsage(values ...*TokenUsage) *TokenUsage {
for _, value := range values {
if value != nil {