Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

Avoid duplicate Codex usage snapshots

shukieshah1cf36d04cea8
diff --git a/cli/beacon-hooks/cmd/codex_usage_sync.go b/cli/beacon-hooks/cmd/codex_usage_sync.go
index 9a1dfab3..78ed87a3 100644
--- a/cli/beacon-hooks/cmd/codex_usage_sync.go
+++ b/cli/beacon-hooks/cmd/codex_usage_sync.go
@@ -42,18 +42,15 @@ func maybeReconcileCodexUsage(logger *logging.Logger) {
}
func reconcileCodexUsage(logger *logging.Logger) error {
- result, err := codexusage.Reconcile(codexusage.ReconcileOptions{})
- if err != nil {
- return err
- }
- for _, event := range result.Events {
+ result, err := codexusage.ReconcileAndWrite(codexusage.ReconcileOptions{}, func(event codexusage.UsageEvent) error {
fields := codexUsageFields(event)
if err := logger.EndpointEvent("token.usage", "metric", "info", "Codex token usage observed", fields); err != nil {
return err
}
- if err := codexusage.MarkEventSeen(event, ""); err != nil {
- return err
- }
+ return nil
+ })
+ if err != nil {
+ return err
}
logger.Debug("Codex usage reconciliation completed", "events", len(result.Events), "scanned", result.Scanned)
return nil
diff --git a/cli/beacon/cmd/endpoint_codex_usage.go b/cli/beacon/cmd/endpoint_codex_usage.go
index a5f139d4..23c93d4f 100644
--- a/cli/beacon/cmd/endpoint_codex_usage.go
+++ b/cli/beacon/cmd/endpoint_codex_usage.go
@@ -50,20 +50,13 @@ func runEndpointCodexUsageSync(cmd *cobra.Command, args []string) error {
}
func syncCodexUsageToLog(logPath string, userMode bool) (codexusage.ReconcileResult, error) {
- result, err := codexusage.Reconcile(codexusage.ReconcileOptions{})
- if err != nil {
- return codexusage.ReconcileResult{}, err
- }
- for _, usage := range result.Events {
+ return codexusage.ReconcileAndWrite(codexusage.ReconcileOptions{}, func(usage codexusage.UsageEvent) error {
event := codexUsageSchemaEvent(usage)
if _, err := writer.AppendEvent(event, writer.Options{Path: logPath, UserMode: userMode}); err != nil {
- return codexusage.ReconcileResult{}, err
- }
- if err := codexusage.MarkEventSeen(usage, ""); err != nil {
- return codexusage.ReconcileResult{}, err
+ return err
}
- }
- return result, nil
+ return nil
+ })
}
func codexUsageSchemaEvent(usage codexusage.UsageEvent) schema.Event {
diff --git a/pkg/asymptoteobserve/codexusage/codexusage.go b/pkg/asymptoteobserve/codexusage/codexusage.go
index 154d35cb..b0531c40 100644
--- a/pkg/asymptoteobserve/codexusage/codexusage.go
+++ b/pkg/asymptoteobserve/codexusage/codexusage.go
@@ -48,6 +48,8 @@ type ReconcileResult struct {
Scanned int
}
+type WriteFunc func(UsageEvent) error
+
type rawLine struct {
Type string `json:"type"`
Timestamp string `json:"timestamp"`
@@ -142,8 +144,16 @@ func ParseFile(path string, opts ParseOptions) ([]UsageEvent, error) {
model string
turnID string
lastUsageRaw string
+ pending *UsageEvent
gate forkGate
)
+ flushPending := func() {
+ if pending == nil {
+ return
+ }
+ events = append(events, *pending)
+ pending = nil
+ }
for scanner.Scan() {
line := strings.TrimSpace(scanner.Text())
if line == "" {
@@ -156,6 +166,7 @@ func ParseFile(path string, opts ParseOptions) ([]UsageEvent, error) {
ts := parseTimestamp(raw.Timestamp)
switch raw.Type {
case "session_meta":
+ flushPending()
if gate.active {
continue
}
@@ -171,6 +182,7 @@ func ParseFile(path string, opts ParseOptions) ([]UsageEvent, error) {
}
gate.arm(meta, ts)
case "turn_context":
+ flushPending()
var payload turnContextPayload
if err := json.Unmarshal(raw.Payload, &payload); err != nil {
continue
@@ -228,12 +240,13 @@ func ParseFile(path string, opts ParseOptions) ([]UsageEvent, error) {
SourcePath: path,
}
event.DedupKey = DedupKey(event, usageRaw)
- events = append(events, event)
+ pending = &event
}
}
if err := scanner.Err(); err != nil {
return nil, err
}
+ flushPending()
return events, nil
}
@@ -269,6 +282,54 @@ func Reconcile(opts ReconcileOptions) (ReconcileResult, error) {
return result, nil
}
+func ReconcileAndWrite(opts ReconcileOptions, write WriteFunc) (ReconcileResult, error) {
+ if write == nil {
+ return ReconcileResult{}, fmt.Errorf("nil Codex usage writer")
+ }
+ statePath := resolveStatePath(opts.StatePath)
+ unlock, err := lockState(statePath)
+ if err != nil {
+ return ReconcileResult{}, err
+ }
+ defer unlock()
+
+ files, err := DiscoverFiles(opts.Roots)
+ if err != nil {
+ return ReconcileResult{}, err
+ }
+ seen, err := loadStateFile(statePath)
+ if err != nil {
+ return ReconcileResult{}, err
+ }
+ result := ReconcileResult{Scanned: len(files)}
+ for _, path := range files {
+ if !opts.ModifiedSince.IsZero() {
+ info, err := os.Stat(path)
+ if err != nil || info.ModTime().Before(opts.ModifiedSince) {
+ continue
+ }
+ }
+ events, err := ParseFile(path, ParseOptions{})
+ if err != nil {
+ continue
+ }
+ for _, event := range events {
+ if event.DedupKey == "" || seen[event.DedupKey] {
+ continue
+ }
+ if err := write(event); err != nil {
+ return result, err
+ }
+ seen[event.DedupKey] = true
+ if err := saveStateFile(statePath, seen); err != nil {
+ return result, err
+ }
+ result.Events = append(result.Events, event)
+ }
+ }
+ return result, nil
+}
+
func MarkEventsSeen(events []UsageEvent, statePath string) error {
if len(events) == 0 {
return nil
@@ -353,9 +414,10 @@ func DefaultStatePath() string {
}
func LoadState(path string) (map[string]bool, error) {
- if path == "" {
- path = DefaultStatePath()
- }
+ return loadStateFile(resolveStatePath(path))
+}
+
+func loadStateFile(path string) (map[string]bool, error) {
data, err := os.ReadFile(path)
if err != nil {
if os.IsNotExist(err) {
@@ -374,9 +436,10 @@ func LoadState(path string) (map[string]bool, error) {
}
func SaveState(path string, seen map[string]bool) error {
- if path == "" {
- path = DefaultStatePath()
- }
+ return saveStateFile(resolveStatePath(path), seen)
+}
+
+func saveStateFile(path string, seen map[string]bool) error {
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
@@ -387,6 +450,13 @@ func SaveState(path string, seen map[string]bool) error {
return os.WriteFile(path, data, 0600)
}
+func resolveStatePath(path string) string {
+ if strings.TrimSpace(path) != "" {
+ return path
+ }
+ return DefaultStatePath()
+}
+
func SourcePathHash(path string) string {
sum := sha256.Sum256([]byte(filepath.Clean(path)))
return hex.EncodeToString(sum[:])
diff --git a/pkg/asymptoteobserve/codexusage/codexusage_test.go b/pkg/asymptoteobserve/codexusage/codexusage_test.go
index b4e92a3e..2cd2ef78 100644
--- a/pkg/asymptoteobserve/codexusage/codexusage_test.go
+++ b/pkg/asymptoteobserve/codexusage/codexusage_test.go
@@ -39,6 +39,30 @@ func TestParseFileNormalizesAndDedupesTokenCount(t *testing.T) {
}
}
+func TestParseFileKeepsLatestTokenSnapshotPerTurn(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "session.jsonl")
+ lines := []string{
+ `{"type":"session_meta","timestamp":"2026-06-29T10:00:00Z","payload":{"id":"codex-session-1"}}`,
+ `{"type":"turn_context","timestamp":"2026-06-29T10:00:01Z","payload":{"model":"gpt-5","turn_id":"turn-1"}}`,
+ `{"type":"event_msg","timestamp":"2026-06-29T10:00:02Z","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":100,"output_tokens":5,"cached_input_tokens":40}}}}`,
+ `{"type":"event_msg","timestamp":"2026-06-29T10:00:03Z","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":200,"output_tokens":9,"cached_input_tokens":80}}}}`,
+ }
+ if err := os.WriteFile(path, []byte(strings.Join(lines, "\n")+"\n"), 0644); err != nil {
+ t.Fatal(err)
+ }
+
+ events, err := ParseFile(path, ParseOptions{})
+ if err != nil {
+ t.Fatalf("ParseFile returned error: %v", err)
+ }
+ if len(events) != 1 {
+ t.Fatalf("events = %#v, want latest snapshot only", events)
+ }
+ if got := events[0]; got.InputTokens != 120 || got.CacheReadTokens != 80 || got.OutputTokens != 9 {
+ t.Fatalf("latest snapshot tokens = %#v", got)
+ }
+}
+
func TestReconcileUsesStateForIdempotence(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "session.jsonl")
@@ -69,6 +93,40 @@ func TestReconcileUsesStateForIdempotence(t *testing.T) {
}
}
+func TestReconcileAndWriteMarksAfterSuccessfulWrite(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "session.jsonl")
+ statePath := filepath.Join(dir, "state.json")
+ line := `{"type":"session_meta","timestamp":"2026-06-29T10:00:00Z","payload":{"id":"codex-session-1"}}` + "\n" +
+ `{"type":"turn_context","timestamp":"2026-06-29T10:00:01Z","payload":{"model":"gpt-5","turn_id":"turn-1"}}` + "\n" +
+ `{"type":"event_msg","timestamp":"2026-06-29T10:00:03Z","payload":{"type":"token_count","info":{"last_token_usage":{"input_tokens":10,"output_tokens":2,"cached_input_tokens":4}}}}` + "\n"
+ if err := os.WriteFile(path, []byte(line), 0644); err != nil {
+ t.Fatal(err)
+ }
+
+ var writes int
+ first, err := ReconcileAndWrite(ReconcileOptions{Roots: []string{dir}, StatePath: statePath}, func(event UsageEvent) error {
+ writes++
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("first reconcile write: %v", err)
+ }
+ if len(first.Events) != 1 || writes != 1 {
+ t.Fatalf("first result events=%#v writes=%d, want one write", first.Events, writes)
+ }
+ second, err := ReconcileAndWrite(ReconcileOptions{Roots: []string{dir}, StatePath: statePath}, func(event UsageEvent) error {
+ writes++
+ return nil
+ })
+ if err != nil {
+ t.Fatalf("second reconcile write: %v", err)
+ }
+ if len(second.Events) != 0 || writes != 1 {
+ t.Fatalf("second result events=%#v writes=%d, want no duplicate write", second.Events, writes)
+ }
+}
+
func TestLoadStateRejectsCorruptJSON(t *testing.T) {
statePath := filepath.Join(t.TempDir(), "state.json")
if err := os.WriteFile(statePath, []byte("{not json"), 0600); err != nil {
diff --git a/pkg/asymptoteobserve/codexusage/lock_unix.go b/pkg/asymptoteobserve/codexusage/lock_unix.go
new file mode 100644
index 00000000..299ab6d2
--- /dev/null
+++ b/pkg/asymptoteobserve/codexusage/lock_unix.go
@@ -0,0 +1,27 @@
+//go:build !windows
+
+package codexusage
+
+import (
+ "os"
+ "path/filepath"
+ "syscall"
+)
+
+func lockState(path string) (func(), error) {
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ return nil, err
+ }
+ lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, 0600)
+ if err != nil {
+ return nil, err
+ }
+ if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
+ _ = lock.Close()
+ return nil, err
+ }
+ return func() {
+ _ = syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)
+ _ = lock.Close()
+ }, nil
+}
diff --git a/pkg/asymptoteobserve/codexusage/lock_windows.go b/pkg/asymptoteobserve/codexusage/lock_windows.go
new file mode 100644
index 00000000..f84fb824
--- /dev/null
+++ b/pkg/asymptoteobserve/codexusage/lock_windows.go
@@ -0,0 +1,7 @@
+//go:build windows
+
+package codexusage
+
+func lockState(path string) (func(), error) {
+ return func() {}, nil
+}