Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(fx): prevent duplicate events on partial emit, fix skip for storage-only events, add snake_case JSON tags to Summary

Cursor Agent78b312541417
diff --git a/cli/beacon/internal/fxsession/collect.go b/cli/beacon/internal/fxsession/collect.go
index 72bdf0aa..4eaf9d91 100644
--- a/cli/beacon/internal/fxsession/collect.go
+++ b/cli/beacon/internal/fxsession/collect.go
@@ -140,12 +140,12 @@ type CollectOptions struct {
// Summary reports what a sweep did. Malformed and PartialSessions are reported rather than logged
// because "Beacon collected nothing" and "Beacon could not read this" look identical from outside.
type Summary struct {
- Sessions int
- SessionsChanged int
- EventsEmitted int
- Errors int
- MalformedLines int
- PartialSessions int
+ Sessions int `json:"sessions"`
+ SessionsChanged int `json:"sessions_changed"`
+ EventsEmitted int `json:"events_emitted"`
+ Errors int `json:"errors"`
+ MalformedLines int `json:"malformed_lines"`
+ PartialSessions int `json:"partial_sessions"`
}
// CollectOnce performs one sweep: list fx's sessions, read the ones that moved since the last
@@ -247,15 +247,18 @@ func collectSession(store *Store, ref SessionRef, state *State, opts CollectOpti
mapped := MapSession(ref, events, MapOptions{MinSeq: minSeq, SkipSessionStarted: cursor.Started})
if len(mapped) == 0 {
- advanceCursor(cursor, ref, events)
+ advanceCursor(cursor, ref, events, stats)
return false, nil
}
for _, item := range mapped {
if err := emit(item.Event, opts); err != nil {
- // The cursor is deliberately not advanced here. The events already written stay
- // written, and the next sweep re-reads from the last confirmed position -- which the
- // writer's duplicate window and the deterministic event ids make safe.
+ // Advance the cursor to cover events that were fully emitted so the
+ // next sweep does not re-append them. Only source events whose entire
+ // set of mapped events was written are advanced past; the partially
+ // written source event is re-emitted in full on the next sweep, and
+ // its coordinate-derived event IDs make the duplicates identifiable.
+ advanceCursorPartial(cursor, ref, events, mapped, &item)
return true, err
}
summary.EventsEmitted++
@@ -263,7 +266,7 @@ func collectSession(store *Store, ref SessionRef, state *State, opts CollectOpti
cursor.Started = true
}
}
- advanceCursor(cursor, ref, events)
+ advanceCursor(cursor, ref, events, stats)
return true, nil
}
@@ -272,20 +275,27 @@ func collectSession(store *Store, ref SessionRef, state *State, opts CollectOpti
// It uses the events actually decoded rather than the manifest's numbers, so a manifest that runs
// ahead of the log -- a copied session directory, an interrupted sync -- cannot advance the cursor
// past records that were never read.
-func advanceCursor(cursor *Cursor, ref SessionRef, events []Event) {
+func advanceCursor(cursor *Cursor, ref SessionRef, events []Event, stats Stats) {
if len(events) == 0 {
return
}
last := events[len(events)-1]
cursor.Generation = last.LogGeneration
cursor.LastSeq = last.Seq
+ // The decoder sees every committed envelope, including non-mapped kinds like
+ // recovery checkpoints that it skips. When those land after the last mapped
+ // event, the cursor's seq falls behind the manifest's LastEventSeq and the
+ // cheap skip in collectSession never fires. Advancing to the highest decoded
+ // seq fixes the gap without trusting the manifest's claim, which can overrun
+ // the log in a copied or interrupted-sync directory.
+ if stats.MaxSeq > cursor.LastSeq {
+ cursor.LastSeq = stats.MaxSeq
+ }
if ref.Manifest != nil {
cursor.UpdatedAtMS = ref.Manifest.UpdatedAtMS
} else {
cursor.UpdatedAtMS = ref.ModTimeUnixMS
}
- // A session whose start was collected before this cursor existed still counts as started: the
- // records are in the log, and re-reporting the start on the next generation would double it.
for i := range events {
if events[i].Kind == KindSessionStarted && events[i].Seq <= cursor.LastSeq {
cursor.Started = true
@@ -294,6 +304,43 @@ func advanceCursor(cursor *Cursor, ref SessionRef, events []Event) {
}
}
+// advanceCursorPartial moves the cursor past source events whose mapped events
+// were all emitted, so the next sweep retries only from the source event that
+// failed. failedItem is the mapped event whose emit returned an error.
+func advanceCursorPartial(cursor *Cursor, ref SessionRef, events []Event, mapped []MappedEvent, failedItem *MappedEvent) {
+ // Walk backwards from the failed item to find the last source event that
+ // was completely emitted (all its mapped events were written).
+ var lastCompleteSeq uint64
+ found := false
+ for i := range mapped {
+ if &mapped[i] == failedItem {
+ break
+ }
+ // When the next mapped event is from a different (higher) source seq,
+ // the current one's source seq is fully emitted.
+ if i+1 < len(mapped) && mapped[i+1].SourceSeq != mapped[i].SourceSeq {
+ lastCompleteSeq = mapped[i].SourceSeq
+ found = true
+ }
+ }
+ if !found {
+ return
+ }
+ cursor.Generation = generationOf(events)
+ cursor.LastSeq = lastCompleteSeq
+ if ref.Manifest != nil {
+ cursor.UpdatedAtMS = ref.Manifest.UpdatedAtMS
+ } else {
+ cursor.UpdatedAtMS = ref.ModTimeUnixMS
+ }
+ for i := range events {
+ if events[i].Kind == KindSessionStarted && events[i].Seq <= lastCompleteSeq {
+ cursor.Started = true
+ break
+ }
+ }
+}
+
func generationOf(events []Event) string {
if len(events) == 0 {
return ""
diff --git a/cli/beacon/internal/fxsession/log.go b/cli/beacon/internal/fxsession/log.go
index 05526bce..bfa425e7 100644
--- a/cli/beacon/internal/fxsession/log.go
+++ b/cli/beacon/internal/fxsession/log.go
@@ -27,6 +27,10 @@ type Stats struct {
Decoded int // envelopes decoded, including kinds with no mapped payload
Skipped int // envelopes skipped: unknown kind or unsupported schema version
Malformed int // lines that were not a decodable envelope, or exceeded MaxFrameBytes
+ // MaxSeq is the highest sequence number across all decoded envelopes, including
+ // non-mapped kinds the decoder skipped. advanceCursor uses it to advance past
+ // storage-only events so the cheap skip in collectSession stays effective.
+ MaxSeq uint64
// PartialTail is set when the file ended without a newline, which means fx was mid-append.
// The incomplete bytes are not decoded and not counted as malformed: they are a frame that
// has not been written yet, and the next sweep will read it whole.
@@ -96,6 +100,9 @@ func (d *Decoder) Next() (*Event, error) {
continue
}
d.stats.Decoded++
+ if event.Seq > d.stats.MaxSeq {
+ d.stats.MaxSeq = event.Seq
+ }
if !mapsToPayload(event.Kind) {
d.stats.Skipped++
continue
diff --git a/cli/beacon/internal/fxsession/mapper.go b/cli/beacon/internal/fxsession/mapper.go
index 4afeae18..2d83b47a 100644
--- a/cli/beacon/internal/fxsession/mapper.go
+++ b/cli/beacon/internal/fxsession/mapper.go
@@ -1,10 +1,12 @@
package fxsession
import (
+ "crypto/sha1"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
+ "io"
"path/filepath"
"strings"
"time"
@@ -41,8 +43,9 @@ var fxMCPControlTools = map[string]bool{
// position within the turn -- rather than from the event's content, so the same fx record maps to
// the same id on every sweep no matter how many times the log is read.
type MappedEvent struct {
- DedupID string
- Event schema.Event
+ DedupID string
+ SourceSeq uint64
+ Event schema.Event
}
// MapOptions controls one mapping pass over a session's decoded log.
@@ -741,9 +744,12 @@ func (m *mapper) emitUsage(event *Event) {
}
func (m *mapper) append(event *Event, suffix string, ev schema.Event) {
+ dedupID := fmt.Sprintf("%s:%s:%d:%s", m.sessionID(), event.LogGeneration, event.Seq, suffix)
+ ev.Event.ID = fxEventID(dedupID)
m.out = append(m.out, MappedEvent{
- DedupID: fmt.Sprintf("%s:%s:%d:%s", m.sessionID(), event.LogGeneration, event.Seq, suffix),
- Event: ev,
+ DedupID: dedupID,
+ SourceSeq: event.Seq,
+ Event: ev,
})
}
@@ -901,5 +907,30 @@ func sha256Hex(value string) string {
return hex.EncodeToString(sum[:])
}
+// fxIDNamespace is the fixed UUID namespace for fx dedup-coordinate event IDs.
+// It is distinct from the content-based namespace in asymptoteobserve so the two
+// derivations can never collide.
+var fxIDNamespace = [16]byte{
+ 0x3a, 0x7c, 0x1f, 0x4b, 0x5a, 0x2c, 0x8e, 0x6d,
+ 0x0d, 0x3f, 0x61, 0x9b, 0x2a, 0x8c, 0x5e, 0x14,
+}
+
+// fxEventID derives a deterministic UUID v5 from a dedup coordinate, so the same
+// fx record always produces the same event ID regardless of how many times it is
+// read. The coordinate encodes session, generation, sequence and position within
+// the turn, which is everything that identifies one mapped event.
+func fxEventID(dedupID string) string {
+ digest := sha1.New()
+ digest.Write(fxIDNamespace[:])
+ io.WriteString(digest, dedupID)
+ var id [16]byte
+ copy(id[:], digest.Sum(nil))
+ id[6] = (id[6] & 0x0f) | 0x50 // version 5
+ id[8] = (id[8] & 0x3f) | 0x80 // RFC 4122 variant
+ out := make([]byte, 32)
+ hex.Encode(out, id[:])
+ return fmt.Sprintf("%s-%s-%s-%s-%s", out[0:8], out[8:12], out[12:16], out[16:20], out[20:32])
+}
+
// millis converts fx's millisecond timestamps into the instant the event writers stamp events with.
func millis(value int64) time.Time { return time.UnixMilli(value) }