The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
package fxsession
import (
"errors"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/writer"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionsync"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionwindow"
)
// StateVersion is the on-disk version of the collector's cursor file. It exists so a future change
// to what a cursor holds can reset the file rather than misread it: re-collecting a session is
// cheap and duplicates are suppressed downstream, while misreading a cursor loses events silently.
const StateVersion = 1
// Cursor is what the collector remembers about one fx session between sweeps.
//
// Generation is fx's log generation and LastSeq is the last sequence collected within it. Both are
// needed because fx restarts the sequence at 1 whenever it compacts a session's log -- a cursor
// holding only a sequence would silently skip the first events of every compacted session.
type Cursor struct {
Generation string `json:"generation"`
LastSeq uint64 `json:"last_seq"`
// Started records that a session.started has already been reported. fx writes a fresh
// session_started at sequence 1 of every generation, including the one compaction creates,
// and a session that never restarted must not be reported as starting twice.
Started bool `json:"started"`
// UpdatedAtMS is the manifest's updated_at at the last full sweep, kept only as a cheap
// "nothing changed" check. It is never the thing that decides what to emit.
UpdatedAtMS int64 `json:"updated_at_ms,omitempty"`
}
// State is the collector's cursor file: one cursor per fx session.
type State struct {
Version int `json:"version"`
Sessions map[string]*Cursor `json:"sessions"`
}
var stateFile = sessionsync.StateFile[State]{
Name: "fx",
New: func() *State { return &State{Version: StateVersion, Sessions: map[string]*Cursor{}} },
Valid: func(s *State) bool { return s.Version == StateVersion && s.Sessions != nil },
Stamp: func(s *State) { s.Version = StateVersion },
}
// LoadState reads the cursor file. A missing file is an empty state, which is what a first run
// looks like. An empty path yields state that is never persisted, which is what --print uses.
//
// A file from another state version is discarded rather than parsed: re-collecting a session
// appends events that the writer's own duplicate window and the deterministic event ids already
// make harmless, while reading a cursor under the wrong meaning skips events for good.
func LoadState(path string) (*State, error) { return stateFile.Load(path) }
// Save writes the cursor file atomically. An empty path is a no-op.
func (s *State) Save(path string) error { return stateFile.Save(path, s) }
func (s *State) cursor(sessionID string) *Cursor {
if s.Sessions == nil {
s.Sessions = map[string]*Cursor{}
}
c := s.Sessions[sessionID]
if c == nil {
c = &Cursor{}
s.Sessions[sessionID] = c
}
return c
}
// DefaultStatePath is where the collector keeps its cursor for the current user.
//
// It lives beside Beacon's other endpoint state rather than in fx's directory: it is Beacon's
// record of what Beacon has read, and writing it into ~/.fx would put a file fx does not know about
// inside the store fx owns.
//
// Falls back to the OS temp directory when there is no home to put it in. Some cron and service
// environments have none, and a sweep with a cursor somewhere volatile still beats a sweep with no
// cursor at all -- the latter re-appends every session's whole history on every run.
func DefaultStatePath() string {
base := filepath.Join(os.TempDir(), "beacon")
if home, err := os.UserHomeDir(); err == nil && strings.TrimSpace(home) != "" {
base = filepath.Join(home, ".beacon")
}
return filepath.Join(base, "endpoint", "state", "fx.json")
}
// CollectOptions configures one sweep over fx's session store.
type CollectOptions struct {
// SessionsDir is fx's session directory. Empty means fx's default location.
SessionsDir string
// StatePath is the cursor file. Empty disables persistence, which makes the sweep a dry run
// that re-reads everything next time.
StatePath string
// Write appends mapped events to the runtime JSONL at LogPath.
Write bool
// LogPath is the runtime JSONL. Empty means the writer's default for UserMode.
LogPath string
UserMode bool
// Print writes each mapped event to Out as JSON. Used by --print, which pairs with an empty
// StatePath so a dry run shows the same events every time.
Print bool
Out io.Writer
// Since, when set, limits the sweep to sessions modified at or after it, newest first. Older
// sessions are not read and their cursors do not move, so a later sync still collects them.
// The install-time backfill sets it; an ordinary sync leaves it zero.
Since time.Time
// Budget, when set, caps the bytes this sweep may append. The sweep stops at the first event
// that does not fit and returns writer.ErrBudgetSpent, with cursors past only what was written.
Budget *writer.Budget
// RotateBytes is the size the runtime log rotates at while this sweep writes it; zero is
// writer.DefaultRotateBytes.
RotateBytes int64
// guard keeps the sweep from rotating its own output out of the log; CollectOnce sets it.
guard *writer.RetentionGuard
}
// Summary reports what a sweep did. Malformed and PartialSessions are reported rather than logged
// because "Beacon collected nothing" and "Beacon could not read this" look identical from outside.
type Summary struct {
Sessions int `json:"sessions"`
SessionsChanged int `json:"sessions_changed"`
EventsEmitted int `json:"events_emitted"`
Errors int `json:"errors"`
MalformedLines int `json:"malformed_lines"`
PartialSessions int `json:"partial_sessions"`
sessionsync.Retention
}
// CollectOnce performs one sweep: list fx's sessions, read the ones that moved since the last
// sweep, map their new records, and emit them.
//
// Idempotent by construction. The cursor advances only over records that were actually emitted, so
// a sweep that fails partway leaves the rest to be retried; and because the cursor is a position in
// fx's own log rather than a set of ids, its size does not grow with the session.
//
// One session's failure does not end the sweep. A session directory can be unreadable for ordinary
// reasons -- another user owns it, fx is mid-write, a disk error -- and none of them is a reason to
// collect nothing from the other sessions on the machine.
func CollectOnce(opts CollectOptions) (summary Summary, err error) {
store, err := NewStore(opts.SessionsDir)
if err != nil {
return summary, err
}
refs, err := store.List()
if err != nil {
return summary, err
}
refs = sessionwindow.Recent(refs, opts.Since, func(r SessionRef) int64 { return r.ModTimeUnixMS })
summary.Sessions = len(refs)
if len(refs) == 0 {
return summary, nil
}
state, err := LoadState(opts.StatePath)
if err != nil {
return summary, err
}
// Progress is saved however this returns. Events are appended as each session is processed, so
// a later session's failure must not throw away the cursor advances already earned -- otherwise
// the next sweep re-emits everything the failed sweep already wrote. err is a named return so a
// failure to persist the cursor is reported rather than swallowed: an unsaved sweep re-emits
// everything it just wrote.
defer func() {
if saveErr := state.Save(opts.StatePath); saveErr != nil && err == nil {
err = fmt.Errorf("save fx collector state: %w", saveErr)
}
}()
var errs []error
opts.guard = sessionsync.NewRetentionGuard(opts.Print, opts.Write)
var stopErr error
for _, ref := range refs {
changed, collectErr := collectSession(store, ref, state, opts, &summary)
if sessionsync.Stopped(collectErr) {
// Not a failed session: this one and the rest are left for a later sync, with cursors
// that have not moved past anything that was not written.
if changed {
summary.SessionsChanged++
}
stopErr = collectErr
errs = append(errs, collectErr)
break
}
if collectErr != nil {
summary.Errors++
errs = append(errs, fmt.Errorf("fx session %s: %w", ref.ID, collectErr))
continue
}
if changed {
summary.SessionsChanged++
}
}
if rotatedOut := summary.Finish(opts.sink(), stopErr); rotatedOut != nil {
errs = append(errs, rotatedOut)
}
if len(errs) > 0 {
return summary, errors.Join(errs...)
}
return summary, nil
}
func collectSession(store *Store, ref SessionRef, state *State, opts CollectOptions, summary *Summary) (bool, error) {
cursor := state.cursor(ref.ID)
// The manifest is fx's own summary of what it has committed, so when it says the session has
// not moved past the cursor there is nothing to read. Skipping on it keeps a sweep over a
// machine with hundreds of old sessions proportional to what changed rather than to what
// exists. A session with no readable manifest is always read: there is nothing cheaper to
// trust, and reading it is correct, just not free.
if ref.Manifest != nil && ref.Manifest.LogGeneration == cursor.Generation &&
ref.Manifest.LastEventSeq <= cursor.LastSeq {
return false, nil
}
events, stats, err := store.Read(ref)
summary.MalformedLines += stats.Malformed
if stats.PartialTail {
summary.PartialSessions++
}
if err != nil {
return false, err
}
if len(events) == 0 {
return false, nil
}
minSeq := cursor.LastSeq
if cursor.Generation != "" && cursor.Generation != generationOf(events) {
// fx compacted this session's log: the sequence restarted, and the turns that were in the
// old log are not in the new one -- compaction rewrites it as a session_started plus an
// opaque state blob. So everything in the new generation is new, and starting from zero
// re-reads nothing that was already collected.
//
// The honest limitation of reading a log rather than being handed events: a turn committed
// after the last sweep and before a compaction is folded into that blob and is gone. fx
// compacts on log growth, so this needs a long gap between sweeps and a lot of activity in
// it; sweeping on a schedule (the watch mode of `beacon endpoint fx sync`) is what keeps
// the window small.
minSeq = 0
}
mapped := MapSession(ref, events, MapOptions{MinSeq: minSeq, SkipSessionStarted: cursor.Started})
if len(mapped) == 0 {
advanceCursor(cursor, ref, events, stats)
return false, nil
}
for i, item := range mapped {
if err := opts.sink().Emit(item.Event); err != nil {
// Advance the cursor to cover events that were fully emitted so the
// next sweep does not re-append them. Only source events whose entire
// set of mapped events was written are advanced past; the partially
// written source event is re-emitted in full on the next sweep, and
// its coordinate-derived event IDs make the duplicates identifiable.
advanceCursorPartial(cursor, ref, events, mapped, i)
return true, err
}
summary.EventsEmitted++
if item.Event.Event.Action == "session.started" {
cursor.Started = true
}
}
advanceCursor(cursor, ref, events, stats)
return true, nil
}
// advanceCursor moves the cursor to the last record this sweep read.
//
// It uses the events actually decoded rather than the manifest's numbers, so a manifest that runs
// ahead of the log -- a copied session directory, an interrupted sync -- cannot advance the cursor
// past records that were never read.
func advanceCursor(cursor *Cursor, ref SessionRef, events []Event, stats Stats) {
if len(events) == 0 {
return
}
last := events[len(events)-1]
cursor.Generation = last.LogGeneration
cursor.LastSeq = last.Seq
// The decoder sees every committed envelope, including non-mapped kinds like
// recovery checkpoints that it skips. When those land after the last mapped
// event, the cursor's seq falls behind the manifest's LastEventSeq and the
// cheap skip in collectSession never fires. Advancing to the highest decoded
// seq fixes the gap without trusting the manifest's claim, which can overrun
// the log in a copied or interrupted-sync directory.
if stats.MaxSeq > cursor.LastSeq {
cursor.LastSeq = stats.MaxSeq
}
if ref.Manifest != nil {
cursor.UpdatedAtMS = ref.Manifest.UpdatedAtMS
} else {
cursor.UpdatedAtMS = ref.ModTimeUnixMS
}
for i := range events {
if events[i].Kind == KindSessionStarted && events[i].Seq <= cursor.LastSeq {
cursor.Started = true
break
}
}
}
// advanceCursorPartial moves the cursor past source events whose mapped events
// were all emitted, so the next sweep retries only from the source event that
// failed. failedIdx is the index into mapped of the event whose emit returned
// an error.
func advanceCursorPartial(cursor *Cursor, ref SessionRef, events []Event, mapped []MappedEvent, failedIdx int) {
// Walk backwards from the failed item to find the last source event that
// was completely emitted (all its mapped events were written).
var lastCompleteSeq uint64
found := false
for i := range mapped {
if i == failedIdx {
break
}
// When the next mapped event is from a different (higher) source seq,
// the current one's source seq is fully emitted.
if i+1 < len(mapped) && mapped[i+1].SourceSeq != mapped[i].SourceSeq {
lastCompleteSeq = mapped[i].SourceSeq
found = true
}
}
if !found {
return
}
cursor.Generation = generationOf(events)
cursor.LastSeq = lastCompleteSeq
if ref.Manifest != nil {
cursor.UpdatedAtMS = ref.Manifest.UpdatedAtMS
} else {
cursor.UpdatedAtMS = ref.ModTimeUnixMS
}
for i := range events {
if events[i].Kind == KindSessionStarted && events[i].Seq <= lastCompleteSeq {
cursor.Started = true
break
}
}
}
func generationOf(events []Event) string {
if len(events) == 0 {
return ""
}
return events[len(events)-1].LogGeneration
}
func (o CollectOptions) sink() sessionsync.Sink {
return sessionsync.Sink{Print: o.Print, Out: o.Out, Write: o.Write, Log: writer.Options{Path: o.LogPath, UserMode: o.UserMode, RotateSize: o.RotateBytes, Budget: o.Budget, Guard: o.guard}}
}