Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

13.8 KB
Raw Blame
package fxsession

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/writer"
	"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionsync"
	"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionwindow"
)

// StateVersion is the on-disk version of the collector's cursor file. It exists so a future change
// to what a cursor holds can reset the file rather than misread it: re-collecting a session is
// cheap and duplicates are suppressed downstream, while misreading a cursor loses events silently.
const StateVersion = 1

// Cursor is what the collector remembers about one fx session between sweeps.
//
// Generation is fx's log generation and LastSeq is the last sequence collected within it. Both are
// needed because fx restarts the sequence at 1 whenever it compacts a session's log -- a cursor
// holding only a sequence would silently skip the first events of every compacted session.
type Cursor struct {
	Generation string `json:"generation"`
	LastSeq    uint64 `json:"last_seq"`
	// Started records that a session.started has already been reported. fx writes a fresh
	// session_started at sequence 1 of every generation, including the one compaction creates,
	// and a session that never restarted must not be reported as starting twice.
	Started bool `json:"started"`
	// UpdatedAtMS is the manifest's updated_at at the last full sweep, kept only as a cheap
	// "nothing changed" check. It is never the thing that decides what to emit.
	UpdatedAtMS int64 `json:"updated_at_ms,omitempty"`
}

// State is the collector's cursor file: one cursor per fx session.
type State struct {
	Version  int                `json:"version"`
	Sessions map[string]*Cursor `json:"sessions"`
}

var stateFile = sessionsync.StateFile[State]{
	Name:  "fx",
	New:   func() *State { return &State{Version: StateVersion, Sessions: map[string]*Cursor{}} },
	Valid: func(s *State) bool { return s.Version == StateVersion && s.Sessions != nil },
	Stamp: func(s *State) { s.Version = StateVersion },
}

// LoadState reads the cursor file. A missing file is an empty state, which is what a first run
// looks like. An empty path yields state that is never persisted, which is what --print uses.
//
// A file from another state version is discarded rather than parsed: re-collecting a session
// appends events that the writer's own duplicate window and the deterministic event ids already
// make harmless, while reading a cursor under the wrong meaning skips events for good.
func LoadState(path string) (*State, error) { return stateFile.Load(path) }

// Save writes the cursor file atomically. An empty path is a no-op.
func (s *State) Save(path string) error { return stateFile.Save(path, s) }

func (s *State) cursor(sessionID string) *Cursor {
	if s.Sessions == nil {
		s.Sessions = map[string]*Cursor{}
	}
	c := s.Sessions[sessionID]
	if c == nil {
		c = &Cursor{}
		s.Sessions[sessionID] = c
	}
	return c
}

// DefaultStatePath is where the collector keeps its cursor for the current user.
//
// It lives beside Beacon's other endpoint state rather than in fx's directory: it is Beacon's
// record of what Beacon has read, and writing it into ~/.fx would put a file fx does not know about
// inside the store fx owns.
//
// Falls back to the OS temp directory when there is no home to put it in. Some cron and service
// environments have none, and a sweep with a cursor somewhere volatile still beats a sweep with no
// cursor at all -- the latter re-appends every session's whole history on every run.
func DefaultStatePath() string {
	base := filepath.Join(os.TempDir(), "beacon")
	if home, err := os.UserHomeDir(); err == nil && strings.TrimSpace(home) != "" {
		base = filepath.Join(home, ".beacon")
	}
	return filepath.Join(base, "endpoint", "state", "fx.json")
}

// CollectOptions configures one sweep over fx's session store.
type CollectOptions struct {
	// SessionsDir is fx's session directory. Empty means fx's default location.
	SessionsDir string
	// StatePath is the cursor file. Empty disables persistence, which makes the sweep a dry run
	// that re-reads everything next time.
	StatePath string
	// Write appends mapped events to the runtime JSONL at LogPath.
	Write bool
	// LogPath is the runtime JSONL. Empty means the writer's default for UserMode.
	LogPath  string
	UserMode bool
	// Print writes each mapped event to Out as JSON. Used by --print, which pairs with an empty
	// StatePath so a dry run shows the same events every time.
	Print bool
	Out   io.Writer
	// Since, when set, limits the sweep to sessions modified at or after it, newest first. Older
	// sessions are not read and their cursors do not move, so a later sync still collects them.
	// The install-time backfill sets it; an ordinary sync leaves it zero.
	Since time.Time
	// Budget, when set, caps the bytes this sweep may append. The sweep stops at the first event
	// that does not fit and returns writer.ErrBudgetSpent, with cursors past only what was written.
	Budget *writer.Budget
	// RotateBytes is the size the runtime log rotates at while this sweep writes it; zero is
	// writer.DefaultRotateBytes.
	RotateBytes int64

	// guard keeps the sweep from rotating its own output out of the log; CollectOnce sets it.
	guard *writer.RetentionGuard
}

// Summary reports what a sweep did. Malformed and PartialSessions are reported rather than logged
// because "Beacon collected nothing" and "Beacon could not read this" look identical from outside.
type Summary struct {
	Sessions        int `json:"sessions"`
	SessionsChanged int `json:"sessions_changed"`
	EventsEmitted   int `json:"events_emitted"`
	Errors          int `json:"errors"`
	MalformedLines  int `json:"malformed_lines"`
	PartialSessions int `json:"partial_sessions"`
	sessionsync.Retention
}

// CollectOnce performs one sweep: list fx's sessions, read the ones that moved since the last
// sweep, map their new records, and emit them.
//
// Idempotent by construction. The cursor advances only over records that were actually emitted, so
// a sweep that fails partway leaves the rest to be retried; and because the cursor is a position in
// fx's own log rather than a set of ids, its size does not grow with the session.
//
// One session's failure does not end the sweep. A session directory can be unreadable for ordinary
// reasons -- another user owns it, fx is mid-write, a disk error -- and none of them is a reason to
// collect nothing from the other sessions on the machine.
func CollectOnce(opts CollectOptions) (summary Summary, err error) {
	store, err := NewStore(opts.SessionsDir)
	if err != nil {
		return summary, err
	}
	refs, err := store.List()
	if err != nil {
		return summary, err
	}
	refs = sessionwindow.Recent(refs, opts.Since, func(r SessionRef) int64 { return r.ModTimeUnixMS })
	summary.Sessions = len(refs)
	if len(refs) == 0 {
		return summary, nil
	}

	state, err := LoadState(opts.StatePath)
	if err != nil {
		return summary, err
	}
	// Progress is saved however this returns. Events are appended as each session is processed, so
	// a later session's failure must not throw away the cursor advances already earned -- otherwise
	// the next sweep re-emits everything the failed sweep already wrote. err is a named return so a
	// failure to persist the cursor is reported rather than swallowed: an unsaved sweep re-emits
	// everything it just wrote.
	defer func() {
		if saveErr := state.Save(opts.StatePath); saveErr != nil && err == nil {
			err = fmt.Errorf("save fx collector state: %w", saveErr)
		}
	}()

	var errs []error
	opts.guard = sessionsync.NewRetentionGuard(opts.Print, opts.Write)
	var stopErr error
	for _, ref := range refs {
		changed, collectErr := collectSession(store, ref, state, opts, &summary)
		if sessionsync.Stopped(collectErr) {
			// Not a failed session: this one and the rest are left for a later sync, with cursors
			// that have not moved past anything that was not written.
			if changed {
				summary.SessionsChanged++
			}
			stopErr = collectErr
			errs = append(errs, collectErr)
			break
		}
		if collectErr != nil {
			summary.Errors++
			errs = append(errs, fmt.Errorf("fx session %s: %w", ref.ID, collectErr))
			continue
		}
		if changed {
			summary.SessionsChanged++
		}
	}
	if rotatedOut := summary.Finish(opts.sink(), stopErr); rotatedOut != nil {
		errs = append(errs, rotatedOut)
	}
	if len(errs) > 0 {
		return summary, errors.Join(errs...)
	}
	return summary, nil
}

func collectSession(store *Store, ref SessionRef, state *State, opts CollectOptions, summary *Summary) (bool, error) {
	cursor := state.cursor(ref.ID)

	// The manifest is fx's own summary of what it has committed, so when it says the session has
	// not moved past the cursor there is nothing to read. Skipping on it keeps a sweep over a
	// machine with hundreds of old sessions proportional to what changed rather than to what
	// exists. A session with no readable manifest is always read: there is nothing cheaper to
	// trust, and reading it is correct, just not free.
	if ref.Manifest != nil && ref.Manifest.LogGeneration == cursor.Generation &&
		ref.Manifest.LastEventSeq <= cursor.LastSeq {
		return false, nil
	}

	events, stats, err := store.Read(ref)
	summary.MalformedLines += stats.Malformed
	if stats.PartialTail {
		summary.PartialSessions++
	}
	if err != nil {
		return false, err
	}
	if len(events) == 0 {
		return false, nil
	}

	minSeq := cursor.LastSeq
	if cursor.Generation != "" && cursor.Generation != generationOf(events) {
		// fx compacted this session's log: the sequence restarted, and the turns that were in the
		// old log are not in the new one -- compaction rewrites it as a session_started plus an
		// opaque state blob. So everything in the new generation is new, and starting from zero
		// re-reads nothing that was already collected.
		//
		// The honest limitation of reading a log rather than being handed events: a turn committed
		// after the last sweep and before a compaction is folded into that blob and is gone. fx
		// compacts on log growth, so this needs a long gap between sweeps and a lot of activity in
		// it; sweeping on a schedule (the watch mode of `beacon endpoint fx sync`) is what keeps
		// the window small.
		minSeq = 0
	}

	mapped := MapSession(ref, events, MapOptions{MinSeq: minSeq, SkipSessionStarted: cursor.Started})
	if len(mapped) == 0 {
		advanceCursor(cursor, ref, events, stats)
		return false, nil
	}

	for i, item := range mapped {
		if err := opts.sink().Emit(item.Event); err != nil {
			// Advance the cursor to cover events that were fully emitted so the
			// next sweep does not re-append them. Only source events whose entire
			// set of mapped events was written are advanced past; the partially
			// written source event is re-emitted in full on the next sweep, and
			// its coordinate-derived event IDs make the duplicates identifiable.
			advanceCursorPartial(cursor, ref, events, mapped, i)
			return true, err
		}
		summary.EventsEmitted++
		if item.Event.Event.Action == "session.started" {
			cursor.Started = true
		}
	}
	advanceCursor(cursor, ref, events, stats)
	return true, nil
}

// advanceCursor moves the cursor to the last record this sweep read.
//
// It uses the events actually decoded rather than the manifest's numbers, so a manifest that runs
// ahead of the log -- a copied session directory, an interrupted sync -- cannot advance the cursor
// past records that were never read.
func advanceCursor(cursor *Cursor, ref SessionRef, events []Event, stats Stats) {
	if len(events) == 0 {
		return
	}
	last := events[len(events)-1]
	cursor.Generation = last.LogGeneration
	cursor.LastSeq = last.Seq
	// The decoder sees every committed envelope, including non-mapped kinds like
	// recovery checkpoints that it skips. When those land after the last mapped
	// event, the cursor's seq falls behind the manifest's LastEventSeq and the
	// cheap skip in collectSession never fires. Advancing to the highest decoded
	// seq fixes the gap without trusting the manifest's claim, which can overrun
	// the log in a copied or interrupted-sync directory.
	if stats.MaxSeq > cursor.LastSeq {
		cursor.LastSeq = stats.MaxSeq
	}
	if ref.Manifest != nil {
		cursor.UpdatedAtMS = ref.Manifest.UpdatedAtMS
	} else {
		cursor.UpdatedAtMS = ref.ModTimeUnixMS
	}
	for i := range events {
		if events[i].Kind == KindSessionStarted && events[i].Seq <= cursor.LastSeq {
			cursor.Started = true
			break
		}
	}
}

// advanceCursorPartial moves the cursor past source events whose mapped events
// were all emitted, so the next sweep retries only from the source event that
// failed. failedIdx is the index into mapped of the event whose emit returned
// an error.
func advanceCursorPartial(cursor *Cursor, ref SessionRef, events []Event, mapped []MappedEvent, failedIdx int) {
	// Walk backwards from the failed item to find the last source event that
	// was completely emitted (all its mapped events were written).
	var lastCompleteSeq uint64
	found := false
	for i := range mapped {
		if i == failedIdx {
			break
		}
		// When the next mapped event is from a different (higher) source seq,
		// the current one's source seq is fully emitted.
		if i+1 < len(mapped) && mapped[i+1].SourceSeq != mapped[i].SourceSeq {
			lastCompleteSeq = mapped[i].SourceSeq
			found = true
		}
	}
	if !found {
		return
	}
	cursor.Generation = generationOf(events)
	cursor.LastSeq = lastCompleteSeq
	if ref.Manifest != nil {
		cursor.UpdatedAtMS = ref.Manifest.UpdatedAtMS
	} else {
		cursor.UpdatedAtMS = ref.ModTimeUnixMS
	}
	for i := range events {
		if events[i].Kind == KindSessionStarted && events[i].Seq <= lastCompleteSeq {
			cursor.Started = true
			break
		}
	}
}

func generationOf(events []Event) string {
	if len(events) == 0 {
		return ""
	}
	return events[len(events)-1].LogGeneration
}

func (o CollectOptions) sink() sessionsync.Sink {
	return sessionsync.Sink{Print: o.Print, Out: o.Out, Write: o.Write, Log: writer.Options{Path: o.LogPath, UserMode: o.UserMode, RotateSize: o.RotateBytes, Budget: o.Budget, Guard: o.guard}}
}