The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix(sync): stable ids for Grok chat events; backfill treats a retention stop as limited (#1019)
Justin D'Souzad9892d7a91a6
diff --git a/cli/beacon/cmd/endpoint_backfill.go b/cli/beacon/cmd/endpoint_backfill.goindex 6963110e..11baafe5 100644--- a/cli/beacon/cmd/endpoint_backfill.go+++ b/cli/beacon/cmd/endpoint_backfill.go@@ -29,6 +29,7 @@ import ("github.com/asymptote-labs/agent-beacon/cli/beacon/internal/pisession""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/primesession""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionfs"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/sessionsync")// The install-time session backfill reads recent history from every agent's own session store,@@ -175,7 +176,10 @@ type endpointBackfillRuntimeResult struct {Sync stringEvents intLimited bool- Err error+ // RetentionLimited says the runtime stopped before log rotation would discard what it had+ // written, rather than at its share of the size budget.+ RetentionLimited bool+ Err error// Oversized names the databases a system backfill would not copy because they are too large;// Sync, run as the user, reads them in place.Oversized []string@@ -263,11 +267,12 @@ func sweepEndpointBackfill(logPath string, userMode bool) endpointBackfillResult// runtime's before the next one runs. A no-op otherwise.sessionfs.DropSnapshots()runtime := endpointBackfillRuntimeResult{Name: runner.Name, Sync: runner.Sync, Events: events, Oversized: sessionfs.TakeOversized()}- if errors.Is(err, writer.ErrBudgetSpent) || share.Refused() {+ if sessionsync.Stopped(err) || share.Refused() {runtime.Limited = trueresult.Limited = true}- if err != nil && !onlyBudgetSpent(err) {+ runtime.RetentionLimited = errors.Is(err, writer.ErrRetentionWindowFull)+ if err != nil && !onlyStopped(err) {runtime.Err = err}result.Events += events@@ -279,17 +284,19 @@ func sweepEndpointBackfill(logPath string, userMode bool) endpointBackfillResultreturn result}-// onlyBudgetSpent reports whether err says nothing but that the budget ran out.-func onlyBudgetSpent(err error) bool {+// onlyStopped reports whether err says nothing but that the sweep stopped early: the budget ran+// out, or writing more would have rotated the sweep's own output out of the log. Either way the+// collector's cursors stop at what was written and the runtime's sync reads the rest.+func onlyStopped(err error) bool {if joined, ok := err.(interface{ Unwrap() []error }); ok {for _, e := range joined.Unwrap() {- if !onlyBudgetSpent(e) {+ if !onlyStopped(e) {return false}}return true}- return errors.Is(err, writer.ErrBudgetSpent)+ return sessionsync.Stopped(err)}// reportEndpointBackfill prints what a backfill wrote: one line when it went cleanly, plus a line@@ -325,6 +332,8 @@ func reportEndpointBackfill(out io.Writer, result endpointBackfillResult) {// The error is the collector failing to open the store named just above.case r.Err != nil:fmt.Fprintf(out, " %s: %v; retry with `%s`.\n", r.Name, r.Err, r.Sync)+ case r.RetentionLimited:+ fmt.Fprintf(out, " %s: stopped before log rotation would discard what it wrote; `%s` reads the rest.\n", r.Name, r.Sync)case r.Limited:fmt.Fprintf(out, " %s: stopped at the backfill size limit; `%s` reads the rest.\n", r.Name, r.Sync)}diff --git a/cli/beacon/cmd/endpoint_backfill_test.go b/cli/beacon/cmd/endpoint_backfill_test.goindex 08b95d61..4675973a 100644--- a/cli/beacon/cmd/endpoint_backfill_test.go+++ b/cli/beacon/cmd/endpoint_backfill_test.go@@ -110,6 +110,31 @@ func TestEndpointBackfillGivesEachRuntimeAShareAndNamesWhereToContinue(t *testin}}+// A collector that stops before rotation would discard its own output has left the rest for its+// sync, exactly as one that spent its share has: limited, not failed, and named as such.+func TestEndpointBackfillReportsARetentionStopAsLimited(t *testing.T) {+ stubBackfill(t, 1<<20, 1<<20,+ endpointBackfillRunner{Name: "Rotating", Sync: "beacon endpoint rotating sync", Run: func(endpointBackfillRun) (int, error) {+ return 7, errors.Join(writer.ErrRetentionWindowFull)+ }},+ endpointBackfillRunner{Name: "Budget", Sync: "beacon endpoint budget sync", Run: func(endpointBackfillRun) (int, error) {+ return 3, writer.ErrBudgetSpent+ }},+ )+ result := runEndpointBackfill(filepath.Join(t.TempDir(), "runtime.jsonl"), true)+ byName := map[string]endpointBackfillRuntimeResult{}+ for _, r := range result.Runtimes {+ byName[r.Name] = r+ }+ rotating, budget := byName["Rotating"], byName["Budget"]+ if !result.Limited || !rotating.Limited || !rotating.RetentionLimited || rotating.Err != nil || rotating.Events != 7 {+ t.Fatalf("retention stop = %+v, result limited %v", rotating, result.Limited)+ }+ if !budget.Limited || budget.RetentionLimited || budget.Err != nil {+ t.Fatalf("budget stop = %+v", budget)+ }+}+func TestEndpointInstallBackfillsAfterTheInstallSucceeds(t *testing.T) {recordEndpointLifecycle(t)logPath := filepath.Join(t.TempDir(), "runtime.jsonl")diff --git a/cli/beacon/internal/groksession/handoff_test.go b/cli/beacon/internal/groksession/handoff_test.goindex 5400a794..91a53a31 100644--- a/cli/beacon/internal/groksession/handoff_test.go+++ b/cli/beacon/internal/groksession/handoff_test.go@@ -44,8 +44,7 @@ func TestMapSessionRecordsAHandoffLink(t *testing.T) {if link.Event.Prompt != nil || link.Event.Content != nil || link.Event.GenAI != nil {t.Fatalf("the link must not repeat the prompt: %+v", link.Event)}- // Every other Grok event is named by the writer from its bytes; the link names itself, from- // where it was read, because its bytes carry the time of the sync.+ // The link names itself from where it was read, because its bytes carry the time of the sync.if link.Event.Event.ID == "" {t.Fatal("the link has no id")}diff --git a/cli/beacon/internal/groksession/mapper.go b/cli/beacon/internal/groksession/mapper.goindex d6320fff..a702ca22 100644--- a/cli/beacon/internal/groksession/mapper.go+++ b/cli/beacon/internal/groksession/mapper.go@@ -53,6 +53,10 @@ type mapper struct {out []MappedEventtoolByID map[string]ToolCalltoolOutput map[string]string+ // chatLine and chatOrdinal number the events mapped from one chat_history.jsonl line, so each+ // is named by where it was read (see chatEventID).+ chatLine int+ chatOrdinal int}type orderedItem struct {@@ -288,12 +292,29 @@ func (m *mapper) base(ts, action, category string, severity schema.Severity, fid// The cursor is per source file and holds the last line written, so a line is skipped at or below// it. session.started has no line of its own and is gated by its own flag.func (m *mapper) append(ev schema.Event, kind SourceKind, line int) {+ if kind == SourceChat && ev.Event.ID == "" {+ ev.Event.ID = m.chatEventID(ev, line)+ }if m.collected(kind, line) {return}m.out = append(m.out, MappedEvent{Event: ev, SourceKind: kind, SourceLine: line})}+// chatEventID names an event mapped from a chat_history.jsonl line by where it was read: the+// session, the line, and its place among that line's events. A chat line carries no time, so the+// event is stamped with the time of the sync, and the id the writer would derive from its bytes+// would differ on every sync; a resend after a sync stopped partway through a session would then+// be a second event rather than the same one. The line is immutable and the whole session is+// mapped on every sweep, so the same line always yields the same events in the same order.+func (m *mapper) chatEventID(ev schema.Event, line int) string {+ if line != m.chatLine {+ m.chatLine, m.chatOrdinal = line, 0+ }+ m.chatOrdinal+++ return grokEventID(fmt.Sprintf("%s:%s:%d:%d:%s", m.data.Ref.ID, m.data.Ref.SourcePath, line, m.chatOrdinal, ev.Event.Action))+}+func (m *mapper) collected(kind SourceKind, line int) bool {switch kind {case SourceChat:diff --git a/cli/beacon/internal/groksession/retention_test.go b/cli/beacon/internal/groksession/retention_test.goindex f81fe046..611613ab 100644--- a/cli/beacon/internal/groksession/retention_test.go+++ b/cli/beacon/internal/groksession/retention_test.go@@ -18,9 +18,7 @@ func TestCollectOnceKeepsWhatItWritesAcrossTheRotationWindow(t *testing.T) {writeFixtureSession(t, root, "/tmp/grok project", fmt.Sprintf("session-%03d", i))}opts := CollectOptions{SessionsDir: root, Write: true, UserMode: true}- // chat_history.jsonl records carry no time, so their events are stamped when read and their- // ids differ between sweeps; compare them without it.- sessionsynctest.CheckSweepKeepsItsOutputBy(t, sessionsynctest.IgnoringTimestamp, func(statePath, logPath string, rotateBytes int64) (sessionsynctest.Result, error) {+ sessionsynctest.CheckSweepKeepsItsOutput(t, func(statePath, logPath string, rotateBytes int64) (sessionsynctest.Result, error) {opts.StatePath, opts.LogPath, opts.RotateBytes = statePath, logPath, rotateBytess, err := CollectOnce(opts)return sessionsynctest.Result{EventsEmitted: s.EventsEmitted, Errors: s.Errors, Retention: s.Retention}, errdiff --git a/cli/beacon/internal/sessionsync/sessionsynctest/sessionsynctest.go b/cli/beacon/internal/sessionsync/sessionsynctest/sessionsynctest.goindex c837bc7d..02060a13 100644--- a/cli/beacon/internal/sessionsync/sessionsynctest/sessionsynctest.go+++ b/cli/beacon/internal/sessionsync/sessionsynctest/sessionsynctest.go@@ -92,28 +92,6 @@ func shipAway(t *testing.T, logPath string, batch int) []schema.Event {// ids the forwarder and ingest deduplicate on; an event lost, or written again under a// different id, fails.func CheckSweepKeepsItsOutput(t *testing.T, sweep Sweep) {- t.Helper()- CheckSweepKeepsItsOutputBy(t, EventID, sweep)-}--// EventID identifies an event by its deterministic event.id.-func EventID(ev schema.Event) string { return ev.Event.ID }--// IgnoringTimestamp identifies an event by everything but its timestamp and the id derived from-// it, for a collector whose records carry no time of their own, so each sweep stamps its events-// with the time it read them.-func IgnoringTimestamp(ev schema.Event) string {- ev.Timestamp = ""- ev.Event.ID = ""- data, err := json.Marshal(ev)- if err != nil {- return ""- }- return string(data)-}--// CheckSweepKeepsItsOutputBy is CheckSweepKeepsItsOutput comparing events by key.-func CheckSweepKeepsItsOutputBy(t *testing.T, key func(schema.Event) string, sweep Sweep) {t.Helper()dir := t.TempDir()@@ -124,7 +102,7 @@ func CheckSweepKeepsItsOutputBy(t *testing.T, key func(schema.Event) string, swe}want := map[string]bool{}for _, ev := range RetainedEvents(t, refLog) {- want[key(ev)] = true+ want[ev.Event.ID] = true}if ref.RetentionLimited || len(want) == 0 || len(want) != ref.EventsEmitted {t.Fatalf("reference sweep: emitted=%d distinct events=%d limited=%v", ref.EventsEmitted, len(want), ref.RetentionLimited)@@ -154,7 +132,7 @@ func CheckSweepKeepsItsOutputBy(t *testing.T, key func(schema.Event) string, swebatch, result.EventsEmitted, result.EventsRetained, len(shipped))}for _, ev := range shipped {- got[key(ev)] = true+ got[ev.Event.ID] = true}if batch == 0 && !result.RetentionLimited {t.Fatal("the first sweep did not fill the window; the fixture proves nothing")diff --git a/docs/cli/endpoint-install.mdx b/docs/cli/endpoint-install.mdxindex 397770dc..044277e9 100644--- a/docs/cli/endpoint-install.mdx+++ b/docs/cli/endpoint-install.mdx@@ -73,7 +73,9 @@ runtime log's rotation window:Nothing past a limit is lost. The collector cursors move only past what was written, and sessionsleft out are not touched, so the runtime's sync command reads the rest; install names the command-for each runtime that hit the limit. A runtime whose store cannot be read is reported as a warning+for each runtime that hit the limit. A runtime also stops early, and is reported the same way,+if writing more would rotate its own backfill out of the log, which takes heavy live activity+during the backfill. A runtime whose store cannot be read is reported as a warningand never fails the install. Re-running install reads only what changed since the last sweep.A system, package or MDM install runs as root, whose home holds no agent sessions, so it reads thediff --git a/docs/runtimes/grok-build.mdx b/docs/runtimes/grok-build.mdxindex 4ee83d02..b9d6f2a5 100644--- a/docs/runtimes/grok-build.mdx+++ b/docs/runtimes/grok-build.mdx@@ -78,6 +78,8 @@ Session sync keeps its cursor in Beacon endpoint state rather than in `~/.grok`,The cursor is a line position in each of a session's two append-only logs, so a session that is still being written is collected incrementally: a later sweep, including each `--watch` tick, emits only the records added since the last one. `--user` and `--system` keep separate cursors beside the runtime log each one writes to, so the two modes never claim each other's progress.+`chat_history.jsonl` records carry no time of their own, so the events Beacon maps from them (prompts, assistant messages and reasoning, tool calls and results) are stamped with the time of the sync that read them.+Restart Grok after installing or removing hooks so new sessions pick up the updated hook configuration.## Related