Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(grok): never let a failed read look like a compacted session

Claude100eb6a5308a
diff --git a/cli/beacon/internal/groksession/collect.go b/cli/beacon/internal/groksession/collect.go
index d4734f23..f2bce8b7 100644
--- a/cli/beacon/internal/groksession/collect.go
+++ b/cli/beacon/internal/groksession/collect.go
@@ -188,13 +188,19 @@ func collectSession(store *Store, ref SessionRef, state *State, opts CollectOpti
// parsed out of it. They are different numbers -- one blank or half-written line makes the
// record count smaller than the line count -- and comparing the record count against a line
// cursor would read an ordinary malformed line as a rewrite and re-emit the whole file.
+ //
+ // A file that yielded no lines at all never resets anything. Zero is what an absent file reads
+ // as, and it is also what a file truncated to nothing reads as; treating that as a rewrite
+ // would throw away a good cursor over a log that has nothing in it to re-collect. Compaction
+ // is not missed by waiting: the check runs every sweep, so the first one that sees the
+ // rewritten file with content in it resets then.
minChat := cursor.ChatLine
minLifecycle := cursor.LifecycleLine
- if data.ChatLines < cursor.ChatLine {
+ if data.ChatLines > 0 && data.ChatLines < cursor.ChatLine {
minChat = 0
cursor.ChatLine = 0
}
- if data.LifecycleLines < cursor.LifecycleLine {
+ if data.LifecycleLines > 0 && data.LifecycleLines < cursor.LifecycleLine {
minLifecycle = 0
cursor.LifecycleLine = 0
}
diff --git a/cli/beacon/internal/groksession/collect_test.go b/cli/beacon/internal/groksession/collect_test.go
index 9d0c848b..4a53888b 100644
--- a/cli/beacon/internal/groksession/collect_test.go
+++ b/cli/beacon/internal/groksession/collect_test.go
@@ -500,10 +500,123 @@ func TestAMalformedLineIsNotMistakenForARewrite(t *testing.T) {
// check must NOT be made against.
func readChatRecords(t *testing.T, path string) []ChatMessage {
t.Helper()
- records, _, _ := readJSONLines[ChatMessage](path)
+ records, _, _, err := readJSONLines[ChatMessage](path)
+ if err != nil {
+ t.Fatalf("readJSONLines(%s): %v", path, err)
+ }
return records
}
+// A sweep that cannot read one of a session's logs must not be mistaken for a sweep that read a
+// short one.
+//
+// A line past the scanner's buffer limit -- Grok writes command output and file contents into the
+// chat history, so it happens -- stops the scan partway. Counting that as one malformed line would
+// drop the rest of the file and report a line count below the cursor, which the shrink check reads
+// as a compacted rewrite: it would clear the cursor and append the whole session to the runtime log
+// again on the next sweep that could read it.
+func TestAnUnreadableLogDoesNotResetTheCursor(t *testing.T) {
+ root := t.TempDir()
+ sessionDir := writeFixtureSession(t, root, "/tmp/grok project", "session-1")
+ chatPath := filepath.Join(sessionDir, "chat_history.jsonl")
+ statePath := filepath.Join(t.TempDir(), "state.json")
+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
+
+ opts := CollectOptions{SessionsDir: root, StatePath: statePath, LogPath: logPath, Write: true, UserMode: true}
+ first, err := CollectOnce(opts)
+ if err != nil {
+ t.Fatalf("first CollectOnce returned error: %v", err)
+ }
+ if first.EventsEmitted == 0 {
+ t.Fatal("first CollectOnce emitted no events")
+ }
+ collected := countLines(t, logPath)
+
+ // A line the scanner cannot hold, with an ordinary record after it.
+ appendLine(t, chatPath, `{"type":"user","content":"`+strings.Repeat("x", 2*1024*1024)+`"}`)
+ appendLine(t, chatPath, `{"type":"user","content":[{"type":"text","text":"after the huge line"}]}`)
+ touchNewer(t, sessionDir)
+
+ // The sweep says it failed rather than reporting a successful read of a file it could not
+ // finish, and writes nothing.
+ if _, err := CollectOnce(opts); err == nil {
+ t.Fatal("CollectOnce reported success for a session whose chat log could not be read through")
+ }
+ if lines := countLines(t, logPath); lines != collected {
+ t.Fatalf("runtime lines = %d, want %d -- nothing should have been written", lines, collected)
+ }
+
+ // Grok replaces the oversized line with an ordinary one, in place, so the lines after it keep
+ // their numbers.
+ rewriteLine(t, chatPath, 5, `{"type":"user","content":[{"type":"text","text":"the replaced line"}]}`)
+ touchNewer(t, sessionDir)
+
+ third, err := CollectOnce(opts)
+ if err != nil {
+ t.Fatalf("CollectOnce after the log became readable: %v", err)
+ }
+ if third.EventsEmitted != 2 {
+ t.Fatalf("sweep after the read failure emitted %d events, want the 2 new lines -- a reset cursor would re-emit the session", third.EventsEmitted)
+ }
+ if actions := logActions(t, logPath); actions["session.started"] != 1 {
+ t.Fatalf("session.started written %d times, want 1 -- the cursor was reset", actions["session.started"])
+ }
+}
+
+// A log that reads as zero lines must not clear the cursor either. Zero is what an absent file
+// reads as, what a file truncated to nothing reads as, and what a sweep sees for a moment while
+// Grok replaces one. Treating it as a compacted rewrite would throw away a good cursor over a log
+// with nothing in it to re-collect, and the session would be appended to the runtime log a second
+// time once its content came back.
+func TestAnEmptyLogDoesNotResetTheCursor(t *testing.T) {
+ root := t.TempDir()
+ sessionDir := writeFixtureSession(t, root, "/tmp/grok project", "session-1")
+ chatPath := filepath.Join(sessionDir, "chat_history.jsonl")
+ statePath := filepath.Join(t.TempDir(), "state.json")
+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
+
+ opts := CollectOptions{SessionsDir: root, StatePath: statePath, LogPath: logPath, Write: true, UserMode: true}
+ first, err := CollectOnce(opts)
+ if err != nil {
+ t.Fatalf("first CollectOnce returned error: %v", err)
+ }
+ if first.EventsEmitted == 0 {
+ t.Fatal("first CollectOnce emitted no events")
+ }
+ saved, err := os.ReadFile(chatPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ collected := countLines(t, logPath)
+
+ // The chat log is momentarily empty.
+ writeFile(t, chatPath, "")
+ touchNewer(t, sessionDir)
+
+ second, err := CollectOnce(opts)
+ if err != nil {
+ t.Fatalf("sweep over the empty log returned error: %v", err)
+ }
+ if second.EventsEmitted != 0 {
+ t.Fatalf("sweep over the empty log emitted %d events, want none", second.EventsEmitted)
+ }
+
+ // Its content comes back unchanged.
+ writeFile(t, chatPath, string(saved))
+ touchNewer(t, sessionDir)
+
+ third, err := CollectOnce(opts)
+ if err != nil {
+ t.Fatalf("sweep after the log came back returned error: %v", err)
+ }
+ if third.EventsEmitted != 0 {
+ t.Fatalf("sweep after the log came back emitted %d events, want none -- the cursor was reset", third.EventsEmitted)
+ }
+ if lines := countLines(t, logPath); lines != collected {
+ t.Fatalf("runtime lines = %d, want %d -- the session was collected twice", lines, collected)
+ }
+}
+
func urlPathEscape(path string) string {
replacer := strings.NewReplacer("/", "%2F", " ", "%20")
return replacer.Replace(path)
diff --git a/cli/beacon/internal/groksession/store.go b/cli/beacon/internal/groksession/store.go
index cedb02a8..5eab5ce2 100644
--- a/cli/beacon/internal/groksession/store.go
+++ b/cli/beacon/internal/groksession/store.go
@@ -100,9 +100,17 @@ func (s *Store) Read(ref SessionRef) (SessionData, Stats, error) {
data.Ref.Workspace = prompt.WorkingDirectory
}
}
- data.Chat, data.ChatLines, stats.Malformed = readJSONLines[ChatMessage](filepath.Join(ref.SourcePath, "chat_history.jsonl"))
- lifecycle, lifecycleLines, malformed := readJSONLines[LifecycleEvent](filepath.Join(ref.SourcePath, "events.jsonl"))
+ chat, chatLines, malformed, err := readJSONLines[ChatMessage](filepath.Join(ref.SourcePath, "chat_history.jsonl"))
+ stats.Malformed = malformed
+ if err != nil {
+ return data, stats, err
+ }
+ data.Chat, data.ChatLines = chat, chatLines
+ lifecycle, lifecycleLines, malformed, err := readJSONLines[LifecycleEvent](filepath.Join(ref.SourcePath, "events.jsonl"))
stats.Malformed += malformed
+ if err != nil {
+ return data, stats, err
+ }
data.Lifecycle = lifecycle
data.LifecycleLines = lifecycleLines
if ref.Summary == nil {
@@ -129,16 +137,25 @@ func readJSONFile[T any](path string) (*T, error) {
}
// readJSONLines parses one of a session's JSONL files, returning the records, the number of lines
-// it scanned, and how many of them did not parse.
+// it scanned, how many of them did not parse, and whether the file could be read at all.
//
// The line count is reported separately from len(records) and the two are not interchangeable: a
// blank or unparseable line advances the count without producing a record. The collector compares
// the count against its line cursor to notice a file that was rewritten shorter, and using the
// record count there would read a single malformed line as a rewrite.
-func readJSONLines[T any](path string) ([]T, int, int) {
+//
+// A file that is simply not there is not an error -- events.jsonl is optional, and a session with
+// no lifecycle log is an ordinary session, not a broken one. Any other open failure is returned, so
+// a sweep that could not read a log says so instead of reporting a successful read of nothing: the
+// collector decides what to do with its cursor from the number of lines a file holds, and a read
+// failure that looks like an empty file is a read failure that looks like a rewrite.
+func readJSONLines[T any](path string) ([]T, int, int, error) {
f, err := os.Open(path)
if err != nil {
- return nil, 0, 0
+ if errors.Is(err, os.ErrNotExist) {
+ return nil, 0, 0, nil
+ }
+ return nil, 0, 0, fmt.Errorf("read %s: %w", path, err)
}
defer f.Close()
var out []T
@@ -170,10 +187,15 @@ func readJSONLines[T any](path string) ([]T, int, int) {
}
out = append(out, item)
}
+ // A scan that stops on an error is a read failure, not a malformed line. It leaves lineNo short
+ // of the file's real length -- a line over the buffer limit, an I/O error partway through --
+ // and a short line count is exactly what the collector reads as a compacted rewrite. Counting
+ // it as one bad line would drop the rest of the file silently and invite that reset, so the
+ // sweep fails for this session instead and its cursor is left untouched.
if err := scanner.Err(); err != nil {
- malformed++
+ return out, lineNo, malformed, fmt.Errorf("read %s: %w", path, err)
}
- return out, lineNo, malformed
+ return out, lineNo, malformed, nil
}
func readTerminalLogs(dir string, logs map[string]string) error {