Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

sync: retry a record whose events were only partly written

Claude658618e7a186
diff --git a/cli/beacon/internal/cursorsession/collect.go b/cli/beacon/internal/cursorsession/collect.go
index 7ae92f3a..31bf73ad 100644
--- a/cli/beacon/internal/cursorsession/collect.go
+++ b/cli/beacon/internal/cursorsession/collect.go
@@ -169,11 +169,10 @@ func collectTrace(store *Store, ref TraceRef, state *State, opts CollectOptions,
}
return false, nil
}
+ start := cursor.LastOrder
for i, item := range mapped {
- if err := emit(item.Event, opts); err != nil {
- if i > 0 {
- cursor.LastOrder = mapped[i-1].SourceOrder
- }
+ if err := emitEvent(item.Event, opts); err != nil {
+ cursor.LastOrder = partialLastOrder(start, mapped, i)
return true, err
}
cursor.LastOrder = item.SourceOrder
@@ -183,6 +182,26 @@ func collectTrace(store *Store, ref TraceRef, state *State, opts CollectOptions,
return true, nil
}
+// partialLastOrder is where the cursor stands after the write of mapped[failedIdx] failed: past
+// every record whose events were all written, so the next sweep retries the record that failed.
+// Stopping at the last written event's record would lose the rest of that record's events for
+// good once a record maps to more than one. start is the cursor before this sweep; the cursor
+// never moves back past it.
+func partialLastOrder(start int, mapped []MappedEvent, failedIdx int) int {
+ for i := failedIdx - 1; i >= 0; i-- {
+ if mapped[i].SourceOrder != mapped[failedIdx].SourceOrder {
+ if mapped[i].SourceOrder > start {
+ return mapped[i].SourceOrder
+ }
+ break
+ }
+ }
+ return start
+}
+
+// emitEvent is emit, swapped by tests to fail a chosen write.
+var emitEvent = emit
+
func emit(event schema.Event, opts CollectOptions) error {
if opts.Print && opts.Out != nil {
data, err := json.Marshal(event)
diff --git a/cli/beacon/internal/cursorsession/partial_test.go b/cli/beacon/internal/cursorsession/partial_test.go
new file mode 100644
index 00000000..0356f02f
--- /dev/null
+++ b/cli/beacon/internal/cursorsession/partial_test.go
@@ -0,0 +1,23 @@
+package cursorsession
+
+import "testing"
+
+// Each record maps to one event today; the rule has to hold once a record maps to more (a prompt
+// and its session.handoff link).
+func TestPartialLastOrderNeverSkipsARecordsRemainingEvents(t *testing.T) {
+ mapped := []MappedEvent{{SourceOrder: 1}, {SourceOrder: 2}, {SourceOrder: 2}, {SourceOrder: 3}, {SourceOrder: 3}}
+ for _, tc := range []struct {
+ failed, start, want int
+ }{
+ {failed: 0, start: 0, want: 0}, // nothing written
+ {failed: 1, start: 0, want: 1}, // record 1 done, record 2 retried
+ {failed: 2, start: 0, want: 1}, // half of record 2 written: retry record 2
+ {failed: 3, start: 0, want: 2},
+ {failed: 4, start: 0, want: 2}, // half of record 3 written: retry record 3
+ {failed: 2, start: 5, want: 5}, // never moves the cursor backwards
+ } {
+ if got := partialLastOrder(tc.start, mapped, tc.failed); got != tc.want {
+ t.Fatalf("failure at %d from %d: LastOrder = %d, want %d", tc.failed, tc.start, got, tc.want)
+ }
+ }
+}
diff --git a/cli/beacon/internal/dshsession/collect.go b/cli/beacon/internal/dshsession/collect.go
index d253b321..23dcc48d 100644
--- a/cli/beacon/internal/dshsession/collect.go
+++ b/cli/beacon/internal/dshsession/collect.go
@@ -176,7 +176,7 @@ func collectSession(store *Store, ref SessionRef, state *State, opts CollectOpti
return false, nil
}
for i, item := range mapped {
- if err := emit(item.Event, opts); err != nil {
+ if err := emitEvent(item.Event, opts); err != nil {
advanceCursorPartial(cursor, mapped, i)
return true, err
}
@@ -206,21 +206,30 @@ func advanceCursor(cursor *Cursor, ref SessionRef, records []Record, stats Stats
}
}
+// advanceCursorPartial moves the cursor past source lines whose mapped events were all emitted, so
+// the next sweep retries from the source line that failed. One line maps to several events (an
+// assistant message's reasoning, text and token usage); stopping at the last emitted event's line
+// would skip the rest of that line for good. failedIdx is the index into mapped of the event whose
+// emit failed.
func advanceCursorPartial(cursor *Cursor, mapped []MappedEvent, failedIdx int) {
- var lastLine int
for i := 0; i < failedIdx; i++ {
- if mapped[i].SourceLine > lastLine {
- lastLine = mapped[i].SourceLine
- }
if mapped[i].Event.Event.Action == "session.started" {
cursor.Started = true
}
}
- if lastLine > cursor.LastLine {
- cursor.LastLine = lastLine
+ for i := failedIdx - 1; i >= 0; i-- {
+ if mapped[i].SourceLine != mapped[failedIdx].SourceLine {
+ if mapped[i].SourceLine > cursor.LastLine {
+ cursor.LastLine = mapped[i].SourceLine
+ }
+ return
+ }
}
}
+// emitEvent is emit, swapped by tests to fail a chosen write.
+var emitEvent = emit
+
func emit(event schema.Event, opts CollectOptions) error {
if opts.Print && opts.Out != nil {
data, err := json.Marshal(event)
diff --git a/cli/beacon/internal/dshsession/partial_test.go b/cli/beacon/internal/dshsession/partial_test.go
new file mode 100644
index 00000000..7f1182d8
--- /dev/null
+++ b/cli/beacon/internal/dshsession/partial_test.go
@@ -0,0 +1,73 @@
+package dshsession
+
+import (
+ "bytes"
+ "errors"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
+)
+
+func TestAdvanceCursorPartialNeverSkipsALinesRemainingEvents(t *testing.T) {
+ // Lines 2 and 3 map to two events each, as an assistant message's text and usage do.
+ mapped := []MappedEvent{{SourceLine: 1}, {SourceLine: 2}, {SourceLine: 2}, {SourceLine: 3}, {SourceLine: 3}}
+ for _, tc := range []struct {
+ failed, start, want int
+ }{
+ {failed: 0, start: 0, want: 0}, // nothing written
+ {failed: 1, start: 0, want: 1}, // line 1 done, line 2 retried
+ {failed: 2, start: 0, want: 1}, // half of line 2 written: retry line 2
+ {failed: 3, start: 0, want: 2},
+ {failed: 4, start: 0, want: 2}, // half of line 3 written: retry line 3
+ {failed: 2, start: 5, want: 5}, // never moves the cursor backwards
+ } {
+ cursor := &Cursor{LastLine: tc.start}
+ advanceCursorPartial(cursor, mapped, tc.failed)
+ if cursor.LastLine != tc.want {
+ t.Fatalf("failure at %d from %d: LastLine = %d, want %d", tc.failed, tc.start, cursor.LastLine, tc.want)
+ }
+ }
+}
+
+// A sweep whose write fails partway through one line's events retries that line, so the events
+// after the failure are not lost.
+func TestCollectOnceRetriesALineThatFailedPartway(t *testing.T) {
+ root := t.TempDir()
+ writeSession(t, filepath.Join(root, "sessions", "s1"), SessionFileJSON,
+ record("session", map[string]interface{}{"id": "s1", "cwd": "/repo"}),
+ record("assistant/message", map[string]interface{}{"message": map[string]interface{}{
+ "content": []interface{}{map[string]interface{}{"type": "text", "text": "done"}},
+ "usage": map[string]interface{}{"inputTokens": 11, "outputTokens": 7},
+ }}),
+ )
+ var out bytes.Buffer
+ opts := CollectOptions{DSHHome: root, StatePath: filepath.Join(t.TempDir(), "state.json"), Print: true, Out: &out}
+
+ prev := emitEvent
+ t.Cleanup(func() { emitEvent = prev })
+ emitEvent = func(event schema.Event, opts CollectOptions) error {
+ if event.Event.Action == "token.usage" {
+ return errors.New("disk full")
+ }
+ return prev(event, opts)
+ }
+ if _, err := CollectOnce(opts); err == nil {
+ t.Fatal("the failed write must surface")
+ }
+ if !strings.Contains(out.String(), `"agent.message"`) {
+ t.Fatalf("the first sweep should have written the message before failing:\n%s", out.String())
+ }
+ emitEvent = prev
+ out.Reset()
+ if _, err := CollectOnce(opts); err != nil {
+ t.Fatalf("retry: %v", err)
+ }
+ if n := strings.Count(out.String(), `"token.usage"`); n != 1 {
+ t.Fatalf("token.usage events after the retry = %d, want 1:\n%s", n, out.String())
+ }
+ if strings.Contains(out.String(), `"session.started"`) {
+ t.Fatalf("the retry must not repeat the session start it already wrote:\n%s", out.String())
+ }
+}
diff --git a/cli/beacon/internal/factorysession/collect.go b/cli/beacon/internal/factorysession/collect.go
index 49cfa617..8ec6935b 100644
--- a/cli/beacon/internal/factorysession/collect.go
+++ b/cli/beacon/internal/factorysession/collect.go
@@ -179,7 +179,7 @@ func collectSession(store *Store, ref SessionRef, state *State, opts CollectOpti
EmitSettingsUsage: emitSettingsUsage,
})
for i, item := range mapped {
- if err := emit(item.Event, opts); err != nil {
+ if err := emitEvent(item.Event, opts); err != nil {
advanceCursorPartial(cursor, ref, mapped, i)
return true, err
}
@@ -200,18 +200,31 @@ func advanceCursor(cursor *Cursor, ref SessionRef, stats ReadStats) {
cursor.SettingsUnixMS = ref.SettingsUnixMS
}
+// advanceCursorPartial moves the cursor past source lines whose mapped events were all emitted, so
+// the next sweep retries from the source line that failed. One line maps to several events (a
+// message's text blocks, a tool call and its result); stopping at the last emitted event's line
+// would skip the rest of that line for good. failedIdx is the index into mapped of the event whose
+// emit failed.
func advanceCursorPartial(cursor *Cursor, ref SessionRef, mapped []MappedEvent, failedIdx int) {
cursor.SourcePath = ref.Path
for i := 0; i < failedIdx; i++ {
- if mapped[i].SourceLine > cursor.LastLine {
- cursor.LastLine = mapped[i].SourceLine
- }
if mapped[i].Event.Event.Action == "session.started" {
cursor.Started = true
}
}
+ for i := failedIdx - 1; i >= 0; i-- {
+ if mapped[i].SourceLine != mapped[failedIdx].SourceLine {
+ if mapped[i].SourceLine > cursor.LastLine {
+ cursor.LastLine = mapped[i].SourceLine
+ }
+ return
+ }
+ }
}
+// emitEvent is emit, swapped by tests to fail a chosen write.
+var emitEvent = emit
+
func emit(event schema.Event, opts CollectOptions) error {
if opts.Print && opts.Out != nil {
data, err := json.Marshal(event)
diff --git a/cli/beacon/internal/factorysession/partial_test.go b/cli/beacon/internal/factorysession/partial_test.go
new file mode 100644
index 00000000..9c78d261
--- /dev/null
+++ b/cli/beacon/internal/factorysession/partial_test.go
@@ -0,0 +1,74 @@
+package factorysession
+
+import (
+ "bytes"
+ "errors"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
+)
+
+func TestAdvanceCursorPartialNeverSkipsALinesRemainingEvents(t *testing.T) {
+ // Lines 2 and 3 map to two events each, as a message with two text blocks does.
+ mapped := []MappedEvent{{SourceLine: 1}, {SourceLine: 2}, {SourceLine: 2}, {SourceLine: 3}, {SourceLine: 3}}
+ for _, tc := range []struct {
+ failed, start, want int
+ }{
+ {failed: 0, start: 0, want: 0},
+ {failed: 1, start: 0, want: 1},
+ {failed: 2, start: 0, want: 1}, // half of line 2 written: retry line 2
+ {failed: 3, start: 0, want: 2},
+ {failed: 4, start: 0, want: 2},
+ {failed: 2, start: 5, want: 5}, // never moves the cursor backwards
+ } {
+ cursor := &Cursor{LastLine: tc.start}
+ advanceCursorPartial(cursor, SessionRef{Path: "/s.jsonl"}, mapped, tc.failed)
+ if cursor.LastLine != tc.want {
+ t.Fatalf("failure at %d from %d: LastLine = %d, want %d", tc.failed, tc.start, cursor.LastLine, tc.want)
+ }
+ }
+}
+
+// A sweep whose write fails partway through one line's events retries that line, so the events
+// after the failure are not lost.
+func TestCollectOnceRetriesALineThatFailedPartway(t *testing.T) {
+ root := t.TempDir()
+ project := filepath.Join(root, "-repo")
+ if err := os.MkdirAll(project, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ writeFile(t, filepath.Join(project, "session-1.jsonl"),
+ `{"type":"session_start","id":"session-1","cwd":"/repo"}`+"\n"+
+ `{"type":"message","id":"u1","timestamp":"2026-05-16T22:11:04.611Z","message":{"role":"user","content":[{"type":"text","text":"first block"},{"type":"text","text":"second block"}]}}`+"\n")
+ var out bytes.Buffer
+ opts := CollectOptions{SessionsDir: root, StatePath: filepath.Join(t.TempDir(), "state.json"), Print: true, Out: &out}
+
+ prev := emitEvent
+ t.Cleanup(func() { emitEvent = prev })
+ emitEvent = func(event schema.Event, opts CollectOptions) error {
+ if event.Prompt != nil && event.Prompt.Text == "second block" {
+ return errors.New("disk full")
+ }
+ return prev(event, opts)
+ }
+ if _, err := CollectOnce(opts); err == nil {
+ t.Fatal("the failed write must surface")
+ }
+ if !strings.Contains(out.String(), "first block") {
+ t.Fatalf("the first sweep should have written the first block before failing:\n%s", out.String())
+ }
+ emitEvent = prev
+ out.Reset()
+ if _, err := CollectOnce(opts); err != nil {
+ t.Fatalf("retry: %v", err)
+ }
+ if n := strings.Count(out.String(), `"prompt":{"text":"second block"}`); n != 1 {
+ t.Fatalf("the retry wrote the second block %d times, want 1:\n%s", n, out.String())
+ }
+ if strings.Contains(out.String(), `"session.started"`) {
+ t.Fatalf("the retry must not repeat the session start it already wrote:\n%s", out.String())
+ }
+}
diff --git a/cli/beacon/internal/openclawsession/collect.go b/cli/beacon/internal/openclawsession/collect.go
index 39ba70d7..eab79df4 100644
--- a/cli/beacon/internal/openclawsession/collect.go
+++ b/cli/beacon/internal/openclawsession/collect.go
@@ -163,11 +163,10 @@ func collectTrace(store *Store, ref TraceRef, state *State, opts CollectOptions,
cursor.UpdatedAtMS = ref.UpdatedAtUnixMS
return false, nil
}
+ start := cursor.LastOrder
for i, item := range mapped {
- if err := emit(item.Event, opts); err != nil {
- if i > 0 {
- cursor.LastOrder = mapped[i-1].SourceOrder
- }
+ if err := emitEvent(item.Event, opts); err != nil {
+ cursor.LastOrder = partialLastOrder(start, mapped, i)
return true, err
}
cursor.LastOrder = item.SourceOrder
@@ -177,6 +176,26 @@ func collectTrace(store *Store, ref TraceRef, state *State, opts CollectOptions,
return true, nil
}
+// partialLastOrder is where the cursor stands after the write of mapped[failedIdx] failed: past
+// every record whose events were all written, so the next sweep retries the record that failed.
+// Stopping at the last written event's record would lose the rest of that record's events for
+// good once a record maps to more than one. start is the cursor before this sweep; the cursor
+// never moves back past it.
+func partialLastOrder(start int, mapped []MappedEvent, failedIdx int) int {
+ for i := failedIdx - 1; i >= 0; i-- {
+ if mapped[i].SourceOrder != mapped[failedIdx].SourceOrder {
+ if mapped[i].SourceOrder > start {
+ return mapped[i].SourceOrder
+ }
+ break
+ }
+ }
+ return start
+}
+
+// emitEvent is emit, swapped by tests to fail a chosen write.
+var emitEvent = emit
+
func emit(event schema.Event, opts CollectOptions) error {
if opts.Print && opts.Out != nil {
data, err := json.Marshal(event)
diff --git a/cli/beacon/internal/openclawsession/partial_test.go b/cli/beacon/internal/openclawsession/partial_test.go
new file mode 100644
index 00000000..6d812bd9
--- /dev/null
+++ b/cli/beacon/internal/openclawsession/partial_test.go
@@ -0,0 +1,23 @@
+package openclawsession
+
+import "testing"
+
+// Each record maps to one event today; the rule has to hold once a record maps to more (a prompt
+// and its session.handoff link).
+func TestPartialLastOrderNeverSkipsARecordsRemainingEvents(t *testing.T) {
+ mapped := []MappedEvent{{SourceOrder: 1}, {SourceOrder: 2}, {SourceOrder: 2}, {SourceOrder: 3}, {SourceOrder: 3}}
+ for _, tc := range []struct {
+ failed, start, want int
+ }{
+ {failed: 0, start: 0, want: 0}, // nothing written
+ {failed: 1, start: 0, want: 1}, // record 1 done, record 2 retried
+ {failed: 2, start: 0, want: 1}, // half of record 2 written: retry record 2
+ {failed: 3, start: 0, want: 2},
+ {failed: 4, start: 0, want: 2}, // half of record 3 written: retry record 3
+ {failed: 2, start: 5, want: 5}, // never moves the cursor backwards
+ } {
+ if got := partialLastOrder(tc.start, mapped, tc.failed); got != tc.want {
+ t.Fatalf("failure at %d from %d: LastOrder = %d, want %d", tc.failed, tc.start, got, tc.want)
+ }
+ }
+}