Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix

Justin D'Souza3c1fe1a23cb9
diff --git a/cli/beacon/internal/ingest/endpoint/reader.go b/cli/beacon/internal/ingest/endpoint/reader.go
index d8d75092..e7a8f1e8 100644
--- a/cli/beacon/internal/ingest/endpoint/reader.go
+++ b/cli/beacon/internal/ingest/endpoint/reader.go
@@ -4,10 +4,13 @@ import (
"bufio"
"bytes"
"encoding/json"
+ "fmt"
"os"
"path/filepath"
"sort"
+ "strconv"
"strings"
+ "syscall"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest"
)
@@ -35,7 +38,6 @@ func (s Source) Batches(state ingest.State, maxEvents int, maxBytes int) ([]inge
}
func readFileBatches(path string, activePath string, state ingest.State, maxEvents int, maxBytes int) ([]ingest.Batch, error) {
- startOffset := state.FileOffsets[path]
file, err := os.Open(path)
if err != nil {
if os.IsNotExist(err) {
@@ -44,6 +46,12 @@ func readFileBatches(path string, activePath string, state ingest.State, maxEven
return nil, err
}
defer file.Close()
+ info, err := file.Stat()
+ if err != nil {
+ return nil, err
+ }
+ fileID := fileIdentity(info)
+ startOffset := startOffsetForFile(path, activePath, state, info.Size(), fileID)
if startOffset > 0 {
if _, err := file.Seek(startOffset, 0); err != nil {
@@ -73,6 +81,7 @@ func readFileBatches(path string, activePath string, state ingest.State, maxEven
Offset: batchEndOffset,
Line: batchLine,
Archive: archiveName(activePath, path),
+ FileID: fileID,
},
Events: events,
Rejected: batchRejected,
@@ -130,13 +139,71 @@ func logFiles(path string) ([]string, error) {
archives = append(archives, filepath.Join(dir, name))
}
}
- sort.Strings(archives)
+ sort.Slice(archives, func(i, j int) bool {
+ left, leftOK := archiveIndex(base, filepath.Base(archives[i]))
+ right, rightOK := archiveIndex(base, filepath.Base(archives[j]))
+ if leftOK && rightOK {
+ return left > right
+ }
+ return archives[i] < archives[j]
+ })
return append(archives, path), nil
}
+func archiveIndex(base string, name string) (int, bool) {
+ suffix, ok := strings.CutPrefix(name, base+".")
+ if !ok {
+ return 0, false
+ }
+ index, err := strconv.Atoi(suffix)
+ if err != nil {
+ return 0, false
+ }
+ return index, true
+}
+
func archiveName(activePath string, path string) string {
if activePath == path {
return ""
}
return filepath.Base(path)
}
+
+func startOffsetForFile(path string, activePath string, state ingest.State, fileSize int64, fileID string) int64 {
+ if fileID != "" && len(state.FileIDs) > 0 {
+ if state.FileIDs[path] == fileID {
+ return boundedOffset(state.FileOffsets[path], fileSize)
+ }
+ for savedPath, savedID := range state.FileIDs {
+ if savedID == fileID {
+ return boundedOffset(state.FileOffsets[savedPath], fileSize)
+ }
+ }
+ if _, knownPath := state.FileIDs[path]; knownPath {
+ return 0
+ }
+ }
+
+ if offset := boundedOffset(state.FileOffsets[path], fileSize); offset > 0 {
+ return offset
+ }
+ if path == activePath+".1" && state.FileIDs[activePath] == "" {
+ return boundedOffset(state.FileOffsets[activePath], fileSize)
+ }
+ return 0
+}
+
+func boundedOffset(offset int64, fileSize int64) int64 {
+ if offset <= 0 || offset > fileSize {
+ return 0
+ }
+ return offset
+}
+
+func fileIdentity(info os.FileInfo) string {
+ stat, ok := info.Sys().(*syscall.Stat_t)
+ if !ok {
+ return ""
+ }
+ return fmt.Sprintf("%d:%d", stat.Dev, stat.Ino)
+}
diff --git a/cli/beacon/internal/ingest/endpoint/reader_test.go b/cli/beacon/internal/ingest/endpoint/reader_test.go
index 3ce94082..839a49c1 100644
--- a/cli/beacon/internal/ingest/endpoint/reader_test.go
+++ b/cli/beacon/internal/ingest/endpoint/reader_test.go
@@ -59,3 +59,171 @@ func TestEndpointSourceHonorsSavedOffset(t *testing.T) {
t.Fatalf("event = %s", batches[0].Events[0])
}
}
+
+func TestEndpointSourceAppliesLegacyActiveOffsetToFirstRotatedArchive(t *testing.T) {
+ dir := t.TempDir()
+ logPath := filepath.Join(dir, "runtime.jsonl")
+ archivePath := logPath + ".1"
+ alreadyUploaded := "{\"event\":\"already-uploaded\"}\n"
+ archiveNew := "{\"event\":\"archive-new\"}\n"
+ activeNew := "{\"event\":\"active\"}\n"
+ if err := os.WriteFile(archivePath, []byte(alreadyUploaded+archiveNew), 0600); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(logPath, []byte(activeNew), 0600); err != nil {
+ t.Fatal(err)
+ }
+
+ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)
+ batches, err := source.Batches(ingest.State{
+ FileOffsets: map[string]int64{logPath: int64(len(alreadyUploaded))},
+ }, 500, 1024)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(batches) != 2 {
+ t.Fatalf("len(batches) = %d, want 2: %#v", len(batches), batches)
+ }
+ if batches[0].Cursor.LogPath != archivePath || batches[0].Cursor.Archive != "runtime.jsonl.1" {
+ t.Fatalf("unexpected archive cursor: %#v", batches[0].Cursor)
+ }
+ if string(batches[0].Events[0]) != "{\"event\":\"archive-new\"}" {
+ t.Fatalf("archive event = %s", batches[0].Events[0])
+ }
+ if batches[1].Cursor.LogPath != logPath || string(batches[1].Events[0]) != "{\"event\":\"active\"}" {
+ t.Fatalf("unexpected active batch: %#v", batches[1])
+ }
+}
+
+func TestEndpointSourceAppliesLegacyActiveOffsetWithUnrelatedFileIDs(t *testing.T) {
+ dir := t.TempDir()
+ logPath := filepath.Join(dir, "runtime.jsonl")
+ archivePath := logPath + ".1"
+ alreadyUploaded := "{\"event\":\"already-uploaded\"}\n"
+ archiveNew := "{\"event\":\"archive-new\"}\n"
+ if err := os.WriteFile(archivePath, []byte(alreadyUploaded+archiveNew), 0600); err != nil {
+ t.Fatal(err)
+ }
+
+ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)
+ batches, err := source.Batches(ingest.State{
+ FileOffsets: map[string]int64{logPath: int64(len(alreadyUploaded))},
+ FileIDs: map[string]string{filepath.Join(dir, "other.jsonl"): "dev:ino"},
+ }, 500, 1024)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(batches) != 1 || batches[0].Cursor.LogPath != archivePath {
+ t.Fatalf("unexpected batches: %#v", batches)
+ }
+ if string(batches[0].Events[0]) != "{\"event\":\"archive-new\"}" {
+ t.Fatalf("archive event = %s", batches[0].Events[0])
+ }
+}
+
+func TestEndpointSourceDoesNotApplyLegacyActiveOffsetWhenActiveFileIDConflicts(t *testing.T) {
+ dir := t.TempDir()
+ logPath := filepath.Join(dir, "runtime.jsonl")
+ archivePath := logPath + ".1"
+ alreadyUploaded := "{\"event\":\"already-uploaded\"}\n"
+ archiveNew := "{\"event\":\"archive-new\"}\n"
+ if err := os.WriteFile(archivePath, []byte(alreadyUploaded+archiveNew), 0600); err != nil {
+ t.Fatal(err)
+ }
+
+ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)
+ batches, err := source.Batches(ingest.State{
+ FileOffsets: map[string]int64{logPath: int64(len(alreadyUploaded))},
+ FileIDs: map[string]string{logPath: "different-file"},
+ }, 500, 1024)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(batches) != 1 || batches[0].Cursor.LogPath != archivePath {
+ t.Fatalf("unexpected batches: %#v", batches)
+ }
+ if len(batches[0].Events) != 2 || string(batches[0].Events[0]) != "{\"event\":\"already-uploaded\"}" {
+ t.Fatalf("archive batch should start at zero when active file identity conflicts: %#v", batches[0])
+ }
+}
+
+func TestEndpointSourceReadsRotatedArchivesOldestFirst(t *testing.T) {
+ dir := t.TempDir()
+ logPath := filepath.Join(dir, "runtime.jsonl")
+ files := map[string]string{
+ logPath + ".10": "{\"event\":\"oldest\"}\n",
+ logPath + ".3": "{\"event\":\"older\"}\n",
+ logPath + ".2": "{\"event\":\"middle\"}\n",
+ logPath + ".1": "{\"event\":\"newer\"}\n",
+ logPath: "{\"event\":\"active\"}\n",
+ }
+ for path, content := range files {
+ if err := os.WriteFile(path, []byte(content), 0600); err != nil {
+ t.Fatalf("write %s: %v", path, err)
+ }
+ }
+
+ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)
+ batches, err := source.Batches(ingest.State{FileOffsets: map[string]int64{}}, 500, 1024)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(batches) != 5 {
+ t.Fatalf("len(batches) = %d, want 5: %#v", len(batches), batches)
+ }
+ wantArchives := []string{"runtime.jsonl.10", "runtime.jsonl.3", "runtime.jsonl.2", "runtime.jsonl.1", ""}
+ wantEvents := []string{
+ "{\"event\":\"oldest\"}",
+ "{\"event\":\"older\"}",
+ "{\"event\":\"middle\"}",
+ "{\"event\":\"newer\"}",
+ "{\"event\":\"active\"}",
+ }
+ for i := range batches {
+ if batches[i].Cursor.Archive != wantArchives[i] {
+ t.Fatalf("batch %d archive = %q, want %q", i, batches[i].Cursor.Archive, wantArchives[i])
+ }
+ if string(batches[i].Events[0]) != wantEvents[i] {
+ t.Fatalf("batch %d event = %s, want %s", i, batches[i].Events[0], wantEvents[i])
+ }
+ }
+}
+
+func TestEndpointSourceFollowsSavedOffsetAcrossArchiveRenumbering(t *testing.T) {
+ dir := t.TempDir()
+ logPath := filepath.Join(dir, "runtime.jsonl")
+ savedPath := logPath + ".1"
+ renumberedPath := logPath + ".2"
+ first := "{\"event\":\"one\"}\n"
+ second := "{\"event\":\"two\"}\n"
+ if err := os.WriteFile(renumberedPath, []byte(first+second), 0600); err != nil {
+ t.Fatal(err)
+ }
+
+ info, err := os.Stat(renumberedPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)
+ batches, err := source.Batches(ingest.State{
+ FileOffsets: map[string]int64{savedPath: int64(len(first))},
+ FileIDs: map[string]string{savedPath: fileIdentity(info)},
+ }, 500, 1024)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ if len(batches) != 1 || len(batches[0].Events) != 1 {
+ t.Fatalf("unexpected batches: %#v", batches)
+ }
+ if batches[0].Cursor.LogPath != renumberedPath || string(batches[0].Events[0]) != "{\"event\":\"two\"}" {
+ t.Fatalf("unexpected renumbered archive batch: %#v", batches[0])
+ }
+ if batches[0].Cursor.FileID == "" {
+ t.Fatalf("missing file identity on cursor: %#v", batches[0].Cursor)
+ }
+}
diff --git a/cli/beacon/internal/ingest/status.go b/cli/beacon/internal/ingest/status.go
index d0912831..9be81c76 100644
--- a/cli/beacon/internal/ingest/status.go
+++ b/cli/beacon/internal/ingest/status.go
@@ -20,4 +20,7 @@ func applyRuntimeState(state *State, settings Settings, creds *beaconauth.Creden
if state.FileOffsets == nil {
state.FileOffsets = map[string]int64{}
}
+ if state.FileIDs == nil {
+ state.FileIDs = map[string]string{}
+ }
}
diff --git a/cli/beacon/internal/ingest/store.go b/cli/beacon/internal/ingest/store.go
index a984ff9e..a2f5e6f8 100644
--- a/cli/beacon/internal/ingest/store.go
+++ b/cli/beacon/internal/ingest/store.go
@@ -22,6 +22,9 @@ func (s Store) Load() State {
if state.FileOffsets == nil {
state.FileOffsets = map[string]int64{}
}
+ if state.FileIDs == nil {
+ state.FileIDs = map[string]string{}
+ }
return state
}
@@ -29,6 +32,9 @@ func (s Store) Save(state State) error {
if state.FileOffsets == nil {
state.FileOffsets = map[string]int64{}
}
+ if state.FileIDs == nil {
+ state.FileIDs = map[string]string{}
+ }
if err := os.MkdirAll(filepath.Dir(s.Path), 0700); err != nil {
return err
}
@@ -40,5 +46,5 @@ func (s Store) Save(state State) error {
}
func emptyState() State {
- return State{FileOffsets: map[string]int64{}}
+ return State{FileOffsets: map[string]int64{}, FileIDs: map[string]string{}}
}
diff --git a/cli/beacon/internal/ingest/types.go b/cli/beacon/internal/ingest/types.go
index 43bc4cae..a27fff57 100644
--- a/cli/beacon/internal/ingest/types.go
+++ b/cli/beacon/internal/ingest/types.go
@@ -21,20 +21,21 @@ type Settings struct {
}
type State struct {
- Enabled bool `json:"enabled"`
- Managed bool `json:"managed"`
- LoggedIn bool `json:"logged_in"`
- UserEmail string `json:"user_email,omitempty"`
- OrgName string `json:"org_name,omitempty"`
- SourceID string `json:"source_id,omitempty"`
- LastUploadAt string `json:"last_upload_at,omitempty"`
- LastEventAt string `json:"last_event_at,omitempty"`
- LastCursor Cursor `json:"last_cursor,omitempty"`
- AcceptedCount int `json:"accepted_count"`
- RejectedCount int `json:"rejected_count"`
- LastError string `json:"last_error,omitempty"`
- FileOffsets map[string]int64 `json:"file_offsets,omitempty"`
- UpdatedAt string `json:"updated_at,omitempty"`
+ Enabled bool `json:"enabled"`
+ Managed bool `json:"managed"`
+ LoggedIn bool `json:"logged_in"`
+ UserEmail string `json:"user_email,omitempty"`
+ OrgName string `json:"org_name,omitempty"`
+ SourceID string `json:"source_id,omitempty"`
+ LastUploadAt string `json:"last_upload_at,omitempty"`
+ LastEventAt string `json:"last_event_at,omitempty"`
+ LastCursor Cursor `json:"last_cursor,omitempty"`
+ AcceptedCount int `json:"accepted_count"`
+ RejectedCount int `json:"rejected_count"`
+ LastError string `json:"last_error,omitempty"`
+ FileOffsets map[string]int64 `json:"file_offsets,omitempty"`
+ FileIDs map[string]string `json:"file_ids,omitempty"`
+ UpdatedAt string `json:"updated_at,omitempty"`
}
type Cursor struct {
@@ -42,6 +43,7 @@ type Cursor struct {
Offset int64 `json:"offset,omitempty"`
Line int `json:"line,omitempty"`
Archive string `json:"archive,omitempty"`
+ FileID string `json:"-"`
}
type SourceMetadata struct {
diff --git a/cli/beacon/internal/ingest/upload.go b/cli/beacon/internal/ingest/upload.go
index 142093ef..9f887ef8 100644
--- a/cli/beacon/internal/ingest/upload.go
+++ b/cli/beacon/internal/ingest/upload.go
@@ -46,6 +46,9 @@ func Upload(ctx context.Context, opts Options) Result {
if len(batch.Events) == 0 {
state.RejectedCount += batch.Rejected
state.FileOffsets[batch.Cursor.LogPath] = batch.Cursor.Offset
+ if batch.Cursor.FileID != "" {
+ state.FileIDs[batch.Cursor.LogPath] = batch.Cursor.FileID
+ }
continue
}
response, err := client.UploadBatch(ctx, opts.Creds.Token, uploadRequest{
@@ -65,6 +68,9 @@ func Upload(ctx context.Context, opts Options) Result {
state.LastEventAt = now
state.LastCursor = batch.Cursor
state.FileOffsets[batch.Cursor.LogPath] = batch.Cursor.Offset
+ if batch.Cursor.FileID != "" {
+ state.FileIDs[batch.Cursor.LogPath] = batch.Cursor.FileID
+ }
state.LastError = ""
uploaded = true
}
diff --git a/cli/beacon/internal/ingest/upload_test.go b/cli/beacon/internal/ingest/upload_test.go
index 40afc0ff..39692bf3 100644
--- a/cli/beacon/internal/ingest/upload_test.go
+++ b/cli/beacon/internal/ingest/upload_test.go
@@ -94,7 +94,7 @@ func TestUploadPostsBatchAndPersistsCursor(t *testing.T) {
source := &fakeSource{
metadata: SourceMetadata{SourceID: "source-1", EndpointMode: "user", LogPath: "runtime.jsonl", ContentRetention: "redacted", ManagedMode: true},
batches: []Batch{{
- Cursor: Cursor{LogPath: "runtime.jsonl", Offset: 128, Line: 1},
+ Cursor: Cursor{LogPath: "runtime.jsonl", Offset: 128, Line: 1, FileID: "dev:ino"},
Events: []json.RawMessage{json.RawMessage(`{"event":"ok"}`)},
}},
}
@@ -111,6 +111,9 @@ func TestUploadPostsBatchAndPersistsCursor(t *testing.T) {
if !res.Uploaded || res.State.AcceptedCount != 1 || res.State.FileOffsets["runtime.jsonl"] != 128 {
t.Fatalf("unexpected upload state: %#v", res.State)
}
+ if res.State.FileIDs["runtime.jsonl"] != "dev:ino" {
+ t.Fatalf("file identity not persisted: %#v", res.State.FileIDs)
+ }
if received.Source.SourceID != "source-1" || !received.Source.ManagedMode || len(received.Events) != 1 {
t.Fatalf("unexpected request: %#v", received)
}