The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
fix
Justin D'Souza3c1fe1a23cb9
diff --git a/cli/beacon/internal/ingest/endpoint/reader.go b/cli/beacon/internal/ingest/endpoint/reader.goindex d8d75092..e7a8f1e8 100644--- a/cli/beacon/internal/ingest/endpoint/reader.go+++ b/cli/beacon/internal/ingest/endpoint/reader.go@@ -4,10 +4,13 @@ import ("bufio""bytes""encoding/json"+ "fmt""os""path/filepath""sort"+ "strconv""strings"+ "syscall""github.com/asymptote-labs/agent-beacon/cli/beacon/internal/ingest")@@ -35,7 +38,6 @@ func (s Source) Batches(state ingest.State, maxEvents int, maxBytes int) ([]inge}func readFileBatches(path string, activePath string, state ingest.State, maxEvents int, maxBytes int) ([]ingest.Batch, error) {- startOffset := state.FileOffsets[path]file, err := os.Open(path)if err != nil {if os.IsNotExist(err) {@@ -44,6 +46,12 @@ func readFileBatches(path string, activePath string, state ingest.State, maxEvenreturn nil, err}defer file.Close()+ info, err := file.Stat()+ if err != nil {+ return nil, err+ }+ fileID := fileIdentity(info)+ startOffset := startOffsetForFile(path, activePath, state, info.Size(), fileID)if startOffset > 0 {if _, err := file.Seek(startOffset, 0); err != nil {@@ -73,6 +81,7 @@ func readFileBatches(path string, activePath string, state ingest.State, maxEvenOffset: batchEndOffset,Line: batchLine,Archive: archiveName(activePath, path),+ FileID: fileID,},Events: events,Rejected: batchRejected,@@ -130,13 +139,71 @@ func logFiles(path string) ([]string, error) {archives = append(archives, filepath.Join(dir, name))}}- sort.Strings(archives)+ sort.Slice(archives, func(i, j int) bool {+ left, leftOK := archiveIndex(base, filepath.Base(archives[i]))+ right, rightOK := archiveIndex(base, filepath.Base(archives[j]))+ if leftOK && rightOK {+ return left > right+ }+ return archives[i] < archives[j]+ })return append(archives, path), nil}+func archiveIndex(base string, name string) (int, bool) {+ suffix, ok := strings.CutPrefix(name, base+".")+ if !ok {+ return 0, false+ }+ index, err := strconv.Atoi(suffix)+ if err != nil {+ return 0, false+ }+ return index, true+}+func archiveName(activePath string, path string) string {if activePath == path {return ""}return filepath.Base(path)}++func startOffsetForFile(path string, activePath string, state ingest.State, fileSize int64, fileID string) int64 {+ if fileID != "" && len(state.FileIDs) > 0 {+ if state.FileIDs[path] == fileID {+ return boundedOffset(state.FileOffsets[path], fileSize)+ }+ for savedPath, savedID := range state.FileIDs {+ if savedID == fileID {+ return boundedOffset(state.FileOffsets[savedPath], fileSize)+ }+ }+ if _, knownPath := state.FileIDs[path]; knownPath {+ return 0+ }+ }++ if offset := boundedOffset(state.FileOffsets[path], fileSize); offset > 0 {+ return offset+ }+ if path == activePath+".1" && state.FileIDs[activePath] == "" {+ return boundedOffset(state.FileOffsets[activePath], fileSize)+ }+ return 0+}++func boundedOffset(offset int64, fileSize int64) int64 {+ if offset <= 0 || offset > fileSize {+ return 0+ }+ return offset+}++func fileIdentity(info os.FileInfo) string {+ stat, ok := info.Sys().(*syscall.Stat_t)+ if !ok {+ return ""+ }+ return fmt.Sprintf("%d:%d", stat.Dev, stat.Ino)+}diff --git a/cli/beacon/internal/ingest/endpoint/reader_test.go b/cli/beacon/internal/ingest/endpoint/reader_test.goindex 3ce94082..839a49c1 100644--- a/cli/beacon/internal/ingest/endpoint/reader_test.go+++ b/cli/beacon/internal/ingest/endpoint/reader_test.go@@ -59,3 +59,171 @@ func TestEndpointSourceHonorsSavedOffset(t *testing.T) {t.Fatalf("event = %s", batches[0].Events[0])}}++func TestEndpointSourceAppliesLegacyActiveOffsetToFirstRotatedArchive(t *testing.T) {+ dir := t.TempDir()+ logPath := filepath.Join(dir, "runtime.jsonl")+ archivePath := logPath + ".1"+ alreadyUploaded := "{\"event\":\"already-uploaded\"}\n"+ archiveNew := "{\"event\":\"archive-new\"}\n"+ activeNew := "{\"event\":\"active\"}\n"+ if err := os.WriteFile(archivePath, []byte(alreadyUploaded+archiveNew), 0600); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(logPath, []byte(activeNew), 0600); err != nil {+ t.Fatal(err)+ }++ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)+ batches, err := source.Batches(ingest.State{+ FileOffsets: map[string]int64{logPath: int64(len(alreadyUploaded))},+ }, 500, 1024)+ if err != nil {+ t.Fatal(err)+ }++ if len(batches) != 2 {+ t.Fatalf("len(batches) = %d, want 2: %#v", len(batches), batches)+ }+ if batches[0].Cursor.LogPath != archivePath || batches[0].Cursor.Archive != "runtime.jsonl.1" {+ t.Fatalf("unexpected archive cursor: %#v", batches[0].Cursor)+ }+ if string(batches[0].Events[0]) != "{\"event\":\"archive-new\"}" {+ t.Fatalf("archive event = %s", batches[0].Events[0])+ }+ if batches[1].Cursor.LogPath != logPath || string(batches[1].Events[0]) != "{\"event\":\"active\"}" {+ t.Fatalf("unexpected active batch: %#v", batches[1])+ }+}++func TestEndpointSourceAppliesLegacyActiveOffsetWithUnrelatedFileIDs(t *testing.T) {+ dir := t.TempDir()+ logPath := filepath.Join(dir, "runtime.jsonl")+ archivePath := logPath + ".1"+ alreadyUploaded := "{\"event\":\"already-uploaded\"}\n"+ archiveNew := "{\"event\":\"archive-new\"}\n"+ if err := os.WriteFile(archivePath, []byte(alreadyUploaded+archiveNew), 0600); err != nil {+ t.Fatal(err)+ }++ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)+ batches, err := source.Batches(ingest.State{+ FileOffsets: map[string]int64{logPath: int64(len(alreadyUploaded))},+ FileIDs: map[string]string{filepath.Join(dir, "other.jsonl"): "dev:ino"},+ }, 500, 1024)+ if err != nil {+ t.Fatal(err)+ }++ if len(batches) != 1 || batches[0].Cursor.LogPath != archivePath {+ t.Fatalf("unexpected batches: %#v", batches)+ }+ if string(batches[0].Events[0]) != "{\"event\":\"archive-new\"}" {+ t.Fatalf("archive event = %s", batches[0].Events[0])+ }+}++func TestEndpointSourceDoesNotApplyLegacyActiveOffsetWhenActiveFileIDConflicts(t *testing.T) {+ dir := t.TempDir()+ logPath := filepath.Join(dir, "runtime.jsonl")+ archivePath := logPath + ".1"+ alreadyUploaded := "{\"event\":\"already-uploaded\"}\n"+ archiveNew := "{\"event\":\"archive-new\"}\n"+ if err := os.WriteFile(archivePath, []byte(alreadyUploaded+archiveNew), 0600); err != nil {+ t.Fatal(err)+ }++ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)+ batches, err := source.Batches(ingest.State{+ FileOffsets: map[string]int64{logPath: int64(len(alreadyUploaded))},+ FileIDs: map[string]string{logPath: "different-file"},+ }, 500, 1024)+ if err != nil {+ t.Fatal(err)+ }++ if len(batches) != 1 || batches[0].Cursor.LogPath != archivePath {+ t.Fatalf("unexpected batches: %#v", batches)+ }+ if len(batches[0].Events) != 2 || string(batches[0].Events[0]) != "{\"event\":\"already-uploaded\"}" {+ t.Fatalf("archive batch should start at zero when active file identity conflicts: %#v", batches[0])+ }+}++func TestEndpointSourceReadsRotatedArchivesOldestFirst(t *testing.T) {+ dir := t.TempDir()+ logPath := filepath.Join(dir, "runtime.jsonl")+ files := map[string]string{+ logPath + ".10": "{\"event\":\"oldest\"}\n",+ logPath + ".3": "{\"event\":\"older\"}\n",+ logPath + ".2": "{\"event\":\"middle\"}\n",+ logPath + ".1": "{\"event\":\"newer\"}\n",+ logPath: "{\"event\":\"active\"}\n",+ }+ for path, content := range files {+ if err := os.WriteFile(path, []byte(content), 0600); err != nil {+ t.Fatalf("write %s: %v", path, err)+ }+ }++ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)+ batches, err := source.Batches(ingest.State{FileOffsets: map[string]int64{}}, 500, 1024)+ if err != nil {+ t.Fatal(err)+ }++ if len(batches) != 5 {+ t.Fatalf("len(batches) = %d, want 5: %#v", len(batches), batches)+ }+ wantArchives := []string{"runtime.jsonl.10", "runtime.jsonl.3", "runtime.jsonl.2", "runtime.jsonl.1", ""}+ wantEvents := []string{+ "{\"event\":\"oldest\"}",+ "{\"event\":\"older\"}",+ "{\"event\":\"middle\"}",+ "{\"event\":\"newer\"}",+ "{\"event\":\"active\"}",+ }+ for i := range batches {+ if batches[i].Cursor.Archive != wantArchives[i] {+ t.Fatalf("batch %d archive = %q, want %q", i, batches[i].Cursor.Archive, wantArchives[i])+ }+ if string(batches[i].Events[0]) != wantEvents[i] {+ t.Fatalf("batch %d event = %s, want %s", i, batches[i].Events[0], wantEvents[i])+ }+ }+}++func TestEndpointSourceFollowsSavedOffsetAcrossArchiveRenumbering(t *testing.T) {+ dir := t.TempDir()+ logPath := filepath.Join(dir, "runtime.jsonl")+ savedPath := logPath + ".1"+ renumberedPath := logPath + ".2"+ first := "{\"event\":\"one\"}\n"+ second := "{\"event\":\"two\"}\n"+ if err := os.WriteFile(renumberedPath, []byte(first+second), 0600); err != nil {+ t.Fatal(err)+ }++ info, err := os.Stat(renumberedPath)+ if err != nil {+ t.Fatal(err)+ }+ source := NewSource(endpointconfig.Config{UserMode: true, LogPath: logPath}, logPath, true)+ batches, err := source.Batches(ingest.State{+ FileOffsets: map[string]int64{savedPath: int64(len(first))},+ FileIDs: map[string]string{savedPath: fileIdentity(info)},+ }, 500, 1024)+ if err != nil {+ t.Fatal(err)+ }++ if len(batches) != 1 || len(batches[0].Events) != 1 {+ t.Fatalf("unexpected batches: %#v", batches)+ }+ if batches[0].Cursor.LogPath != renumberedPath || string(batches[0].Events[0]) != "{\"event\":\"two\"}" {+ t.Fatalf("unexpected renumbered archive batch: %#v", batches[0])+ }+ if batches[0].Cursor.FileID == "" {+ t.Fatalf("missing file identity on cursor: %#v", batches[0].Cursor)+ }+}diff --git a/cli/beacon/internal/ingest/status.go b/cli/beacon/internal/ingest/status.goindex d0912831..9be81c76 100644--- a/cli/beacon/internal/ingest/status.go+++ b/cli/beacon/internal/ingest/status.go@@ -20,4 +20,7 @@ func applyRuntimeState(state *State, settings Settings, creds *beaconauth.Credenif state.FileOffsets == nil {state.FileOffsets = map[string]int64{}}+ if state.FileIDs == nil {+ state.FileIDs = map[string]string{}+ }}diff --git a/cli/beacon/internal/ingest/store.go b/cli/beacon/internal/ingest/store.goindex a984ff9e..a2f5e6f8 100644--- a/cli/beacon/internal/ingest/store.go+++ b/cli/beacon/internal/ingest/store.go@@ -22,6 +22,9 @@ func (s Store) Load() State {if state.FileOffsets == nil {state.FileOffsets = map[string]int64{}}+ if state.FileIDs == nil {+ state.FileIDs = map[string]string{}+ }return state}@@ -29,6 +32,9 @@ func (s Store) Save(state State) error {if state.FileOffsets == nil {state.FileOffsets = map[string]int64{}}+ if state.FileIDs == nil {+ state.FileIDs = map[string]string{}+ }if err := os.MkdirAll(filepath.Dir(s.Path), 0700); err != nil {return err}@@ -40,5 +46,5 @@ func (s Store) Save(state State) error {}func emptyState() State {- return State{FileOffsets: map[string]int64{}}+ return State{FileOffsets: map[string]int64{}, FileIDs: map[string]string{}}}diff --git a/cli/beacon/internal/ingest/types.go b/cli/beacon/internal/ingest/types.goindex 43bc4cae..a27fff57 100644--- a/cli/beacon/internal/ingest/types.go+++ b/cli/beacon/internal/ingest/types.go@@ -21,20 +21,21 @@ type Settings struct {}type State struct {- Enabled bool `json:"enabled"`- Managed bool `json:"managed"`- LoggedIn bool `json:"logged_in"`- UserEmail string `json:"user_email,omitempty"`- OrgName string `json:"org_name,omitempty"`- SourceID string `json:"source_id,omitempty"`- LastUploadAt string `json:"last_upload_at,omitempty"`- LastEventAt string `json:"last_event_at,omitempty"`- LastCursor Cursor `json:"last_cursor,omitempty"`- AcceptedCount int `json:"accepted_count"`- RejectedCount int `json:"rejected_count"`- LastError string `json:"last_error,omitempty"`- FileOffsets map[string]int64 `json:"file_offsets,omitempty"`- UpdatedAt string `json:"updated_at,omitempty"`+ Enabled bool `json:"enabled"`+ Managed bool `json:"managed"`+ LoggedIn bool `json:"logged_in"`+ UserEmail string `json:"user_email,omitempty"`+ OrgName string `json:"org_name,omitempty"`+ SourceID string `json:"source_id,omitempty"`+ LastUploadAt string `json:"last_upload_at,omitempty"`+ LastEventAt string `json:"last_event_at,omitempty"`+ LastCursor Cursor `json:"last_cursor,omitempty"`+ AcceptedCount int `json:"accepted_count"`+ RejectedCount int `json:"rejected_count"`+ LastError string `json:"last_error,omitempty"`+ FileOffsets map[string]int64 `json:"file_offsets,omitempty"`+ FileIDs map[string]string `json:"file_ids,omitempty"`+ UpdatedAt string `json:"updated_at,omitempty"`}type Cursor struct {@@ -42,6 +43,7 @@ type Cursor struct {Offset int64 `json:"offset,omitempty"`Line int `json:"line,omitempty"`Archive string `json:"archive,omitempty"`+ FileID string `json:"-"`}type SourceMetadata struct {diff --git a/cli/beacon/internal/ingest/upload.go b/cli/beacon/internal/ingest/upload.goindex 142093ef..9f887ef8 100644--- a/cli/beacon/internal/ingest/upload.go+++ b/cli/beacon/internal/ingest/upload.go@@ -46,6 +46,9 @@ func Upload(ctx context.Context, opts Options) Result {if len(batch.Events) == 0 {state.RejectedCount += batch.Rejectedstate.FileOffsets[batch.Cursor.LogPath] = batch.Cursor.Offset+ if batch.Cursor.FileID != "" {+ state.FileIDs[batch.Cursor.LogPath] = batch.Cursor.FileID+ }continue}response, err := client.UploadBatch(ctx, opts.Creds.Token, uploadRequest{@@ -65,6 +68,9 @@ func Upload(ctx context.Context, opts Options) Result {state.LastEventAt = nowstate.LastCursor = batch.Cursorstate.FileOffsets[batch.Cursor.LogPath] = batch.Cursor.Offset+ if batch.Cursor.FileID != "" {+ state.FileIDs[batch.Cursor.LogPath] = batch.Cursor.FileID+ }state.LastError = ""uploaded = true}diff --git a/cli/beacon/internal/ingest/upload_test.go b/cli/beacon/internal/ingest/upload_test.goindex 40afc0ff..39692bf3 100644--- a/cli/beacon/internal/ingest/upload_test.go+++ b/cli/beacon/internal/ingest/upload_test.go@@ -94,7 +94,7 @@ func TestUploadPostsBatchAndPersistsCursor(t *testing.T) {source := &fakeSource{metadata: SourceMetadata{SourceID: "source-1", EndpointMode: "user", LogPath: "runtime.jsonl", ContentRetention: "redacted", ManagedMode: true},batches: []Batch{{- Cursor: Cursor{LogPath: "runtime.jsonl", Offset: 128, Line: 1},+ Cursor: Cursor{LogPath: "runtime.jsonl", Offset: 128, Line: 1, FileID: "dev:ino"},Events: []json.RawMessage{json.RawMessage(`{"event":"ok"}`)},}},}@@ -111,6 +111,9 @@ func TestUploadPostsBatchAndPersistsCursor(t *testing.T) {if !res.Uploaded || res.State.AcceptedCount != 1 || res.State.FileOffsets["runtime.jsonl"] != 128 {t.Fatalf("unexpected upload state: %#v", res.State)}+ if res.State.FileIDs["runtime.jsonl"] != "dev:ino" {+ t.Fatalf("file identity not persisted: %#v", res.State.FileIDs)+ }if received.Source.SourceID != "source-1" || !received.Source.ManagedMode || len(received.Events) != 1 {t.Fatalf("unexpected request: %#v", received)}