Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

fix(rotation): never drop an event when the runtime log cannot rotate (#1148)

Shukan Shah889a60db07f6
diff --git a/cli/beacon-hooks/internal/logging/logging.go b/cli/beacon-hooks/internal/logging/logging.go
index 552e43a1..4cfbedbd 100644
--- a/cli/beacon-hooks/internal/logging/logging.go
+++ b/cli/beacon-hooks/internal/logging/logging.go
@@ -2,10 +2,8 @@ package logging
import (
"encoding/json"
- "errors"
"fmt"
"io"
- "io/fs"
"os"
osuser "os/user"
"path/filepath"
@@ -30,7 +28,7 @@ var endpointSequence asymptoteobserve.Sequencer
const (
defaultEndpointRotateBytes = 10 * 1024 * 1024
- defaultEndpointRotateArchives = 5
+ defaultEndpointRotateArchives = asymptoteobserve.DefaultRuntimeLogRotateArchives
// endpointRuntimeFileMode is the shared (system-mode) log's mode. A log inside the current
// user's home is private instead; asymptoteobserve.RuntimeLogModes decides, identically in all
// three writers.
@@ -700,7 +698,8 @@ func truncateEndpoint(value string, limit int) string {
return asymptoteobserve.TruncateString(value, limit)
}
-// Keep this rotation and file-mode contract mirrored with the endpoint CLI and beaconjson exporter.
+// Keep this file-mode contract mirrored with the endpoint CLI and beaconjson exporter; the rotation
+// itself is asymptoteobserve.RotateRuntimeLog for all three.
func appendEndpointJSONL(path string, line []byte, rotateBytes int64, rotateArchives int) error {
_, dirMode, _ := asymptoteobserve.RuntimeLogModes(path)
if err := os.MkdirAll(filepath.Dir(path), dirMode); err != nil {
@@ -721,15 +720,15 @@ func appendEndpointJSONL(path string, line []byte, rotateBytes int64, rotateArch
if asymptoteobserve.IsDuplicateEndpointEvent(path, line, asymptoteobserve.EndpointDuplicateWindow) {
return nil
}
- // A hook usually runs as the person at the keyboard, and on a system-mode log it may not rename
- // or delete anything in the directory: the directory is root-owned 0755 on POSIX, and on Windows
- // its ACL gives ordinary users no right to add, rename or delete entries -- the rights that would
- // let one redirect a SYSTEM write. Rotation is the privileged writers' job there. The refusal
- // comes at the first step, so nothing has moved, and the event is appended past the size limit
- // rather than lost; the collector rotates at its next write.
- if err := rotateEndpointLogIfNeeded(path, rotateBytes, rotateArchives, int64(len(line))); err != nil && !errors.Is(err, fs.ErrPermission) {
- return err
- }
+ // A rotation that cannot happen now is deferred, never fatal, and the event is appended past the
+ // size limit for a later write to rotate. A hook usually runs as the person at the keyboard, and
+ // on a system-mode log it may not rename or delete anything in the directory: the directory is
+ // root-owned 0755 on POSIX, and on Windows its ACL gives ordinary users no right to add, rename or
+ // delete entries -- the rights that would let one redirect a SYSTEM write. Rotation is the
+ // privileged writers' job there. And on Windows a reader holding the log open without delete
+ // sharing blocks the rename for every writer until it lets go. asymptoteobserve.RotateRuntimeLog
+ // moves nothing in either case.
+ _ = asymptoteobserve.RotateRuntimeLog(path, rotateBytes, rotateArchives, int64(len(line)))
f, err := openEndpointRuntimeFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY)
if err != nil {
return err
@@ -751,65 +750,6 @@ func openEndpointRuntimeFileAs(path, logPath string, flag int) (*os.File, error)
return asymptoteobserve.OpenRuntimeLogFile(path, logPath, flag)
}
-func rotateEndpointLogIfNeeded(path string, maxSize int64, archives int, nextWriteBytes int64) error {
- if maxSize <= 0 {
- return nil
- }
- if archives < 1 {
- archives = defaultEndpointRotateArchives
- }
- info, err := os.Stat(path)
- if err != nil {
- if os.IsNotExist(err) {
- return nil
- }
- return err
- }
- if info.Size() == 0 || info.Size()+nextWriteBytes <= maxSize {
- return nil
- }
- if err := removeOverflowEndpointArchives(path, archives); err != nil {
- return err
- }
- for i := archives - 1; i >= 1; i-- {
- from := path + fmt.Sprintf(".%d", i)
- to := path + fmt.Sprintf(".%d", i+1)
- if err := os.Rename(from, to); err != nil && !os.IsNotExist(err) {
- return err
- }
- }
- if err := os.Rename(path, path+".1"); err != nil {
- return err
- }
- asymptoteobserve.SettleRuntimeLogArchives(path, archives)
- return nil
-}
-
-func removeOverflowEndpointArchives(path string, archives int) error {
- dir := filepath.Dir(path)
- base := filepath.Base(path)
- entries, err := os.ReadDir(dir)
- if err != nil {
- return err
- }
- prefix := base + "."
- for _, entry := range entries {
- name := entry.Name()
- suffix, ok := strings.CutPrefix(name, prefix)
- if !ok {
- continue
- }
- index, err := strconv.Atoi(suffix)
- if err != nil || index < archives {
- continue
- }
- if err := os.Remove(filepath.Join(dir, name)); err != nil && !os.IsNotExist(err) {
- return err
- }
- }
- return nil
-}
-
// Debug logs a debug message
func (l *Logger) Debug(message string, fields ...interface{}) {
entry := l.formatEntry("debug", message, fields...)
diff --git a/cli/beacon-hooks/internal/logging/rotation_test.go b/cli/beacon-hooks/internal/logging/rotation_test.go
new file mode 100644
index 00000000..4a952d17
--- /dev/null
+++ b/cli/beacon-hooks/internal/logging/rotation_test.go
@@ -0,0 +1,77 @@
+package logging
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+// appendForRotationTest appends with a size limit every line in these tests goes past.
+func appendForRotationTest(path string, line []byte) error {
+ return appendEndpointJSONL(path, line, 8, defaultEndpointRotateArchives)
+}
+
+// rotationBlockedFixture is a log over its size limit with archives .1 to .4 and, where the oldest
+// archive goes, a directory that is not empty. A rotation would have to remove it first, which
+// every platform refuses, so this blocks rotation the way a held file does on Windows.
+func rotationBlockedFixture(t *testing.T) (path, blocker string) {
+ t.Helper()
+ path = filepath.Join(t.TempDir(), "runtime.jsonl")
+ if err := os.WriteFile(path, []byte("{\"existing\":true}\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ for i := 1; i <= 4; i++ {
+ if err := os.WriteFile(fmt.Sprintf("%s.%d", path, i), []byte(fmt.Sprintf("{\"archive\":%d}\n", i)), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ blocker = path + ".5"
+ if err := os.MkdirAll(filepath.Join(blocker, "keep"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return path, blocker
+}
+
+func readRotationFile(t *testing.T, path string) string {
+ t.Helper()
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(data)
+}
+
+// A rotation that cannot happen is deferred and the line still lands, past the size limit. It used
+// to be returned as an error before the line was written, which dropped the event. Once the blocker
+// is gone the next write rotates normally.
+func TestAppendKeepsTheLineWhenRotationCannotHappen(t *testing.T) {
+ path, blocker := rotationBlockedFixture(t)
+ if err := appendForRotationTest(path, []byte("{\"next\":true}\n")); err != nil {
+ t.Fatalf("append with rotation blocked: %v", err)
+ }
+ if got, want := readRotationFile(t, path), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("log = %q, want the new line appended past the limit", got)
+ }
+ for i := 1; i <= 4; i++ {
+ if got, want := readRotationFile(t, fmt.Sprintf("%s.%d", path, i)), fmt.Sprintf("{\"archive\":%d}\n", i); got != want {
+ t.Fatalf("archive %d = %q after a rotation that could not happen, want %q", i, got, want)
+ }
+ }
+
+ if err := os.RemoveAll(blocker); err != nil {
+ t.Fatal(err)
+ }
+ if err := appendForRotationTest(path, []byte("{\"after\":true}\n")); err != nil {
+ t.Fatal(err)
+ }
+ if got, want := readRotationFile(t, path), "{\"after\":true}\n"; got != want {
+ t.Fatalf("log after the blocker cleared = %q, want only the new line", got)
+ }
+ if got, want := readRotationFile(t, path+".1"), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("archive 1 = %q, want the rotated log", got)
+ }
+ if got, want := readRotationFile(t, path+".5"), "{\"archive\":4}\n"; got != want {
+ t.Fatalf("archive 5 = %q, want what was archive 4", got)
+ }
+}
diff --git a/cli/beacon-hooks/internal/logging/rotation_windows_test.go b/cli/beacon-hooks/internal/logging/rotation_windows_test.go
new file mode 100644
index 00000000..06be15ba
--- /dev/null
+++ b/cli/beacon-hooks/internal/logging/rotation_windows_test.go
@@ -0,0 +1,75 @@
+//go:build windows
+
+package logging
+
+import (
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+)
+
+// holdWithoutDeleteSharing opens path the way PowerShell's Get-Content and [IO.File]::OpenRead do,
+// sharing reads and writes but not deletes, which makes Windows refuse to rename it.
+func holdWithoutDeleteSharing(t *testing.T, path string) syscall.Handle {
+ t.Helper()
+ name, err := syscall.UTF16PtrFromString(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ h, err := syscall.CreateFile(name, syscall.GENERIC_READ, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE,
+ nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return h
+}
+
+// The failure behind this fix, for real: a reader holds the live log without delete sharing, so its
+// rename is refused. The line must still land and no archive may move; once the reader lets go the
+// next write rotates.
+func TestAppendKeepsTheLineWhileAReaderHoldsTheLog(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "runtime.jsonl")
+ if err := os.WriteFile(path, []byte("{\"existing\":true}\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path+".1", []byte("{\"archive\":1}\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ h := holdWithoutDeleteSharing(t, path)
+ held := true
+ defer func() {
+ if held {
+ _ = syscall.CloseHandle(h)
+ }
+ }()
+ if err := appendForRotationTest(path, []byte("{\"next\":true}\n")); err != nil {
+ t.Fatalf("append while a reader holds the log: %v", err)
+ }
+ if got, want := readRotationFile(t, path), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("log = %q, want the new line appended past the limit", got)
+ }
+ if got, want := readRotationFile(t, path+".1"), "{\"archive\":1}\n"; got != want {
+ t.Fatalf("archive 1 = %q, want it untouched", got)
+ }
+ if _, err := os.Stat(path + ".2"); !os.IsNotExist(err) {
+ t.Fatalf("an archive moved although the rotation could not finish: %v", err)
+ }
+
+ held = false
+ if err := syscall.CloseHandle(h); err != nil {
+ t.Fatal(err)
+ }
+ if err := appendForRotationTest(path, []byte("{\"after\":true}\n")); err != nil {
+ t.Fatal(err)
+ }
+ if got, want := readRotationFile(t, path+".1"), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("archive 1 after the reader let go = %q, want the rotated log", got)
+ }
+ if got, want := readRotationFile(t, path+".2"), "{\"archive\":1}\n"; got != want {
+ t.Fatalf("archive 2 = %q, want what was archive 1", got)
+ }
+ if got, want := readRotationFile(t, path), "{\"after\":true}\n"; got != want {
+ t.Fatalf("log = %q, want only the new line", got)
+ }
+}
diff --git a/cli/beacon/internal/endpoint/writer/retention.go b/cli/beacon/internal/endpoint/writer/retention.go
index e83808a7..2bcce606 100644
--- a/cli/beacon/internal/endpoint/writer/retention.go
+++ b/cli/beacon/internal/endpoint/writer/retention.go
@@ -115,9 +115,10 @@ func (s retentionSegment) firstLineAt(path string) bool {
}
// rotationWouldDiscardOwnOutput reports whether rotating path now would delete a file holding the
-// guard's output. Rotation deletes the archive at .archives (and anything numbered past it, which
-// no reader looks at), so that is the one file to check. Called under the log's lock, so the answer
-// holds for the rotation that follows it.
+// guard's output. Rotation deletes at most the archive at .archives (and anything numbered past it,
+// which no reader looks at), so that is the one file to check; when an earlier rotation left a gap
+// below it, asymptoteobserve.RotateRuntimeLog fills the gap and deletes nothing, and the answer is
+// merely cautious. Called under the log's lock, so the answer holds for the rotation that follows it.
func (g *RetentionGuard) rotationWouldDiscardOwnOutput(path string, archives int) (bool, error) {
if len(g.segments) == 0 {
return false, nil
diff --git a/cli/beacon/internal/endpoint/writer/rotation_test.go b/cli/beacon/internal/endpoint/writer/rotation_test.go
new file mode 100644
index 00000000..be18b731
--- /dev/null
+++ b/cli/beacon/internal/endpoint/writer/rotation_test.go
@@ -0,0 +1,77 @@
+package writer
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+// appendForRotationTest appends with a size limit every line in these tests goes past.
+func appendForRotationTest(path string, line []byte) error {
+ return appendJSONL(path, line, 8, DefaultRotateArchives, nil, nil)
+}
+
+// rotationBlockedFixture is a log over its size limit with archives .1 to .4 and, where the oldest
+// archive goes, a directory that is not empty. A rotation would have to remove it first, which
+// every platform refuses, so this blocks rotation the way a held file does on Windows.
+func rotationBlockedFixture(t *testing.T) (path, blocker string) {
+ t.Helper()
+ path = filepath.Join(t.TempDir(), "runtime.jsonl")
+ if err := os.WriteFile(path, []byte("{\"existing\":true}\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ for i := 1; i <= 4; i++ {
+ if err := os.WriteFile(fmt.Sprintf("%s.%d", path, i), []byte(fmt.Sprintf("{\"archive\":%d}\n", i)), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ blocker = path + ".5"
+ if err := os.MkdirAll(filepath.Join(blocker, "keep"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return path, blocker
+}
+
+func readRotationFile(t *testing.T, path string) string {
+ t.Helper()
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(data)
+}
+
+// A rotation that cannot happen is deferred and the line still lands, past the size limit. It used
+// to be returned as an error before the line was written, which dropped the event. Once the blocker
+// is gone the next write rotates normally.
+func TestAppendKeepsTheLineWhenRotationCannotHappen(t *testing.T) {
+ path, blocker := rotationBlockedFixture(t)
+ if err := appendForRotationTest(path, []byte("{\"next\":true}\n")); err != nil {
+ t.Fatalf("append with rotation blocked: %v", err)
+ }
+ if got, want := readRotationFile(t, path), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("log = %q, want the new line appended past the limit", got)
+ }
+ for i := 1; i <= 4; i++ {
+ if got, want := readRotationFile(t, fmt.Sprintf("%s.%d", path, i)), fmt.Sprintf("{\"archive\":%d}\n", i); got != want {
+ t.Fatalf("archive %d = %q after a rotation that could not happen, want %q", i, got, want)
+ }
+ }
+
+ if err := os.RemoveAll(blocker); err != nil {
+ t.Fatal(err)
+ }
+ if err := appendForRotationTest(path, []byte("{\"after\":true}\n")); err != nil {
+ t.Fatal(err)
+ }
+ if got, want := readRotationFile(t, path), "{\"after\":true}\n"; got != want {
+ t.Fatalf("log after the blocker cleared = %q, want only the new line", got)
+ }
+ if got, want := readRotationFile(t, path+".1"), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("archive 1 = %q, want the rotated log", got)
+ }
+ if got, want := readRotationFile(t, path+".5"), "{\"archive\":4}\n"; got != want {
+ t.Fatalf("archive 5 = %q, want what was archive 4", got)
+ }
+}
diff --git a/cli/beacon/internal/endpoint/writer/rotation_windows_test.go b/cli/beacon/internal/endpoint/writer/rotation_windows_test.go
new file mode 100644
index 00000000..cde922a8
--- /dev/null
+++ b/cli/beacon/internal/endpoint/writer/rotation_windows_test.go
@@ -0,0 +1,75 @@
+//go:build windows
+
+package writer
+
+import (
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+)
+
+// holdWithoutDeleteSharing opens path the way PowerShell's Get-Content and [IO.File]::OpenRead do,
+// sharing reads and writes but not deletes, which makes Windows refuse to rename it.
+func holdWithoutDeleteSharing(t *testing.T, path string) syscall.Handle {
+ t.Helper()
+ name, err := syscall.UTF16PtrFromString(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ h, err := syscall.CreateFile(name, syscall.GENERIC_READ, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE,
+ nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return h
+}
+
+// The failure behind this fix, for real: a reader holds the live log without delete sharing, so its
+// rename is refused. The line must still land and no archive may move; once the reader lets go the
+// next write rotates.
+func TestAppendKeepsTheLineWhileAReaderHoldsTheLog(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "runtime.jsonl")
+ if err := os.WriteFile(path, []byte("{\"existing\":true}\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path+".1", []byte("{\"archive\":1}\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ h := holdWithoutDeleteSharing(t, path)
+ held := true
+ defer func() {
+ if held {
+ _ = syscall.CloseHandle(h)
+ }
+ }()
+ if err := appendForRotationTest(path, []byte("{\"next\":true}\n")); err != nil {
+ t.Fatalf("append while a reader holds the log: %v", err)
+ }
+ if got, want := readRotationFile(t, path), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("log = %q, want the new line appended past the limit", got)
+ }
+ if got, want := readRotationFile(t, path+".1"), "{\"archive\":1}\n"; got != want {
+ t.Fatalf("archive 1 = %q, want it untouched", got)
+ }
+ if _, err := os.Stat(path + ".2"); !os.IsNotExist(err) {
+ t.Fatalf("an archive moved although the rotation could not finish: %v", err)
+ }
+
+ held = false
+ if err := syscall.CloseHandle(h); err != nil {
+ t.Fatal(err)
+ }
+ if err := appendForRotationTest(path, []byte("{\"after\":true}\n")); err != nil {
+ t.Fatal(err)
+ }
+ if got, want := readRotationFile(t, path+".1"), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("archive 1 after the reader let go = %q, want the rotated log", got)
+ }
+ if got, want := readRotationFile(t, path+".2"), "{\"archive\":1}\n"; got != want {
+ t.Fatalf("archive 2 = %q, want what was archive 1", got)
+ }
+ if got, want := readRotationFile(t, path), "{\"after\":true}\n"; got != want {
+ t.Fatalf("log = %q, want only the new line", got)
+ }
+}
diff --git a/cli/beacon/internal/endpoint/writer/writer.go b/cli/beacon/internal/endpoint/writer/writer.go
index 3972bbd3..f99e7533 100644
--- a/cli/beacon/internal/endpoint/writer/writer.go
+++ b/cli/beacon/internal/endpoint/writer/writer.go
@@ -5,13 +5,10 @@ import (
"encoding/json"
"errors"
"fmt"
- "io/fs"
"os"
"path/filepath"
"reflect"
"runtime"
- "strconv"
- "strings"
"testing"
endpointconfig "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/config"
@@ -25,7 +22,7 @@ const (
UserLogPath = ".beacon/endpoint/logs/runtime.jsonl"
MaxEventBytes = 64 * 1024
DefaultRotateBytes = 10 * 1024 * 1024
- DefaultRotateArchives = 5
+ DefaultRotateArchives = asymptoteobserve.DefaultRuntimeLogRotateArchives
RotateBytes = DefaultRotateBytes
// runtimeFileMode is the shared (system-mode) log's mode. A log inside the current user's home
// is private instead; asymptoteobserve.RuntimeLogModes decides, identically in all three writers.
@@ -311,7 +308,8 @@ func SanitizeEvent(event schema.Event, maxBytes int) schema.Event {
return asymptoteobserve.SanitizeEvent(event, maxBytes)
}
-// Keep this rotation and file-mode contract mirrored with the hook adapter and beaconjson exporter.
+// Keep this file-mode contract mirrored with the hook adapter and beaconjson exporter; the rotation
+// itself is asymptoteobserve.RotateRuntimeLog for all three.
func appendJSONL(path string, line []byte, rotateBytes int64, rotateArchives int, guard *RetentionGuard, budget *Budget) (err error) {
_, dirMode, _ := asymptoteobserve.RuntimeLogModes(path)
if err := os.MkdirAll(filepath.Dir(path), dirMode); err != nil {
@@ -350,7 +348,7 @@ func appendJSONL(path string, line []byte, rotateBytes int64, rotateArchives int
if guard != nil {
// Decided under the lock, so no other Beacon writer can rotate between the check and the
// rotation it permits.
- rotate, err := needsRotation(path, rotateBytes, int64(len(line)))
+ rotate, err := asymptoteobserve.RuntimeLogNeedsRotation(path, rotateBytes, int64(len(line)))
if err != nil {
return err
}
@@ -364,9 +362,13 @@ func appendJSONL(path string, line []byte, rotateBytes int64, rotateArchives int
}
}
}
- if err := rotateIfNeeded(path, rotateBytes, rotateArchives, int64(len(line))); err != nil && !rotationRefused(err) {
- return err
- }
+ // A rotation that cannot happen now is deferred, never fatal, and the line is appended past the
+ // size limit for a later write to rotate. Two cases reach here. An unprivileged writer of a system
+ // log may not rename or delete in its directory: on POSIX it is root-owned 0755, and on Windows its
+ // ACL withholds those rights on purpose, since they would let a user redirect a SYSTEM write. And
+ // on Windows a reader that holds the log open without delete sharing blocks the rename for
+ // everyone until it lets go. asymptoteobserve.RotateRuntimeLog moves nothing in either case.
+ _ = asymptoteobserve.RotateRuntimeLog(path, rotateBytes, rotateArchives, int64(len(line)))
f, err := openRuntimeFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY)
if err != nil {
return err
@@ -402,86 +404,3 @@ func openRuntimeFileAs(path, logPath string, flag int) (*os.File, error) {
func RetainedLogPaths(path string) []string {
return retainedPaths(path, DefaultRotateArchives)
}
-
-func rotateIfNeeded(path string, maxSize int64, archives int, nextWriteBytes int64) error {
- if maxSize <= 0 {
- return nil
- }
- if archives < 1 {
- archives = DefaultRotateArchives
- }
- rotate, err := needsRotation(path, maxSize, nextWriteBytes)
- if err != nil || !rotate {
- return err
- }
- if err := removeOverflowArchives(path, archives); err != nil {
- return err
- }
- for i := archives - 1; i >= 1; i-- {
- from := path + fmt.Sprintf(".%d", i)
- to := path + fmt.Sprintf(".%d", i+1)
- if err := os.Rename(from, to); err != nil && !os.IsNotExist(err) {
- return err
- }
- }
- rotated := path + ".1"
- if err := os.Rename(path, rotated); err != nil {
- return err
- }
- asymptoteobserve.SettleRuntimeLogArchives(path, archives)
- return nil
-}
-
-// rotationRefused reports whether a rotation failed because this process may not rename or delete
-// in the log directory, which is the expected outcome for an unprivileged writer of a system log.
-//
-// Rotation belongs to the privileged writers there: on POSIX the directory is root-owned 0755, and
-// on Windows its ACL gives ordinary users no right to add, rename or delete entries -- which is the
-// point, since those are the rights that let a user redirect a SYSTEM write. The first step of a
-// rotation is therefore the one refused, nothing has moved, and the event is better appended past
-// the size limit than lost; the collector rotates the log at its next write.
-func rotationRefused(err error) bool {
- return errors.Is(err, fs.ErrPermission)
-}
-
-// needsRotation reports whether appending nextWriteBytes to path would take it past maxSize, which
-// is when rotateIfNeeded rotates. An empty or missing file never rotates, so one oversized line
-// still lands somewhere.
-func needsRotation(path string, maxSize int64, nextWriteBytes int64) (bool, error) {
- if maxSize <= 0 {
- return false, nil
- }
- info, err := os.Stat(path)
- if err != nil {
- if os.IsNotExist(err) {
- return false, nil
- }
- return false, err
- }
- return info.Size() > 0 && info.Size()+nextWriteBytes > maxSize, nil
-}
-
-func removeOverflowArchives(path string, archives int) error {
- dir := filepath.Dir(path)
- base := filepath.Base(path)
- entries, err := os.ReadDir(dir)
- if err != nil {
- return err
- }
- prefix := base + "."
- for _, entry := range entries {
- name := entry.Name()
- suffix, ok := strings.CutPrefix(name, prefix)
- if !ok {
- continue
- }
- index, err := strconv.Atoi(suffix)
- if err != nil || index < archives {
- continue
- }
- if err := os.Remove(filepath.Join(dir, name)); err != nil && !os.IsNotExist(err) {
- return err
- }
- }
- return nil
-}
diff --git a/collector-builder/exporter/beaconjsonexporter/internal/runtimelog/rotation_test.go b/collector-builder/exporter/beaconjsonexporter/internal/runtimelog/rotation_test.go
new file mode 100644
index 00000000..19945cb4
--- /dev/null
+++ b/collector-builder/exporter/beaconjsonexporter/internal/runtimelog/rotation_test.go
@@ -0,0 +1,77 @@
+package runtimelog
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+// appendForRotationTest appends with a size limit every line in these tests goes past.
+func appendForRotationTest(path string, line []byte) error {
+ return Append(path, line, 8, DefaultRotateArchives)
+}
+
+// rotationBlockedFixture is a log over its size limit with archives .1 to .4 and, where the oldest
+// archive goes, a directory that is not empty. A rotation would have to remove it first, which
+// every platform refuses, so this blocks rotation the way a held file does on Windows.
+func rotationBlockedFixture(t *testing.T) (path, blocker string) {
+ t.Helper()
+ path = filepath.Join(t.TempDir(), "runtime.jsonl")
+ if err := os.WriteFile(path, []byte("{\"existing\":true}\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ for i := 1; i <= 4; i++ {
+ if err := os.WriteFile(fmt.Sprintf("%s.%d", path, i), []byte(fmt.Sprintf("{\"archive\":%d}\n", i)), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ blocker = path + ".5"
+ if err := os.MkdirAll(filepath.Join(blocker, "keep"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return path, blocker
+}
+
+func readRotationFile(t *testing.T, path string) string {
+ t.Helper()
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return string(data)
+}
+
+// A rotation that cannot happen is deferred and the line still lands, past the size limit. It used
+// to be returned as an error before the line was written, which dropped the event. Once the blocker
+// is gone the next write rotates normally.
+func TestAppendKeepsTheLineWhenRotationCannotHappen(t *testing.T) {
+ path, blocker := rotationBlockedFixture(t)
+ if err := appendForRotationTest(path, []byte("{\"next\":true}\n")); err != nil {
+ t.Fatalf("append with rotation blocked: %v", err)
+ }
+ if got, want := readRotationFile(t, path), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("log = %q, want the new line appended past the limit", got)
+ }
+ for i := 1; i <= 4; i++ {
+ if got, want := readRotationFile(t, fmt.Sprintf("%s.%d", path, i)), fmt.Sprintf("{\"archive\":%d}\n", i); got != want {
+ t.Fatalf("archive %d = %q after a rotation that could not happen, want %q", i, got, want)
+ }
+ }
+
+ if err := os.RemoveAll(blocker); err != nil {
+ t.Fatal(err)
+ }
+ if err := appendForRotationTest(path, []byte("{\"after\":true}\n")); err != nil {
+ t.Fatal(err)
+ }
+ if got, want := readRotationFile(t, path), "{\"after\":true}\n"; got != want {
+ t.Fatalf("log after the blocker cleared = %q, want only the new line", got)
+ }
+ if got, want := readRotationFile(t, path+".1"), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("archive 1 = %q, want the rotated log", got)
+ }
+ if got, want := readRotationFile(t, path+".5"), "{\"archive\":4}\n"; got != want {
+ t.Fatalf("archive 5 = %q, want what was archive 4", got)
+ }
+}
diff --git a/collector-builder/exporter/beaconjsonexporter/internal/runtimelog/rotation_windows_test.go b/collector-builder/exporter/beaconjsonexporter/internal/runtimelog/rotation_windows_test.go
new file mode 100644
index 00000000..26b2f3d9
--- /dev/null
+++ b/collector-builder/exporter/beaconjsonexporter/internal/runtimelog/rotation_windows_test.go
@@ -0,0 +1,75 @@
+//go:build windows
+
+package runtimelog
+
+import (
+ "os"
+ "path/filepath"
+ "syscall"
+ "testing"
+)
+
+// holdWithoutDeleteSharing opens path the way PowerShell's Get-Content and [IO.File]::OpenRead do,
+// sharing reads and writes but not deletes, which makes Windows refuse to rename it.
+func holdWithoutDeleteSharing(t *testing.T, path string) syscall.Handle {
+ t.Helper()
+ name, err := syscall.UTF16PtrFromString(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ h, err := syscall.CreateFile(name, syscall.GENERIC_READ, syscall.FILE_SHARE_READ|syscall.FILE_SHARE_WRITE,
+ nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return h
+}
+
+// The failure behind this fix, for real: a reader holds the live log without delete sharing, so its
+// rename is refused. The line must still land and no archive may move; once the reader lets go the
+// next write rotates.
+func TestAppendKeepsTheLineWhileAReaderHoldsTheLog(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "runtime.jsonl")
+ if err := os.WriteFile(path, []byte("{\"existing\":true}\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(path+".1", []byte("{\"archive\":1}\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ h := holdWithoutDeleteSharing(t, path)
+ held := true
+ defer func() {
+ if held {
+ _ = syscall.CloseHandle(h)
+ }
+ }()
+ if err := appendForRotationTest(path, []byte("{\"next\":true}\n")); err != nil {
+ t.Fatalf("append while a reader holds the log: %v", err)
+ }
+ if got, want := readRotationFile(t, path), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("log = %q, want the new line appended past the limit", got)
+ }
+ if got, want := readRotationFile(t, path+".1"), "{\"archive\":1}\n"; got != want {
+ t.Fatalf("archive 1 = %q, want it untouched", got)
+ }
+ if _, err := os.Stat(path + ".2"); !os.IsNotExist(err) {
+ t.Fatalf("an archive moved although the rotation could not finish: %v", err)
+ }
+
+ held = false
+ if err := syscall.CloseHandle(h); err != nil {
+ t.Fatal(err)
+ }
+ if err := appendForRotationTest(path, []byte("{\"after\":true}\n")); err != nil {
+ t.Fatal(err)
+ }
+ if got, want := readRotationFile(t, path+".1"), "{\"existing\":true}\n{\"next\":true}\n"; got != want {
+ t.Fatalf("archive 1 after the reader let go = %q, want the rotated log", got)
+ }
+ if got, want := readRotationFile(t, path+".2"), "{\"archive\":1}\n"; got != want {
+ t.Fatalf("archive 2 = %q, want what was archive 1", got)
+ }
+ if got, want := readRotationFile(t, path), "{\"after\":true}\n"; got != want {
+ t.Fatalf("log = %q, want only the new line", got)
+ }
+}
diff --git a/collector-builder/exporter/beaconjsonexporter/internal/runtimelog/runtimelog.go b/collector-builder/exporter/beaconjsonexporter/internal/runtimelog/runtimelog.go
index a67e23e6..9f0b1c79 100644
--- a/collector-builder/exporter/beaconjsonexporter/internal/runtimelog/runtimelog.go
+++ b/collector-builder/exporter/beaconjsonexporter/internal/runtimelog/runtimelog.go
@@ -4,23 +4,21 @@
package runtimelog
import (
- "fmt"
"os"
"path/filepath"
- "strconv"
- "strings"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"
)
// DefaultRotateArchives is how many archives a log keeps when the caller passes none.
-const DefaultRotateArchives = 5
+const DefaultRotateArchives = asymptoteobserve.DefaultRuntimeLogRotateArchives
// Append writes one line to the runtime log at path under its lock, rotating first when the line
// would take the log past rotateBytes and dropping a line that duplicates a recent one.
//
-// Keep this rotation and file-mode contract mirrored with the endpoint CLI and hook writer.
+// Keep this file-mode contract mirrored with the endpoint CLI and hook writer; the rotation itself
+// is asymptoteobserve.RotateRuntimeLog for all three.
func Append(path string, line []byte, rotateBytes int64, rotateArchives int) error {
_, dirMode, _ := asymptoteobserve.RuntimeLogModes(path)
if err := os.MkdirAll(filepath.Dir(path), dirMode); err != nil {
@@ -41,9 +39,10 @@ func Append(path string, line []byte, rotateBytes int64, rotateArchives int) err
if asymptoteobserve.IsDuplicateEndpointEvent(path, line, asymptoteobserve.EndpointDuplicateWindow) {
return nil
}
- if err := rotateIfNeeded(path, rotateBytes, rotateArchives, int64(len(line))); err != nil {
- return err
- }
+ // A rotation that cannot happen now (another process holds the log open without delete sharing,
+ // say) is deferred, not fatal: the line goes into the live file past the size limit and a later
+ // write rotates. Returning the error here used to drop the line.
+ _ = asymptoteobserve.RotateRuntimeLog(path, rotateBytes, rotateArchives, int64(len(line)))
f, err := openRuntimeFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY)
if err != nil {
return err
@@ -64,62 +63,3 @@ func openRuntimeFile(path string, flag int) (*os.File, error) {
func openRuntimeFileAs(path, logPath string, flag int) (*os.File, error) {
return asymptoteobserve.OpenRuntimeLogFile(path, logPath, flag)
}
-
-func rotateIfNeeded(path string, maxSize int64, archives int, nextWriteBytes int64) error {
- if maxSize <= 0 {
- return nil
- }
- if archives < 1 {
- archives = DefaultRotateArchives
- }
- info, err := os.Stat(path)
- if err != nil {
- if os.IsNotExist(err) {
- return nil
- }
- return err
- }
- if info.Size() == 0 || info.Size()+nextWriteBytes <= maxSize {
- return nil
- }
- if err := removeOverflowArchives(path, archives); err != nil {
- return err
- }
- for i := archives - 1; i >= 1; i-- {
- from := path + fmt.Sprintf(".%d", i)
- to := path + fmt.Sprintf(".%d", i+1)
- if err := os.Rename(from, to); err != nil && !os.IsNotExist(err) {
- return err
- }
- }
- if err := os.Rename(path, path+".1"); err != nil {
- return err
- }
- asymptoteobserve.SettleRuntimeLogArchives(path, archives)
- return nil
-}
-
-func removeOverflowArchives(path string, archives int) error {
- dir := filepath.Dir(path)
- base := filepath.Base(path)
- entries, err := os.ReadDir(dir)
- if err != nil {
- return err
- }
- prefix := base + "."
- for _, entry := range entries {
- name := entry.Name()
- suffix, ok := strings.CutPrefix(name, prefix)
- if !ok {
- continue
- }
- index, err := strconv.Atoi(suffix)
- if err != nil || index < archives {
- continue
- }
- if err := os.Remove(filepath.Join(dir, name)); err != nil && !os.IsNotExist(err) {
- return err
- }
- }
- return nil
-}
diff --git a/docs/log-forwarding/local-jsonl.mdx b/docs/log-forwarding/local-jsonl.mdx
index 3f5708ab..d1bb9189 100644
--- a/docs/log-forwarding/local-jsonl.mdx
+++ b/docs/log-forwarding/local-jsonl.mdx
@@ -29,6 +29,18 @@ Beacon uses the same JSONL source for:
Beacon rotates the active log at 10 MiB and keeps five numbered archives such as `runtime.jsonl.1`. The active `runtime.jsonl` path remains the stable handoff point for the local dashboard and external shippers.
+### Reading the log on Windows
+
+Rotation renames the log. Windows refuses to rename a file that another process has open, unless that process allowed deletes when it opened the file. Vector allows it. PowerShell's `Get-Content` and `[IO.File]::OpenRead` do not, and if you use another agent, check how it opens files.
+
+If you write your own shipper, open the log with read, write and delete sharing:
+
+```powershell title="Open the log the way rotation needs"
+$stream = [System.IO.File]::Open($path, 'Open', 'Read', [System.IO.FileShare]'ReadWrite, Delete')
+```
+
+A reader that does not share deletes holds rotation up for as long as it keeps the file open. Beacon keeps appending to `runtime.jsonl` past 10 MiB and rotates once the file is closed, so no events are lost, but the log grows in the meantime. After a rotation, a handle you already hold reads the renamed `runtime.jsonl.1`, so reopen `runtime.jsonl` when you reach the end of the old file.
+
## Inspect locally
Confirm the endpoint is healthy and writing events:
diff --git a/docs/platforms/windows.mdx b/docs/platforms/windows.mdx
index dba4db48..9a151090 100644
--- a/docs/platforms/windows.mdx
+++ b/docs/platforms/windows.mdx
@@ -146,7 +146,7 @@ So the install sets these ACLs, by well-known SID, with inheritance from `%Progr
| `%ProgramData%\Beacon` and everything below it | Administrators | SYSTEM and Administrators: full control. Users: read and execute |
| `%ProgramData%\Beacon\Endpoint\logs` | Administrators | The same, plus `INTERACTIVE`: read and write data on files directly in the directory (inherit-only). No right to create, delete or rename entries, and no right to change a file's permissions or owner |
-Because users cannot create files in the log directory, the install creates `runtime.jsonl` and `runtime.jsonl.lock` itself; files the collector creates later, such as the log it starts after a rotation, inherit the same append grant. Rotation is done by the collector and by `beacon` commands run as an administrator. A hook that finds the log over its size limit appends anyway and leaves the rename to them.
+Because users cannot create files in the log directory, the install creates `runtime.jsonl` and `runtime.jsonl.lock` itself; files the collector creates later, such as the log it starts after a rotation, inherit the same append grant. Rotation is done by the collector and by `beacon` commands run as an administrator. A hook that finds the log over its size limit appends anyway and leaves the rename to them. A program that reads the log without allowing deletes, such as a PowerShell `Get-Content` loop, holds rotation up until it closes the file. The log grows past its size limit in the meantime and no events are lost. See [Reading the log on Windows](/log-forwarding/local-jsonl#reading-the-log-on-windows).
`%ProgramData%\Beacon` is reset even when it already exists. `%ProgramData%` lets any user create a subdirectory, so one found there before the package was installed may belong to someone else: the install takes ownership, replaces its ACL, removes any junction, symbolic link or hard link inside it (Beacon creates none), and resets anything below it that grants an ordinary user write, delete or ownership rights. Anything already locked down further than this is left as it is.
diff --git a/docs/security/restricted-log-access.mdx b/docs/security/restricted-log-access.mdx
index 2c1325d3..f05abab3 100644
--- a/docs/security/restricted-log-access.mdx
+++ b/docs/security/restricted-log-access.mdx
@@ -157,7 +157,7 @@ Vector keeps its read position in `/var/lib/vector/beacon-sentinel`, so it resum
**rsyslog.** When rsyslog [forwards the log to your collector](/log-forwarding/rsyslog), it reads the file as root on RHEL, Debian and most other distributions, so nothing changes there. On Ubuntu it runs as the `syslog` user: run `sudo usermod -aG beacon-log syslog` and restart rsyslog. See [With restricted log access](/log-forwarding/rsyslog#with-restricted-log-access).
-The same rule applies to Splunk Universal Forwarder, Filebeat or Elastic Agent, the Datadog Agent, Wazuh, Rapid7, Sumo Logic and any customer-managed shipper reading the [local JSONL log](/log-forwarding/local-jsonl): running as root, nothing changes; running as its own account, add that account to `beacon-log` and restart it. Splunk HEC and Falcon LogScale HEC are sent by the collector itself and are not affected.
+The same rule applies to Splunk Universal Forwarder, Filebeat or Elastic Agent, the Datadog Agent, Wazuh, Rapid7, Sumo Logic and any customer-managed shipper reading the [local JSONL log](/log-forwarding/local-jsonl): running as root, nothing changes; running as its own account, add that account to `beacon-log` and restart it. Splunk HEC and Falcon LogScale HEC are sent by the collector itself and are not affected. On Windows, a shipper you write yourself should also open the log with delete sharing, or it holds rotation up while it reads; see [Reading the log on Windows](/log-forwarding/local-jsonl#reading-the-log-on-windows).
## Who can do what on a restricted endpoint
diff --git a/pkg/asymptoteobserve/runtimelogrotate.go b/pkg/asymptoteobserve/runtimelogrotate.go
new file mode 100644
index 00000000..d0cf1f1f
--- /dev/null
+++ b/pkg/asymptoteobserve/runtimelogrotate.go
@@ -0,0 +1,155 @@
+package asymptoteobserve
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+)
+
+// DefaultRuntimeLogRotateArchives is how many archives a runtime log keeps when a writer passes
+// none: path.1 through path.5 beside the live file.
+const DefaultRuntimeLogRotateArchives = 5
+
+// ErrRuntimeLogRotationDeferred reports a rotation that was not started, because a file it would
+// rename or remove cannot be renamed or removed right now. Nothing moved, and the next write tries
+// again.
+var ErrRuntimeLogRotationDeferred = errors.New("runtime log rotation deferred")
+
+// runtimeLogRotationProbe and runtimeLogRotationRename are the two operations a rotation depends
+// on, replaceable so tests can refuse either at a chosen file.
+var (
+ runtimeLogRotationProbe = runtimeLogRenamable
+ runtimeLogRotationRename = os.Rename
+)
+
+// RuntimeLogNeedsRotation reports whether appending nextWriteBytes to path would take it past
+// maxSize, which is when RotateRuntimeLog rotates. An empty or missing file never rotates, so one
+// oversized line still lands somewhere.
+func RuntimeLogNeedsRotation(path string, maxSize, nextWriteBytes int64) (bool, error) {
+ if maxSize <= 0 {
+ return false, nil
+ }
+ info, err := os.Stat(path)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return false, nil
+ }
+ return false, err
+ }
+ return info.Size() > 0 && info.Size()+nextWriteBytes > maxSize, nil
+}
+
+// RotateRuntimeLog rotates the runtime log at path when appending nextWriteBytes would take it past
+// maxSize: the live file becomes path.1, each archive moves up one, and the archive at path.archives
+// (and anything numbered past it, which no reader looks at) is removed. The caller holds the log's
+// lock.
+//
+// Every writer of the log -- the endpoint CLI, the hook adapter and the collector -- rotates through
+// this one function and appends its line whatever it returns. A rotation that cannot happen now is
+// deferred rather than forced: the log grows past maxSize until a later write can rotate it. The
+// writers used to return the error instead, so a rotation that failed dropped the event it was
+// making room for.
+//
+// Nothing moves unless the whole rotation can complete. Windows refuses to rename or delete a file
+// another process holds open without delete sharing, which is how PowerShell's Get-Content and
+// [IO.File]::OpenRead open it, so every file the rotation would rename or remove is checked first
+// and the rotation is skipped when one is held. Before that check existed, the archives had already
+// moved up when the rename of the live file was refused, so each further attempt deleted one more
+// archive. Elsewhere a rename ignores open handles and the check passes.
+//
+// No archive is renamed over another. The rotation moves archives up only as far as the lowest free
+// slot, so a gap left by a rotation that failed partway (a reader opened a file between the check
+// and its rename) is closed by the next rotation instead of costing an archive.
+func RotateRuntimeLog(path string, maxSize int64, archives int, nextWriteBytes int64) error {
+ rotate, err := RuntimeLogNeedsRotation(path, maxSize, nextWriteBytes)
+ if err != nil || !rotate {
+ return err
+ }
+ if archives < 1 {
+ archives = DefaultRuntimeLogRotateArchives
+ }
+ archive := func(i int) string { return fmt.Sprintf("%s.%d", path, i) }
+
+ remove, err := runtimeLogArchivesPast(path, archives)
+ if err != nil {
+ return err
+ }
+ free := 0
+ for i := 1; i <= archives; i++ {
+ _, err := os.Lstat(archive(i))
+ if os.IsNotExist(err) {
+ free = i
+ break
+ }
+ if err != nil {
+ return err
+ }
+ }
+ if free == 0 {
+ remove = append(remove, archive(archives))
+ free = archives
+ }
+
+ touched := []string{path}
+ for i := 1; i < free; i++ {
+ touched = append(touched, archive(i))
+ }
+ for _, p := range append(touched, remove...) {
+ if err := runtimeLogRotationProbe(p); err != nil {
+ return fmt.Errorf("%w: %v", ErrRuntimeLogRotationDeferred, err)
+ }
+ }
+
+ for _, p := range remove {
+ if err := os.Remove(p); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ }
+ for i := free - 1; i >= 1; i-- {
+ if err := renameRuntimeLogFile(archive(i), archive(i+1)); err != nil {
+ return err
+ }
+ }
+ if err := renameRuntimeLogFile(path, archive(1)); err != nil {
+ return err
+ }
+ SettleRuntimeLogArchives(path, archives)
+ return nil
+}
+
+// renameRuntimeLogFile renames from to to, refusing when to exists. os.Rename replaces an existing
+// file on every platform, and an archive that is replaced is lost.
+func renameRuntimeLogFile(from, to string) error {
+ if _, err := os.Lstat(to); err == nil {
+ return fmt.Errorf("%w: %s already exists", ErrRuntimeLogRotationDeferred, to)
+ } else if !os.IsNotExist(err) {
+ return err
+ }
+ return runtimeLogRotationRename(from, to)
+}
+
+// runtimeLogArchivesPast lists path's archives numbered past archives, which a rotation removes.
+func runtimeLogArchivesPast(path string, archives int) ([]string, error) {
+ dir := filepath.Dir(path)
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return nil, err
+ }
+ prefix := filepath.Base(path) + "."
+ var past []string
+ for _, entry := range entries {
+ suffix, ok := strings.CutPrefix(entry.Name(), prefix)
+ if !ok {
+ continue
+ }
+ index, err := strconv.Atoi(suffix)
+ if err != nil || index <= archives {
+ continue
+ }
+ past = append(past, filepath.Join(dir, entry.Name()))
+ }
+ return past, nil
+}
diff --git a/pkg/asymptoteobserve/runtimelogrotate_other.go b/pkg/asymptoteobserve/runtimelogrotate_other.go
new file mode 100644
index 00000000..7bf6d431
--- /dev/null
+++ b/pkg/asymptoteobserve/runtimelogrotate_other.go
@@ -0,0 +1,7 @@
+//go:build !windows
+
+package asymptoteobserve
+
+// runtimeLogRenamable is always nil off Windows, where a rename or delete does not care who has the
+// file open.
+func runtimeLogRenamable(string) error { return nil }
diff --git a/pkg/asymptoteobserve/runtimelogrotate_test.go b/pkg/asymptoteobserve/runtimelogrotate_test.go
new file mode 100644
index 00000000..1819b6f5
--- /dev/null
+++ b/pkg/asymptoteobserve/runtimelogrotate_test.go
@@ -0,0 +1,157 @@
+package asymptoteobserve
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+// rotationFixture writes a live log holding "live" and archives 1..n holding their own number, so
+// every file's content says where it started.
+func rotationFixture(t *testing.T, n int) string {
+ t.Helper()
+ path := filepath.Join(t.TempDir(), "runtime.jsonl")
+ if err := os.WriteFile(path, []byte("live"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ for i := 1; i <= n; i++ {
+ if err := os.WriteFile(fmt.Sprintf("%s.%d", path, i), []byte(fmt.Sprint(i)), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return path
+}
+
+// rotationState maps "live", "1" .. "archives" (and any later number present) to file contents.
+func rotationState(t *testing.T, path string, archives int) map[string]string {
+ t.Helper()
+ state := map[string]string{}
+ read := func(name, file string) {
+ data, err := os.ReadFile(file)
+ if err == nil {
+ state[name] = string(data)
+ } else if !os.IsNotExist(err) {
+ t.Fatal(err)
+ }
+ }
+ read("live", path)
+ for i := 1; i <= archives+2; i++ {
+ read(fmt.Sprint(i), fmt.Sprintf("%s.%d", path, i))
+ }
+ return state
+}
+
+func assertRotationState(t *testing.T, got, want map[string]string) {
+ t.Helper()
+ if fmt.Sprint(got) != fmt.Sprint(want) {
+ t.Fatalf("files after rotation = %v, want %v", got, want)
+ }
+}
+
+func withRotationSeams(t *testing.T, probe func(string) error, rename func(string, string) error) {
+ t.Helper()
+ oldProbe, oldRename := runtimeLogRotationProbe, runtimeLogRotationRename
+ if probe != nil {
+ runtimeLogRotationProbe = probe
+ }
+ if rename != nil {
+ runtimeLogRotationRename = rename
+ }
+ t.Cleanup(func() { runtimeLogRotationProbe, runtimeLogRotationRename = oldProbe, oldRename })
+}
+
+func TestRotateRuntimeLogMovesEveryArchiveUpAndDropsTheOldest(t *testing.T) {
+ path := rotationFixture(t, 5)
+ if err := os.WriteFile(fmt.Sprintf("%s.7", path), []byte("stale"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := RotateRuntimeLog(path, 4, 5, 1); err != nil {
+ t.Fatal(err)
+ }
+ assertRotationState(t, rotationState(t, path, 5), map[string]string{
+ "1": "live", "2": "1", "3": "2", "4": "3", "5": "4",
+ })
+}
+
+func TestRotateRuntimeLogLeavesASmallOrMissingLogAlone(t *testing.T) {
+ path := rotationFixture(t, 2)
+ if err := RotateRuntimeLog(path, 100, 5, 1); err != nil {
+ t.Fatal(err)
+ }
+ assertRotationState(t, rotationState(t, path, 5), map[string]string{"live": "live", "1": "1", "2": "2"})
+ if err := RotateRuntimeLog(path+".missing", 1, 5, 1); err != nil {
+ t.Fatalf("a missing log is not an error: %v", err)
+ }
+}
+
+// A file the rotation would move but cannot (on Windows, one held open without delete sharing)
+// defers the whole rotation. Before the check, the archives had already moved up when the live
+// file's rename was refused, so each further attempt deleted one more archive and the event was
+// dropped.
+func TestRotateRuntimeLogMovesNothingWhenAFileIsHeld(t *testing.T) {
+ for _, held := range []string{"", ".3", ".5"} {
+ t.Run("held"+held, func(t *testing.T) {
+ path := rotationFixture(t, 5)
+ before := rotationState(t, path, 5)
+ withRotationSeams(t, func(p string) error {
+ if p == path+held {
+ return errors.New("held open")
+ }
+ return nil
+ }, nil)
+ err := RotateRuntimeLog(path, 4, 5, 1)
+ if !errors.Is(err, ErrRuntimeLogRotationDeferred) {
+ t.Fatalf("RotateRuntimeLog = %v, want ErrRuntimeLogRotationDeferred", err)
+ }
+ assertRotationState(t, rotationState(t, path, 5), before)
+ })
+ }
+}
+
+// A rename that fails after the check passed (a reader opened the file in between) leaves a gap.
+// The next rotation fills it rather than removing the oldest archive again, so the two together end
+// exactly where one clean rotation would.
+func TestRotateRuntimeLogFillsTheGapAFailedRotationLeft(t *testing.T) {
+ path := rotationFixture(t, 5)
+ withRotationSeams(t, nil, func(from, to string) error {
+ if from == path+".2" {
+ return errors.New("opened by a reader")
+ }
+ return os.Rename(from, to)
+ })
+ if err := RotateRuntimeLog(path, 4, 5, 1); err == nil {
+ t.Fatal("a refused rename must be reported")
+ }
+ assertRotationState(t, rotationState(t, path, 5), map[string]string{
+ "live": "live", "1": "1", "2": "2", "4": "3", "5": "4",
+ })
+
+ runtimeLogRotationRename = os.Rename
+ if err := RotateRuntimeLog(path, 4, 5, 1); err != nil {
+ t.Fatal(err)
+ }
+ assertRotationState(t, rotationState(t, path, 5), map[string]string{
+ "1": "live", "2": "1", "3": "2", "4": "3", "5": "4",
+ })
+}
+
+// os.Rename replaces its destination on every platform, so a rotation renames only into a slot that
+// is free when it gets there. A name that appears in between is kept and the rotation waits.
+func TestRotateRuntimeLogNeverRenamesOverAnArchive(t *testing.T) {
+ path := rotationFixture(t, 0)
+ withRotationSeams(t, func(p string) error {
+ if p == path {
+ if err := os.WriteFile(path+".1", []byte("appeared"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return nil
+ }, nil)
+ err := RotateRuntimeLog(path, 2, 5, 1)
+ if !errors.Is(err, ErrRuntimeLogRotationDeferred) {
+ t.Fatalf("RotateRuntimeLog = %v, want ErrRuntimeLogRotationDeferred", err)
+ }
+ assertRotationState(t, rotationState(t, path, 5), map[string]string{"live": "live", "1": "appeared"})
+}
diff --git a/pkg/asymptoteobserve/runtimelogrotate_windows.go b/pkg/asymptoteobserve/runtimelogrotate_windows.go
new file mode 100644
index 00000000..8e06c6a3
--- /dev/null
+++ b/pkg/asymptoteobserve/runtimelogrotate_windows.go
@@ -0,0 +1,33 @@
+//go:build windows
+
+package asymptoteobserve
+
+import (
+ "errors"
+ "os"
+
+ "golang.org/x/sys/windows"
+)
+
+// runtimeLogRenamable reports whether path could be renamed or removed now, by opening it for DELETE
+// while sharing everything. Windows refuses a rename or delete of a file another process holds open
+// without FILE_SHARE_DELETE, and it refuses a DELETE open by the same sharing check, so this gives
+// the rename's own answer without moving anything. A missing file has nothing to refuse.
+//
+// The link itself is opened, not its target, because that is what a rename moves.
+func runtimeLogRenamable(path string) error {
+ name, err := windows.UTF16PtrFromString(path)
+ if err != nil {
+ return err
+ }
+ h, err := windows.CreateFile(name, windows.DELETE,
+ windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE,
+ nil, windows.OPEN_EXISTING, windows.FILE_FLAG_OPEN_REPARSE_POINT, 0)
+ if err != nil {
+ if errors.Is(err, windows.ERROR_FILE_NOT_FOUND) || errors.Is(err, windows.ERROR_PATH_NOT_FOUND) {
+ return nil
+ }
+ return &os.PathError{Op: "open for rename", Path: path, Err: err}
+ }
+ return windows.CloseHandle(h)
+}
diff --git a/pkg/asymptoteobserve/runtimelogrotate_windows_test.go b/pkg/asymptoteobserve/runtimelogrotate_windows_test.go
new file mode 100644
index 00000000..6d56ed67
--- /dev/null
+++ b/pkg/asymptoteobserve/runtimelogrotate_windows_test.go
@@ -0,0 +1,74 @@
+//go:build windows
+
+package asymptoteobserve
+
+import (
+ "errors"
+ "testing"
+
+ "golang.org/x/sys/windows"
+)
+
+// holdWithoutDeleteSharing opens path the way PowerShell's Get-Content and [IO.File]::OpenRead do:
+// sharing reads and writes but not deletes, which is what makes Windows refuse a rename.
+func holdWithoutDeleteSharing(t *testing.T, path string) windows.Handle {
+ t.Helper()
+ name, err := windows.UTF16PtrFromString(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ h, err := windows.CreateFile(name, windows.GENERIC_READ, windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE,
+ nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return h
+}
+
+// The real thing the seam stands in for: an archive held open by another reader defers the
+// rotation with every file where it was, and the rotation goes through once the reader lets go.
+func TestRotateRuntimeLogWaitsForAReaderWithoutDeleteSharing(t *testing.T) {
+ for _, held := range []string{"", ".2"} {
+ t.Run("held"+held, func(t *testing.T) {
+ path := rotationFixture(t, 5)
+ before := rotationState(t, path, 5)
+ h := holdWithoutDeleteSharing(t, path+held)
+ err := RotateRuntimeLog(path, 4, 5, 1)
+ if !errors.Is(err, ErrRuntimeLogRotationDeferred) {
+ _ = windows.CloseHandle(h)
+ t.Fatalf("RotateRuntimeLog = %v, want ErrRuntimeLogRotationDeferred", err)
+ }
+ assertRotationState(t, rotationState(t, path, 5), before)
+
+ if err := windows.CloseHandle(h); err != nil {
+ t.Fatal(err)
+ }
+ if err := RotateRuntimeLog(path, 4, 5, 1); err != nil {
+ t.Fatal(err)
+ }
+ assertRotationState(t, rotationState(t, path, 5), map[string]string{
+ "1": "live", "2": "1", "3": "2", "4": "3", "5": "4",
+ })
+ })
+ }
+}
+
+// A reader that does share deletes, as Vector and Beacon's own dashboard do, never stands in the way.
+func TestRotateRuntimeLogIgnoresAReaderThatSharesDeletes(t *testing.T) {
+ path := rotationFixture(t, 1)
+ name, err := windows.UTF16PtrFromString(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ h, err := windows.CreateFile(name, windows.GENERIC_READ,
+ windows.FILE_SHARE_READ|windows.FILE_SHARE_WRITE|windows.FILE_SHARE_DELETE,
+ nil, windows.OPEN_EXISTING, windows.FILE_ATTRIBUTE_NORMAL, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer windows.CloseHandle(h)
+ if err := RotateRuntimeLog(path, 4, 5, 1); err != nil {
+ t.Fatal(err)
+ }
+ assertRotationState(t, rotationState(t, path, 5), map[string]string{"1": "live", "2": "1"})
+}