Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

add log rotation

Justin D'Souzaf13e1c7974fc
diff --git a/README.md b/README.md
index 7ccefb32..1094b6a1 100644
--- a/README.md
+++ b/README.md
@@ -133,6 +133,13 @@ without a hosted backend. The overview screen summarizes recent runtime events
and collection status, while log search helps teams inspect normalized event
records during rollout, testing, and investigations.
+Beacon writes endpoint activity to a stable local `runtime.jsonl` file. The
+active file rotates at 10 MiB with five numbered local archives, keeping the
+endpoint handoff file bounded while external SIEM forwarders continue tailing
+the active path. The dashboard and local validation commands read the active log
+in this release; SIEM destinations remain the source of truth for long-term
+retention and search.
+
<p align="center">
<img src="images/dashboard-overview.png" alt="Beacon dashboard overview" width="860">
</p>
diff --git a/cli/beacon-hooks/internal/logging/endpoint_event_test.go b/cli/beacon-hooks/internal/logging/endpoint_event_test.go
index b8cb85b7..ffb0507b 100644
--- a/cli/beacon-hooks/internal/logging/endpoint_event_test.go
+++ b/cli/beacon-hooks/internal/logging/endpoint_event_test.go
@@ -3,6 +3,7 @@ package logging
import (
"os"
"path/filepath"
+ "strings"
"testing"
)
@@ -36,3 +37,21 @@ func TestEndpointEventStillWritesStructuredTelemetry(t *testing.T) {
t.Fatalf("expected structured endpoint event, len=%d err=%v", len(data), err)
}
}
+
+func TestEndpointEventRotatesRuntimeLog(t *testing.T) {
+ logPath := filepath.Join(t.TempDir(), "runtime.jsonl")
+ t.Setenv("BEACON_ENDPOINT_LOG", logPath)
+ if err := os.WriteFile(logPath, []byte(strings.Repeat("old", 4*1024*1024)), 0644); err != nil {
+ t.Fatalf("write existing log: %v", err)
+ }
+
+ logger := NewLoggerForPlatform("pre-tool", "test")
+ logger.EndpointEvent("approval.allowed", "approval", "info", "Pre-tool observed", nil)
+
+ if rotated, err := os.ReadFile(logPath + ".1"); err != nil || len(rotated) == 0 {
+ t.Fatalf("expected rotated archive, len=%d err=%v", len(rotated), err)
+ }
+ if current, err := os.ReadFile(logPath); err != nil || !strings.Contains(string(current), "Pre-tool observed") {
+ t.Fatalf("expected current log to contain new event, data=%q err=%v", string(current), err)
+ }
+}
diff --git a/cli/beacon-hooks/internal/logging/logging.go b/cli/beacon-hooks/internal/logging/logging.go
index 1d4c173f..4725428d 100644
--- a/cli/beacon-hooks/internal/logging/logging.go
+++ b/cli/beacon-hooks/internal/logging/logging.go
@@ -7,6 +7,7 @@ import (
"path/filepath"
"regexp"
"runtime"
+ "strconv"
"strings"
"syscall"
"time"
@@ -21,6 +22,11 @@ var endpointSecretPatterns = []*regexp.Regexp{
regexp.MustCompile(`sk-[a-zA-Z0-9]{20,}`),
}
+const (
+ defaultEndpointRotateBytes = 10 * 1024 * 1024
+ defaultEndpointRotateArchives = 5
+)
+
// Logger provides structured JSON logging for hook events
type Logger struct {
hookName string
@@ -173,17 +179,7 @@ func writeEndpointJSON(path string, event map[string]interface{}) {
return
}
}
- f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
- if err != nil {
- return
- }
- if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
- _ = f.Close()
- return
- }
- defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
- defer f.Close()
- _, _ = f.Write(append(data, '\n'))
+ _ = appendEndpointJSONL(path, append(data, '\n'), defaultEndpointRotateBytes, defaultEndpointRotateArchives)
}
func endpointLogPath() string {
@@ -266,6 +262,88 @@ func truncateEndpoint(value string, limit int) string {
return value[:limit-15] + "...[truncated]"
}
+// Keep this rotation contract mirrored with the endpoint CLI and beaconjson exporter.
+func appendEndpointJSONL(path string, line []byte, rotateBytes int64, rotateArchives int) error {
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ return err
+ }
+ lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, 0644)
+ if err != nil {
+ return err
+ }
+ if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
+ _ = lock.Close()
+ return err
+ }
+ defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)
+ defer lock.Close()
+ if err := rotateEndpointLogIfNeeded(path, rotateBytes, rotateArchives, int64(len(line))); err != nil {
+ return err
+ }
+ f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ _, err = f.Write(line)
+ return err
+}
+
+func rotateEndpointLogIfNeeded(path string, maxSize int64, archives int, nextWriteBytes int64) error {
+ if maxSize <= 0 {
+ return nil
+ }
+ if archives < 1 {
+ archives = defaultEndpointRotateArchives
+ }
+ info, err := os.Stat(path)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil
+ }
+ return err
+ }
+ if info.Size() == 0 || info.Size()+nextWriteBytes <= maxSize {
+ return nil
+ }
+ if err := removeOverflowEndpointArchives(path, archives); err != nil {
+ return err
+ }
+ for i := archives - 1; i >= 1; i-- {
+ from := path + fmt.Sprintf(".%d", i)
+ to := path + fmt.Sprintf(".%d", i+1)
+ if err := os.Rename(from, to); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ }
+ return os.Rename(path, path+".1")
+}
+
+func removeOverflowEndpointArchives(path string, archives int) error {
+ dir := filepath.Dir(path)
+ base := filepath.Base(path)
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return err
+ }
+ prefix := base + "."
+ for _, entry := range entries {
+ name := entry.Name()
+ suffix, ok := strings.CutPrefix(name, prefix)
+ if !ok {
+ continue
+ }
+ index, err := strconv.Atoi(suffix)
+ if err != nil || index < archives {
+ continue
+ }
+ if err := os.Remove(filepath.Join(dir, name)); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ }
+ return nil
+}
+
// Debug logs a debug message
func (l *Logger) Debug(message string, fields ...interface{}) {
entry := l.formatEntry("debug", message, fields...)
diff --git a/cli/beacon/internal/endpoint/collector/collector.go b/cli/beacon/internal/endpoint/collector/collector.go
index 66818b2a..cd56c77e 100644
--- a/cli/beacon/internal/endpoint/collector/collector.go
+++ b/cli/beacon/internal/endpoint/collector/collector.go
@@ -144,6 +144,7 @@ exporters:
path: %q
max_event_bytes: 65536
rotate_bytes: 10485760
+ rotate_archives: 5
redact_secrets: true
content_retention: %q
%s
diff --git a/cli/beacon/internal/endpoint/collector/collector_test.go b/cli/beacon/internal/endpoint/collector/collector_test.go
index 0efbbe19..7f6cc330 100644
--- a/cli/beacon/internal/endpoint/collector/collector_test.go
+++ b/cli/beacon/internal/endpoint/collector/collector_test.go
@@ -40,6 +40,7 @@ func TestConfigYAMLIncludesReleaseContractFields(t *testing.T) {
"path: " + `"` + cfg.LogPath + `"`,
"max_event_bytes: 65536",
"rotate_bytes: 10485760",
+ "rotate_archives: 5",
"redact_secrets: true",
"level: none",
"receivers: [otlp]",
diff --git a/cli/beacon/internal/endpoint/writer/writer.go b/cli/beacon/internal/endpoint/writer/writer.go
index cd929dd2..12c9d25b 100644
--- a/cli/beacon/internal/endpoint/writer/writer.go
+++ b/cli/beacon/internal/endpoint/writer/writer.go
@@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"regexp"
+ "strconv"
"strings"
"syscall"
@@ -14,10 +15,12 @@ import (
)
const (
- SystemLogPath = "/var/log/beacon-agent/runtime.jsonl"
- UserLogPath = ".beacon/endpoint/logs/runtime.jsonl"
- MaxEventBytes = 64 * 1024
- RotateBytes = 10 * 1024 * 1024
+ SystemLogPath = "/var/log/beacon-agent/runtime.jsonl"
+ UserLogPath = ".beacon/endpoint/logs/runtime.jsonl"
+ MaxEventBytes = 64 * 1024
+ DefaultRotateBytes = 10 * 1024 * 1024
+ DefaultRotateArchives = 5
+ RotateBytes = DefaultRotateBytes
)
var secretPatterns = []*regexp.Regexp{
@@ -28,10 +31,11 @@ var secretPatterns = []*regexp.Regexp{
}
type Options struct {
- Path string
- UserMode bool
- MaxBytes int
- RotateSize int64
+ Path string
+ UserMode bool
+ MaxBytes int
+ RotateSize int64
+ RotateArchives int
}
func DefaultPath(userMode bool) string {
@@ -52,19 +56,16 @@ func AppendEvent(event schema.Event, opts Options) (string, error) {
if opts.MaxBytes == 0 {
opts.MaxBytes = MaxEventBytes
}
- if opts.RotateSize == 0 {
- opts.RotateSize = RotateBytes
+ if opts.RotateSize <= 0 {
+ opts.RotateSize = DefaultRotateBytes
+ }
+ if opts.RotateArchives < 1 {
+ opts.RotateArchives = DefaultRotateArchives
}
event = SanitizeEvent(event, opts.MaxBytes)
if err := event.Validate(); err != nil {
return "", err
}
- if err := os.MkdirAll(filepath.Dir(opts.Path), 0755); err != nil {
- return "", err
- }
- if err := rotateIfNeeded(opts.Path, opts.RotateSize); err != nil {
- return "", err
- }
data, err := json.Marshal(event)
if err != nil {
return "", err
@@ -81,17 +82,7 @@ func AppendEvent(event schema.Event, opts Options) (string, error) {
if len(data) > opts.MaxBytes {
return "", fmt.Errorf("event exceeds maximum size after truncation: %d bytes", len(data))
}
- f, err := os.OpenFile(opts.Path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
- if err != nil {
- return "", err
- }
- if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
- _ = f.Close()
- return "", err
- }
- defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
- defer f.Close()
- if _, err := f.Write(append(data, '\n')); err != nil {
+ if err := appendJSONL(opts.Path, append(data, '\n'), opts.RotateSize, opts.RotateArchives); err != nil {
return "", err
}
return opts.Path, nil
@@ -201,7 +192,39 @@ func truncate(value string, limit int) string {
return value[:limit-15] + "...[truncated]"
}
-func rotateIfNeeded(path string, maxSize int64) error {
+func appendJSONL(path string, line []byte, rotateBytes int64, rotateArchives int) error {
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ return err
+ }
+ lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, 0644)
+ if err != nil {
+ return err
+ }
+ if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
+ _ = lock.Close()
+ return err
+ }
+ defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)
+ defer lock.Close()
+ if err := rotateIfNeeded(path, rotateBytes, rotateArchives, int64(len(line))); err != nil {
+ return err
+ }
+ f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ _, err = f.Write(line)
+ return err
+}
+
+func rotateIfNeeded(path string, maxSize int64, archives int, nextWriteBytes int64) error {
+ if maxSize <= 0 {
+ return nil
+ }
+ if archives < 1 {
+ archives = DefaultRotateArchives
+ }
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
@@ -209,10 +232,44 @@ func rotateIfNeeded(path string, maxSize int64) error {
}
return err
}
- if info.Size() <= maxSize {
+ if info.Size() == 0 || info.Size()+nextWriteBytes <= maxSize {
return nil
}
+ if err := removeOverflowArchives(path, archives); err != nil {
+ return err
+ }
+ for i := archives - 1; i >= 1; i-- {
+ from := path + fmt.Sprintf(".%d", i)
+ to := path + fmt.Sprintf(".%d", i+1)
+ if err := os.Rename(from, to); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ }
rotated := path + ".1"
- _ = os.Remove(rotated)
return os.Rename(path, rotated)
}
+
+func removeOverflowArchives(path string, archives int) error {
+ dir := filepath.Dir(path)
+ base := filepath.Base(path)
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return err
+ }
+ prefix := base + "."
+ for _, entry := range entries {
+ name := entry.Name()
+ suffix, ok := strings.CutPrefix(name, prefix)
+ if !ok {
+ continue
+ }
+ index, err := strconv.Atoi(suffix)
+ if err != nil || index < archives {
+ continue
+ }
+ if err := os.Remove(filepath.Join(dir, name)); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/cli/beacon/internal/endpoint/writer/writer_test.go b/cli/beacon/internal/endpoint/writer/writer_test.go
index d0adcb6b..f24d4671 100644
--- a/cli/beacon/internal/endpoint/writer/writer_test.go
+++ b/cli/beacon/internal/endpoint/writer/writer_test.go
@@ -4,7 +4,9 @@ import (
"encoding/json"
"os"
"path/filepath"
+ "strconv"
"strings"
+ "sync"
"testing"
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
@@ -115,6 +117,78 @@ func TestAppendEventRotatesExistingLog(t *testing.T) {
}
}
+func TestAppendEventPrunesNumberedArchives(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "runtime.jsonl")
+ for i := 0; i <= 3; i++ {
+ target := path
+ if i > 0 {
+ target = path + "." + strconv.Itoa(i)
+ }
+ if err := os.WriteFile(target, []byte("log-"+strconv.Itoa(i)), 0644); err != nil {
+ t.Fatalf("write %s: %v", target, err)
+ }
+ }
+ event := schema.NewEvent(schema.NewEventOptions{
+ Action: "agent.detected",
+ Harness: schema.HarnessInfo{Name: "test"},
+ Message: "new event",
+ })
+
+ if _, err := AppendEvent(event, Options{Path: path, RotateSize: 1, RotateArchives: 2}); err != nil {
+ t.Fatalf("AppendEvent returned error: %v", err)
+ }
+ if _, err := os.Stat(path + ".3"); !os.IsNotExist(err) {
+ t.Fatalf("expected .3 archive to be pruned, err=%v", err)
+ }
+ if data, err := os.ReadFile(path + ".1"); err != nil || string(data) != "log-0" {
+ t.Fatalf(".1 = %q err=%v, want prior active log", string(data), err)
+ }
+ if data, err := os.ReadFile(path + ".2"); err != nil || string(data) != "log-1" {
+ t.Fatalf(".2 = %q err=%v, want prior .1 archive", string(data), err)
+ }
+}
+
+func TestAppendEventSerializesConcurrentWrites(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "runtime.jsonl")
+ event := schema.NewEvent(schema.NewEventOptions{
+ Action: "agent.detected",
+ Harness: schema.HarnessInfo{Name: "test"},
+ Message: "concurrent event",
+ })
+ var wg sync.WaitGroup
+ errs := make(chan error, 20)
+ for i := 0; i < 20; i++ {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ _, err := AppendEvent(event, Options{Path: path, RotateSize: 64 * 1024})
+ errs <- err
+ }()
+ }
+ wg.Wait()
+ close(errs)
+ for err := range errs {
+ if err != nil {
+ t.Fatalf("AppendEvent returned error: %v", err)
+ }
+ }
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatalf("read log: %v", err)
+ }
+ lines := strings.Split(strings.TrimSpace(string(data)), "\n")
+ if len(lines) != 20 {
+ t.Fatalf("expected 20 complete lines, got %d: %s", len(lines), string(data))
+ }
+ for _, line := range lines {
+ var decoded map[string]interface{}
+ if err := json.Unmarshal([]byte(line), &decoded); err != nil {
+ t.Fatalf("line is not JSON: %v line=%q", err, line)
+ }
+ }
+}
+
func TestLastLine(t *testing.T) {
path := filepath.Join(t.TempDir(), "runtime.jsonl")
if _, err := LastLine(path); err == nil {
diff --git a/collector-builder/exporter/beaconjsonexporter/config.go b/collector-builder/exporter/beaconjsonexporter/config.go
index 9a827c89..b91065cb 100644
--- a/collector-builder/exporter/beaconjsonexporter/config.go
+++ b/collector-builder/exporter/beaconjsonexporter/config.go
@@ -3,8 +3,9 @@ package beaconjsonexporter
import "fmt"
const (
- defaultMaxEventBytes = 64 * 1024
- defaultRotateBytes = 10 * 1024 * 1024
+ defaultMaxEventBytes = 64 * 1024
+ defaultRotateBytes = 10 * 1024 * 1024
+ defaultRotateArchives = 5
)
// Config controls local JSONL export for Beacon endpoint events.
@@ -12,6 +13,7 @@ type Config struct {
Path string `mapstructure:"path"`
MaxEventBytes int `mapstructure:"max_event_bytes"`
RotateBytes int64 `mapstructure:"rotate_bytes"`
+ RotateArchives int `mapstructure:"rotate_archives"`
RedactSecrets bool `mapstructure:"redact_secrets"`
ContentRetention string `mapstructure:"content_retention"`
IncludeRuntimeMetrics bool `mapstructure:"include_runtime_metrics"`
@@ -22,6 +24,7 @@ func createDefaultConfig() *Config {
return &Config{
MaxEventBytes: defaultMaxEventBytes,
RotateBytes: defaultRotateBytes,
+ RotateArchives: defaultRotateArchives,
RedactSecrets: true,
ContentRetention: "full",
}
@@ -34,9 +37,6 @@ func (c *Config) Validate() error {
if c.MaxEventBytes <= 0 {
return fmt.Errorf("max_event_bytes must be positive")
}
- if c.RotateBytes < 0 {
- return fmt.Errorf("rotate_bytes cannot be negative")
- }
switch c.ContentRetention {
case "", "metadata", "redacted", "full":
return nil
diff --git a/collector-builder/exporter/beaconjsonexporter/exporter.go b/collector-builder/exporter/beaconjsonexporter/exporter.go
index d6aa08f9..849ab271 100644
--- a/collector-builder/exporter/beaconjsonexporter/exporter.go
+++ b/collector-builder/exporter/beaconjsonexporter/exporter.go
@@ -50,9 +50,12 @@ func newExporter(raw component.Config, set exporter.Settings) (*beaconExporter,
if cfg.MaxEventBytes == 0 {
cfg.MaxEventBytes = defaultMaxEventBytes
}
- if cfg.RotateBytes == 0 {
+ if cfg.RotateBytes <= 0 {
cfg.RotateBytes = defaultRotateBytes
}
+ if cfg.RotateArchives <= 0 {
+ cfg.RotateArchives = defaultRotateArchives
+ }
if cfg.ContentRetention == "" {
cfg.ContentRetention = "full"
}
@@ -62,10 +65,11 @@ func newExporter(raw component.Config, set exporter.Settings) (*beaconExporter,
return &beaconExporter{
cfg: cfg,
writer: jsonlWriter{
- path: cfg.Path,
- maxEventBytes: cfg.MaxEventBytes,
- rotateBytes: cfg.RotateBytes,
- redactSecrets: cfg.RedactSecrets,
+ path: cfg.Path,
+ maxEventBytes: cfg.MaxEventBytes,
+ rotateBytes: cfg.RotateBytes,
+ rotateArchives: cfg.RotateArchives,
+ redactSecrets: cfg.RedactSecrets,
},
logger: set.Logger,
}, nil
diff --git a/collector-builder/exporter/beaconjsonexporter/exporter_test.go b/collector-builder/exporter/beaconjsonexporter/exporter_test.go
index dad6394e..42d5c9fe 100644
--- a/collector-builder/exporter/beaconjsonexporter/exporter_test.go
+++ b/collector-builder/exporter/beaconjsonexporter/exporter_test.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"os"
"path/filepath"
+ "strconv"
"strings"
"testing"
"time"
@@ -33,6 +34,16 @@ func TestDefaultConfigUsesFullRetention(t *testing.T) {
}
}
+func TestDefaultConfigUsesBoundedRotation(t *testing.T) {
+ cfg := createDefaultConfig()
+ if cfg.RotateBytes != defaultRotateBytes {
+ t.Fatalf("RotateBytes = %d, want %d", cfg.RotateBytes, defaultRotateBytes)
+ }
+ if cfg.RotateArchives != defaultRotateArchives {
+ t.Fatalf("RotateArchives = %d, want %d", cfg.RotateArchives, defaultRotateArchives)
+ }
+}
+
func TestNewExporterDefaultsEmptyRetentionToFull(t *testing.T) {
cfg := &Config{
Path: filepath.Join(t.TempDir(), "runtime.jsonl"),
@@ -49,6 +60,62 @@ func TestNewExporterDefaultsEmptyRetentionToFull(t *testing.T) {
}
}
+func TestNewExporterNormalizesRotationDefaults(t *testing.T) {
+ cfg := &Config{
+ Path: filepath.Join(t.TempDir(), "runtime.jsonl"),
+ MaxEventBytes: defaultMaxEventBytes,
+ RotateBytes: -1,
+ RotateArchives: -1,
+ RedactSecrets: true,
+ }
+ exp, err := newExporter(cfg, exporter.Settings{})
+ if err != nil {
+ t.Fatalf("newExporter returned error: %v", err)
+ }
+ if exp.writer.rotateBytes != defaultRotateBytes {
+ t.Fatalf("writer rotateBytes = %d, want %d", exp.writer.rotateBytes, defaultRotateBytes)
+ }
+ if exp.writer.rotateArchives != defaultRotateArchives {
+ t.Fatalf("writer rotateArchives = %d, want %d", exp.writer.rotateArchives, defaultRotateArchives)
+ }
+}
+
+func TestJSONLWriterRotatesAndPrunesArchives(t *testing.T) {
+ dir := t.TempDir()
+ path := filepath.Join(dir, "runtime.jsonl")
+ for i := 0; i <= 3; i++ {
+ target := path
+ if i > 0 {
+ target = path + "." + strconv.Itoa(i)
+ }
+ if err := os.WriteFile(target, []byte("log-"+strconv.Itoa(i)), 0644); err != nil {
+ t.Fatalf("write %s: %v", target, err)
+ }
+ }
+ writer := jsonlWriter{
+ path: path,
+ maxEventBytes: defaultMaxEventBytes,
+ rotateBytes: 1,
+ rotateArchives: 2,
+ redactSecrets: true,
+ }
+ event := newBeaconEvent("tool.invoked", "tool", "info", "test", time.Now())
+ event.Message = "new event"
+
+ if err := writer.append(event); err != nil {
+ t.Fatalf("append returned error: %v", err)
+ }
+ if _, err := os.Stat(path + ".3"); !os.IsNotExist(err) {
+ t.Fatalf("expected .3 archive to be pruned, err=%v", err)
+ }
+ if data, err := os.ReadFile(path + ".1"); err != nil || string(data) != "log-0" {
+ t.Fatalf(".1 = %q err=%v, want prior active log", string(data), err)
+ }
+ if data, err := os.ReadFile(path); err != nil || !strings.Contains(string(data), "new event") {
+ t.Fatalf("current log = %q err=%v, want new event", string(data), err)
+ }
+}
+
func TestConsumeLogsWritesBeaconJSONL(t *testing.T) {
path := filepath.Join(t.TempDir(), "runtime.jsonl")
exp, err := newExporter(&Config{
diff --git a/collector-builder/exporter/beaconjsonexporter/writer.go b/collector-builder/exporter/beaconjsonexporter/writer.go
index 5353b1f1..84e7411d 100644
--- a/collector-builder/exporter/beaconjsonexporter/writer.go
+++ b/collector-builder/exporter/beaconjsonexporter/writer.go
@@ -6,6 +6,7 @@ import (
"os"
"path/filepath"
"regexp"
+ "strconv"
"strings"
"syscall"
)
@@ -18,10 +19,11 @@ var secretPatterns = []*regexp.Regexp{
}
type jsonlWriter struct {
- path string
- maxEventBytes int
- rotateBytes int64
- redactSecrets bool
+ path string
+ maxEventBytes int
+ rotateBytes int64
+ rotateArchives int
+ redactSecrets bool
}
func (w jsonlWriter) append(event beaconEvent) error {
@@ -45,21 +47,7 @@ func (w jsonlWriter) append(event beaconEvent) error {
if err := os.MkdirAll(filepath.Dir(w.path), 0755); err != nil {
return err
}
- if err := rotateIfNeeded(w.path, w.rotateBytes); err != nil {
- return err
- }
- f, err := os.OpenFile(w.path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
- if err != nil {
- return err
- }
- if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX); err != nil {
- _ = f.Close()
- return err
- }
- defer syscall.Flock(int(f.Fd()), syscall.LOCK_UN)
- defer f.Close()
- _, err = f.Write(append(data, '\n'))
- return err
+ return appendJSONL(w.path, append(data, '\n'), w.rotateBytes, w.rotateArchives)
}
func (w jsonlWriter) sanitize(event beaconEvent) beaconEvent {
@@ -153,10 +141,40 @@ func truncate(value string, limit int) string {
return value[:limit-15] + "...[truncated]"
}
-func rotateIfNeeded(path string, maxSize int64) error {
- if maxSize == 0 {
+// Keep this rotation contract mirrored with the endpoint CLI and hook writer.
+func appendJSONL(path string, line []byte, rotateBytes int64, rotateArchives int) error {
+ if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
+ return err
+ }
+ lock, err := os.OpenFile(path+".lock", os.O_CREATE|os.O_RDWR, 0644)
+ if err != nil {
+ return err
+ }
+ if err := syscall.Flock(int(lock.Fd()), syscall.LOCK_EX); err != nil {
+ _ = lock.Close()
+ return err
+ }
+ defer syscall.Flock(int(lock.Fd()), syscall.LOCK_UN)
+ defer lock.Close()
+ if err := rotateIfNeeded(path, rotateBytes, rotateArchives, int64(len(line))); err != nil {
+ return err
+ }
+ f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ _, err = f.Write(line)
+ return err
+}
+
+func rotateIfNeeded(path string, maxSize int64, archives int, nextWriteBytes int64) error {
+ if maxSize <= 0 {
return nil
}
+ if archives < 1 {
+ archives = defaultRotateArchives
+ }
info, err := os.Stat(path)
if err != nil {
if os.IsNotExist(err) {
@@ -164,10 +182,43 @@ func rotateIfNeeded(path string, maxSize int64) error {
}
return err
}
- if info.Size() <= maxSize {
+ if info.Size() == 0 || info.Size()+nextWriteBytes <= maxSize {
return nil
}
- rotated := path + ".1"
- _ = os.Remove(rotated)
- return os.Rename(path, rotated)
+ if err := removeOverflowArchives(path, archives); err != nil {
+ return err
+ }
+ for i := archives - 1; i >= 1; i-- {
+ from := path + fmt.Sprintf(".%d", i)
+ to := path + fmt.Sprintf(".%d", i+1)
+ if err := os.Rename(from, to); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ }
+ return os.Rename(path, path+".1")
+}
+
+func removeOverflowArchives(path string, archives int) error {
+ dir := filepath.Dir(path)
+ base := filepath.Base(path)
+ entries, err := os.ReadDir(dir)
+ if err != nil {
+ return err
+ }
+ prefix := base + "."
+ for _, entry := range entries {
+ name := entry.Name()
+ suffix, ok := strings.CutPrefix(name, prefix)
+ if !ok {
+ continue
+ }
+ index, err := strconv.Atoi(suffix)
+ if err != nil || index < archives {
+ continue
+ }
+ if err := os.Remove(filepath.Join(dir, name)); err != nil && !os.IsNotExist(err) {
+ return err
+ }
+ }
+ return nil
}