Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

hooks, CLI: send system-log events through the collector when it listens

Claude59034a1a4ccd
diff --git a/cli/beacon-hooks/cmd/main_test.go b/cli/beacon-hooks/cmd/main_test.go
index 6dd82eb4..3fe5eb93 100644
--- a/cli/beacon-hooks/cmd/main_test.go
+++ b/cli/beacon-hooks/cmd/main_test.go
@@ -7,6 +7,7 @@ import (
"github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/logging"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/activesession"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest"
)
// TestMain keeps the active-session record every endpoint event updates out of the real home
@@ -20,6 +21,12 @@ func TestMain(m *testing.M) {
// Auto-share is off unless a test turns it on in a settings file of its own.
settingsPath := filepath.Join(dir, "share-settings.json")
logging.ShareSettingsPath = func() string { return settingsPath }
+ // Never send to a real collector's ingest socket, which a developer's restricted-mode machine
+ // has.
+ logging.EndpointIngest = localingest.Deliverer{
+ Client: localingest.Client{SocketPath: filepath.Join(dir, "no-ingest.sock")},
+ Spool: localingest.Spool{Dir: filepath.Join(dir, "spool")},
+ }
// A suite run from inside a Claude Code session must not append to that session's env file.
os.Unsetenv("CLAUDE_ENV_FILE")
code := m.Run()
diff --git a/cli/beacon-hooks/internal/logging/ingest_unix_test.go b/cli/beacon-hooks/internal/logging/ingest_unix_test.go
new file mode 100644
index 00000000..266c22a3
--- /dev/null
+++ b/cli/beacon-hooks/internal/logging/ingest_unix_test.go
@@ -0,0 +1,239 @@
+//go:build linux || darwin
+
+package logging
+
+import (
+ "bufio"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest"
+)
+
+type ingestFixture struct {
+ sock string
+ logPath string
+ mu sync.Mutex
+ got [][]byte
+ server *localingest.Server
+}
+
+func (f *ingestFixture) received() [][]byte {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return append([][]byte(nil), f.got...)
+}
+
+func shortSocketDir(t *testing.T) string {
+ t.Helper()
+ dir, err := os.MkdirTemp("", "bh")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.RemoveAll(dir) })
+ if err := os.Chmod(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ return dir
+}
+
+// useIngest points the hook writer at a socket of the test's own, as an ordinary account, and at a
+// system log outside the test's home.
+func useIngest(t *testing.T) *ingestFixture {
+ t.Helper()
+ setHome(t, t.TempDir())
+ base := shortSocketDir(t)
+ f := &ingestFixture{
+ sock: filepath.Join(base, "run", "ingest.sock"),
+ logPath: filepath.Join(t.TempDir(), "beacon-agent", "runtime.jsonl"),
+ }
+ prev := EndpointIngest
+ EndpointIngest = localingest.Deliverer{
+ Client: localingest.Client{SocketPath: f.sock, TrustedOwner: os.Geteuid(), Timeout: time.Second},
+ Spool: localingest.Spool{Dir: filepath.Join(t.TempDir(), "spool")},
+ Euid: func() int { return 1000 },
+ // Each emit stands in for a separate hook process, which never remembers an earlier
+ // process's failed send.
+ FailureBackoff: -1,
+ }
+ t.Cleanup(func() { EndpointIngest = prev })
+ t.Setenv("BEACON_ENDPOINT_LOG", f.logPath)
+ return f
+}
+
+func setHome(t *testing.T, home string) {
+ t.Helper()
+ t.Setenv("HOME", home)
+}
+
+func (f *ingestFixture) start(t *testing.T) {
+ t.Helper()
+ l, err := localingest.Listen(f.sock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.server = &localingest.Server{Append: func(line []byte) error {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.got = append(f.got, append([]byte(nil), line...))
+ return nil
+ }}
+ go func() { _ = f.server.Serve(l) }()
+ t.Cleanup(func() { _ = f.server.Close() })
+}
+
+// leaveStaleSocket leaves the socket a collector that died without cleaning up would leave.
+func (f *ingestFixture) leaveStaleSocket(t *testing.T) {
+ t.Helper()
+ l, err := localingest.Listen(f.sock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ l.SetUnlinkOnClose(false)
+ _ = l.Close()
+}
+
+func emit(t *testing.T, message string) {
+ t.Helper()
+ logger := NewLoggerForPlatform("pre-tool", "claude_code")
+ if err := logger.EndpointEvent("tool.invoked", "tool", "info", message, map[string]interface{}{
+ "session": map[string]interface{}{"id": "sess-" + message},
+ }); err != nil {
+ t.Fatalf("EndpointEvent(%s): %v", message, err)
+ }
+}
+
+func messages(t *testing.T, lines [][]byte) []string {
+ t.Helper()
+ var out []string
+ for _, line := range lines {
+ var e map[string]interface{}
+ if err := json.Unmarshal(line, &e); err != nil {
+ t.Fatalf("not JSON: %s", line)
+ }
+ out = append(out, fmt.Sprint(e["message"]))
+ }
+ return out
+}
+
+func TestHookEventsGoThroughTheIngestSocket(t *testing.T) {
+ f := useIngest(t)
+ f.start(t)
+ emit(t, "one")
+ emit(t, "two")
+ got := messages(t, f.received())
+ if len(got) != 2 || got[0] != "one" || got[1] != "two" {
+ t.Fatalf("collector received %v", got)
+ }
+ if _, err := os.Stat(f.logPath); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("the hook touched the log directly: %v", err)
+ }
+ // The line is the same complete event the hook would have appended: it carries an id.
+ var e map[string]interface{}
+ _ = json.Unmarshal(f.received()[0], &e)
+ if id, _ := e["event"].(map[string]interface{})["id"].(string); id == "" {
+ t.Fatal("the event reached the collector without its event.id")
+ }
+}
+
+func TestHookEventsWithoutASocketAreWrittenDirectly(t *testing.T) {
+ f := useIngest(t)
+ emit(t, "direct")
+ lines := readLines(t, f.logPath)
+ if len(lines) != 1 || messages(t, lines)[0] != "direct" {
+ t.Fatalf("log = %s", lines)
+ }
+}
+
+// A socket a crashed collector left behind must not swallow a shared endpoint's events.
+func TestHookEventsThroughAStaleSocketOnASharedLogAreWrittenDirectly(t *testing.T) {
+ f := useIngest(t)
+ f.leaveStaleSocket(t)
+ emit(t, "stale")
+ if lines := readLines(t, f.logPath); len(lines) != 1 {
+ t.Fatalf("log has %d lines", len(lines))
+ }
+ if n, _ := EndpointIngest.Pending(f.logPath); n != 0 {
+ t.Fatalf("Pending = %d; a writable log must not spool", n)
+ }
+}
+
+// A restricted log refuses this account's direct append. While the collector is down the event is
+// spooled rather than lost, and it arrives with the next event once the collector is back.
+func TestHookEventsSpoolWhenARestrictedLogRefusesThemAndArriveAfter(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("root is never refused by a file mode")
+ }
+ f := useIngest(t)
+ if err := os.MkdirAll(filepath.Dir(f.logPath), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(f.logPath, nil, 0o440); err != nil {
+ t.Fatal(err)
+ }
+ f.leaveStaleSocket(t)
+ emit(t, "while-down")
+ if n, _ := EndpointIngest.Pending(f.logPath); n != 1 {
+ t.Fatalf("Pending = %d, want the refused event spooled", n)
+ }
+ f.start(t)
+ emit(t, "after")
+ got := messages(t, f.received())
+ if len(got) != 2 || got[0] != "after" || got[1] != "while-down" {
+ t.Fatalf("collector received %v; want the new event and the spooled one", got)
+ }
+ if n, _ := EndpointIngest.Pending(f.logPath); n != 0 {
+ t.Fatalf("Pending after recovery = %d", n)
+ }
+}
+
+func TestAPrivateLogNeverUsesTheSocket(t *testing.T) {
+ f := useIngest(t)
+ f.start(t)
+ home := t.TempDir()
+ setHome(t, home)
+ private := filepath.Join(home, ".beacon", "endpoint", "logs", "runtime.jsonl")
+ t.Setenv("BEACON_ENDPOINT_LOG", private)
+ emit(t, "mine")
+ if len(f.received()) != 0 {
+ t.Fatal("a per-user log's event went to the collector")
+ }
+ if lines := readLines(t, private); len(lines) != 1 {
+ t.Fatalf("private log has %d lines", len(lines))
+ }
+}
+
+func TestRootHooksWriteDirectly(t *testing.T) {
+ f := useIngest(t)
+ f.start(t)
+ EndpointIngest.Euid = func() int { return 0 }
+ emit(t, "as-root")
+ if len(f.received()) != 0 {
+ t.Fatal("root's event went through the socket")
+ }
+ if lines := readLines(t, f.logPath); len(lines) != 1 {
+ t.Fatalf("log has %d lines", len(lines))
+ }
+}
+
+func readLines(t *testing.T, path string) [][]byte {
+ t.Helper()
+ file, err := os.Open(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer file.Close()
+ var out [][]byte
+ scanner := bufio.NewScanner(file)
+ scanner.Buffer(make([]byte, 0, 1<<20), 1<<20)
+ for scanner.Scan() {
+ out = append(out, append([]byte(nil), scanner.Bytes()...))
+ }
+ return out
+}
diff --git a/cli/beacon-hooks/internal/logging/logging.go b/cli/beacon-hooks/internal/logging/logging.go
index 0c5440b3..d561f88a 100644
--- a/cli/beacon-hooks/internal/logging/logging.go
+++ b/cli/beacon-hooks/internal/logging/logging.go
@@ -19,6 +19,7 @@ import (
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/activesession"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/shareauto"
)
@@ -521,7 +522,27 @@ func writeEndpointJSONFile(path string, event map[string]interface{}) error {
if data, err = withEndpointEventID(event, data); err != nil {
return err
}
- return appendEndpointJSONL(path, append(data, '\n'), defaultEndpointRotateBytes, defaultEndpointRotateArchives)
+ return deliverEndpointLine(path, data)
+}
+
+// EndpointIngest delivers to the system log through the collector when the endpoint is in the
+// restricted log access mode. Tests point it at a socket and spool of their own; nothing else
+// changes it.
+var EndpointIngest = localingest.Deliverer{}
+
+// deliverEndpointLine writes one finished line to the runtime log at path.
+//
+// A hook writing the system log as an ordinary account hands the line to the collector over the
+// local ingest socket when the endpoint has one (the restricted log access mode, in which the log
+// is not writable by this account at all), and otherwise appends directly, as hooks always have.
+// Only a direct append the log refuses -- a restricted endpoint whose collector is restarting --
+// puts the line in this account's private spool, and it reaches the log with a later event
+// instead of being lost. Root and per-user logs are always written directly. See
+// localingest.Deliverer.Write.
+func deliverEndpointLine(path string, data []byte) error {
+ return EndpointIngest.Write(path, data, func(line []byte) error {
+ return appendEndpointJSONL(path, append(line[:len(line):len(line)], '\n'), defaultEndpointRotateBytes, defaultEndpointRotateArchives)
+ })
}
// withEndpointEventID fills event.id on an event assembled as a map and returns
diff --git a/cli/beacon-hooks/internal/logging/main_test.go b/cli/beacon-hooks/internal/logging/main_test.go
index 7a263bfb..a7f07f8c 100644
--- a/cli/beacon-hooks/internal/logging/main_test.go
+++ b/cli/beacon-hooks/internal/logging/main_test.go
@@ -6,6 +6,7 @@ import (
"testing"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/activesession"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest"
)
// TestMain keeps the active-session record every endpoint event updates out of the real home
@@ -19,6 +20,12 @@ func TestMain(m *testing.M) {
// Auto-share is off unless a test turns it on in a settings file of its own.
settingsPath := filepath.Join(dir, "share-settings.json")
ShareSettingsPath = func() string { return settingsPath }
+ // Never send to a real collector's ingest socket, which a developer's restricted-mode machine
+ // has; tests that exercise the socket start a server of their own.
+ EndpointIngest = localingest.Deliverer{
+ Client: localingest.Client{SocketPath: filepath.Join(dir, "no-ingest.sock")},
+ Spool: localingest.Spool{Dir: filepath.Join(dir, "spool")},
+ }
code := m.Run()
os.RemoveAll(dir)
os.Exit(code)
diff --git a/cli/beacon/internal/endpoint/writer/ingest_unix_test.go b/cli/beacon/internal/endpoint/writer/ingest_unix_test.go
new file mode 100644
index 00000000..114b7326
--- /dev/null
+++ b/cli/beacon/internal/endpoint/writer/ingest_unix_test.go
@@ -0,0 +1,164 @@
+//go:build linux || darwin
+
+package writer
+
+import (
+ "encoding/json"
+ "errors"
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/testenv"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest"
+)
+
+type collectorStub struct {
+ mu sync.Mutex
+ got [][]byte
+}
+
+func (c *collectorStub) lines() [][]byte {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return append([][]byte(nil), c.got...)
+}
+
+// useIngest makes the writer an ordinary account writing a system log (outside the test's home)
+// on an endpoint whose collector serves logPath, and returns that log's path.
+func useIngest(t *testing.T, serve bool) (string, *collectorStub) {
+ t.Helper()
+ testenv.SetHome(t, t.TempDir())
+ base, err := os.MkdirTemp("", "bw")
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.RemoveAll(base) })
+ _ = os.Chmod(base, 0o755)
+ sock := filepath.Join(base, "run", "ingest.sock")
+ logPath := filepath.Join(t.TempDir(), "beacon-agent", "runtime.jsonl")
+
+ prev := Ingest
+ Ingest = localingest.Deliverer{
+ Client: localingest.Client{SocketPath: sock, TrustedOwner: os.Geteuid(), Timeout: time.Second},
+ Spool: localingest.Spool{Dir: filepath.Join(t.TempDir(), "spool")},
+ Euid: func() int { return 1000 },
+ FailureBackoff: -1,
+ }
+ t.Cleanup(func() { Ingest = prev })
+
+ stub := &collectorStub{}
+ l, err := localingest.Listen(sock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !serve {
+ // A collector that died without removing its socket.
+ l.SetUnlinkOnClose(false)
+ _ = l.Close()
+ return logPath, stub
+ }
+ server := &localingest.Server{LogPath: logPath, Append: func(line []byte) error {
+ stub.mu.Lock()
+ defer stub.mu.Unlock()
+ stub.got = append(stub.got, append([]byte(nil), line...))
+ return nil
+ }}
+ go func() { _ = server.Serve(l) }()
+ t.Cleanup(func() { _ = server.Close() })
+ return logPath, stub
+}
+
+func ingestEvent(message string) schema.Event {
+ return schema.NewEvent(schema.NewEventOptions{
+ Action: "agent.detected",
+ Harness: schema.HarnessInfo{Name: "test"},
+ Message: message,
+ })
+}
+
+func TestAppendEventGoesThroughTheCollectorOnARestrictedEndpoint(t *testing.T) {
+ logPath, stub := useIngest(t, true)
+ path, err := AppendEvent(ingestEvent("via socket"), Options{Path: logPath})
+ if err != nil || path != logPath {
+ t.Fatalf("AppendEvent = %q, %v", path, err)
+ }
+ got := stub.lines()
+ if len(got) != 1 {
+ t.Fatalf("collector received %d lines", len(got))
+ }
+ var e map[string]interface{}
+ if err := json.Unmarshal(got[0], &e); err != nil {
+ t.Fatal(err)
+ }
+ if e["message"] != "via socket" {
+ t.Fatalf("message = %v", e["message"])
+ }
+ if _, err := os.Stat(logPath); !errors.Is(err, os.ErrNotExist) {
+ t.Fatal("the writer also touched the log directly")
+ }
+}
+
+func TestAppendEventForAnotherLogIsWrittenDirectly(t *testing.T) {
+ _, stub := useIngest(t, true)
+ other := filepath.Join(t.TempDir(), "elsewhere", "runtime.jsonl")
+ if _, err := AppendEvent(ingestEvent("custom path"), Options{Path: other}); err != nil {
+ t.Fatal(err)
+ }
+ if len(stub.lines()) != 0 {
+ t.Fatal("an event for another log went to the collector")
+ }
+ if data, err := os.ReadFile(other); err != nil || len(data) == 0 {
+ t.Fatalf("the other log was not written: %v", err)
+ }
+}
+
+func TestAppendEventThroughAStaleSocketToASharedLogIsWrittenDirectly(t *testing.T) {
+ logPath, _ := useIngest(t, false)
+ if _, err := AppendEvent(ingestEvent("stale"), Options{Path: logPath}); err != nil {
+ t.Fatal(err)
+ }
+ if data, err := os.ReadFile(logPath); err != nil || len(data) == 0 {
+ t.Fatalf("the shared log was not written: %v", err)
+ }
+ if n, _ := Ingest.Pending(logPath); n != 0 {
+ t.Fatalf("Pending = %d", n)
+ }
+}
+
+func TestAppendEventSpoolsWhenARestrictedLogRefusesIt(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("root is never refused by a file mode")
+ }
+ logPath, _ := useIngest(t, false)
+ if err := os.MkdirAll(filepath.Dir(logPath), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(logPath, nil, 0o440); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := AppendEvent(ingestEvent("later"), Options{Path: logPath}); err != nil {
+ t.Fatalf("AppendEvent with the collector down: %v", err)
+ }
+ if n, _ := Ingest.Pending(logPath); n != 1 {
+ t.Fatalf("Pending = %d", n)
+ }
+}
+
+// A backfill sets a guard and a budget, which only a direct writer holding the log's lock can
+// honor, so it never goes through the socket.
+func TestGuardedAppendsAreWrittenDirectly(t *testing.T) {
+ logPath, stub := useIngest(t, true)
+ if _, err := AppendEvent(ingestEvent("budgeted"), Options{Path: logPath, Budget: NewBudget(1 << 20)}); err != nil {
+ t.Fatal(err)
+ }
+ if len(stub.lines()) != 0 {
+ t.Fatal("a budgeted append went through the socket")
+ }
+ if _, err := os.Stat(logPath); err != nil {
+ t.Fatalf("the budgeted append was not written directly: %v", err)
+ }
+}
diff --git a/cli/beacon/internal/endpoint/writer/writer.go b/cli/beacon/internal/endpoint/writer/writer.go
index 6d6c9519..fe35a5ca 100644
--- a/cli/beacon/internal/endpoint/writer/writer.go
+++ b/cli/beacon/internal/endpoint/writer/writer.go
@@ -16,6 +16,7 @@ import (
"github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest"
)
const (
@@ -121,12 +122,34 @@ func AppendEvent(event schema.Event, opts Options) (string, error) {
}
}
line := append(data, '\n')
+ if opts.Guard == nil && opts.Budget == nil && Ingest.Applies(opts.Path) {
+ if err := deliver(opts, data); err != nil {
+ return "", err
+ }
+ return opts.Path, nil
+ }
if err := appendJSONL(opts.Path, line, opts.RotateSize, opts.RotateArchives, opts.Guard, opts.Budget); err != nil {
return "", err
}
return opts.Path, nil
}
+// Ingest delivers to the system log through the collector when the endpoint is in the restricted
+// log access mode. Tests point it at a socket and spool of their own; nothing else changes it.
+var Ingest = localingest.Deliverer{}
+
+// deliver writes one line to the system log as an account that is not root, the way the hook
+// adapter does: through the collector's ingest socket when there is one, directly otherwise, and
+// into this account's private spool only when the restricted log refuses the direct append (see
+// localingest.Deliverer.Write). A guarded or budgeted append (a session backfill) never comes
+// here: those limits are decided under the log's lock, which only a direct writer holds, and the
+// backfill that sets them runs as root on a system log.
+func deliver(opts Options, data []byte) error {
+ return Ingest.Write(opts.Path, data, func(line []byte) error {
+ return appendJSONL(opts.Path, append(line[:len(line):len(line)], '\n'), opts.RotateSize, opts.RotateArchives, nil, nil)
+ })
+}
+
// compactRetainedContent drops an event's retained text and records that it did, so a reader can
// still tell what the event was about and that its content was held back rather than never
// captured. Mirrors the collector exporter's step of the same name.
diff --git a/pkg/asymptoteobserve/localingest/deliver.go b/pkg/asymptoteobserve/localingest/deliver.go
new file mode 100644
index 00000000..c4a0e751
--- /dev/null
+++ b/pkg/asymptoteobserve/localingest/deliver.go
@@ -0,0 +1,187 @@
+package localingest
+
+import (
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "runtime"
+ "sync/atomic"
+ "time"
+
+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"
+)
+
+// spoolDirInHome is where a writer's spools live, under its account's home directory.
+const spoolDirInHome = ".beacon/endpoint/spool"
+
+// DefaultSpoolDir is the current account's spool directory, or "" when it has no home.
+func DefaultSpoolDir() string {
+ home, err := os.UserHomeDir()
+ if err != nil || home == "" {
+ return ""
+ }
+ return filepath.Join(home, spoolDirInHome)
+}
+
+// defaultFailureBackoff is how long a process skips the socket after a failed send.
+const defaultFailureBackoff = 10 * time.Second
+
+// lastSendFailure is when this process last failed to send, in Unix nanoseconds; zero when the last
+// attempt succeeded or none has failed.
+var lastSendFailure atomic.Int64
+
+// Deliverer is how a writer that is not root gets a line into the system runtime log: through the
+// collector's ingest socket when one answers, by appending directly when the log lets it, and into
+// its private spool when neither works -- a restricted log whose collector is stopped or
+// restarting. A spooled line is late, never lost: the next line that gets through, either way,
+// takes the spool with it.
+type Deliverer struct {
+ Client Client
+ // Spool holds lines that could not be written. An empty Dir means DefaultSpoolDir. Each log
+ // has its own spool beneath it, so a line is only ever written to the log it was meant for.
+ Spool Spool
+ // Euid reports the process's effective uid; nil means os.Geteuid. Tests replace it.
+ Euid func() int
+ // FailureBackoff is how long, after a failed send, this process goes straight to the direct
+ // append without trying the socket. Zero means 10s; negative turns it off.
+ FailureBackoff time.Duration
+}
+
+// Applies reports whether a write to logPath goes through Write's socket path at all. Root writes
+// the log directly: it owns the file in every mode, and the events it writes -- a backfill of the
+// console user's sessions, the collector's own -- name their account themselves, which the ingest
+// server would replace with root. A per-user log is private to its account and always written
+// directly. Windows has no ingest socket.
+func (d Deliverer) Applies(logPath string) bool {
+ if runtime.GOOS == "windows" {
+ return false
+ }
+ euid := os.Geteuid()
+ if d.Euid != nil {
+ euid = d.Euid()
+ }
+ return euid != 0 && !asymptoteobserve.RuntimeLogIsPrivate(logPath)
+}
+
+// Write gets one line, meant for logPath, into that log. appendLine appends one line directly, the
+// way the writer always has.
+//
+// - Where Applies is false, it is appendLine and nothing else.
+// - Otherwise the line goes to the collector's ingest socket when there is one, the collector
+// writes logPath, and it takes the line. A rejected line is reported, never retried.
+// - If there is no socket, or the send fails -- a socket a crashed collector left behind, a
+// collector that is hung -- the line is appended directly. On a shared log that simply works.
+// - Only when the direct append is refused for permission, as a restricted log refuses it, is the
+// line spooled.
+//
+// Every line that gets through takes up to a batch of this account's spool for logPath with it, by
+// the same route. After a failed send the process skips the socket for FailureBackoff, so a hook
+// writing several events waits out a hung collector once.
+func (d Deliverer) Write(logPath string, line []byte, appendLine func([]byte) error) error {
+ if !d.Applies(logPath) {
+ return appendLine(line)
+ }
+ if !d.inBackoff() {
+ c := d.Client
+ c.LogPath = logPath
+ resp, err := c.Send([][]byte{line})
+ switch {
+ case err == nil && resp.Rejected > 0:
+ return fmt.Errorf("the collector rejected the event: %w", ErrRejected)
+ case err == nil:
+ lastSendFailure.Store(0)
+ d.drain(logPath, func(lines [][]byte) (int, error) {
+ resp, err := c.Send(lines)
+ return resp.Processed, err
+ })
+ return nil
+ case errors.Is(err, ErrNoSocket), errors.Is(err, ErrUntrustedSocket), errors.Is(err, ErrLogMismatch):
+ // Not a collector this line can go through; the endpoint is shared, or the line is for
+ // some other log.
+ default:
+ lastSendFailure.Store(time.Now().UnixNano())
+ }
+ }
+ err := appendLine(line)
+ if err == nil {
+ d.drain(logPath, func(lines [][]byte) (int, error) {
+ for i, line := range lines {
+ if err := appendLine(line); err != nil {
+ return i, err
+ }
+ }
+ return len(lines), nil
+ })
+ return nil
+ }
+ if !errors.Is(err, fs.ErrPermission) {
+ return err
+ }
+ if spoolErr := d.spoolLine(logPath, line); spoolErr != nil {
+ return fmt.Errorf("%w; spool: %v", err, spoolErr)
+ }
+ return nil
+}
+
+func (d Deliverer) inBackoff() bool {
+ backoff := d.FailureBackoff
+ if backoff == 0 {
+ backoff = defaultFailureBackoff
+ }
+ last := lastSendFailure.Load()
+ return backoff > 0 && last != 0 && time.Since(time.Unix(0, last)) < backoff
+}
+
+// spoolFor is the spool for logPath: its own directory under the account's spool, named by a
+// digest of the log's path, so a drain can only ever deliver a line to the log it was meant for.
+func (d Deliverer) spoolFor(logPath string) Spool {
+ s := d.Spool
+ if s.Dir == "" {
+ s.Dir = DefaultSpoolDir()
+ }
+ if s.Dir == "" {
+ return s
+ }
+ sum := sha256.Sum256([]byte(filepath.Clean(logPath)))
+ s.Dir = filepath.Join(s.Dir, hex.EncodeToString(sum[:8]))
+ return s
+}
+
+func (d Deliverer) spoolLine(logPath string, line []byte) error {
+ s := d.spoolFor(logPath)
+ if s.Dir == "" {
+ return errors.New("no home directory to spool in")
+ }
+ dropped, err := s.Append([][]byte{line})
+ if err != nil {
+ return err
+ }
+ if dropped > 0 {
+ return fmt.Errorf("the spool in %s is full", s.Dir)
+ }
+ return nil
+}
+
+// drain sends one batch of logPath's spool through send. Best effort: whatever is left goes with a
+// later line.
+func (d Deliverer) drain(logPath string, send func([][]byte) (int, error)) {
+ s := d.spoolFor(logPath)
+ if s.Dir == "" {
+ return
+ }
+ _, _ = s.Drain(MaxLinesPerConn, send)
+}
+
+// Pending reports how many events this account has spooled for logPath, for doctor.
+func (d Deliverer) Pending(logPath string) (int, error) {
+ s := d.spoolFor(logPath)
+ if s.Dir == "" {
+ return 0, nil
+ }
+ n, _, err := s.Pending()
+ return n, err
+}
diff --git a/pkg/asymptoteobserve/localingest/deliver_unix_test.go b/pkg/asymptoteobserve/localingest/deliver_unix_test.go
new file mode 100644
index 00000000..2a520f6e
--- /dev/null
+++ b/pkg/asymptoteobserve/localingest/deliver_unix_test.go
@@ -0,0 +1,348 @@
+//go:build linux || darwin
+
+package localingest
+
+import (
+ "errors"
+ "io/fs"
+ "net"
+ "os"
+ "path/filepath"
+ "sync"
+ "testing"
+ "time"
+)
+
+// testLog is the log the deliverer tests write; their servers accept any.
+const testLog = "/var/log/beacon-agent/runtime.jsonl"
+
+func deliverer(t *testing.T, socketPath string) Deliverer {
+ t.Helper()
+ lastSendFailure.Store(0)
+ t.Cleanup(func() { lastSendFailure.Store(0) })
+ return Deliverer{
+ Client: Client{SocketPath: socketPath, TrustedOwner: os.Geteuid(), Timeout: time.Second},
+ Spool: Spool{Dir: filepath.Join(t.TempDir(), "spool")},
+ Euid: func() int { return 1000 },
+ // Each Write stands in for its own hook process unless a test says otherwise.
+ FailureBackoff: -1,
+ }
+}
+
+// directLog stands in for the writer's own append: it records lines, or refuses them the way a
+// restricted log refuses an ordinary account.
+type directLog struct {
+ mu sync.Mutex
+ lines []string
+ refuse bool
+ failErr error
+}
+
+func (l *directLog) append(line []byte) error {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ if l.refuse {
+ return &fs.PathError{Op: "open", Path: testLog, Err: fs.ErrPermission}
+ }
+ if l.failErr != nil {
+ return l.failErr
+ }
+ l.lines = append(l.lines, string(line))
+ return nil
+}
+
+func (l *directLog) count() int {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ return len(l.lines)
+}
+
+func staleSocket(t *testing.T) string {
+ t.Helper()
+ sock := filepath.Join(shortTempDir(t), "run", "ingest.sock")
+ l, err := Listen(sock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ l.SetUnlinkOnClose(false)
+ _ = l.Close()
+ return sock
+}
+
+func TestDeliverApplies(t *testing.T) {
+ home := t.TempDir()
+ t.Setenv("HOME", home)
+ system := filepath.Join(t.TempDir(), "runtime.jsonl")
+ private := filepath.Join(home, ".beacon", "endpoint", "logs", "runtime.jsonl")
+
+ user := Deliverer{Euid: func() int { return 501 }}
+ root := Deliverer{Euid: func() int { return 0 }}
+ if !user.Applies(system) {
+ t.Fatal("an ordinary account writing the system log must go through the collector")
+ }
+ if user.Applies(private) {
+ t.Fatal("a per-user log is written directly")
+ }
+ if root.Applies(system) {
+ t.Fatal("root writes the system log directly")
+ }
+}
+
+func TestWriteWithoutASocketAppendsDirectly(t *testing.T) {
+ d := deliverer(t, filepath.Join(shortTempDir(t), "none.sock"))
+ log := &directLog{}
+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {
+ t.Fatal(err)
+ }
+ if log.count() != 1 {
+ t.Fatalf("direct log has %d lines", log.count())
+ }
+ if n, _ := d.Pending(testLog); n != 0 {
+ t.Fatal("spooled with no socket at all")
+ }
+}
+
+func TestWriteThatDoesNotApplyNeverTouchesTheSocket(t *testing.T) {
+ h := startServer(t, nil)
+ d := deliverer(t, h.path)
+ d.Euid = func() int { return 0 }
+ log := &directLog{}
+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {
+ t.Fatal(err)
+ }
+ if h.rec.count() != 0 || log.count() != 1 {
+ t.Fatalf("collector %d, direct %d; root writes directly", h.rec.count(), log.count())
+ }
+}
+
+func TestWriteSendsThroughTheSocket(t *testing.T) {
+ h := startServer(t, nil)
+ d := deliverer(t, h.path)
+ log := &directLog{}
+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {
+ t.Fatal(err)
+ }
+ if h.rec.count() != 1 || log.count() != 0 {
+ t.Fatalf("collector %d, direct %d", h.rec.count(), log.count())
+ }
+}
+
+// A socket a crashed collector left behind must not swallow a shared endpoint's events: the direct
+// append works there, and nothing should be spooled.
+func TestWriteThroughAStaleSocketToASharedLogAppendsDirectly(t *testing.T) {
+ d := deliverer(t, staleSocket(t))
+ log := &directLog{}
+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {
+ t.Fatal(err)
+ }
+ if log.count() != 1 {
+ t.Fatalf("direct log has %d lines", log.count())
+ }
+ if n, _ := d.Pending(testLog); n != 0 {
+ t.Fatalf("Pending = %d; a writable log must not spool", n)
+ }
+}
+
+func TestWriteSpoolsWhenARestrictedLogRefusesAndDrainsAfter(t *testing.T) {
+ sock := staleSocket(t)
+ d := deliverer(t, sock)
+ log := &directLog{refuse: true}
+ batch := lines(t, 3)
+ for _, line := range batch[:2] {
+ if err := d.Write(testLog, line, log.append); err != nil {
+ t.Fatalf("Write with the collector down = %v; want it spooled", err)
+ }
+ }
+ if n, _ := d.Pending(testLog); n != 2 {
+ t.Fatalf("Pending = %d, want 2", n)
+ }
+
+ // The collector comes back on the same path.
+ l, err := Listen(sock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rec := &recorder{}
+ s := &Server{Append: rec.append}
+ go func() { _ = s.Serve(l) }()
+ t.Cleanup(func() { _ = s.Close() })
+
+ if err := d.Write(testLog, batch[2], log.append); err != nil {
+ t.Fatal(err)
+ }
+ if rec.count() != 3 {
+ t.Fatalf("collector received %d lines, want the new one and both spooled", rec.count())
+ }
+ if n, _ := d.Pending(testLog); n != 0 {
+ t.Fatalf("Pending after drain = %d", n)
+ }
+ got := map[string]bool{}
+ for _, line := range rec.lines {
+ got[decode(t, line)["message"].(string)] = true
+ }
+ for _, want := range []string{"m-0", "m-1", "m-2"} {
+ if !got[want] {
+ t.Fatalf("event %s never reached the collector: %v", want, got)
+ }
+ }
+}
+
+// Switching a restricted endpoint back to shared must not strand what was spooled meanwhile: the
+// next direct append drains it.
+func TestASpoolDrainsByDirectAppendOnceTheLogIsWritable(t *testing.T) {
+ d := deliverer(t, filepath.Join(shortTempDir(t), "none.sock"))
+ log := &directLog{refuse: true}
+ _ = d.Write(testLog, lines(t, 1)[0], log.append)
+ if n, _ := d.Pending(testLog); n != 1 {
+ t.Fatalf("Pending = %d", n)
+ }
+ log.refuse = false
+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {
+ t.Fatal(err)
+ }
+ if log.count() != 2 {
+ t.Fatalf("direct log has %d lines; want the new one and the spooled one", log.count())
+ }
+ if n, _ := d.Pending(testLog); n != 0 {
+ t.Fatalf("Pending = %d", n)
+ }
+}
+
+func TestEachLogHasItsOwnSpool(t *testing.T) {
+ d := deliverer(t, filepath.Join(shortTempDir(t), "none.sock"))
+ refused := &directLog{refuse: true}
+ other := "/tmp/elsewhere/runtime.jsonl"
+ _ = d.Write(other, lines(t, 1)[0], refused.append)
+ if n, _ := d.Pending(other); n != 1 {
+ t.Fatalf("Pending(other) = %d", n)
+ }
+ if n, _ := d.Pending(testLog); n != 0 {
+ t.Fatalf("a line meant for another log is waiting for the system log: %d", n)
+ }
+ system := &directLog{}
+ _ = d.Write(testLog, lines(t, 1)[0], system.append)
+ if system.count() != 1 {
+ t.Fatalf("system log got %d lines; another log's spool must not drain into it", system.count())
+ }
+}
+
+func TestWriteReportsOtherAppendErrors(t *testing.T) {
+ d := deliverer(t, filepath.Join(shortTempDir(t), "none.sock"))
+ log := &directLog{failErr: errors.New("disk full")}
+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err == nil {
+ t.Fatal("an append error that is not a refusal was swallowed")
+ }
+ if n, _ := d.Pending(testLog); n != 0 {
+ t.Fatal("spooled an error that spooling cannot fix")
+ }
+}
+
+func TestWriteReportsARejectedEvent(t *testing.T) {
+ h := startServer(t, nil)
+ d := deliverer(t, h.path)
+ log := &directLog{}
+ err := d.Write(testLog, []byte(`{"not":"an event"}`), log.append)
+ if !errors.Is(err, ErrRejected) {
+ t.Fatalf("Write = %v, want ErrRejected", err)
+ }
+ if n, _ := d.Pending(testLog); n != 0 || log.count() != 0 {
+ t.Fatal("a rejected event was spooled or written directly")
+ }
+}
+
+func TestWriteFailsWhenItCanNeitherWriteNorSpool(t *testing.T) {
+ d := deliverer(t, staleSocket(t))
+ d.Spool.MaxBytes = 1
+ log := &directLog{refuse: true}
+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err == nil {
+ t.Fatal("Write succeeded with nowhere to put the line")
+ }
+}
+
+func TestWriteDoesNotSendToAnUntrustedSocket(t *testing.T) {
+ h := startServer(t, nil)
+ d := deliverer(t, h.path)
+ d.Client.TrustedOwner = os.Geteuid() + 1
+ log := &directLog{}
+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {
+ t.Fatal(err)
+ }
+ if h.rec.count() != 0 || log.count() != 1 {
+ t.Fatalf("collector %d, direct %d", h.rec.count(), log.count())
+ }
+}
+
+func TestWriteForAnotherLogIsLeftToTheDirectAppend(t *testing.T) {
+ h := startServer(t, func(s *Server) { s.LogPath = testLog })
+ d := deliverer(t, h.path)
+ log := &directLog{}
+ if err := d.Write("/tmp/elsewhere/runtime.jsonl", lines(t, 1)[0], log.append); err != nil {
+ t.Fatal(err)
+ }
+ if h.rec.count() != 0 || log.count() != 1 {
+ t.Fatalf("collector %d, direct %d", h.rec.count(), log.count())
+ }
+}
+
+func TestWriteWaitsOutAHungCollectorOnce(t *testing.T) {
+ sock := filepath.Join(shortTempDir(t), "run", "ingest.sock")
+ l, err := Listen(sock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer l.Close()
+ go func() { // accepts and never answers
+ var held []net.Conn
+ for {
+ c, err := l.Accept()
+ if err != nil {
+ for _, h := range held {
+ _ = h.Close()
+ }
+ return
+ }
+ held = append(held, c)
+ }
+ }()
+ d := deliverer(t, sock)
+ d.Client.Timeout = 200 * time.Millisecond
+ d.FailureBackoff = 0 // the default: one process, several events
+ log := &directLog{refuse: true}
+ for i := 0; i < 3; i++ {
+ start := time.Now()
+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {
+ t.Fatal(err)
+ }
+ limit := 2 * time.Second
+ if i > 0 {
+ limit = 100 * time.Millisecond
+ }
+ if elapsed := time.Since(start); elapsed > limit {
+ t.Fatalf("write %d took %v; only the first may wait for the hung collector", i, elapsed)
+ }
+ }
+ if n, _ := d.Pending(testLog); n != 3 {
+ t.Fatalf("Pending = %d", n)
+ }
+}
+
+func TestWriteRetriesTheSocketOnceTheBackoffPasses(t *testing.T) {
+ h := startServer(t, nil)
+ d := deliverer(t, h.path)
+ d.FailureBackoff = 0
+ lastSendFailure.Store(time.Now().Add(-defaultFailureBackoff - time.Second).UnixNano())
+ if err := d.Write(testLog, lines(t, 1)[0], (&directLog{}).append); err != nil {
+ t.Fatal(err)
+ }
+ if h.rec.count() != 1 || lastSendFailure.Load() != 0 {
+ t.Fatal("did not retry the socket after the backoff, or did not clear the failure")
+ }
+}
+
+func TestDefaultSpoolDir(t *testing.T) {
+ home := t.TempDir()
+ t.Setenv("HOME", home)
+ if got, want := DefaultSpoolDir(), filepath.Join(home, ".beacon", "endpoint", "spool"); got != want {
+ t.Fatalf("DefaultSpoolDir = %q, want %q", got, want)
+ }
+}