The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
hooks, CLI: send system-log events through the collector when it listens
Claude59034a1a4ccd
diff --git a/cli/beacon-hooks/cmd/main_test.go b/cli/beacon-hooks/cmd/main_test.goindex 6dd82eb4..3fe5eb93 100644--- a/cli/beacon-hooks/cmd/main_test.go+++ b/cli/beacon-hooks/cmd/main_test.go@@ -7,6 +7,7 @@ import ("github.com/asymptote-labs/agent-beacon/cli/beacon-hooks/internal/logging""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/activesession"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest")// TestMain keeps the active-session record every endpoint event updates out of the real home@@ -20,6 +21,12 @@ func TestMain(m *testing.M) {// Auto-share is off unless a test turns it on in a settings file of its own.settingsPath := filepath.Join(dir, "share-settings.json")logging.ShareSettingsPath = func() string { return settingsPath }+ // Never send to a real collector's ingest socket, which a developer's restricted-mode machine+ // has.+ logging.EndpointIngest = localingest.Deliverer{+ Client: localingest.Client{SocketPath: filepath.Join(dir, "no-ingest.sock")},+ Spool: localingest.Spool{Dir: filepath.Join(dir, "spool")},+ }// A suite run from inside a Claude Code session must not append to that session's env file.os.Unsetenv("CLAUDE_ENV_FILE")code := m.Run()diff --git a/cli/beacon-hooks/internal/logging/ingest_unix_test.go b/cli/beacon-hooks/internal/logging/ingest_unix_test.gonew file mode 100644index 00000000..266c22a3--- /dev/null+++ b/cli/beacon-hooks/internal/logging/ingest_unix_test.go@@ -0,0 +1,239 @@+//go:build linux || darwin++package logging++import (+ "bufio"+ "encoding/json"+ "errors"+ "fmt"+ "os"+ "path/filepath"+ "sync"+ "testing"+ "time"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest"+)++type ingestFixture struct {+ sock string+ logPath string+ mu sync.Mutex+ got [][]byte+ server *localingest.Server+}++func (f *ingestFixture) received() [][]byte {+ f.mu.Lock()+ defer f.mu.Unlock()+ return append([][]byte(nil), f.got...)+}++func shortSocketDir(t *testing.T) string {+ t.Helper()+ dir, err := os.MkdirTemp("", "bh")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = os.RemoveAll(dir) })+ if err := os.Chmod(dir, 0o755); err != nil {+ t.Fatal(err)+ }+ return dir+}++// useIngest points the hook writer at a socket of the test's own, as an ordinary account, and at a+// system log outside the test's home.+func useIngest(t *testing.T) *ingestFixture {+ t.Helper()+ setHome(t, t.TempDir())+ base := shortSocketDir(t)+ f := &ingestFixture{+ sock: filepath.Join(base, "run", "ingest.sock"),+ logPath: filepath.Join(t.TempDir(), "beacon-agent", "runtime.jsonl"),+ }+ prev := EndpointIngest+ EndpointIngest = localingest.Deliverer{+ Client: localingest.Client{SocketPath: f.sock, TrustedOwner: os.Geteuid(), Timeout: time.Second},+ Spool: localingest.Spool{Dir: filepath.Join(t.TempDir(), "spool")},+ Euid: func() int { return 1000 },+ // Each emit stands in for a separate hook process, which never remembers an earlier+ // process's failed send.+ FailureBackoff: -1,+ }+ t.Cleanup(func() { EndpointIngest = prev })+ t.Setenv("BEACON_ENDPOINT_LOG", f.logPath)+ return f+}++func setHome(t *testing.T, home string) {+ t.Helper()+ t.Setenv("HOME", home)+}++func (f *ingestFixture) start(t *testing.T) {+ t.Helper()+ l, err := localingest.Listen(f.sock)+ if err != nil {+ t.Fatal(err)+ }+ f.server = &localingest.Server{Append: func(line []byte) error {+ f.mu.Lock()+ defer f.mu.Unlock()+ f.got = append(f.got, append([]byte(nil), line...))+ return nil+ }}+ go func() { _ = f.server.Serve(l) }()+ t.Cleanup(func() { _ = f.server.Close() })+}++// leaveStaleSocket leaves the socket a collector that died without cleaning up would leave.+func (f *ingestFixture) leaveStaleSocket(t *testing.T) {+ t.Helper()+ l, err := localingest.Listen(f.sock)+ if err != nil {+ t.Fatal(err)+ }+ l.SetUnlinkOnClose(false)+ _ = l.Close()+}++func emit(t *testing.T, message string) {+ t.Helper()+ logger := NewLoggerForPlatform("pre-tool", "claude_code")+ if err := logger.EndpointEvent("tool.invoked", "tool", "info", message, map[string]interface{}{+ "session": map[string]interface{}{"id": "sess-" + message},+ }); err != nil {+ t.Fatalf("EndpointEvent(%s): %v", message, err)+ }+}++func messages(t *testing.T, lines [][]byte) []string {+ t.Helper()+ var out []string+ for _, line := range lines {+ var e map[string]interface{}+ if err := json.Unmarshal(line, &e); err != nil {+ t.Fatalf("not JSON: %s", line)+ }+ out = append(out, fmt.Sprint(e["message"]))+ }+ return out+}++func TestHookEventsGoThroughTheIngestSocket(t *testing.T) {+ f := useIngest(t)+ f.start(t)+ emit(t, "one")+ emit(t, "two")+ got := messages(t, f.received())+ if len(got) != 2 || got[0] != "one" || got[1] != "two" {+ t.Fatalf("collector received %v", got)+ }+ if _, err := os.Stat(f.logPath); !errors.Is(err, os.ErrNotExist) {+ t.Fatalf("the hook touched the log directly: %v", err)+ }+ // The line is the same complete event the hook would have appended: it carries an id.+ var e map[string]interface{}+ _ = json.Unmarshal(f.received()[0], &e)+ if id, _ := e["event"].(map[string]interface{})["id"].(string); id == "" {+ t.Fatal("the event reached the collector without its event.id")+ }+}++func TestHookEventsWithoutASocketAreWrittenDirectly(t *testing.T) {+ f := useIngest(t)+ emit(t, "direct")+ lines := readLines(t, f.logPath)+ if len(lines) != 1 || messages(t, lines)[0] != "direct" {+ t.Fatalf("log = %s", lines)+ }+}++// A socket a crashed collector left behind must not swallow a shared endpoint's events.+func TestHookEventsThroughAStaleSocketOnASharedLogAreWrittenDirectly(t *testing.T) {+ f := useIngest(t)+ f.leaveStaleSocket(t)+ emit(t, "stale")+ if lines := readLines(t, f.logPath); len(lines) != 1 {+ t.Fatalf("log has %d lines", len(lines))+ }+ if n, _ := EndpointIngest.Pending(f.logPath); n != 0 {+ t.Fatalf("Pending = %d; a writable log must not spool", n)+ }+}++// A restricted log refuses this account's direct append. While the collector is down the event is+// spooled rather than lost, and it arrives with the next event once the collector is back.+func TestHookEventsSpoolWhenARestrictedLogRefusesThemAndArriveAfter(t *testing.T) {+ if os.Geteuid() == 0 {+ t.Skip("root is never refused by a file mode")+ }+ f := useIngest(t)+ if err := os.MkdirAll(filepath.Dir(f.logPath), 0o755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(f.logPath, nil, 0o440); err != nil {+ t.Fatal(err)+ }+ f.leaveStaleSocket(t)+ emit(t, "while-down")+ if n, _ := EndpointIngest.Pending(f.logPath); n != 1 {+ t.Fatalf("Pending = %d, want the refused event spooled", n)+ }+ f.start(t)+ emit(t, "after")+ got := messages(t, f.received())+ if len(got) != 2 || got[0] != "after" || got[1] != "while-down" {+ t.Fatalf("collector received %v; want the new event and the spooled one", got)+ }+ if n, _ := EndpointIngest.Pending(f.logPath); n != 0 {+ t.Fatalf("Pending after recovery = %d", n)+ }+}++func TestAPrivateLogNeverUsesTheSocket(t *testing.T) {+ f := useIngest(t)+ f.start(t)+ home := t.TempDir()+ setHome(t, home)+ private := filepath.Join(home, ".beacon", "endpoint", "logs", "runtime.jsonl")+ t.Setenv("BEACON_ENDPOINT_LOG", private)+ emit(t, "mine")+ if len(f.received()) != 0 {+ t.Fatal("a per-user log's event went to the collector")+ }+ if lines := readLines(t, private); len(lines) != 1 {+ t.Fatalf("private log has %d lines", len(lines))+ }+}++func TestRootHooksWriteDirectly(t *testing.T) {+ f := useIngest(t)+ f.start(t)+ EndpointIngest.Euid = func() int { return 0 }+ emit(t, "as-root")+ if len(f.received()) != 0 {+ t.Fatal("root's event went through the socket")+ }+ if lines := readLines(t, f.logPath); len(lines) != 1 {+ t.Fatalf("log has %d lines", len(lines))+ }+}++func readLines(t *testing.T, path string) [][]byte {+ t.Helper()+ file, err := os.Open(path)+ if err != nil {+ t.Fatal(err)+ }+ defer file.Close()+ var out [][]byte+ scanner := bufio.NewScanner(file)+ scanner.Buffer(make([]byte, 0, 1<<20), 1<<20)+ for scanner.Scan() {+ out = append(out, append([]byte(nil), scanner.Bytes()...))+ }+ return out+}diff --git a/cli/beacon-hooks/internal/logging/logging.go b/cli/beacon-hooks/internal/logging/logging.goindex 0c5440b3..d561f88a 100644--- a/cli/beacon-hooks/internal/logging/logging.go+++ b/cli/beacon-hooks/internal/logging/logging.go@@ -19,6 +19,7 @@ import ("github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/activesession""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/shareauto")@@ -521,7 +522,27 @@ func writeEndpointJSONFile(path string, event map[string]interface{}) error {if data, err = withEndpointEventID(event, data); err != nil {return err}- return appendEndpointJSONL(path, append(data, '\n'), defaultEndpointRotateBytes, defaultEndpointRotateArchives)+ return deliverEndpointLine(path, data)+}++// EndpointIngest delivers to the system log through the collector when the endpoint is in the+// restricted log access mode. Tests point it at a socket and spool of their own; nothing else+// changes it.+var EndpointIngest = localingest.Deliverer{}++// deliverEndpointLine writes one finished line to the runtime log at path.+//+// A hook writing the system log as an ordinary account hands the line to the collector over the+// local ingest socket when the endpoint has one (the restricted log access mode, in which the log+// is not writable by this account at all), and otherwise appends directly, as hooks always have.+// Only a direct append the log refuses -- a restricted endpoint whose collector is restarting --+// puts the line in this account's private spool, and it reaches the log with a later event+// instead of being lost. Root and per-user logs are always written directly. See+// localingest.Deliverer.Write.+func deliverEndpointLine(path string, data []byte) error {+ return EndpointIngest.Write(path, data, func(line []byte) error {+ return appendEndpointJSONL(path, append(line[:len(line):len(line)], '\n'), defaultEndpointRotateBytes, defaultEndpointRotateArchives)+ })}// withEndpointEventID fills event.id on an event assembled as a map and returnsdiff --git a/cli/beacon-hooks/internal/logging/main_test.go b/cli/beacon-hooks/internal/logging/main_test.goindex 7a263bfb..a7f07f8c 100644--- a/cli/beacon-hooks/internal/logging/main_test.go+++ b/cli/beacon-hooks/internal/logging/main_test.go@@ -6,6 +6,7 @@ import ("testing""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/activesession"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest")// TestMain keeps the active-session record every endpoint event updates out of the real home@@ -19,6 +20,12 @@ func TestMain(m *testing.M) {// Auto-share is off unless a test turns it on in a settings file of its own.settingsPath := filepath.Join(dir, "share-settings.json")ShareSettingsPath = func() string { return settingsPath }+ // Never send to a real collector's ingest socket, which a developer's restricted-mode machine+ // has; tests that exercise the socket start a server of their own.+ EndpointIngest = localingest.Deliverer{+ Client: localingest.Client{SocketPath: filepath.Join(dir, "no-ingest.sock")},+ Spool: localingest.Spool{Dir: filepath.Join(dir, "spool")},+ }code := m.Run()os.RemoveAll(dir)os.Exit(code)diff --git a/cli/beacon/internal/endpoint/writer/ingest_unix_test.go b/cli/beacon/internal/endpoint/writer/ingest_unix_test.gonew file mode 100644index 00000000..114b7326--- /dev/null+++ b/cli/beacon/internal/endpoint/writer/ingest_unix_test.go@@ -0,0 +1,164 @@+//go:build linux || darwin++package writer++import (+ "encoding/json"+ "errors"+ "os"+ "path/filepath"+ "sync"+ "testing"+ "time"++ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema"+ "github.com/asymptote-labs/agent-beacon/cli/beacon/internal/testenv"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest"+)++type collectorStub struct {+ mu sync.Mutex+ got [][]byte+}++func (c *collectorStub) lines() [][]byte {+ c.mu.Lock()+ defer c.mu.Unlock()+ return append([][]byte(nil), c.got...)+}++// useIngest makes the writer an ordinary account writing a system log (outside the test's home)+// on an endpoint whose collector serves logPath, and returns that log's path.+func useIngest(t *testing.T, serve bool) (string, *collectorStub) {+ t.Helper()+ testenv.SetHome(t, t.TempDir())+ base, err := os.MkdirTemp("", "bw")+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = os.RemoveAll(base) })+ _ = os.Chmod(base, 0o755)+ sock := filepath.Join(base, "run", "ingest.sock")+ logPath := filepath.Join(t.TempDir(), "beacon-agent", "runtime.jsonl")++ prev := Ingest+ Ingest = localingest.Deliverer{+ Client: localingest.Client{SocketPath: sock, TrustedOwner: os.Geteuid(), Timeout: time.Second},+ Spool: localingest.Spool{Dir: filepath.Join(t.TempDir(), "spool")},+ Euid: func() int { return 1000 },+ FailureBackoff: -1,+ }+ t.Cleanup(func() { Ingest = prev })++ stub := &collectorStub{}+ l, err := localingest.Listen(sock)+ if err != nil {+ t.Fatal(err)+ }+ if !serve {+ // A collector that died without removing its socket.+ l.SetUnlinkOnClose(false)+ _ = l.Close()+ return logPath, stub+ }+ server := &localingest.Server{LogPath: logPath, Append: func(line []byte) error {+ stub.mu.Lock()+ defer stub.mu.Unlock()+ stub.got = append(stub.got, append([]byte(nil), line...))+ return nil+ }}+ go func() { _ = server.Serve(l) }()+ t.Cleanup(func() { _ = server.Close() })+ return logPath, stub+}++func ingestEvent(message string) schema.Event {+ return schema.NewEvent(schema.NewEventOptions{+ Action: "agent.detected",+ Harness: schema.HarnessInfo{Name: "test"},+ Message: message,+ })+}++func TestAppendEventGoesThroughTheCollectorOnARestrictedEndpoint(t *testing.T) {+ logPath, stub := useIngest(t, true)+ path, err := AppendEvent(ingestEvent("via socket"), Options{Path: logPath})+ if err != nil || path != logPath {+ t.Fatalf("AppendEvent = %q, %v", path, err)+ }+ got := stub.lines()+ if len(got) != 1 {+ t.Fatalf("collector received %d lines", len(got))+ }+ var e map[string]interface{}+ if err := json.Unmarshal(got[0], &e); err != nil {+ t.Fatal(err)+ }+ if e["message"] != "via socket" {+ t.Fatalf("message = %v", e["message"])+ }+ if _, err := os.Stat(logPath); !errors.Is(err, os.ErrNotExist) {+ t.Fatal("the writer also touched the log directly")+ }+}++func TestAppendEventForAnotherLogIsWrittenDirectly(t *testing.T) {+ _, stub := useIngest(t, true)+ other := filepath.Join(t.TempDir(), "elsewhere", "runtime.jsonl")+ if _, err := AppendEvent(ingestEvent("custom path"), Options{Path: other}); err != nil {+ t.Fatal(err)+ }+ if len(stub.lines()) != 0 {+ t.Fatal("an event for another log went to the collector")+ }+ if data, err := os.ReadFile(other); err != nil || len(data) == 0 {+ t.Fatalf("the other log was not written: %v", err)+ }+}++func TestAppendEventThroughAStaleSocketToASharedLogIsWrittenDirectly(t *testing.T) {+ logPath, _ := useIngest(t, false)+ if _, err := AppendEvent(ingestEvent("stale"), Options{Path: logPath}); err != nil {+ t.Fatal(err)+ }+ if data, err := os.ReadFile(logPath); err != nil || len(data) == 0 {+ t.Fatalf("the shared log was not written: %v", err)+ }+ if n, _ := Ingest.Pending(logPath); n != 0 {+ t.Fatalf("Pending = %d", n)+ }+}++func TestAppendEventSpoolsWhenARestrictedLogRefusesIt(t *testing.T) {+ if os.Geteuid() == 0 {+ t.Skip("root is never refused by a file mode")+ }+ logPath, _ := useIngest(t, false)+ if err := os.MkdirAll(filepath.Dir(logPath), 0o755); err != nil {+ t.Fatal(err)+ }+ if err := os.WriteFile(logPath, nil, 0o440); err != nil {+ t.Fatal(err)+ }+ if _, err := AppendEvent(ingestEvent("later"), Options{Path: logPath}); err != nil {+ t.Fatalf("AppendEvent with the collector down: %v", err)+ }+ if n, _ := Ingest.Pending(logPath); n != 1 {+ t.Fatalf("Pending = %d", n)+ }+}++// A backfill sets a guard and a budget, which only a direct writer holding the log's lock can+// honor, so it never goes through the socket.+func TestGuardedAppendsAreWrittenDirectly(t *testing.T) {+ logPath, stub := useIngest(t, true)+ if _, err := AppendEvent(ingestEvent("budgeted"), Options{Path: logPath, Budget: NewBudget(1 << 20)}); err != nil {+ t.Fatal(err)+ }+ if len(stub.lines()) != 0 {+ t.Fatal("a budgeted append went through the socket")+ }+ if _, err := os.Stat(logPath); err != nil {+ t.Fatalf("the budgeted append was not written directly: %v", err)+ }+}diff --git a/cli/beacon/internal/endpoint/writer/writer.go b/cli/beacon/internal/endpoint/writer/writer.goindex 6d6c9519..fe35a5ca 100644--- a/cli/beacon/internal/endpoint/writer/writer.go+++ b/cli/beacon/internal/endpoint/writer/writer.go@@ -16,6 +16,7 @@ import ("github.com/asymptote-labs/agent-beacon/cli/beacon/internal/endpoint/schema""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve""github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/filelock"+ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest")const (@@ -121,12 +122,34 @@ func AppendEvent(event schema.Event, opts Options) (string, error) {}}line := append(data, '\n')+ if opts.Guard == nil && opts.Budget == nil && Ingest.Applies(opts.Path) {+ if err := deliver(opts, data); err != nil {+ return "", err+ }+ return opts.Path, nil+ }if err := appendJSONL(opts.Path, line, opts.RotateSize, opts.RotateArchives, opts.Guard, opts.Budget); err != nil {return "", err}return opts.Path, nil}+// Ingest delivers to the system log through the collector when the endpoint is in the restricted+// log access mode. Tests point it at a socket and spool of their own; nothing else changes it.+var Ingest = localingest.Deliverer{}++// deliver writes one line to the system log as an account that is not root, the way the hook+// adapter does: through the collector's ingest socket when there is one, directly otherwise, and+// into this account's private spool only when the restricted log refuses the direct append (see+// localingest.Deliverer.Write). A guarded or budgeted append (a session backfill) never comes+// here: those limits are decided under the log's lock, which only a direct writer holds, and the+// backfill that sets them runs as root on a system log.+func deliver(opts Options, data []byte) error {+ return Ingest.Write(opts.Path, data, func(line []byte) error {+ return appendJSONL(opts.Path, append(line[:len(line):len(line)], '\n'), opts.RotateSize, opts.RotateArchives, nil, nil)+ })+}+// compactRetainedContent drops an event's retained text and records that it did, so a reader can// still tell what the event was about and that its content was held back rather than never// captured. Mirrors the collector exporter's step of the same name.diff --git a/pkg/asymptoteobserve/localingest/deliver.go b/pkg/asymptoteobserve/localingest/deliver.gonew file mode 100644index 00000000..c4a0e751--- /dev/null+++ b/pkg/asymptoteobserve/localingest/deliver.go@@ -0,0 +1,187 @@+package localingest++import (+ "crypto/sha256"+ "encoding/hex"+ "errors"+ "fmt"+ "io/fs"+ "os"+ "path/filepath"+ "runtime"+ "sync/atomic"+ "time"++ "github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve"+)++// spoolDirInHome is where a writer's spools live, under its account's home directory.+const spoolDirInHome = ".beacon/endpoint/spool"++// DefaultSpoolDir is the current account's spool directory, or "" when it has no home.+func DefaultSpoolDir() string {+ home, err := os.UserHomeDir()+ if err != nil || home == "" {+ return ""+ }+ return filepath.Join(home, spoolDirInHome)+}++// defaultFailureBackoff is how long a process skips the socket after a failed send.+const defaultFailureBackoff = 10 * time.Second++// lastSendFailure is when this process last failed to send, in Unix nanoseconds; zero when the last+// attempt succeeded or none has failed.+var lastSendFailure atomic.Int64++// Deliverer is how a writer that is not root gets a line into the system runtime log: through the+// collector's ingest socket when one answers, by appending directly when the log lets it, and into+// its private spool when neither works -- a restricted log whose collector is stopped or+// restarting. A spooled line is late, never lost: the next line that gets through, either way,+// takes the spool with it.+type Deliverer struct {+ Client Client+ // Spool holds lines that could not be written. An empty Dir means DefaultSpoolDir. Each log+ // has its own spool beneath it, so a line is only ever written to the log it was meant for.+ Spool Spool+ // Euid reports the process's effective uid; nil means os.Geteuid. Tests replace it.+ Euid func() int+ // FailureBackoff is how long, after a failed send, this process goes straight to the direct+ // append without trying the socket. Zero means 10s; negative turns it off.+ FailureBackoff time.Duration+}++// Applies reports whether a write to logPath goes through Write's socket path at all. Root writes+// the log directly: it owns the file in every mode, and the events it writes -- a backfill of the+// console user's sessions, the collector's own -- name their account themselves, which the ingest+// server would replace with root. A per-user log is private to its account and always written+// directly. Windows has no ingest socket.+func (d Deliverer) Applies(logPath string) bool {+ if runtime.GOOS == "windows" {+ return false+ }+ euid := os.Geteuid()+ if d.Euid != nil {+ euid = d.Euid()+ }+ return euid != 0 && !asymptoteobserve.RuntimeLogIsPrivate(logPath)+}++// Write gets one line, meant for logPath, into that log. appendLine appends one line directly, the+// way the writer always has.+//+// - Where Applies is false, it is appendLine and nothing else.+// - Otherwise the line goes to the collector's ingest socket when there is one, the collector+// writes logPath, and it takes the line. A rejected line is reported, never retried.+// - If there is no socket, or the send fails -- a socket a crashed collector left behind, a+// collector that is hung -- the line is appended directly. On a shared log that simply works.+// - Only when the direct append is refused for permission, as a restricted log refuses it, is the+// line spooled.+//+// Every line that gets through takes up to a batch of this account's spool for logPath with it, by+// the same route. After a failed send the process skips the socket for FailureBackoff, so a hook+// writing several events waits out a hung collector once.+func (d Deliverer) Write(logPath string, line []byte, appendLine func([]byte) error) error {+ if !d.Applies(logPath) {+ return appendLine(line)+ }+ if !d.inBackoff() {+ c := d.Client+ c.LogPath = logPath+ resp, err := c.Send([][]byte{line})+ switch {+ case err == nil && resp.Rejected > 0:+ return fmt.Errorf("the collector rejected the event: %w", ErrRejected)+ case err == nil:+ lastSendFailure.Store(0)+ d.drain(logPath, func(lines [][]byte) (int, error) {+ resp, err := c.Send(lines)+ return resp.Processed, err+ })+ return nil+ case errors.Is(err, ErrNoSocket), errors.Is(err, ErrUntrustedSocket), errors.Is(err, ErrLogMismatch):+ // Not a collector this line can go through; the endpoint is shared, or the line is for+ // some other log.+ default:+ lastSendFailure.Store(time.Now().UnixNano())+ }+ }+ err := appendLine(line)+ if err == nil {+ d.drain(logPath, func(lines [][]byte) (int, error) {+ for i, line := range lines {+ if err := appendLine(line); err != nil {+ return i, err+ }+ }+ return len(lines), nil+ })+ return nil+ }+ if !errors.Is(err, fs.ErrPermission) {+ return err+ }+ if spoolErr := d.spoolLine(logPath, line); spoolErr != nil {+ return fmt.Errorf("%w; spool: %v", err, spoolErr)+ }+ return nil+}++func (d Deliverer) inBackoff() bool {+ backoff := d.FailureBackoff+ if backoff == 0 {+ backoff = defaultFailureBackoff+ }+ last := lastSendFailure.Load()+ return backoff > 0 && last != 0 && time.Since(time.Unix(0, last)) < backoff+}++// spoolFor is the spool for logPath: its own directory under the account's spool, named by a+// digest of the log's path, so a drain can only ever deliver a line to the log it was meant for.+func (d Deliverer) spoolFor(logPath string) Spool {+ s := d.Spool+ if s.Dir == "" {+ s.Dir = DefaultSpoolDir()+ }+ if s.Dir == "" {+ return s+ }+ sum := sha256.Sum256([]byte(filepath.Clean(logPath)))+ s.Dir = filepath.Join(s.Dir, hex.EncodeToString(sum[:8]))+ return s+}++func (d Deliverer) spoolLine(logPath string, line []byte) error {+ s := d.spoolFor(logPath)+ if s.Dir == "" {+ return errors.New("no home directory to spool in")+ }+ dropped, err := s.Append([][]byte{line})+ if err != nil {+ return err+ }+ if dropped > 0 {+ return fmt.Errorf("the spool in %s is full", s.Dir)+ }+ return nil+}++// drain sends one batch of logPath's spool through send. Best effort: whatever is left goes with a+// later line.+func (d Deliverer) drain(logPath string, send func([][]byte) (int, error)) {+ s := d.spoolFor(logPath)+ if s.Dir == "" {+ return+ }+ _, _ = s.Drain(MaxLinesPerConn, send)+}++// Pending reports how many events this account has spooled for logPath, for doctor.+func (d Deliverer) Pending(logPath string) (int, error) {+ s := d.spoolFor(logPath)+ if s.Dir == "" {+ return 0, nil+ }+ n, _, err := s.Pending()+ return n, err+}diff --git a/pkg/asymptoteobserve/localingest/deliver_unix_test.go b/pkg/asymptoteobserve/localingest/deliver_unix_test.gonew file mode 100644index 00000000..2a520f6e--- /dev/null+++ b/pkg/asymptoteobserve/localingest/deliver_unix_test.go@@ -0,0 +1,348 @@+//go:build linux || darwin++package localingest++import (+ "errors"+ "io/fs"+ "net"+ "os"+ "path/filepath"+ "sync"+ "testing"+ "time"+)++// testLog is the log the deliverer tests write; their servers accept any.+const testLog = "/var/log/beacon-agent/runtime.jsonl"++func deliverer(t *testing.T, socketPath string) Deliverer {+ t.Helper()+ lastSendFailure.Store(0)+ t.Cleanup(func() { lastSendFailure.Store(0) })+ return Deliverer{+ Client: Client{SocketPath: socketPath, TrustedOwner: os.Geteuid(), Timeout: time.Second},+ Spool: Spool{Dir: filepath.Join(t.TempDir(), "spool")},+ Euid: func() int { return 1000 },+ // Each Write stands in for its own hook process unless a test says otherwise.+ FailureBackoff: -1,+ }+}++// directLog stands in for the writer's own append: it records lines, or refuses them the way a+// restricted log refuses an ordinary account.+type directLog struct {+ mu sync.Mutex+ lines []string+ refuse bool+ failErr error+}++func (l *directLog) append(line []byte) error {+ l.mu.Lock()+ defer l.mu.Unlock()+ if l.refuse {+ return &fs.PathError{Op: "open", Path: testLog, Err: fs.ErrPermission}+ }+ if l.failErr != nil {+ return l.failErr+ }+ l.lines = append(l.lines, string(line))+ return nil+}++func (l *directLog) count() int {+ l.mu.Lock()+ defer l.mu.Unlock()+ return len(l.lines)+}++func staleSocket(t *testing.T) string {+ t.Helper()+ sock := filepath.Join(shortTempDir(t), "run", "ingest.sock")+ l, err := Listen(sock)+ if err != nil {+ t.Fatal(err)+ }+ l.SetUnlinkOnClose(false)+ _ = l.Close()+ return sock+}++func TestDeliverApplies(t *testing.T) {+ home := t.TempDir()+ t.Setenv("HOME", home)+ system := filepath.Join(t.TempDir(), "runtime.jsonl")+ private := filepath.Join(home, ".beacon", "endpoint", "logs", "runtime.jsonl")++ user := Deliverer{Euid: func() int { return 501 }}+ root := Deliverer{Euid: func() int { return 0 }}+ if !user.Applies(system) {+ t.Fatal("an ordinary account writing the system log must go through the collector")+ }+ if user.Applies(private) {+ t.Fatal("a per-user log is written directly")+ }+ if root.Applies(system) {+ t.Fatal("root writes the system log directly")+ }+}++func TestWriteWithoutASocketAppendsDirectly(t *testing.T) {+ d := deliverer(t, filepath.Join(shortTempDir(t), "none.sock"))+ log := &directLog{}+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {+ t.Fatal(err)+ }+ if log.count() != 1 {+ t.Fatalf("direct log has %d lines", log.count())+ }+ if n, _ := d.Pending(testLog); n != 0 {+ t.Fatal("spooled with no socket at all")+ }+}++func TestWriteThatDoesNotApplyNeverTouchesTheSocket(t *testing.T) {+ h := startServer(t, nil)+ d := deliverer(t, h.path)+ d.Euid = func() int { return 0 }+ log := &directLog{}+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {+ t.Fatal(err)+ }+ if h.rec.count() != 0 || log.count() != 1 {+ t.Fatalf("collector %d, direct %d; root writes directly", h.rec.count(), log.count())+ }+}++func TestWriteSendsThroughTheSocket(t *testing.T) {+ h := startServer(t, nil)+ d := deliverer(t, h.path)+ log := &directLog{}+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {+ t.Fatal(err)+ }+ if h.rec.count() != 1 || log.count() != 0 {+ t.Fatalf("collector %d, direct %d", h.rec.count(), log.count())+ }+}++// A socket a crashed collector left behind must not swallow a shared endpoint's events: the direct+// append works there, and nothing should be spooled.+func TestWriteThroughAStaleSocketToASharedLogAppendsDirectly(t *testing.T) {+ d := deliverer(t, staleSocket(t))+ log := &directLog{}+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {+ t.Fatal(err)+ }+ if log.count() != 1 {+ t.Fatalf("direct log has %d lines", log.count())+ }+ if n, _ := d.Pending(testLog); n != 0 {+ t.Fatalf("Pending = %d; a writable log must not spool", n)+ }+}++func TestWriteSpoolsWhenARestrictedLogRefusesAndDrainsAfter(t *testing.T) {+ sock := staleSocket(t)+ d := deliverer(t, sock)+ log := &directLog{refuse: true}+ batch := lines(t, 3)+ for _, line := range batch[:2] {+ if err := d.Write(testLog, line, log.append); err != nil {+ t.Fatalf("Write with the collector down = %v; want it spooled", err)+ }+ }+ if n, _ := d.Pending(testLog); n != 2 {+ t.Fatalf("Pending = %d, want 2", n)+ }++ // The collector comes back on the same path.+ l, err := Listen(sock)+ if err != nil {+ t.Fatal(err)+ }+ rec := &recorder{}+ s := &Server{Append: rec.append}+ go func() { _ = s.Serve(l) }()+ t.Cleanup(func() { _ = s.Close() })++ if err := d.Write(testLog, batch[2], log.append); err != nil {+ t.Fatal(err)+ }+ if rec.count() != 3 {+ t.Fatalf("collector received %d lines, want the new one and both spooled", rec.count())+ }+ if n, _ := d.Pending(testLog); n != 0 {+ t.Fatalf("Pending after drain = %d", n)+ }+ got := map[string]bool{}+ for _, line := range rec.lines {+ got[decode(t, line)["message"].(string)] = true+ }+ for _, want := range []string{"m-0", "m-1", "m-2"} {+ if !got[want] {+ t.Fatalf("event %s never reached the collector: %v", want, got)+ }+ }+}++// Switching a restricted endpoint back to shared must not strand what was spooled meanwhile: the+// next direct append drains it.+func TestASpoolDrainsByDirectAppendOnceTheLogIsWritable(t *testing.T) {+ d := deliverer(t, filepath.Join(shortTempDir(t), "none.sock"))+ log := &directLog{refuse: true}+ _ = d.Write(testLog, lines(t, 1)[0], log.append)+ if n, _ := d.Pending(testLog); n != 1 {+ t.Fatalf("Pending = %d", n)+ }+ log.refuse = false+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {+ t.Fatal(err)+ }+ if log.count() != 2 {+ t.Fatalf("direct log has %d lines; want the new one and the spooled one", log.count())+ }+ if n, _ := d.Pending(testLog); n != 0 {+ t.Fatalf("Pending = %d", n)+ }+}++func TestEachLogHasItsOwnSpool(t *testing.T) {+ d := deliverer(t, filepath.Join(shortTempDir(t), "none.sock"))+ refused := &directLog{refuse: true}+ other := "/tmp/elsewhere/runtime.jsonl"+ _ = d.Write(other, lines(t, 1)[0], refused.append)+ if n, _ := d.Pending(other); n != 1 {+ t.Fatalf("Pending(other) = %d", n)+ }+ if n, _ := d.Pending(testLog); n != 0 {+ t.Fatalf("a line meant for another log is waiting for the system log: %d", n)+ }+ system := &directLog{}+ _ = d.Write(testLog, lines(t, 1)[0], system.append)+ if system.count() != 1 {+ t.Fatalf("system log got %d lines; another log's spool must not drain into it", system.count())+ }+}++func TestWriteReportsOtherAppendErrors(t *testing.T) {+ d := deliverer(t, filepath.Join(shortTempDir(t), "none.sock"))+ log := &directLog{failErr: errors.New("disk full")}+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err == nil {+ t.Fatal("an append error that is not a refusal was swallowed")+ }+ if n, _ := d.Pending(testLog); n != 0 {+ t.Fatal("spooled an error that spooling cannot fix")+ }+}++func TestWriteReportsARejectedEvent(t *testing.T) {+ h := startServer(t, nil)+ d := deliverer(t, h.path)+ log := &directLog{}+ err := d.Write(testLog, []byte(`{"not":"an event"}`), log.append)+ if !errors.Is(err, ErrRejected) {+ t.Fatalf("Write = %v, want ErrRejected", err)+ }+ if n, _ := d.Pending(testLog); n != 0 || log.count() != 0 {+ t.Fatal("a rejected event was spooled or written directly")+ }+}++func TestWriteFailsWhenItCanNeitherWriteNorSpool(t *testing.T) {+ d := deliverer(t, staleSocket(t))+ d.Spool.MaxBytes = 1+ log := &directLog{refuse: true}+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err == nil {+ t.Fatal("Write succeeded with nowhere to put the line")+ }+}++func TestWriteDoesNotSendToAnUntrustedSocket(t *testing.T) {+ h := startServer(t, nil)+ d := deliverer(t, h.path)+ d.Client.TrustedOwner = os.Geteuid() + 1+ log := &directLog{}+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {+ t.Fatal(err)+ }+ if h.rec.count() != 0 || log.count() != 1 {+ t.Fatalf("collector %d, direct %d", h.rec.count(), log.count())+ }+}++func TestWriteForAnotherLogIsLeftToTheDirectAppend(t *testing.T) {+ h := startServer(t, func(s *Server) { s.LogPath = testLog })+ d := deliverer(t, h.path)+ log := &directLog{}+ if err := d.Write("/tmp/elsewhere/runtime.jsonl", lines(t, 1)[0], log.append); err != nil {+ t.Fatal(err)+ }+ if h.rec.count() != 0 || log.count() != 1 {+ t.Fatalf("collector %d, direct %d", h.rec.count(), log.count())+ }+}++func TestWriteWaitsOutAHungCollectorOnce(t *testing.T) {+ sock := filepath.Join(shortTempDir(t), "run", "ingest.sock")+ l, err := Listen(sock)+ if err != nil {+ t.Fatal(err)+ }+ defer l.Close()+ go func() { // accepts and never answers+ var held []net.Conn+ for {+ c, err := l.Accept()+ if err != nil {+ for _, h := range held {+ _ = h.Close()+ }+ return+ }+ held = append(held, c)+ }+ }()+ d := deliverer(t, sock)+ d.Client.Timeout = 200 * time.Millisecond+ d.FailureBackoff = 0 // the default: one process, several events+ log := &directLog{refuse: true}+ for i := 0; i < 3; i++ {+ start := time.Now()+ if err := d.Write(testLog, lines(t, 1)[0], log.append); err != nil {+ t.Fatal(err)+ }+ limit := 2 * time.Second+ if i > 0 {+ limit = 100 * time.Millisecond+ }+ if elapsed := time.Since(start); elapsed > limit {+ t.Fatalf("write %d took %v; only the first may wait for the hung collector", i, elapsed)+ }+ }+ if n, _ := d.Pending(testLog); n != 3 {+ t.Fatalf("Pending = %d", n)+ }+}++func TestWriteRetriesTheSocketOnceTheBackoffPasses(t *testing.T) {+ h := startServer(t, nil)+ d := deliverer(t, h.path)+ d.FailureBackoff = 0+ lastSendFailure.Store(time.Now().Add(-defaultFailureBackoff - time.Second).UnixNano())+ if err := d.Write(testLog, lines(t, 1)[0], (&directLog{}).append); err != nil {+ t.Fatal(err)+ }+ if h.rec.count() != 1 || lastSendFailure.Load() != 0 {+ t.Fatal("did not retry the socket after the backoff, or did not clear the failure")+ }+}++func TestDefaultSpoolDir(t *testing.T) {+ home := t.TempDir()+ t.Setenv("HOME", home)+ if got, want := DefaultSpoolDir(), filepath.Join(home, ".beacon", "endpoint", "spool"); got != want {+ t.Fatalf("DefaultSpoolDir = %q, want %q", got, want)+ }+}