Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

beacon_ingest: report a serve failure, bound its logging, check the burst

Claude9856e9d275d8
diff --git a/collector-builder/exporter/beaconjsonexporter/beaconingestextension/config.go b/collector-builder/exporter/beaconjsonexporter/beaconingestextension/config.go
index ecd7025f..ca6db560 100644
--- a/collector-builder/exporter/beaconjsonexporter/beaconingestextension/config.go
+++ b/collector-builder/exporter/beaconjsonexporter/beaconingestextension/config.go
@@ -2,6 +2,7 @@ package beaconingestextension
import (
"errors"
+ "fmt"
"github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve/localingest"
)
@@ -48,5 +49,11 @@ func (c *Config) Validate() error {
if c.RotateBytes < 0 || c.RotateArchives < 0 || c.MaxConnections < 0 || c.MaxConnectionsPerUID < 0 || c.RateBurstBytes < 0 {
return errors.New("rotate_bytes, rotate_archives, max_connections, max_connections_per_uid and rate_burst_bytes must not be negative")
}
+ // Each line costs its size plus one against the burst, which is also the most the bucket ever
+ // holds: a burst below that could never let a maximum-size event through, and a spool whose
+ // oldest line is one would never drain.
+ if c.RateBurstBytes > 0 && c.RateBurstBytes < localingest.MaxLineBytes+1 {
+ return fmt.Errorf("rate_burst_bytes must be 0 (the default) or at least %d, the cost of one maximum-size event", localingest.MaxLineBytes+1)
+ }
return nil
}
diff --git a/collector-builder/exporter/beaconjsonexporter/beaconingestextension/extension.go b/collector-builder/exporter/beaconjsonexporter/beaconingestextension/extension.go
index 55eb3e56..27ca3586 100644
--- a/collector-builder/exporter/beaconjsonexporter/beaconingestextension/extension.go
+++ b/collector-builder/exporter/beaconjsonexporter/beaconingestextension/extension.go
@@ -15,6 +15,7 @@ import (
"sync"
"go.opentelemetry.io/collector/component"
+ "go.opentelemetry.io/collector/component/componentstatus"
"go.uber.org/zap"
"github.com/asymptote-labs/agent-beacon/collector-builder/exporter/beaconjsonexporter/internal/runtimelog"
@@ -27,9 +28,10 @@ type ingest struct {
logger *zap.Logger
names *nameCache
- // listen and appendLine are the seams tests replace.
- listen func(path string) (*net.UnixListener, error)
- appendLine func(path string, line []byte, rotateBytes int64, rotateArchives int) error
+ // listen, serveListener and appendLine are the seams tests replace.
+ listen func(path string) (*net.UnixListener, error)
+ serveListener func(*net.UnixListener) net.Listener
+ appendLine func(path string, line []byte, rotateBytes int64, rotateArchives int) error
mu sync.Mutex
server *localingest.Server
@@ -52,7 +54,7 @@ func newIngest(cfg *Config, logger *zap.Logger) *ingest {
// Start creates the socket and serves it in the background. A socket another running collector
// still answers on, or a non-socket at the path, fails the start: two writers serving one log, or
// a path someone put something else at, is a configuration problem to surface, not to work around.
-func (e *ingest) Start(context.Context, component.Host) error {
+func (e *ingest) Start(_ context.Context, host component.Host) error {
if !asymptoteobserve.RuntimeLogAccessRestricted(e.cfg.LogPath) {
// Harmless -- the log is simply also writable directly -- but not what install sets up,
// so it is worth a line in the collector's log.
@@ -71,12 +73,30 @@ func (e *ingest) Start(context.Context, component.Host) error {
MaxConnsPerUID: e.cfg.MaxConnectionsPerUID,
RateBytesPerSecond: e.cfg.RateBytesPerSecond,
RateBurstBytes: e.cfg.RateBurstBytes,
+ // One fixed message with the detail as a field, so the collector's log sampler groups
+ // these; the server already limits how often it reports any one account.
Logf: func(format string, args ...interface{}) {
- e.logger.Warn(fmt.Sprintf(format, args...))
+ e.logger.Warn("beacon_ingest refused input", zap.String("detail", fmt.Sprintf(format, args...)))
},
}
done := make(chan error, 1)
- go func() { done <- server.Serve(l) }()
+ go func() {
+ var served net.Listener = l
+ if e.serveListener != nil {
+ served = e.serveListener(l)
+ }
+ err := server.Serve(served)
+ if err != nil && !errors.Is(err, localingest.ErrServerClosed) {
+ // Serving stopped on its own. The server has closed its listener, so the socket is
+ // gone and hooks fall back to spooling at once; saying so loudly, and as a fatal
+ // status, gets the collector restarted instead of running on without its socket.
+ e.logger.Error("beacon_ingest stopped serving", zap.Error(err), zap.String("socket", e.socketPath()))
+ if host != nil {
+ componentstatus.ReportStatus(host, componentstatus.NewFatalErrorEvent(fmt.Errorf("beacon_ingest: %w", err)))
+ }
+ }
+ done <- err
+ }()
e.mu.Lock()
e.server, e.done = server, done
e.mu.Unlock()
@@ -95,9 +115,7 @@ func (e *ingest) Shutdown(context.Context) error {
return nil
}
err := server.Close()
- if serveErr := <-done; serveErr != nil && !errors.Is(serveErr, localingest.ErrServerClosed) && err == nil {
- err = serveErr
- }
+ <-done // a serve error was reported when it happened
return err
}
diff --git a/collector-builder/exporter/beaconjsonexporter/beaconingestextension/extension_unix_test.go b/collector-builder/exporter/beaconjsonexporter/beaconingestextension/extension_unix_test.go
index 689db613..81f21e37 100644
--- a/collector-builder/exporter/beaconjsonexporter/beaconingestextension/extension_unix_test.go
+++ b/collector-builder/exporter/beaconjsonexporter/beaconingestextension/extension_unix_test.go
@@ -12,10 +12,13 @@ import (
"os"
"path/filepath"
"strings"
+ "sync"
"sync/atomic"
"testing"
+ "time"
"go.opentelemetry.io/collector/component"
+ "go.opentelemetry.io/collector/component/componentstatus"
"go.opentelemetry.io/collector/extension"
"go.uber.org/zap"
@@ -324,3 +327,77 @@ func TestConfigValidate(t *testing.T) {
t.Errorf("a negative rate turns the bound off and must validate: %v", err)
}
}
+
+type statusHost struct {
+ mu sync.Mutex
+ events []*componentstatus.Event
+}
+
+func (h *statusHost) GetExtensions() map[component.ID]component.Component { return nil }
+
+func (h *statusHost) Report(e *componentstatus.Event) {
+ h.mu.Lock()
+ h.events = append(h.events, e)
+ h.mu.Unlock()
+}
+
+func (h *statusHost) fatal() bool {
+ h.mu.Lock()
+ defer h.mu.Unlock()
+ for _, e := range h.events {
+ if e.Status() == componentstatus.StatusFatalError {
+ return true
+ }
+ }
+ return false
+}
+
+type brokenListener struct{ *net.UnixListener }
+
+func (l brokenListener) Accept() (net.Conn, error) {
+ return nil, &net.OpError{Op: "accept", Net: "unix", Err: errors.New("permanently broken")}
+}
+
+// If serving stops on its own, the collector says so as a fatal status and the socket goes away,
+// so hooks fall back at once instead of queueing on a socket nothing accepts.
+func TestAServeFailureIsReportedAndTheSocketRemoved(t *testing.T) {
+ dir := shortDir(t)
+ cfg := createDefaultConfig()
+ cfg.SocketPath = filepath.Join(dir, "run", "ingest.sock")
+ cfg.LogPath = filepath.Join(dir, "log", "runtime.jsonl")
+ ext := newIngest(cfg, zap.NewNop())
+ ext.listen = func(path string) (*net.UnixListener, error) {
+ return localingest.Listen(path)
+ }
+ ext.serveListener = func(l *net.UnixListener) net.Listener { return brokenListener{l} }
+ host := &statusHost{}
+ if err := ext.Start(context.Background(), host); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = ext.Shutdown(context.Background()) })
+ deadline := time.Now().Add(3 * time.Second)
+ for !host.fatal() && time.Now().Before(deadline) {
+ time.Sleep(10 * time.Millisecond)
+ }
+ if !host.fatal() {
+ t.Fatal("no fatal status was reported when serving stopped")
+ }
+ if _, err := os.Lstat(cfg.SocketPath); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("the socket was left behind: %v", err)
+ }
+ if err := ext.Shutdown(context.Background()); err != nil {
+ t.Fatalf("Shutdown after a serve failure = %v", err)
+ }
+}
+
+func TestRateBurstMustFitOneEvent(t *testing.T) {
+ for burst, ok := range map[float64]bool{
+ 0: true, float64(localingest.MaxLineBytes + 1): true, 16 << 20: true,
+ 65536: false, 1: false,
+ } {
+ err := (&Config{LogPath: "x", RateBurstBytes: burst}).Validate()
+ if (err == nil) != ok {
+ t.Errorf("rate_burst_bytes %v: Validate = %v", burst, err)
+ }
+ }
+}
diff --git a/collector-builder/exporter/beaconjsonexporter/beaconingestextension/names.go b/collector-builder/exporter/beaconjsonexporter/beaconingestextension/names.go
index b3eb4cbd..0340fa43 100644
--- a/collector-builder/exporter/beaconjsonexporter/beaconingestextension/names.go
+++ b/collector-builder/exporter/beaconjsonexporter/beaconingestextension/names.go
@@ -2,6 +2,7 @@ package beaconingestextension
import (
"context"
+ "os"
"os/exec"
"os/user"
"strconv"
@@ -62,21 +63,35 @@ func (c *nameCache) lookup(uid uint32) string {
// consulted.
var idCommands = []string{"/usr/bin/id", "/bin/id"}
+// firstExecutable is the first of paths that exists as an executable file.
+func firstExecutable(paths []string) string {
+ for _, path := range paths {
+ if info, err := os.Stat(path); err == nil && info.Mode().IsRegular() && info.Mode().Perm()&0o111 != 0 {
+ return path
+ }
+ }
+ return ""
+}
+
func resolveName(uid uint32) string {
id := strconv.FormatUint(uint64(uid), 10)
if u, err := user.LookupId(id); err == nil && validName(u.Username) {
return u.Username
}
- for _, path := range idCommands {
- ctx, cancel := context.WithTimeout(context.Background(), nameLookupTimeout)
- out, err := exec.CommandContext(ctx, path, "-nu", id).Output()
- cancel()
- if err != nil {
- continue
- }
- if name := strings.TrimSpace(string(out)); validName(name) {
- return name
- }
+ // id(1) is tried once: on merged-/usr systems both paths are the same binary, and running it
+ // again behind a directory service that did not answer would only double the wait.
+ path := firstExecutable(idCommands)
+ if path == "" {
+ return ""
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), nameLookupTimeout)
+ out, err := exec.CommandContext(ctx, path, "-nu", id).Output()
+ cancel()
+ if err != nil {
+ return ""
+ }
+ if name := strings.TrimSpace(string(out)); validName(name) {
+ return name
}
return ""
}
diff --git a/collector-builder/exporter/beaconjsonexporter/beaconingestextension/names_test.go b/collector-builder/exporter/beaconjsonexporter/beaconingestextension/names_test.go
index 489f006a..2fbc507d 100644
--- a/collector-builder/exporter/beaconjsonexporter/beaconingestextension/names_test.go
+++ b/collector-builder/exporter/beaconjsonexporter/beaconingestextension/names_test.go
@@ -80,3 +80,21 @@ func TestValidName(t *testing.T) {
}
}
}
+
+func TestFirstExecutable(t *testing.T) {
+ if runtime.GOOS == "windows" {
+ t.Skip("no execute bits")
+ }
+ dir := t.TempDir()
+ plain := dir + "/plain"
+ exe := dir + "/exe"
+ _ = os.WriteFile(plain, nil, 0o644)
+ _ = os.WriteFile(exe, nil, 0o755)
+ _ = os.Chmod(exe, 0o755)
+ if got := firstExecutable([]string{dir + "/missing", plain, exe}); got != exe {
+ t.Fatalf("firstExecutable = %q, want %q", got, exe)
+ }
+ if got := firstExecutable([]string{dir + "/missing"}); got != "" {
+ t.Fatalf("firstExecutable = %q, want none", got)
+ }
+}
diff --git a/collector-builder/exporter/beaconjsonexporter/go.mod b/collector-builder/exporter/beaconjsonexporter/go.mod
index b3a53e5c..a80378e8 100644
--- a/collector-builder/exporter/beaconjsonexporter/go.mod
+++ b/collector-builder/exporter/beaconjsonexporter/go.mod
@@ -7,6 +7,7 @@ toolchain go1.27.3
require (
github.com/asymptote-labs/agent-beacon/pkg/asymptoteobserve v0.0.0
go.opentelemetry.io/collector/component v1.27.0
+ go.opentelemetry.io/collector/component/componentstatus v0.121.0
go.opentelemetry.io/collector/config/configretry v1.27.0
go.opentelemetry.io/collector/consumer v1.27.0
go.opentelemetry.io/collector/consumer/consumertest v0.121.0
@@ -41,7 +42,6 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.11.1 // indirect
go.opentelemetry.io/auto/sdk v1.2.1 // indirect
- go.opentelemetry.io/collector/component/componentstatus v0.121.0 // indirect
go.opentelemetry.io/collector/component/componenttest v0.121.0 // indirect
go.opentelemetry.io/collector/confmap v1.27.0 // indirect
go.opentelemetry.io/collector/consumer/consumererror v0.121.0 // indirect