Skip to content

The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu

localingest: address review on the protocol and spool

Clauded3e50ee312f7
diff --git a/pkg/asymptoteobserve/localingest/accept_other.go b/pkg/asymptoteobserve/localingest/accept_other.go
new file mode 100644
index 00000000..c99171d1
--- /dev/null
+++ b/pkg/asymptoteobserve/localingest/accept_other.go
@@ -0,0 +1,6 @@
+//go:build !unix
+
+package localingest
+
+// temporaryAcceptError is never true off Unix, where nothing serves the socket.
+func temporaryAcceptError(error) bool { return false }
diff --git a/pkg/asymptoteobserve/localingest/accept_unix.go b/pkg/asymptoteobserve/localingest/accept_unix.go
new file mode 100644
index 00000000..5f04ae2a
--- /dev/null
+++ b/pkg/asymptoteobserve/localingest/accept_unix.go
@@ -0,0 +1,19 @@
+//go:build unix
+
+package localingest
+
+import (
+ "errors"
+ "syscall"
+)
+
+// temporaryAcceptError reports an accept failure that passes on its own: the process or system is
+// out of descriptors or buffers for a moment, or a client gave up before it was accepted.
+func temporaryAcceptError(err error) bool {
+ for _, errno := range []syscall.Errno{syscall.EMFILE, syscall.ENFILE, syscall.ENOBUFS, syscall.ENOMEM, syscall.ECONNABORTED, syscall.EINTR} {
+ if errors.Is(err, errno) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/pkg/asymptoteobserve/localingest/client.go b/pkg/asymptoteobserve/localingest/client.go
index d45873b5..55593d7a 100644
--- a/pkg/asymptoteobserve/localingest/client.go
+++ b/pkg/asymptoteobserve/localingest/client.go
@@ -42,7 +42,20 @@ func (c Client) path() string {
// writer decides whether the endpoint is in the restricted mode at all; ErrNoSocket means it is
// not.
func (c Client) Available() error {
- return checkSocket(c.path(), c.TrustedOwner)
+ _, err := checkSocket(c.path(), c.TrustedOwner)
+ return err
+}
+
+// Probe asks the collector to answer an empty batch for LogPath. It is what tells a live
+// collector from a socket a killed one left behind, which Available cannot: that only looks at the
+// file. ErrNoSocket and ErrLogMismatch keep their meaning.
+func (c Client) Probe() error {
+ socket, err := checkSocket(c.path(), c.TrustedOwner)
+ if err != nil {
+ return err
+ }
+ _, err = c.sendBatch(socket, nil)
+ return err
}
// Send delivers lines in batches of at most MaxLinesPerConn and returns how far it got: lines
@@ -60,7 +73,8 @@ func (c Client) Send(lines [][]byte) (Response, error) {
}
cleaned[i] = line
}
- if err := checkSocket(c.path(), c.TrustedOwner); err != nil {
+ socket, err := checkSocket(c.path(), c.TrustedOwner)
+ if err != nil {
return Response{}, err
}
var total Response
@@ -69,7 +83,7 @@ func (c Client) Send(lines [][]byte) (Response, error) {
if end > len(cleaned) {
end = len(cleaned)
}
- resp, err := c.sendBatch(cleaned[start:end])
+ resp, err := c.sendBatch(socket, cleaned[start:end])
total.Accepted += resp.Accepted
total.Rejected += resp.Rejected
total.Processed += resp.Processed
@@ -81,12 +95,12 @@ func (c Client) Send(lines [][]byte) (Response, error) {
return total, nil
}
-func (c Client) sendBatch(lines [][]byte) (Response, error) {
+func (c Client) sendBatch(socket string, lines [][]byte) (Response, error) {
timeout := c.Timeout
if timeout <= 0 {
timeout = DefaultClientTimeout
}
- conn, err := net.DialTimeout("unix", c.path(), timeout)
+ conn, err := net.DialTimeout("unix", socket, timeout)
if err != nil {
return Response{}, fmt.Errorf("connect to %s: %w", c.path(), err)
}
diff --git a/pkg/asymptoteobserve/localingest/localingest.go b/pkg/asymptoteobserve/localingest/localingest.go
index 9f4eaa37..d64e52bb 100644
--- a/pkg/asymptoteobserve/localingest/localingest.go
+++ b/pkg/asymptoteobserve/localingest/localingest.go
@@ -36,9 +36,12 @@ package localingest
import (
"bytes"
+ "crypto/sha1"
+ "encoding/hex"
"encoding/json"
"errors"
"fmt"
+ "io"
"strconv"
"strings"
@@ -117,8 +120,10 @@ type Response struct {
//
// The returned line is re-encoded, so whatever bytes the peer sent, exactly one JSON object on one
// line reaches the log: nothing the sender writes can break the log into extra lines or forge a
-// second event inside the first. Numbers are kept exactly as sent. event.id is always derived
-// from the stamped line, never taken from the sender.
+// second event inside the first. Numbers are kept exactly as sent.
+//
+// event.id follows the rules in stampEventID. A line that would only exceed MaxLineBytes because
+// of the id is written without one, as every writer does with an event at the size limit.
//
// lookupName resolves a uid to an account name and may be nil; a name it cannot resolve is left
// out rather than taken from the event, which the sender wrote.
@@ -138,6 +143,15 @@ func StampLine(line []byte, peer Peer, lookupName func(uid uint32) string) ([]by
return nil, fmt.Errorf("%w: %v", ErrRejected, err)
}
+ info := event["event"].(map[string]interface{})
+ sentID, _ := info["id"].(string)
+ sentID = strings.TrimSpace(sentID)
+ delete(info, "id")
+ claimed, err := marshalLine(event)
+ if err != nil {
+ return nil, fmt.Errorf("%w: %v", ErrRejected, err)
+ }
+
user := map[string]interface{}{"uid": strconv.FormatUint(uint64(peer.UID), 10)}
if lookupName != nil {
if name := strings.TrimSpace(lookupName(peer.UID)); name != "" {
@@ -145,27 +159,73 @@ func StampLine(line []byte, peer Peer, lookupName func(uid uint32) string) ([]by
}
}
event["user"] = user
-
- // The id is derived here, from the line as it will be written, never taken from the sender:
- // otherwise one account could reuse another's event.id, and consumers that deduplicate by id
- // would drop the real event as a repeat. Deterministic, so a resent line keeps its id.
- info := event["event"].(map[string]interface{})
- delete(info, "id")
- unstamped, err := marshalLine(event)
+ withoutID, err := marshalLine(event)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrRejected, err)
}
- info["id"] = asymptoteobserve.EventIDForLine(unstamped)
+ if len(withoutID) > MaxLineBytes {
+ return nil, fmt.Errorf("%w: %d bytes after stamping is over the %d byte limit", ErrRejected, len(withoutID), MaxLineBytes)
+ }
+ info["id"] = stampEventID(sentID, claimed, withoutID, peer)
out, err := marshalLine(event)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrRejected, err)
}
if len(out) > MaxLineBytes {
- return nil, fmt.Errorf("%w: %d bytes after stamping is over the %d byte limit", ErrRejected, len(out), MaxLineBytes)
+ // Forty-odd bytes of identity are not worth an event: the line goes without an id, which
+ // is what the hook and CLI writers do with an event at the limit.
+ return withoutID, nil
}
return out, nil
}
+// stampEventID decides the event.id of a stamped line. claimed is the line as the sender wrote it
+// and withoutID the stamped line, both without an id.
+//
+// - No id, or the id derived from the stamped line: that derived id. This is every hook event
+// whose sender claimed the account it is, and every call-keyed id (EventIDForLine names a tool
+// call by its session, action, target and call id, not by account, so the hook and OTLP paths
+// give one call one id).
+// - The id derived from the sender's own claims but not from the stamped line: the line named an
+// account it is not, so the id is derived again from what is written.
+// - Any other id is one the sender chose, as session-store importers do from their source
+// records. It is kept stable for that account, so a re-swept record keeps its id, but scoped to
+// the verified uid, so one account cannot write under an id another account's events carry.
+//
+// A call-keyed id stays as strong as the session and call ids it is built from: an account that
+// knows another session's ids can name the same call, as it could over the loopback OTLP port.
+func stampEventID(sentID string, claimed, withoutID []byte, peer Peer) string {
+ derived := asymptoteobserve.EventIDForLine(withoutID)
+ switch {
+ case sentID == "" || sentID == derived:
+ return derived
+ case sentID == asymptoteobserve.EventIDForLine(claimed):
+ return derived
+ default:
+ return scopedEventID(peer.UID, sentID)
+ }
+}
+
+// scopedIDNamespace scopes a sender-chosen event.id to the account that sent it. Like the event id
+// namespace, it is a contract: changing it renames every scoped id written through the socket.
+var scopedIDNamespace = [16]byte{0x6b, 0x2d, 0x1f, 0x9a, 0x4c, 0x37, 0x4e, 0x0b, 0x9f, 0x51, 0x8e, 0x2a, 0x77, 0xc3, 0x05, 0xd4}
+
+// scopedEventID is a version 5 UUID of the uid and the id the sender chose.
+func scopedEventID(uid uint32, sentID string) string {
+ digest := sha1.New()
+ digest.Write(scopedIDNamespace[:])
+ io.WriteString(digest, strconv.FormatUint(uint64(uid), 10))
+ digest.Write([]byte{0})
+ io.WriteString(digest, sentID)
+ var id [16]byte
+ copy(id[:], digest.Sum(nil))
+ id[6] = (id[6] & 0x0f) | 0x50
+ id[8] = (id[8] & 0x3f) | 0x80
+ out := make([]byte, 32)
+ hex.Encode(out, id[:])
+ return fmt.Sprintf("%s-%s-%s-%s-%s", out[0:8], out[8:12], out[12:16], out[16:20], out[20:32])
+}
+
func decodeObject(line []byte) (map[string]interface{}, error) {
dec := json.NewDecoder(bytes.NewReader(line))
dec.UseNumber()
diff --git a/pkg/asymptoteobserve/localingest/localingest_test.go b/pkg/asymptoteobserve/localingest/localingest_test.go
index 3789f75a..afdc2b1f 100644
--- a/pkg/asymptoteobserve/localingest/localingest_test.go
+++ b/pkg/asymptoteobserve/localingest/localingest_test.go
@@ -213,15 +213,132 @@ func TestStampLineAcceptsATrailingCarriageReturn(t *testing.T) {
}
}
-func TestStampLineNeverKeepsTheSendersEventID(t *testing.T) {
+func stampedID(t *testing.T, out []byte) string {
+ t.Helper()
+ id, _ := decode(t, out)["event"].(map[string]interface{})["id"].(string)
+ return id
+}
+
+// A hook names its event by EventIDForLine before sending. When it claimed the account it is, the
+// server keeps that id: it is the one the shared mode would have written.
+func TestStampLineKeepsAnIDDerivedFromWhatIsWritten(t *testing.T) {
+ line := validLine(t, func(e map[string]interface{}) {
+ e["user"] = map[string]interface{}{"uid": "501", "name": "alice"}
+ delete(e["event"].(map[string]interface{}), "id")
+ })
+ want := asymptoteobserve.EventIDForLine(line)
+ withID := validLine(t, func(e map[string]interface{}) {
+ e["user"] = map[string]interface{}{"uid": "501", "name": "alice"}
+ e["event"].(map[string]interface{})["id"] = want
+ })
+ out, err := StampLine(withID, Peer{UID: 501}, func(uint32) string { return "alice" })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := stampedID(t, out); got != want {
+ t.Fatalf("event.id = %s, want the sender's derived %s", got, want)
+ }
+}
+
+// A call-keyed id names a tool call, not an account, so it survives stamping: the hook and the OTLP
+// path give one call one id.
+func TestStampLineKeepsACallKeyedID(t *testing.T) {
+ call := func(e map[string]interface{}) {
+ e["event"] = map[string]interface{}{"action": "tool.invoked", "category": "tool"}
+ e["session"] = map[string]interface{}{"id": "s-1"}
+ e["tool"] = map[string]interface{}{"name": "Bash"}
+ e["command"] = map[string]interface{}{"command": "ls"}
+ e["gen_ai"] = map[string]interface{}{"tool": map[string]interface{}{"name": "Bash", "call": map[string]interface{}{"id": "toolu_1"}}}
+ e["user"] = map[string]interface{}{"uid": "0", "name": "root"}
+ }
+ plain := validLine(t, call)
+ sent := asymptoteobserve.EventIDForLine(plain)
line := validLine(t, func(e map[string]interface{}) {
- e["event"].(map[string]interface{})["id"] = "someone-elses-event-id"
+ call(e)
+ e["event"].(map[string]interface{})["id"] = sent
})
- out, err := StampLine(line, Peer{UID: 501}, nil)
+ out, err := StampLine(line, Peer{UID: 501}, func(uint32) string { return "alice" })
+ if err != nil {
+ t.Fatal(err)
+ }
+ stamped := decode(t, out)
+ delete(stamped["event"].(map[string]interface{}), "id")
+ again, _ := json.Marshal(stamped)
+ if got, want := stampedID(t, out), asymptoteobserve.EventIDForLine(again); got != want {
+ t.Fatalf("event.id = %s, want %s derived from what is written", got, want)
+ }
+}
+
+// A line that claimed another account carries an id derived from that claim; once the account is
+// corrected, the id is derived again from what is written.
+func TestStampLineRederivesTheIDOfALineThatClaimedAnotherAccount(t *testing.T) {
+ claimed := validLine(t, func(e map[string]interface{}) {
+ e["user"] = map[string]interface{}{"uid": "0", "name": "root"}
+ delete(e["event"].(map[string]interface{}), "id")
+ })
+ sent := asymptoteobserve.EventIDForLine(claimed)
+ line := validLine(t, func(e map[string]interface{}) {
+ e["user"] = map[string]interface{}{"uid": "0", "name": "root"}
+ e["event"].(map[string]interface{})["id"] = sent
+ })
+ out, err := StampLine(line, Peer{UID: 501}, func(uint32) string { return "alice" })
if err != nil {
t.Fatal(err)
}
- if id := decode(t, out)["event"].(map[string]interface{})["id"]; id == "someone-elses-event-id" || id == "" {
- t.Fatalf("event.id = %v; a sender must not choose it", id)
+ if got := stampedID(t, out); got == sent || got == "" {
+ t.Fatalf("event.id = %q; an id derived from a false claim must not be kept", got)
+ }
+}
+
+// An id the sender chose itself (a session-store importer's source coordinate) stays stable for
+// that account, so a re-swept record keeps its id, but cannot be another account's.
+func TestStampLineScopesASenderChosenIDToItsAccount(t *testing.T) {
+ line := validLine(t, func(e map[string]interface{}) {
+ e["event"].(map[string]interface{})["id"] = "claude:session-1:record-42"
+ })
+ first, err := StampLine(line, Peer{UID: 501}, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ again, _ := StampLine(line, Peer{UID: 501}, nil)
+ other, _ := StampLine(line, Peer{UID: 502}, nil)
+ a, b, c := stampedID(t, first), stampedID(t, again), stampedID(t, other)
+ if a == "claude:session-1:record-42" || a == "" {
+ t.Fatalf("event.id = %q; a sender-chosen id must be scoped", a)
+ }
+ if a != b {
+ t.Fatal("a scoped id is not stable across resends")
+ }
+ if a == c {
+ t.Fatal("two accounts sending the same chosen id got the same event.id")
+ }
+ if a != scopedEventID(501, "claude:session-1:record-42") {
+ t.Fatal("scoped id does not match its definition")
+ }
+}
+
+// An event at the limit, which only the id would push over, is written without one, the way the
+// writers handle it, rather than lost.
+func TestStampLineDropsTheIDRatherThanTheEvent(t *testing.T) {
+ user := map[string]interface{}{"uid": "501", "name": "alice"}
+ base := validLine(t, func(e map[string]interface{}) {
+ e["user"] = user
+ e["message"] = ""
+ delete(e["event"].(map[string]interface{}), "id")
+ })
+ line := validLine(t, func(e map[string]interface{}) {
+ e["user"] = user
+ e["message"] = strings.Repeat("m", MaxLineBytes-len(base)-5)
+ delete(e["event"].(map[string]interface{}), "id")
+ })
+ if len(line) > MaxLineBytes {
+ t.Fatal("fixture over the limit")
+ }
+ out, err := StampLine(line, Peer{UID: 501}, func(uint32) string { return "alice" })
+ if err != nil {
+ t.Fatalf("an event the id alone would push over the limit was rejected: %v", err)
+ }
+ if len(out) > MaxLineBytes || stampedID(t, out) != "" {
+ t.Fatalf("got %d bytes, id %q; want the line within the limit and without an id", len(out), stampedID(t, out))
}
}
diff --git a/pkg/asymptoteobserve/localingest/server.go b/pkg/asymptoteobserve/localingest/server.go
index e58f840a..9dc5048a 100644
--- a/pkg/asymptoteobserve/localingest/server.go
+++ b/pkg/asymptoteobserve/localingest/server.go
@@ -72,6 +72,7 @@ type Server struct {
wg sync.WaitGroup
buckets map[uint32]*bucket
perUID map[uint32]int
+ logStates map[uint32]*logState
}
// ErrServerClosed is returned by Serve after Close.
@@ -91,6 +92,7 @@ func (s *Server) Serve(l net.Listener) error {
return ErrServerClosed
}
defer s.untrack(l)
+ var retry time.Duration
for {
conn, err := l.Accept()
if err != nil {
@@ -101,12 +103,28 @@ func (s *Server) Serve(l net.Listener) error {
if errors.As(err, &ne) && ne.Timeout() {
continue
}
+ if temporaryAcceptError(err) {
+ // Out of descriptors or memory for a moment, as net/http treats it: wait and accept
+ // again rather than stop serving for good.
+ if retry == 0 {
+ retry = 5 * time.Millisecond
+ } else if retry *= 2; retry > time.Second {
+ retry = time.Second
+ }
+ s.logf(noUID, "accept failed, retrying in %v: %v", retry, err)
+ time.Sleep(retry)
+ continue
+ }
+ // Anything else ends serving. The listener is closed so the socket goes away and
+ // clients fall back at once, rather than queueing in a backlog nothing accepts.
+ _ = l.Close()
return err
}
+ retry = 0
select {
case s.slots <- struct{}{}:
default:
- s.logf("refusing connection: %d already open", cap(s.slots))
+ s.logf(noUID, "refusing connection: %d already open", cap(s.slots))
_ = conn.Close()
continue
}
@@ -197,10 +215,46 @@ func (s *Server) isClosed() bool {
return s.closed
}
-func (s *Server) logf(format string, args ...interface{}) {
- if s.Logf != nil {
- s.Logf(format, args...)
+// noUID keys log lines that belong to no one connection.
+const noUID = ^uint32(0)
+
+// logInterval is the least time between two log lines about one account. Any account can connect,
+// and each refused connection or rejected line is something to log, so without a bound one
+// account could fill the root collector's log as fast as it can send.
+const logInterval = 10 * time.Second
+
+type logState struct {
+ last time.Time
+ suppressed int
+}
+
+func (s *Server) logf(uid uint32, format string, args ...interface{}) {
+ if s.Logf == nil {
+ return
+ }
+ now := time.Now()
+ s.mu.Lock()
+ if s.logStates == nil {
+ s.logStates = map[uint32]*logState{}
}
+ st := s.logStates[uid]
+ if st == nil {
+ st = &logState{}
+ s.logStates[uid] = st
+ }
+ if !st.last.IsZero() && now.Sub(st.last) < logInterval {
+ st.suppressed++
+ s.mu.Unlock()
+ return
+ }
+ suppressed := st.suppressed
+ st.last, st.suppressed = now, 0
+ s.mu.Unlock()
+ msg := fmt.Sprintf(format, args...)
+ if suppressed > 0 {
+ msg = fmt.Sprintf("%s (and %d more like it from this account since the last report)", msg, suppressed)
+ }
+ s.Logf("%s", msg)
}
func (s *Server) handle(conn net.Conn) {
@@ -208,21 +262,24 @@ func (s *Server) handle(conn net.Conn) {
if timeout <= 0 {
timeout = defaultConnTimeout
}
+ // Reads may run past the budget by replyGrace: a read that fails at the budget itself would
+ // end the connection with no reply after lines were already written, and the client would
+ // send them all again. Past the budget the loop below stops handling lines and replies.
budget := time.Now().Add(timeout)
- _ = conn.SetDeadline(budget)
+ _ = conn.SetDeadline(budget.Add(replyGrace))
unixConn, ok := conn.(*net.UnixConn)
if !ok {
- s.logf("refusing connection: not a Unix socket")
+ s.logf(noUID, "refusing connection: not a Unix socket")
return
}
peer, err := peerCredentials(unixConn)
if err != nil {
- s.logf("refusing connection: %v", err)
+ s.logf(noUID, "refusing connection: %v", err)
return
}
if !s.acquireUID(peer.UID) {
- s.logf("refusing connection from uid %d: it already has the most connections one account may hold", peer.UID)
+ s.logf(peer.UID, "refusing connection from uid %d: it already has the most connections one account may hold", peer.UID)
return
}
defer s.releaseUID(peer.UID)
@@ -237,7 +294,7 @@ func (s *Server) handle(conn net.Conn) {
r := bufio.NewReaderSize(conn, MaxLineBytes+2)
preamble, tooLong, err := readLine(r, len(Preamble)+1)
if err != nil && !errors.Is(err, io.EOF) || tooLong || string(preamble) != Preamble {
- s.logf("refusing connection from uid %d: no %s preamble", peer.UID, Preamble)
+ s.logf(peer.UID, "refusing connection from uid %d: no %s preamble", peer.UID, Preamble)
writeResponse(conn, Response{Error: "expected " + Preamble})
return
}
@@ -248,7 +305,7 @@ func (s *Server) handle(conn net.Conn) {
var header Header
switch {
case err != nil && !errors.Is(err, io.EOF), tooLong, json.Unmarshal(headerLine, &header) != nil:
- s.logf("refusing connection from uid %d: no valid header", peer.UID)
+ s.logf(peer.UID, "refusing connection from uid %d: no valid header", peer.UID)
writeResponse(conn, Response{Error: "expected a header"})
return
case s.LogPath != "" && !sameLog(header.Log, s.LogPath):
@@ -260,11 +317,18 @@ func (s *Server) handle(conn net.Conn) {
for {
line, tooLong, err := readLine(r, MaxLineBytes)
if err != nil && !errors.Is(err, io.EOF) {
- // A read that failed part way (a timeout, a reset) may have cut the last line short, so
- // it is not handled, and the client gets no reply: it keeps the whole batch and sends it
- // again. A line written before the failure may then appear twice, under the same
- // deterministic event.id, which is the trade the log already makes on any retried write.
- s.logf("connection from uid %d ended early: %v", peer.UID, err)
+ var ne net.Error
+ if errors.As(err, &ne) && ne.Timeout() {
+ // Out of time with the rest of the batch unread, or a line cut short: whatever is
+ // unread is not handled, and the reply says how far this got so the client resends
+ // only the rest.
+ if resp.Error == "" {
+ resp.Error = "busy"
+ }
+ break
+ }
+ // The connection broke (a reset): there is nobody to reply to.
+ s.logf(peer.UID, "connection from uid %d ended early: %v", peer.UID, err)
return
}
if len(line) == 0 && !tooLong {
@@ -285,13 +349,12 @@ func (s *Server) handle(conn net.Conn) {
// Out of time: stop writing and say how far this got, while the client still waits.
resp.Error = "busy"
stopped = true
- _ = conn.SetDeadline(time.Now().Add(replyGrace))
case lines > MaxLinesPerConn:
resp.Error = fmt.Sprintf("batch is over %d lines", MaxLinesPerConn)
stopped = true
case tooLong:
resp.Rejected++
- s.logf("rejected an event from uid %d: over %d bytes", peer.UID, MaxLineBytes)
+ s.logf(peer.UID, "rejected an event from uid %d: over %d bytes", peer.UID, MaxLineBytes)
case !s.allow(peer.UID, len(line)):
resp.Error = "rate limited"
stopped = true
@@ -299,7 +362,7 @@ func (s *Server) handle(conn net.Conn) {
stamped, stampErr := StampLine(line, peer, lookup)
if stampErr != nil {
resp.Rejected++
- s.logf("rejected an event from uid %d: %v", peer.UID, stampErr)
+ s.logf(peer.UID, "rejected an event from uid %d: %v", peer.UID, stampErr)
break
}
if appendErr := s.Append(stamped); appendErr != nil {
@@ -308,7 +371,7 @@ func (s *Server) handle(conn net.Conn) {
break
}
resp.Error = "write failed"
- s.logf("could not write an event from uid %d: %v", peer.UID, appendErr)
+ s.logf(peer.UID, "could not write an event from uid %d: %v", peer.UID, appendErr)
stopped = true
break
}
@@ -319,9 +382,7 @@ func (s *Server) handle(conn net.Conn) {
}
}
resp.Processed = resp.Accepted + resp.Rejected
- if time.Now().After(budget) {
- _ = conn.SetWriteDeadline(time.Now().Add(replyGrace))
- }
+ _ = conn.SetWriteDeadline(time.Now().Add(replyGrace))
writeResponse(conn, resp)
}
diff --git a/pkg/asymptoteobserve/localingest/server_unix_test.go b/pkg/asymptoteobserve/localingest/server_unix_test.go
index e513a2bb..97d1dcc7 100644
--- a/pkg/asymptoteobserve/localingest/server_unix_test.go
+++ b/pkg/asymptoteobserve/localingest/server_unix_test.go
@@ -345,7 +345,9 @@ func TestServerTakesAFinalLineWithoutANewline(t *testing.T) {
}
}
-func TestServerDropsAConnectionThatStalls(t *testing.T) {
+// A client that stalls mid-batch gets a reply naming how far the server got, so it resends only the
+// rest; the line it cut short is not handled.
+func TestServerAnswersAConnectionThatStalls(t *testing.T) {
h := startServer(t, func(s *Server) { s.ConnTimeout = 150 * time.Millisecond })
conn, err := net.Dial("unix", h.path)
if err != nil {
@@ -359,14 +361,61 @@ func TestServerDropsAConnectionThatStalls(t *testing.T) {
}
_ = conn.SetReadDeadline(time.Now().Add(3 * time.Second))
reply, err := bufio.NewReader(conn).ReadBytes('\n')
- if err == nil {
- t.Fatalf("got reply %q from a stalled connection; want it closed without one", reply)
+ if err != nil {
+ t.Fatalf("no reply from a server whose client stalled: %v", err)
+ }
+ var resp Response
+ if err := json.Unmarshal(reply, &resp); err != nil {
+ t.Fatal(err)
+ }
+ if resp.Processed != 1 || resp.Error != "busy" {
+ t.Fatalf("resp = %+v, want the one complete line reported and the rest left", resp)
}
if h.rec.count() != 1 {
t.Fatalf("wrote %d lines, want only the complete one", h.rec.count())
}
}
+// The last line's write running past the budget must not cost the reply: the client would resend
+// a line already written.
+func TestASlowLastWriteStillGetsAReply(t *testing.T) {
+ h := startServer(t, func(s *Server) { s.ConnTimeout = 100 * time.Millisecond })
+ h.rec.setFail(func(int, []byte) error {
+ time.Sleep(250 * time.Millisecond)
+ return nil
+ })
+ resp, err := h.client.Send(lines(t, 1))
+ if err != nil || resp.Accepted != 1 {
+ t.Fatalf("Send = %+v, %v; want the written line acknowledged", resp, err)
+ }
+}
+
+// A batch bigger than the server's read buffer, whose budget runs out after the buffer is used,
+// gets a reply that matches what was written.
+func TestABusyServerReportsHowFarItGotPastItsReadBuffer(t *testing.T) {
+ h := startServer(t, func(s *Server) {
+ s.ConnTimeout = 150 * time.Millisecond
+ s.RateBytesPerSecond = -1
+ })
+ h.rec.setFail(func(int, []byte) error {
+ time.Sleep(40 * time.Millisecond)
+ return nil
+ })
+ big := make([][]byte, 12)
+ for i := range big {
+ big[i] = validLine(t, func(e map[string]interface{}) {
+ e["message"] = fmt.Sprintf("m-%d-%s", i, strings.Repeat("x", 20*1024))
+ })
+ }
+ resp, err := h.client.Send(big)
+ if err == nil || resp.Error != "busy" {
+ t.Fatalf("Send = %+v, %v; want the server to stop and report", resp, err)
+ }
+ if resp.Processed == 0 || resp.Processed != h.rec.count() {
+ t.Fatalf("processed %d, wrote %d; the reply must match what was written", resp.Processed, h.rec.count())
+ }
+}
+
func TestRateLimitStopsAFloodingAccount(t *testing.T) {
now := time.Unix(1000, 0)
h := startServer(t, func(s *Server) {
@@ -856,3 +905,135 @@ func TestRemoveStale(t *testing.T) {
t.Fatal(err)
}
}
+
+func TestProbe(t *testing.T) {
+ h := startServer(t, func(s *Server) { s.LogPath = "/var/log/beacon-agent/runtime.jsonl" })
+ c := h.client
+ c.LogPath = "/var/log/beacon-agent/runtime.jsonl"
+ if err := c.Probe(); err != nil {
+ t.Fatalf("Probe of a live collector = %v", err)
+ }
+ if h.rec.count() != 0 {
+ t.Fatal("a probe wrote something")
+ }
+ c.LogPath = "/tmp/other.jsonl"
+ if err := c.Probe(); !errors.Is(err, ErrLogMismatch) {
+ t.Fatalf("Probe for another log = %v", err)
+ }
+ stale := filepath.Join(shortTempDir(t), "run", "ingest.sock")
+ l, err := Listen(stale)
+ if err != nil {
+ t.Fatal(err)
+ }
+ l.SetUnlinkOnClose(false)
+ _ = l.Close()
+ dead := Client{SocketPath: stale, TrustedOwner: os.Geteuid(), Timeout: time.Second}
+ if err := dead.Available(); err != nil {
+ t.Fatalf("fixture: Available on a dead socket = %v", err)
+ }
+ if err := dead.Probe(); err == nil || errors.Is(err, ErrNoSocket) {
+ t.Fatalf("Probe of a dead socket = %v; want a connection failure", err)
+ }
+}
+
+// The client dials the path it checked, with links resolved, so a link it passed through cannot
+// be repointed between the check and the dial.
+func TestClientDialsTheResolvedSocket(t *testing.T) {
+ h := startServer(t, nil)
+ link := filepath.Join(shortTempDir(t), "linkdir")
+ if err := os.Symlink(filepath.Dir(h.path), link); err != nil {
+ t.Fatal(err)
+ }
+ c := h.client
+ c.SocketPath = filepath.Join(link, filepath.Base(h.path))
+ socket, err := checkSocket(c.SocketPath, os.Geteuid())
+ if err != nil {
+ t.Fatal(err)
+ }
+ want, _ := filepath.EvalSymlinks(h.path)
+ if socket != want {
+ t.Fatalf("checkSocket returned %q, want the resolved %q", socket, want)
+ }
+ if _, err := c.Send(lines(t, 1)); err != nil || h.rec.count() != 1 {
+ t.Fatalf("Send through a linked directory = %v", err)
+ }
+}
+
+func TestServeClosesItsListenerWhenAcceptFails(t *testing.T) {
+ dir := filepath.Join(shortTempDir(t), "run")
+ l, err := Listen(filepath.Join(dir, "ingest.sock"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ failing := &failingListener{Listener: l, err: errors.New("accept: something permanent")}
+ s := &Server{Append: func([]byte) error { return nil }}
+ if err := s.Serve(failing); err == nil || errors.Is(err, ErrServerClosed) {
+ t.Fatalf("Serve = %v", err)
+ }
+ if _, err := os.Lstat(filepath.Join(dir, "ingest.sock")); !errors.Is(err, os.ErrNotExist) {
+ t.Fatal("a server that stopped serving left its socket for clients to queue on")
+ }
+}
+
+func TestServeRetriesATemporaryAcceptError(t *testing.T) {
+ dir := filepath.Join(shortTempDir(t), "run")
+ sock := filepath.Join(dir, "ingest.sock")
+ l, err := Listen(sock)
+ if err != nil {
+ t.Fatal(err)
+ }
+ failing := &failingListener{Listener: l, err: syscall.EMFILE, times: 3}
+ rec := &recorder{}
+ s := &Server{Append: rec.append}
+ go func() { _ = s.Serve(failing) }()
+ t.Cleanup(func() { _ = s.Close() })
+ c := Client{SocketPath: sock, TrustedOwner: os.Geteuid(), Timeout: 2 * time.Second}
+ if _, err := c.Send(lines(t, 1)); err != nil || rec.count() != 1 {
+ t.Fatalf("Send after temporary accept errors = %v", err)
+ }
+}
+
+type failingListener struct {
+ net.Listener
+ mu sync.Mutex
+ err error
+ times int // 0 means always
+}
+
+func (l *failingListener) Accept() (net.Conn, error) {
+ l.mu.Lock()
+ if l.times == 0 || l.times > 0 {
+ if l.times > 0 {
+ l.times--
+ if l.times == 0 {
+ l.times = -1
+ }
+ }
+ l.mu.Unlock()
+ return nil, &net.OpError{Op: "accept", Net: "unix", Err: l.err}
+ }
+ l.mu.Unlock()
+ return l.Listener.Accept()
+}
+
+func TestServerLogsAnAccountAtMostOncePerInterval(t *testing.T) {
+ var mu sync.Mutex
+ var logged []string
+ h := startServer(t, func(s *Server) {
+ s.Logf = func(format string, args ...interface{}) {
+ mu.Lock()
+ logged = append(logged, fmt.Sprintf(format, args...))
+ mu.Unlock()
+ }
+ })
+ for i := 0; i < 20; i++ {
+ if _, err := rawExchange(t, h.path, "no preamble\n"); err != nil {
+ t.Fatal(err)
+ }
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if len(logged) != 1 {
+ t.Fatalf("logged %d lines for one account's flood, want 1: %q", len(logged), logged)
+ }
+}
diff --git a/pkg/asymptoteobserve/localingest/spool.go b/pkg/asymptoteobserve/localingest/spool.go
index 22d7c38d..68c2705c 100644
--- a/pkg/asymptoteobserve/localingest/spool.go
+++ b/pkg/asymptoteobserve/localingest/spool.go
@@ -70,12 +70,17 @@ func (s Spool) Append(lines [][]byte) (dropped int, err error) {
if len(lines) == 0 {
return 0, nil
}
- err = s.withLock(func() error {
+ err = s.withLock(func() (err error) {
f, err := os.OpenFile(s.filePath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, SpoolFileMode)
if err != nil {
return err
}
- defer f.Close()
+ // A close can report a write the kernel deferred; losing it would lose the line silently.
+ defer func() {
+ if cerr := f.Close(); cerr != nil && err == nil {
+ err = cerr
+ }
+ }()
info, err := f.Stat()
if err != nil {
return err
@@ -117,7 +122,7 @@ func (s Spool) Append(lines [][]byte) (dropped int, err error) {
// batch and to remove what was sent, not while sending, so a hook spooling an event meanwhile is
// not held up by a slow collector: appends only ever add to the end, and only the drainer removes
// from the start.
-func (s Spool) Drain(maxLines int, send func(lines [][]byte) (int, error)) (int, error) {
+func (s Spool) Drain(maxLines int, send func(lines [][]byte) (int, error)) (n int, err error) {
if maxLines <= 0 {
return 0, nil
}
@@ -128,7 +133,7 @@ func (s Spool) Drain(maxLines int, send func(lines [][]byte) (int, error)) (int,
if err != nil {
return 0, err
}
- defer drainFile.Close()
+ defer closeLock(drainFile, &err)
draining, err := filelock.TryExclusive(drainFile)
if err != nil {
return 0, nil
@@ -181,12 +186,12 @@ func (s Spool) Drain(maxLines int, send func(lines [][]byte) (int, error)) (int,
return processed, sendErr
}
-func (s Spool) withLock(fn func() error) error {
+func (s Spool) withLock(fn func() error) (err error) {
lockFile, err := s.openLock()
if err != nil {
return err
}
- defer lockFile.Close()
+ defer closeLock(lockFile, &err)
wait := s.LockWait
if wait <= 0 {
wait = defaultSpoolLockWait
@@ -254,3 +259,11 @@ func splitLines(data []byte) [][]byte {
}
return out
}
+
+// closeLock closes a lock file and reports a failure to close it when nothing else failed first.
+// Lock files are never written, so a close error loses no data, but it is still not dropped.
+func closeLock(f *os.File, err *error) {
+ if cerr := f.Close(); cerr != nil && *err == nil {
+ *err = cerr
+ }
+}
diff --git a/pkg/asymptoteobserve/localingest/trust_other.go b/pkg/asymptoteobserve/localingest/trust_other.go
index 22a42c1b..5ec9c3c1 100644
--- a/pkg/asymptoteobserve/localingest/trust_other.go
+++ b/pkg/asymptoteobserve/localingest/trust_other.go
@@ -9,7 +9,7 @@ import (
// checkSocket always reports no socket off Unix: the restricted log access mode is not available
// there, so writers keep appending to the log as before.
-func checkSocket(string, int) error { return ErrNoSocket }
+func checkSocket(string, int) (string, error) { return "", ErrNoSocket }
// RemoveStale has nothing to remove off Unix.
func RemoveStale(string) error { return nil }
diff --git a/pkg/asymptoteobserve/localingest/trust_unix.go b/pkg/asymptoteobserve/localingest/trust_unix.go
index d2a0745b..976aef9a 100644
--- a/pkg/asymptoteobserve/localingest/trust_unix.go
+++ b/pkg/asymptoteobserve/localingest/trust_unix.go
@@ -24,24 +24,41 @@ const SocketMode os.FileMode = 0o666
// must its directory, which nobody else may write; every directory above it must belong to root or
// owner and be writable by nobody else unless it is sticky (as /tmp is), so the directory holding
// the socket cannot be swapped for another.
-func checkSocket(path string, owner int) error {
- info, err := os.Lstat(path)
+//
+// It returns the socket's path with every link in its directories resolved, and the caller dials
+// that path, never the one it was given: a link anywhere in the given path could otherwise be
+// repointed between this check and the dial, at a socket of someone else's choosing. Resolving
+// keeps the default path working, whose /var/run is a root-owned link.
+func checkSocket(path string, owner int) (string, error) {
+ dir, err := filepath.EvalSymlinks(filepath.Dir(path))
+ if err != nil {
+ if errors.Is(err, os.ErrNotExist) {
+ return "", ErrNoSocket
+ }
+ return "", fmt.Errorf("%w: %v", ErrUntrustedSocket, err)
+ }
+ resolved := filepath.Join(dir, filepath.Base(path))
+ info, err := os.Lstat(resolved)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
- return ErrNoSocket
+ return "", ErrNoSocket
}
- return fmt.Errorf("%w: %v", ErrUntrustedSocket, err)
+ return "", fmt.Errorf("%w: %v", ErrUntrustedSocket, err)
}
if info.Mode()&os.ModeSocket == 0 {
- return fmt.Errorf("%w: %s is not a socket", ErrUntrustedSocket, path)
+ return "", fmt.Errorf("%w: %s is not a socket", ErrUntrustedSocket, path)
}
if uid, ok := fileOwner(info); !ok || uid != owner {
- return fmt.Errorf("%w: %s is not owned by uid %d", ErrUntrustedSocket, path, owner)
+ return "", fmt.Errorf("%w: %s is not owned by uid %d", ErrUntrustedSocket, path, owner)
}
- dir, err := filepath.EvalSymlinks(filepath.Dir(path))
- if err != nil {
- return fmt.Errorf("%w: %v", ErrUntrustedSocket, err)
+ if err := checkSocketDirs(dir, owner); err != nil {
+ return "", err
}
+ return resolved, nil
+}
+
+// checkSocketDirs checks dir, which holds the socket, and every directory above it.
+func checkSocketDirs(dir string, owner int) error {
for first := true; ; first = false {
info, err := os.Lstat(dir)
if err != nil {