The cross-harness, self-improving memory layer for AI agents. Join our community: https://discord.com/invite/zdNChS2fBu
Go agent-ai-cliagent-harnessesagent-securitybrowser-extensionciclaude-codecloudcodexcursordetection-engineeringjevknowledge-basememorymobile-device-managementobservabilitysecuritysecurity-information-and-event-managementsecurity-toolstelemetrytraces
localingest: address review on the protocol and spool
Clauded3e50ee312f7
diff --git a/pkg/asymptoteobserve/localingest/accept_other.go b/pkg/asymptoteobserve/localingest/accept_other.gonew file mode 100644index 00000000..c99171d1--- /dev/null+++ b/pkg/asymptoteobserve/localingest/accept_other.go@@ -0,0 +1,6 @@+//go:build !unix++package localingest++// temporaryAcceptError is never true off Unix, where nothing serves the socket.+func temporaryAcceptError(error) bool { return false }diff --git a/pkg/asymptoteobserve/localingest/accept_unix.go b/pkg/asymptoteobserve/localingest/accept_unix.gonew file mode 100644index 00000000..5f04ae2a--- /dev/null+++ b/pkg/asymptoteobserve/localingest/accept_unix.go@@ -0,0 +1,19 @@+//go:build unix++package localingest++import (+ "errors"+ "syscall"+)++// temporaryAcceptError reports an accept failure that passes on its own: the process or system is+// out of descriptors or buffers for a moment, or a client gave up before it was accepted.+func temporaryAcceptError(err error) bool {+ for _, errno := range []syscall.Errno{syscall.EMFILE, syscall.ENFILE, syscall.ENOBUFS, syscall.ENOMEM, syscall.ECONNABORTED, syscall.EINTR} {+ if errors.Is(err, errno) {+ return true+ }+ }+ return false+}diff --git a/pkg/asymptoteobserve/localingest/client.go b/pkg/asymptoteobserve/localingest/client.goindex d45873b5..55593d7a 100644--- a/pkg/asymptoteobserve/localingest/client.go+++ b/pkg/asymptoteobserve/localingest/client.go@@ -42,7 +42,20 @@ func (c Client) path() string {// writer decides whether the endpoint is in the restricted mode at all; ErrNoSocket means it is// not.func (c Client) Available() error {- return checkSocket(c.path(), c.TrustedOwner)+ _, err := checkSocket(c.path(), c.TrustedOwner)+ return err+}++// Probe asks the collector to answer an empty batch for LogPath. It is what tells a live+// collector from a socket a killed one left behind, which Available cannot: that only looks at the+// file. ErrNoSocket and ErrLogMismatch keep their meaning.+func (c Client) Probe() error {+ socket, err := checkSocket(c.path(), c.TrustedOwner)+ if err != nil {+ return err+ }+ _, err = c.sendBatch(socket, nil)+ return err}// Send delivers lines in batches of at most MaxLinesPerConn and returns how far it got: lines@@ -60,7 +73,8 @@ func (c Client) Send(lines [][]byte) (Response, error) {}cleaned[i] = line}- if err := checkSocket(c.path(), c.TrustedOwner); err != nil {+ socket, err := checkSocket(c.path(), c.TrustedOwner)+ if err != nil {return Response{}, err}var total Response@@ -69,7 +83,7 @@ func (c Client) Send(lines [][]byte) (Response, error) {if end > len(cleaned) {end = len(cleaned)}- resp, err := c.sendBatch(cleaned[start:end])+ resp, err := c.sendBatch(socket, cleaned[start:end])total.Accepted += resp.Acceptedtotal.Rejected += resp.Rejectedtotal.Processed += resp.Processed@@ -81,12 +95,12 @@ func (c Client) Send(lines [][]byte) (Response, error) {return total, nil}-func (c Client) sendBatch(lines [][]byte) (Response, error) {+func (c Client) sendBatch(socket string, lines [][]byte) (Response, error) {timeout := c.Timeoutif timeout <= 0 {timeout = DefaultClientTimeout}- conn, err := net.DialTimeout("unix", c.path(), timeout)+ conn, err := net.DialTimeout("unix", socket, timeout)if err != nil {return Response{}, fmt.Errorf("connect to %s: %w", c.path(), err)}diff --git a/pkg/asymptoteobserve/localingest/localingest.go b/pkg/asymptoteobserve/localingest/localingest.goindex 9f4eaa37..d64e52bb 100644--- a/pkg/asymptoteobserve/localingest/localingest.go+++ b/pkg/asymptoteobserve/localingest/localingest.go@@ -36,9 +36,12 @@ package localingestimport ("bytes"+ "crypto/sha1"+ "encoding/hex""encoding/json""errors""fmt"+ "io""strconv""strings"@@ -117,8 +120,10 @@ type Response struct {//// The returned line is re-encoded, so whatever bytes the peer sent, exactly one JSON object on one// line reaches the log: nothing the sender writes can break the log into extra lines or forge a-// second event inside the first. Numbers are kept exactly as sent. event.id is always derived-// from the stamped line, never taken from the sender.+// second event inside the first. Numbers are kept exactly as sent.+//+// event.id follows the rules in stampEventID. A line that would only exceed MaxLineBytes because+// of the id is written without one, as every writer does with an event at the size limit.//// lookupName resolves a uid to an account name and may be nil; a name it cannot resolve is left// out rather than taken from the event, which the sender wrote.@@ -138,6 +143,15 @@ func StampLine(line []byte, peer Peer, lookupName func(uid uint32) string) ([]byreturn nil, fmt.Errorf("%w: %v", ErrRejected, err)}+ info := event["event"].(map[string]interface{})+ sentID, _ := info["id"].(string)+ sentID = strings.TrimSpace(sentID)+ delete(info, "id")+ claimed, err := marshalLine(event)+ if err != nil {+ return nil, fmt.Errorf("%w: %v", ErrRejected, err)+ }+user := map[string]interface{}{"uid": strconv.FormatUint(uint64(peer.UID), 10)}if lookupName != nil {if name := strings.TrimSpace(lookupName(peer.UID)); name != "" {@@ -145,27 +159,73 @@ func StampLine(line []byte, peer Peer, lookupName func(uid uint32) string) ([]by}}event["user"] = user-- // The id is derived here, from the line as it will be written, never taken from the sender:- // otherwise one account could reuse another's event.id, and consumers that deduplicate by id- // would drop the real event as a repeat. Deterministic, so a resent line keeps its id.- info := event["event"].(map[string]interface{})- delete(info, "id")- unstamped, err := marshalLine(event)+ withoutID, err := marshalLine(event)if err != nil {return nil, fmt.Errorf("%w: %v", ErrRejected, err)}- info["id"] = asymptoteobserve.EventIDForLine(unstamped)+ if len(withoutID) > MaxLineBytes {+ return nil, fmt.Errorf("%w: %d bytes after stamping is over the %d byte limit", ErrRejected, len(withoutID), MaxLineBytes)+ }+ info["id"] = stampEventID(sentID, claimed, withoutID, peer)out, err := marshalLine(event)if err != nil {return nil, fmt.Errorf("%w: %v", ErrRejected, err)}if len(out) > MaxLineBytes {- return nil, fmt.Errorf("%w: %d bytes after stamping is over the %d byte limit", ErrRejected, len(out), MaxLineBytes)+ // Forty-odd bytes of identity are not worth an event: the line goes without an id, which+ // is what the hook and CLI writers do with an event at the limit.+ return withoutID, nil}return out, nil}+// stampEventID decides the event.id of a stamped line. claimed is the line as the sender wrote it+// and withoutID the stamped line, both without an id.+//+// - No id, or the id derived from the stamped line: that derived id. This is every hook event+// whose sender claimed the account it is, and every call-keyed id (EventIDForLine names a tool+// call by its session, action, target and call id, not by account, so the hook and OTLP paths+// give one call one id).+// - The id derived from the sender's own claims but not from the stamped line: the line named an+// account it is not, so the id is derived again from what is written.+// - Any other id is one the sender chose, as session-store importers do from their source+// records. It is kept stable for that account, so a re-swept record keeps its id, but scoped to+// the verified uid, so one account cannot write under an id another account's events carry.+//+// A call-keyed id stays as strong as the session and call ids it is built from: an account that+// knows another session's ids can name the same call, as it could over the loopback OTLP port.+func stampEventID(sentID string, claimed, withoutID []byte, peer Peer) string {+ derived := asymptoteobserve.EventIDForLine(withoutID)+ switch {+ case sentID == "" || sentID == derived:+ return derived+ case sentID == asymptoteobserve.EventIDForLine(claimed):+ return derived+ default:+ return scopedEventID(peer.UID, sentID)+ }+}++// scopedIDNamespace scopes a sender-chosen event.id to the account that sent it. Like the event id+// namespace, it is a contract: changing it renames every scoped id written through the socket.+var scopedIDNamespace = [16]byte{0x6b, 0x2d, 0x1f, 0x9a, 0x4c, 0x37, 0x4e, 0x0b, 0x9f, 0x51, 0x8e, 0x2a, 0x77, 0xc3, 0x05, 0xd4}++// scopedEventID is a version 5 UUID of the uid and the id the sender chose.+func scopedEventID(uid uint32, sentID string) string {+ digest := sha1.New()+ digest.Write(scopedIDNamespace[:])+ io.WriteString(digest, strconv.FormatUint(uint64(uid), 10))+ digest.Write([]byte{0})+ io.WriteString(digest, sentID)+ var id [16]byte+ copy(id[:], digest.Sum(nil))+ id[6] = (id[6] & 0x0f) | 0x50+ id[8] = (id[8] & 0x3f) | 0x80+ out := make([]byte, 32)+ hex.Encode(out, id[:])+ return fmt.Sprintf("%s-%s-%s-%s-%s", out[0:8], out[8:12], out[12:16], out[16:20], out[20:32])+}+func decodeObject(line []byte) (map[string]interface{}, error) {dec := json.NewDecoder(bytes.NewReader(line))dec.UseNumber()diff --git a/pkg/asymptoteobserve/localingest/localingest_test.go b/pkg/asymptoteobserve/localingest/localingest_test.goindex 3789f75a..afdc2b1f 100644--- a/pkg/asymptoteobserve/localingest/localingest_test.go+++ b/pkg/asymptoteobserve/localingest/localingest_test.go@@ -213,15 +213,132 @@ func TestStampLineAcceptsATrailingCarriageReturn(t *testing.T) {}}-func TestStampLineNeverKeepsTheSendersEventID(t *testing.T) {+func stampedID(t *testing.T, out []byte) string {+ t.Helper()+ id, _ := decode(t, out)["event"].(map[string]interface{})["id"].(string)+ return id+}++// A hook names its event by EventIDForLine before sending. When it claimed the account it is, the+// server keeps that id: it is the one the shared mode would have written.+func TestStampLineKeepsAnIDDerivedFromWhatIsWritten(t *testing.T) {+ line := validLine(t, func(e map[string]interface{}) {+ e["user"] = map[string]interface{}{"uid": "501", "name": "alice"}+ delete(e["event"].(map[string]interface{}), "id")+ })+ want := asymptoteobserve.EventIDForLine(line)+ withID := validLine(t, func(e map[string]interface{}) {+ e["user"] = map[string]interface{}{"uid": "501", "name": "alice"}+ e["event"].(map[string]interface{})["id"] = want+ })+ out, err := StampLine(withID, Peer{UID: 501}, func(uint32) string { return "alice" })+ if err != nil {+ t.Fatal(err)+ }+ if got := stampedID(t, out); got != want {+ t.Fatalf("event.id = %s, want the sender's derived %s", got, want)+ }+}++// A call-keyed id names a tool call, not an account, so it survives stamping: the hook and the OTLP+// path give one call one id.+func TestStampLineKeepsACallKeyedID(t *testing.T) {+ call := func(e map[string]interface{}) {+ e["event"] = map[string]interface{}{"action": "tool.invoked", "category": "tool"}+ e["session"] = map[string]interface{}{"id": "s-1"}+ e["tool"] = map[string]interface{}{"name": "Bash"}+ e["command"] = map[string]interface{}{"command": "ls"}+ e["gen_ai"] = map[string]interface{}{"tool": map[string]interface{}{"name": "Bash", "call": map[string]interface{}{"id": "toolu_1"}}}+ e["user"] = map[string]interface{}{"uid": "0", "name": "root"}+ }+ plain := validLine(t, call)+ sent := asymptoteobserve.EventIDForLine(plain)line := validLine(t, func(e map[string]interface{}) {- e["event"].(map[string]interface{})["id"] = "someone-elses-event-id"+ call(e)+ e["event"].(map[string]interface{})["id"] = sent})- out, err := StampLine(line, Peer{UID: 501}, nil)+ out, err := StampLine(line, Peer{UID: 501}, func(uint32) string { return "alice" })+ if err != nil {+ t.Fatal(err)+ }+ stamped := decode(t, out)+ delete(stamped["event"].(map[string]interface{}), "id")+ again, _ := json.Marshal(stamped)+ if got, want := stampedID(t, out), asymptoteobserve.EventIDForLine(again); got != want {+ t.Fatalf("event.id = %s, want %s derived from what is written", got, want)+ }+}++// A line that claimed another account carries an id derived from that claim; once the account is+// corrected, the id is derived again from what is written.+func TestStampLineRederivesTheIDOfALineThatClaimedAnotherAccount(t *testing.T) {+ claimed := validLine(t, func(e map[string]interface{}) {+ e["user"] = map[string]interface{}{"uid": "0", "name": "root"}+ delete(e["event"].(map[string]interface{}), "id")+ })+ sent := asymptoteobserve.EventIDForLine(claimed)+ line := validLine(t, func(e map[string]interface{}) {+ e["user"] = map[string]interface{}{"uid": "0", "name": "root"}+ e["event"].(map[string]interface{})["id"] = sent+ })+ out, err := StampLine(line, Peer{UID: 501}, func(uint32) string { return "alice" })if err != nil {t.Fatal(err)}- if id := decode(t, out)["event"].(map[string]interface{})["id"]; id == "someone-elses-event-id" || id == "" {- t.Fatalf("event.id = %v; a sender must not choose it", id)+ if got := stampedID(t, out); got == sent || got == "" {+ t.Fatalf("event.id = %q; an id derived from a false claim must not be kept", got)+ }+}++// An id the sender chose itself (a session-store importer's source coordinate) stays stable for+// that account, so a re-swept record keeps its id, but cannot be another account's.+func TestStampLineScopesASenderChosenIDToItsAccount(t *testing.T) {+ line := validLine(t, func(e map[string]interface{}) {+ e["event"].(map[string]interface{})["id"] = "claude:session-1:record-42"+ })+ first, err := StampLine(line, Peer{UID: 501}, nil)+ if err != nil {+ t.Fatal(err)+ }+ again, _ := StampLine(line, Peer{UID: 501}, nil)+ other, _ := StampLine(line, Peer{UID: 502}, nil)+ a, b, c := stampedID(t, first), stampedID(t, again), stampedID(t, other)+ if a == "claude:session-1:record-42" || a == "" {+ t.Fatalf("event.id = %q; a sender-chosen id must be scoped", a)+ }+ if a != b {+ t.Fatal("a scoped id is not stable across resends")+ }+ if a == c {+ t.Fatal("two accounts sending the same chosen id got the same event.id")+ }+ if a != scopedEventID(501, "claude:session-1:record-42") {+ t.Fatal("scoped id does not match its definition")+ }+}++// An event at the limit, which only the id would push over, is written without one, the way the+// writers handle it, rather than lost.+func TestStampLineDropsTheIDRatherThanTheEvent(t *testing.T) {+ user := map[string]interface{}{"uid": "501", "name": "alice"}+ base := validLine(t, func(e map[string]interface{}) {+ e["user"] = user+ e["message"] = ""+ delete(e["event"].(map[string]interface{}), "id")+ })+ line := validLine(t, func(e map[string]interface{}) {+ e["user"] = user+ e["message"] = strings.Repeat("m", MaxLineBytes-len(base)-5)+ delete(e["event"].(map[string]interface{}), "id")+ })+ if len(line) > MaxLineBytes {+ t.Fatal("fixture over the limit")+ }+ out, err := StampLine(line, Peer{UID: 501}, func(uint32) string { return "alice" })+ if err != nil {+ t.Fatalf("an event the id alone would push over the limit was rejected: %v", err)+ }+ if len(out) > MaxLineBytes || stampedID(t, out) != "" {+ t.Fatalf("got %d bytes, id %q; want the line within the limit and without an id", len(out), stampedID(t, out))}}diff --git a/pkg/asymptoteobserve/localingest/server.go b/pkg/asymptoteobserve/localingest/server.goindex e58f840a..9dc5048a 100644--- a/pkg/asymptoteobserve/localingest/server.go+++ b/pkg/asymptoteobserve/localingest/server.go@@ -72,6 +72,7 @@ type Server struct {wg sync.WaitGroupbuckets map[uint32]*bucketperUID map[uint32]int+ logStates map[uint32]*logState}// ErrServerClosed is returned by Serve after Close.@@ -91,6 +92,7 @@ func (s *Server) Serve(l net.Listener) error {return ErrServerClosed}defer s.untrack(l)+ var retry time.Durationfor {conn, err := l.Accept()if err != nil {@@ -101,12 +103,28 @@ func (s *Server) Serve(l net.Listener) error {if errors.As(err, &ne) && ne.Timeout() {continue}+ if temporaryAcceptError(err) {+ // Out of descriptors or memory for a moment, as net/http treats it: wait and accept+ // again rather than stop serving for good.+ if retry == 0 {+ retry = 5 * time.Millisecond+ } else if retry *= 2; retry > time.Second {+ retry = time.Second+ }+ s.logf(noUID, "accept failed, retrying in %v: %v", retry, err)+ time.Sleep(retry)+ continue+ }+ // Anything else ends serving. The listener is closed so the socket goes away and+ // clients fall back at once, rather than queueing in a backlog nothing accepts.+ _ = l.Close()return err}+ retry = 0select {case s.slots <- struct{}{}:default:- s.logf("refusing connection: %d already open", cap(s.slots))+ s.logf(noUID, "refusing connection: %d already open", cap(s.slots))_ = conn.Close()continue}@@ -197,10 +215,46 @@ func (s *Server) isClosed() bool {return s.closed}-func (s *Server) logf(format string, args ...interface{}) {- if s.Logf != nil {- s.Logf(format, args...)+// noUID keys log lines that belong to no one connection.+const noUID = ^uint32(0)++// logInterval is the least time between two log lines about one account. Any account can connect,+// and each refused connection or rejected line is something to log, so without a bound one+// account could fill the root collector's log as fast as it can send.+const logInterval = 10 * time.Second++type logState struct {+ last time.Time+ suppressed int+}++func (s *Server) logf(uid uint32, format string, args ...interface{}) {+ if s.Logf == nil {+ return+ }+ now := time.Now()+ s.mu.Lock()+ if s.logStates == nil {+ s.logStates = map[uint32]*logState{}}+ st := s.logStates[uid]+ if st == nil {+ st = &logState{}+ s.logStates[uid] = st+ }+ if !st.last.IsZero() && now.Sub(st.last) < logInterval {+ st.suppressed+++ s.mu.Unlock()+ return+ }+ suppressed := st.suppressed+ st.last, st.suppressed = now, 0+ s.mu.Unlock()+ msg := fmt.Sprintf(format, args...)+ if suppressed > 0 {+ msg = fmt.Sprintf("%s (and %d more like it from this account since the last report)", msg, suppressed)+ }+ s.Logf("%s", msg)}func (s *Server) handle(conn net.Conn) {@@ -208,21 +262,24 @@ func (s *Server) handle(conn net.Conn) {if timeout <= 0 {timeout = defaultConnTimeout}+ // Reads may run past the budget by replyGrace: a read that fails at the budget itself would+ // end the connection with no reply after lines were already written, and the client would+ // send them all again. Past the budget the loop below stops handling lines and replies.budget := time.Now().Add(timeout)- _ = conn.SetDeadline(budget)+ _ = conn.SetDeadline(budget.Add(replyGrace))unixConn, ok := conn.(*net.UnixConn)if !ok {- s.logf("refusing connection: not a Unix socket")+ s.logf(noUID, "refusing connection: not a Unix socket")return}peer, err := peerCredentials(unixConn)if err != nil {- s.logf("refusing connection: %v", err)+ s.logf(noUID, "refusing connection: %v", err)return}if !s.acquireUID(peer.UID) {- s.logf("refusing connection from uid %d: it already has the most connections one account may hold", peer.UID)+ s.logf(peer.UID, "refusing connection from uid %d: it already has the most connections one account may hold", peer.UID)return}defer s.releaseUID(peer.UID)@@ -237,7 +294,7 @@ func (s *Server) handle(conn net.Conn) {r := bufio.NewReaderSize(conn, MaxLineBytes+2)preamble, tooLong, err := readLine(r, len(Preamble)+1)if err != nil && !errors.Is(err, io.EOF) || tooLong || string(preamble) != Preamble {- s.logf("refusing connection from uid %d: no %s preamble", peer.UID, Preamble)+ s.logf(peer.UID, "refusing connection from uid %d: no %s preamble", peer.UID, Preamble)writeResponse(conn, Response{Error: "expected " + Preamble})return}@@ -248,7 +305,7 @@ func (s *Server) handle(conn net.Conn) {var header Headerswitch {case err != nil && !errors.Is(err, io.EOF), tooLong, json.Unmarshal(headerLine, &header) != nil:- s.logf("refusing connection from uid %d: no valid header", peer.UID)+ s.logf(peer.UID, "refusing connection from uid %d: no valid header", peer.UID)writeResponse(conn, Response{Error: "expected a header"})returncase s.LogPath != "" && !sameLog(header.Log, s.LogPath):@@ -260,11 +317,18 @@ func (s *Server) handle(conn net.Conn) {for {line, tooLong, err := readLine(r, MaxLineBytes)if err != nil && !errors.Is(err, io.EOF) {- // A read that failed part way (a timeout, a reset) may have cut the last line short, so- // it is not handled, and the client gets no reply: it keeps the whole batch and sends it- // again. A line written before the failure may then appear twice, under the same- // deterministic event.id, which is the trade the log already makes on any retried write.- s.logf("connection from uid %d ended early: %v", peer.UID, err)+ var ne net.Error+ if errors.As(err, &ne) && ne.Timeout() {+ // Out of time with the rest of the batch unread, or a line cut short: whatever is+ // unread is not handled, and the reply says how far this got so the client resends+ // only the rest.+ if resp.Error == "" {+ resp.Error = "busy"+ }+ break+ }+ // The connection broke (a reset): there is nobody to reply to.+ s.logf(peer.UID, "connection from uid %d ended early: %v", peer.UID, err)return}if len(line) == 0 && !tooLong {@@ -285,13 +349,12 @@ func (s *Server) handle(conn net.Conn) {// Out of time: stop writing and say how far this got, while the client still waits.resp.Error = "busy"stopped = true- _ = conn.SetDeadline(time.Now().Add(replyGrace))case lines > MaxLinesPerConn:resp.Error = fmt.Sprintf("batch is over %d lines", MaxLinesPerConn)stopped = truecase tooLong:resp.Rejected++- s.logf("rejected an event from uid %d: over %d bytes", peer.UID, MaxLineBytes)+ s.logf(peer.UID, "rejected an event from uid %d: over %d bytes", peer.UID, MaxLineBytes)case !s.allow(peer.UID, len(line)):resp.Error = "rate limited"stopped = true@@ -299,7 +362,7 @@ func (s *Server) handle(conn net.Conn) {stamped, stampErr := StampLine(line, peer, lookup)if stampErr != nil {resp.Rejected++- s.logf("rejected an event from uid %d: %v", peer.UID, stampErr)+ s.logf(peer.UID, "rejected an event from uid %d: %v", peer.UID, stampErr)break}if appendErr := s.Append(stamped); appendErr != nil {@@ -308,7 +371,7 @@ func (s *Server) handle(conn net.Conn) {break}resp.Error = "write failed"- s.logf("could not write an event from uid %d: %v", peer.UID, appendErr)+ s.logf(peer.UID, "could not write an event from uid %d: %v", peer.UID, appendErr)stopped = truebreak}@@ -319,9 +382,7 @@ func (s *Server) handle(conn net.Conn) {}}resp.Processed = resp.Accepted + resp.Rejected- if time.Now().After(budget) {- _ = conn.SetWriteDeadline(time.Now().Add(replyGrace))- }+ _ = conn.SetWriteDeadline(time.Now().Add(replyGrace))writeResponse(conn, resp)}diff --git a/pkg/asymptoteobserve/localingest/server_unix_test.go b/pkg/asymptoteobserve/localingest/server_unix_test.goindex e513a2bb..97d1dcc7 100644--- a/pkg/asymptoteobserve/localingest/server_unix_test.go+++ b/pkg/asymptoteobserve/localingest/server_unix_test.go@@ -345,7 +345,9 @@ func TestServerTakesAFinalLineWithoutANewline(t *testing.T) {}}-func TestServerDropsAConnectionThatStalls(t *testing.T) {+// A client that stalls mid-batch gets a reply naming how far the server got, so it resends only the+// rest; the line it cut short is not handled.+func TestServerAnswersAConnectionThatStalls(t *testing.T) {h := startServer(t, func(s *Server) { s.ConnTimeout = 150 * time.Millisecond })conn, err := net.Dial("unix", h.path)if err != nil {@@ -359,14 +361,61 @@ func TestServerDropsAConnectionThatStalls(t *testing.T) {}_ = conn.SetReadDeadline(time.Now().Add(3 * time.Second))reply, err := bufio.NewReader(conn).ReadBytes('\n')- if err == nil {- t.Fatalf("got reply %q from a stalled connection; want it closed without one", reply)+ if err != nil {+ t.Fatalf("no reply from a server whose client stalled: %v", err)+ }+ var resp Response+ if err := json.Unmarshal(reply, &resp); err != nil {+ t.Fatal(err)+ }+ if resp.Processed != 1 || resp.Error != "busy" {+ t.Fatalf("resp = %+v, want the one complete line reported and the rest left", resp)}if h.rec.count() != 1 {t.Fatalf("wrote %d lines, want only the complete one", h.rec.count())}}+// The last line's write running past the budget must not cost the reply: the client would resend+// a line already written.+func TestASlowLastWriteStillGetsAReply(t *testing.T) {+ h := startServer(t, func(s *Server) { s.ConnTimeout = 100 * time.Millisecond })+ h.rec.setFail(func(int, []byte) error {+ time.Sleep(250 * time.Millisecond)+ return nil+ })+ resp, err := h.client.Send(lines(t, 1))+ if err != nil || resp.Accepted != 1 {+ t.Fatalf("Send = %+v, %v; want the written line acknowledged", resp, err)+ }+}++// A batch bigger than the server's read buffer, whose budget runs out after the buffer is used,+// gets a reply that matches what was written.+func TestABusyServerReportsHowFarItGotPastItsReadBuffer(t *testing.T) {+ h := startServer(t, func(s *Server) {+ s.ConnTimeout = 150 * time.Millisecond+ s.RateBytesPerSecond = -1+ })+ h.rec.setFail(func(int, []byte) error {+ time.Sleep(40 * time.Millisecond)+ return nil+ })+ big := make([][]byte, 12)+ for i := range big {+ big[i] = validLine(t, func(e map[string]interface{}) {+ e["message"] = fmt.Sprintf("m-%d-%s", i, strings.Repeat("x", 20*1024))+ })+ }+ resp, err := h.client.Send(big)+ if err == nil || resp.Error != "busy" {+ t.Fatalf("Send = %+v, %v; want the server to stop and report", resp, err)+ }+ if resp.Processed == 0 || resp.Processed != h.rec.count() {+ t.Fatalf("processed %d, wrote %d; the reply must match what was written", resp.Processed, h.rec.count())+ }+}+func TestRateLimitStopsAFloodingAccount(t *testing.T) {now := time.Unix(1000, 0)h := startServer(t, func(s *Server) {@@ -856,3 +905,135 @@ func TestRemoveStale(t *testing.T) {t.Fatal(err)}}++func TestProbe(t *testing.T) {+ h := startServer(t, func(s *Server) { s.LogPath = "/var/log/beacon-agent/runtime.jsonl" })+ c := h.client+ c.LogPath = "/var/log/beacon-agent/runtime.jsonl"+ if err := c.Probe(); err != nil {+ t.Fatalf("Probe of a live collector = %v", err)+ }+ if h.rec.count() != 0 {+ t.Fatal("a probe wrote something")+ }+ c.LogPath = "/tmp/other.jsonl"+ if err := c.Probe(); !errors.Is(err, ErrLogMismatch) {+ t.Fatalf("Probe for another log = %v", err)+ }+ stale := filepath.Join(shortTempDir(t), "run", "ingest.sock")+ l, err := Listen(stale)+ if err != nil {+ t.Fatal(err)+ }+ l.SetUnlinkOnClose(false)+ _ = l.Close()+ dead := Client{SocketPath: stale, TrustedOwner: os.Geteuid(), Timeout: time.Second}+ if err := dead.Available(); err != nil {+ t.Fatalf("fixture: Available on a dead socket = %v", err)+ }+ if err := dead.Probe(); err == nil || errors.Is(err, ErrNoSocket) {+ t.Fatalf("Probe of a dead socket = %v; want a connection failure", err)+ }+}++// The client dials the path it checked, with links resolved, so a link it passed through cannot+// be repointed between the check and the dial.+func TestClientDialsTheResolvedSocket(t *testing.T) {+ h := startServer(t, nil)+ link := filepath.Join(shortTempDir(t), "linkdir")+ if err := os.Symlink(filepath.Dir(h.path), link); err != nil {+ t.Fatal(err)+ }+ c := h.client+ c.SocketPath = filepath.Join(link, filepath.Base(h.path))+ socket, err := checkSocket(c.SocketPath, os.Geteuid())+ if err != nil {+ t.Fatal(err)+ }+ want, _ := filepath.EvalSymlinks(h.path)+ if socket != want {+ t.Fatalf("checkSocket returned %q, want the resolved %q", socket, want)+ }+ if _, err := c.Send(lines(t, 1)); err != nil || h.rec.count() != 1 {+ t.Fatalf("Send through a linked directory = %v", err)+ }+}++func TestServeClosesItsListenerWhenAcceptFails(t *testing.T) {+ dir := filepath.Join(shortTempDir(t), "run")+ l, err := Listen(filepath.Join(dir, "ingest.sock"))+ if err != nil {+ t.Fatal(err)+ }+ failing := &failingListener{Listener: l, err: errors.New("accept: something permanent")}+ s := &Server{Append: func([]byte) error { return nil }}+ if err := s.Serve(failing); err == nil || errors.Is(err, ErrServerClosed) {+ t.Fatalf("Serve = %v", err)+ }+ if _, err := os.Lstat(filepath.Join(dir, "ingest.sock")); !errors.Is(err, os.ErrNotExist) {+ t.Fatal("a server that stopped serving left its socket for clients to queue on")+ }+}++func TestServeRetriesATemporaryAcceptError(t *testing.T) {+ dir := filepath.Join(shortTempDir(t), "run")+ sock := filepath.Join(dir, "ingest.sock")+ l, err := Listen(sock)+ if err != nil {+ t.Fatal(err)+ }+ failing := &failingListener{Listener: l, err: syscall.EMFILE, times: 3}+ rec := &recorder{}+ s := &Server{Append: rec.append}+ go func() { _ = s.Serve(failing) }()+ t.Cleanup(func() { _ = s.Close() })+ c := Client{SocketPath: sock, TrustedOwner: os.Geteuid(), Timeout: 2 * time.Second}+ if _, err := c.Send(lines(t, 1)); err != nil || rec.count() != 1 {+ t.Fatalf("Send after temporary accept errors = %v", err)+ }+}++type failingListener struct {+ net.Listener+ mu sync.Mutex+ err error+ times int // 0 means always+}++func (l *failingListener) Accept() (net.Conn, error) {+ l.mu.Lock()+ if l.times == 0 || l.times > 0 {+ if l.times > 0 {+ l.times--+ if l.times == 0 {+ l.times = -1+ }+ }+ l.mu.Unlock()+ return nil, &net.OpError{Op: "accept", Net: "unix", Err: l.err}+ }+ l.mu.Unlock()+ return l.Listener.Accept()+}++func TestServerLogsAnAccountAtMostOncePerInterval(t *testing.T) {+ var mu sync.Mutex+ var logged []string+ h := startServer(t, func(s *Server) {+ s.Logf = func(format string, args ...interface{}) {+ mu.Lock()+ logged = append(logged, fmt.Sprintf(format, args...))+ mu.Unlock()+ }+ })+ for i := 0; i < 20; i++ {+ if _, err := rawExchange(t, h.path, "no preamble\n"); err != nil {+ t.Fatal(err)+ }+ }+ mu.Lock()+ defer mu.Unlock()+ if len(logged) != 1 {+ t.Fatalf("logged %d lines for one account's flood, want 1: %q", len(logged), logged)+ }+}diff --git a/pkg/asymptoteobserve/localingest/spool.go b/pkg/asymptoteobserve/localingest/spool.goindex 22d7c38d..68c2705c 100644--- a/pkg/asymptoteobserve/localingest/spool.go+++ b/pkg/asymptoteobserve/localingest/spool.go@@ -70,12 +70,17 @@ func (s Spool) Append(lines [][]byte) (dropped int, err error) {if len(lines) == 0 {return 0, nil}- err = s.withLock(func() error {+ err = s.withLock(func() (err error) {f, err := os.OpenFile(s.filePath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, SpoolFileMode)if err != nil {return err}- defer f.Close()+ // A close can report a write the kernel deferred; losing it would lose the line silently.+ defer func() {+ if cerr := f.Close(); cerr != nil && err == nil {+ err = cerr+ }+ }()info, err := f.Stat()if err != nil {return err@@ -117,7 +122,7 @@ func (s Spool) Append(lines [][]byte) (dropped int, err error) {// batch and to remove what was sent, not while sending, so a hook spooling an event meanwhile is// not held up by a slow collector: appends only ever add to the end, and only the drainer removes// from the start.-func (s Spool) Drain(maxLines int, send func(lines [][]byte) (int, error)) (int, error) {+func (s Spool) Drain(maxLines int, send func(lines [][]byte) (int, error)) (n int, err error) {if maxLines <= 0 {return 0, nil}@@ -128,7 +133,7 @@ func (s Spool) Drain(maxLines int, send func(lines [][]byte) (int, error)) (int,if err != nil {return 0, err}- defer drainFile.Close()+ defer closeLock(drainFile, &err)draining, err := filelock.TryExclusive(drainFile)if err != nil {return 0, nil@@ -181,12 +186,12 @@ func (s Spool) Drain(maxLines int, send func(lines [][]byte) (int, error)) (int,return processed, sendErr}-func (s Spool) withLock(fn func() error) error {+func (s Spool) withLock(fn func() error) (err error) {lockFile, err := s.openLock()if err != nil {return err}- defer lockFile.Close()+ defer closeLock(lockFile, &err)wait := s.LockWaitif wait <= 0 {wait = defaultSpoolLockWait@@ -254,3 +259,11 @@ func splitLines(data []byte) [][]byte {}return out}++// closeLock closes a lock file and reports a failure to close it when nothing else failed first.+// Lock files are never written, so a close error loses no data, but it is still not dropped.+func closeLock(f *os.File, err *error) {+ if cerr := f.Close(); cerr != nil && *err == nil {+ *err = cerr+ }+}diff --git a/pkg/asymptoteobserve/localingest/trust_other.go b/pkg/asymptoteobserve/localingest/trust_other.goindex 22a42c1b..5ec9c3c1 100644--- a/pkg/asymptoteobserve/localingest/trust_other.go+++ b/pkg/asymptoteobserve/localingest/trust_other.go@@ -9,7 +9,7 @@ import (// checkSocket always reports no socket off Unix: the restricted log access mode is not available// there, so writers keep appending to the log as before.-func checkSocket(string, int) error { return ErrNoSocket }+func checkSocket(string, int) (string, error) { return "", ErrNoSocket }// RemoveStale has nothing to remove off Unix.func RemoveStale(string) error { return nil }diff --git a/pkg/asymptoteobserve/localingest/trust_unix.go b/pkg/asymptoteobserve/localingest/trust_unix.goindex d2a0745b..976aef9a 100644--- a/pkg/asymptoteobserve/localingest/trust_unix.go+++ b/pkg/asymptoteobserve/localingest/trust_unix.go@@ -24,24 +24,41 @@ const SocketMode os.FileMode = 0o666// must its directory, which nobody else may write; every directory above it must belong to root or// owner and be writable by nobody else unless it is sticky (as /tmp is), so the directory holding// the socket cannot be swapped for another.-func checkSocket(path string, owner int) error {- info, err := os.Lstat(path)+//+// It returns the socket's path with every link in its directories resolved, and the caller dials+// that path, never the one it was given: a link anywhere in the given path could otherwise be+// repointed between this check and the dial, at a socket of someone else's choosing. Resolving+// keeps the default path working, whose /var/run is a root-owned link.+func checkSocket(path string, owner int) (string, error) {+ dir, err := filepath.EvalSymlinks(filepath.Dir(path))+ if err != nil {+ if errors.Is(err, os.ErrNotExist) {+ return "", ErrNoSocket+ }+ return "", fmt.Errorf("%w: %v", ErrUntrustedSocket, err)+ }+ resolved := filepath.Join(dir, filepath.Base(path))+ info, err := os.Lstat(resolved)if err != nil {if errors.Is(err, os.ErrNotExist) {- return ErrNoSocket+ return "", ErrNoSocket}- return fmt.Errorf("%w: %v", ErrUntrustedSocket, err)+ return "", fmt.Errorf("%w: %v", ErrUntrustedSocket, err)}if info.Mode()&os.ModeSocket == 0 {- return fmt.Errorf("%w: %s is not a socket", ErrUntrustedSocket, path)+ return "", fmt.Errorf("%w: %s is not a socket", ErrUntrustedSocket, path)}if uid, ok := fileOwner(info); !ok || uid != owner {- return fmt.Errorf("%w: %s is not owned by uid %d", ErrUntrustedSocket, path, owner)+ return "", fmt.Errorf("%w: %s is not owned by uid %d", ErrUntrustedSocket, path, owner)}- dir, err := filepath.EvalSymlinks(filepath.Dir(path))- if err != nil {- return fmt.Errorf("%w: %v", ErrUntrustedSocket, err)+ if err := checkSocketDirs(dir, owner); err != nil {+ return "", err}+ return resolved, nil+}++// checkSocketDirs checks dir, which holds the socket, and every directory above it.+func checkSocketDirs(dir string, owner int) error {for first := true; ; first = false {info, err := os.Lstat(dir)if err != nil {