Skip to content

Agent Mesh

feat(node): add per-request audit and accounting for FinOps telemetry

Antonio Ojeaf2a1c9e51386
diff --git a/internal/node/middleware.go b/internal/node/middleware.go
index c19079d3..db29951f 100644
--- a/internal/node/middleware.go
+++ b/internal/node/middleware.go
@@ -36,24 +36,55 @@ type RequestContext struct {
Target string
}
+// trackingStream wraps a network.Stream to count bytes read and written.
+type trackingStream struct {
+ network.Stream
+ bytesRead int64
+ bytesWritten int64
+}
+
+func (t *trackingStream) Read(p []byte) (n int, err error) {
+ n, err = t.Stream.Read(p)
+ t.bytesRead += int64(n)
+ return n, err
+}
+
+func (t *trackingStream) Write(p []byte) (n int, err error) {
+ n, err = t.Stream.Write(p)
+ t.bytesWritten += int64(n)
+ return n, err
+}
+
// WithBiscuitAuth enforces a Protobuf handshake on a stream before calling the next handler.
func (n *SamNode) WithBiscuitAuth(next func(network.Stream, RequestContext)) network.StreamHandler {
return func(s network.Stream) {
+ ts := &trackingStream{Stream: s}
+ remotePeer := s.Conn().RemotePeer()
+ var reqCtx RequestContext
+
defer func() {
- if err := s.Close(); err != nil {
+ if reqCtx.Target != "" { // Only log if we successfully got a request context
+ logger.Infow("Stream Accounting",
+ "peer_id", remotePeer.String(),
+ "target", reqCtx.Target,
+ "protocol", reqCtx.Protocol,
+ "bytes_read", ts.bytesRead,
+ "bytes_written", ts.bytesWritten,
+ )
+ }
+ if err := ts.Close(); err != nil {
logger.Debugf("[Auth] Failed to close stream: %v", err)
}
}()
- remotePeer := s.Conn().RemotePeer()
if !n.rateLimiter.Allow(remotePeer.String()) {
logger.Warnf("[Auth] Rate limit exceeded for %s, dropping connection", remotePeer)
- _ = s.Reset()
+ _ = ts.Reset()
return
}
// Read AuthFrame
- reader := msgio.NewVarintReaderSize(s, 1024*64)
+ reader := msgio.NewVarintReaderSize(ts, 1024*64)
msg, err := reader.ReadMsg()
if err != nil {
logger.Errorf("[Auth] Failed to read auth frame from %s: %v", remotePeer, err)
@@ -66,14 +97,15 @@ func (n *SamNode) WithBiscuitAuth(next func(network.Stream, RequestContext)) net
logger.Warnf("[Auth] Invalid auth frame from %s", remotePeer)
return
}
- reqCtx := RequestContext{
+
+ reqCtx = RequestContext{
PeerID: remotePeer,
User: "", // Not used in Authorize
- Protocol: string(s.Protocol()),
+ Protocol: string(ts.Protocol()),
Target: authFrame.TargetService,
}
- writer := msgio.NewVarintWriter(s)
+ writer := msgio.NewVarintWriter(ts)
err = n.VerifyBiscuitToken(authFrame.Biscuit, reqCtx)
if err != nil {
@@ -92,7 +124,7 @@ func (n *SamNode) WithBiscuitAuth(next func(network.Stream, RequestContext)) net
return
}
- next(s, reqCtx)
+ next(ts, reqCtx)
}
}
@@ -219,8 +251,48 @@ func (n *SamNode) Authorize(rawToken []byte, req RequestContext, pubKey ed25519.
if err != nil {
logger.Errorf("Authorizer failure: %v, token: %s", err, b.String())
logger.Debugf("Authorizer state: %s", authorizer.PrintWorld())
+ return err
}
- return err
+
+ var userStr, emailStr, roleStr string
+
+ if facts, _ := authorizer.Query(biscuit.Rule{
+ Head: biscuit.Predicate{Name: "get_user", IDs: []biscuit.Term{biscuit.Variable("u")}},
+ Body: []biscuit.Predicate{{Name: api.FactUser, IDs: []biscuit.Term{biscuit.Variable("u")}}},
+ }); len(facts) > 0 {
+ if s, ok := facts[0].Predicate.IDs[0].(biscuit.String); ok {
+ userStr = string(s)
+ }
+ }
+
+ if facts, _ := authorizer.Query(biscuit.Rule{
+ Head: biscuit.Predicate{Name: "get_email", IDs: []biscuit.Term{biscuit.Variable("e")}},
+ Body: []biscuit.Predicate{{Name: api.FactEmail, IDs: []biscuit.Term{biscuit.Variable("e")}}},
+ }); len(facts) > 0 {
+ if s, ok := facts[0].Predicate.IDs[0].(biscuit.String); ok {
+ emailStr = string(s)
+ }
+ }
+
+ if facts, _ := authorizer.Query(biscuit.Rule{
+ Head: biscuit.Predicate{Name: "get_role", IDs: []biscuit.Term{biscuit.Variable("r")}},
+ Body: []biscuit.Predicate{{Name: api.FactRole, IDs: []biscuit.Term{biscuit.Variable("r")}}},
+ }); len(facts) > 0 {
+ if s, ok := facts[0].Predicate.IDs[0].(biscuit.String); ok {
+ roleStr = string(s)
+ }
+ }
+
+ logger.Infow("Audit Traceability",
+ "peer_id", req.PeerID.String(),
+ "user", userStr,
+ "email", emailStr,
+ "role", roleStr,
+ "target", req.Target,
+ "protocol", req.Protocol,
+ )
+
+ return nil
}
func (n *SamNode) injectIdentityFacts(authorizer biscuit.Authorizer, pubKey ed25519.PublicKey) error {
diff --git a/internal/node/middleware_test.go b/internal/node/middleware_test.go
index 1995cf18..d91017c8 100644
--- a/internal/node/middleware_test.go
+++ b/internal/node/middleware_test.go
@@ -15,6 +15,7 @@
package node
import (
+ "bytes"
"context"
"crypto/ed25519"
"fmt"
@@ -693,3 +694,33 @@ func TestMiddlewareTargetChecks(t *testing.T) {
})
}
}
+
+func TestTrackingStream(t *testing.T) {
+ var pr1 bytes.Buffer
+ var pw1 bytes.Buffer
+
+ // Create mock underlying stream
+ mock := &mockStream{r: &pr1, w: &pw1, protocol: protocol.ID("/test")}
+ ts := &trackingStream{Stream: mock}
+
+ // Write data
+ writeData := []byte("hello world")
+ n, err := ts.Write(writeData)
+ if err != nil || n != len(writeData) {
+ t.Fatalf("Write failed: %v", err)
+ }
+ if ts.bytesWritten != int64(len(writeData)) {
+ t.Errorf("Expected bytesWritten to be %d, got %d", len(writeData), ts.bytesWritten)
+ }
+
+ // Read data
+ pr1.Write([]byte("response")) // Pre-load the read buffer
+ readBuf := make([]byte, 10)
+ n, err = ts.Read(readBuf)
+ if err != nil {
+ t.Fatalf("Read failed: %v", err)
+ }
+ if ts.bytesRead != int64(n) {
+ t.Errorf("Expected bytesRead to be %d, got %d", n, ts.bytesRead)
+ }
+}
diff --git a/internal/node/node.go b/internal/node/node.go
index 7ecc0f74..e20841db 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -1712,6 +1712,34 @@ func (n *SamNode) ListLocalServices(typeFilter api.ServiceType) []*api.ServiceIn
return n.services.List(typeFilter)
}
+type readCloserWithCount struct {
+ io.ReadCloser
+ bytesRead int64
+}
+
+func (rc *readCloserWithCount) Read(p []byte) (int, error) {
+ n, err := rc.ReadCloser.Read(p)
+ rc.bytesRead += int64(n)
+ return n, err
+}
+
+type responseWriterWithCount struct {
+ http.ResponseWriter
+ bytesWritten int64
+ statusCode int
+}
+
+func (w *responseWriterWithCount) Write(p []byte) (int, error) {
+ n, err := w.ResponseWriter.Write(p)
+ w.bytesWritten += int64(n)
+ return n, err
+}
+
+func (w *responseWriterWithCount) WriteHeader(statusCode int) {
+ w.statusCode = statusCode
+ w.ResponseWriter.WriteHeader(statusCode)
+}
+
func (n *SamNode) StartIngressServer(ctx context.Context) error {
listener, err := gostream.Listen(n.Host, "/libp2p-http")
if err != nil {
@@ -1798,7 +1826,23 @@ func (n *SamNode) StartIngressServer(ctx context.Context) error {
r.URL.Path = "/" + upstreamPath
}
r.URL.RawPath = ""
- svc.Handler().ServeHTTP(w, r)
+
+ // Wrap for accounting
+ rc := &readCloserWithCount{ReadCloser: r.Body}
+ r.Body = rc
+ wc := &responseWriterWithCount{ResponseWriter: w, statusCode: http.StatusOK}
+
+ defer func() {
+ logger.Infow("Stream Accounting",
+ "peer_id", remotePeer.String(),
+ "target", reqCtx.Target,
+ "protocol", "/libp2p-http",
+ "bytes_read", rc.bytesRead,
+ "bytes_written", wc.bytesWritten,
+ )
+ }()
+
+ svc.Handler().ServeHTTP(wc, r)
}),
}
diff --git a/tests/integration/datapath_test.go b/tests/integration/datapath_test.go
index 0ffb24c9..995dd7a5 100644
--- a/tests/integration/datapath_test.go
+++ b/tests/integration/datapath_test.go
@@ -20,6 +20,7 @@ import (
"io"
"net/http"
"net/http/httptest"
+ "os"
"path/filepath"
"strings"
"testing"
@@ -245,6 +246,7 @@ func TestIntegrationHTTPDatapath(t *testing.T) {
for i := 0; i < 3; i++ {
req, _ := http.NewRequest("GET", url, nil)
req.Header.Set("Authorization", "Bearer "+apiToken)
+ req.Close = true // Force close the connection so the libp2p stream terminates and flushed accounting logs
httpResp, err = client.Do(req)
if err == nil && httpResp.StatusCode == http.StatusOK {
break
@@ -265,8 +267,28 @@ func TestIntegrationHTTPDatapath(t *testing.T) {
if err != nil {
t.Fatal(err)
}
+ client.CloseIdleConnections()
if string(bodyBytes) != expectedBody {
t.Fatalf("Expected body %s, got %s", expectedBody, string(bodyBytes))
}
+
+ // Verify Audit Traceability and Stream Accounting logs were emitted
+ assertLogInFile(t, filepath.Join(homeA, "node.log"), "Audit Traceability")
+ assertLogInFile(t, filepath.Join(homeA, "node.log"), "Stream Accounting")
+ assertLogInFile(t, filepath.Join(homeA, "node.log"), "bytes_read")
+ assertLogInFile(t, filepath.Join(homeA, "node.log"), "bytes_written")
+}
+
+func assertLogInFile(t *testing.T, logFile, expectedMsg string) {
+ deadline := time.Now().Add(2 * time.Second)
+ for time.Now().Before(deadline) {
+ b, err := os.ReadFile(logFile)
+ if err == nil && strings.Contains(string(b), expectedMsg) {
+ return
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ b, _ := os.ReadFile(logFile)
+ t.Errorf("Expected log file %s to contain %q, but it didn't.\nLog end:\n%s", logFile, expectedMsg, string(b[len(b)-1000:])) // print last 1000 bytes
}