Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
sambox,node: measure what the boundary costs
Antonio Ojea24d4a46750a8
diff --git a/cmd/sam-box/main.go b/cmd/sam-box/main.goindex c62e5736..44f613a0 100644--- a/cmd/sam-box/main.go+++ b/cmd/sam-box/main.go@@ -47,6 +47,7 @@ func main() {issuer stringaudience stringinsecure bool+ metricsAddr stringlogLevel string)@@ -92,6 +93,13 @@ func main() {_ = os.Remove(sandboxSocket)}()+ if metricsAddr != "" {+ if _, err := sambox.ServeMetrics(cmd.Context(), metricsAddr); err != nil {+ return fmt.Errorf("serve metrics: %w", err)+ }+ logger.Infof("Serving metrics on http://%s/metrics", metricsAddr)+ }+server := &sambox.SOCKS5Server{Dialer: &sambox.AgentDialer{Router: &sambox.Router{Egress: egress},@@ -124,6 +132,7 @@ func main() {runCmd.Flags().StringVar(&issuer, "credential-issuer", "", "Issuer whose credentials attest an agent's identity, e.g. a cluster's service-account issuer; required with --bundle")runCmd.Flags().StringVar(&audience, "credential-audience", "", "Audience an agent's credential must be scoped to; required with --bundle")runCmd.Flags().BoolVar(&insecure, "insecure-unverified-bundle", false, "Trust the bundle's declared identity without a credential to back it, letting whoever can write the file decide which agent this sandbox is")+ runCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve unauthenticated Prometheus metrics on this address, e.g. 127.0.0.1:9600; off by default")runCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")for _, required := range []string{"socket", "sidecar-socket"} {if err := runCmd.MarkFlagRequired(required); err != nil {diff --git a/internal/node/sidecar.go b/internal/node/sidecar.goindex 39e6b2c5..ef3fee4f 100644--- a/internal/node/sidecar.go+++ b/internal/node/sidecar.go@@ -83,7 +83,7 @@ func StartSidecarServer(node *SamNode, addr, socketPath, token, certFile, keyFilmux.Handle("/", withAuth(token, true, withMeshConnection(node, mcpHandler)))server := &http.Server{- Handler: mux,+ Handler: observeRequests(mux),// Bound header-read time only: bodies/responses can legitimately stream// (MCP sessions, inference completions), so no ReadTimeout/WriteTimeout.ReadHeaderTimeout: 10 * time.Second,diff --git a/internal/node/sidecar_metrics.go b/internal/node/sidecar_metrics.gonew file mode 100644index 00000000..58767451--- /dev/null+++ b/internal/node/sidecar_metrics.go@@ -0,0 +1,152 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "net/http"+ "strconv"+ "strings"+ "time"++ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"+)++// Half of what the sidecar serves streams: an MCP session and a completion+// both hold the response open for as long as the work lasts. Total handler+// time therefore measures the workload, not the mesh. Time to first byte is+// the number that reflects what the mesh added, so both are recorded and the+// distinction is kept in the metric names rather than left to the reader.++var requestBuckets = []float64{+ 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30,+}++var (+ requestTTFBSeconds = promauto.NewHistogramVec(+ prometheus.HistogramOpts{+ Name: "sam_node_request_ttfb_seconds",+ Help: "Time from accepting a sidecar request to its first response byte",+ Buckets: requestBuckets,+ },+ []string{"route"},+ )++ requestDurationSeconds = promauto.NewHistogramVec(+ prometheus.HistogramOpts{+ Name: "sam_node_request_duration_seconds",+ Help: "Time a sidecar request occupied its handler, including streaming",+ Buckets: requestBuckets,+ },+ []string{"route", "code"},+ )++ requestsInFlight = promauto.NewGauge(+ prometheus.GaugeOpts{+ Name: "sam_node_requests_in_flight",+ Help: "Sidecar requests currently being served",+ },+ )+)++// classifyRoute reduces a path to a closed vocabulary. Paths carry peer ids and+// service names chosen off-node, so the raw path can never become a label.+func classifyRoute(path string) string {+ switch {+ case path == "/healthz", path == "/readyz", path == "/metrics":+ return "health"+ case strings.HasPrefix(path, "/sam/service/"):+ return "service-registry"+ case strings.HasPrefix(path, "/sam/"):+ return "egress"+ case path == "/v1/models":+ return "models"+ case path == "/v1/chat/completions", path == "/v1/completions":+ return "completions"+ default:+ return "mcp"+ }+}++// observeRequests times every sidecar request. It wraps the whole mux so a+// route added later is measured without anyone remembering to instrument it.+func observeRequests(next http.Handler) http.Handler {+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ route := classifyRoute(r.URL.Path)+ if route == "health" {+ next.ServeHTTP(w, r)+ return+ }++ start := time.Now()+ requestsInFlight.Inc()+ defer requestsInFlight.Dec()++ rec := &timedWriter{ResponseWriter: w, route: route, start: start}+ next.ServeHTTP(rec, r)++ requestDurationSeconds.WithLabelValues(route, strconv.Itoa(rec.status())).Observe(time.Since(start).Seconds())+ })+}++// timedWriter records when the first byte reached the client. It deliberately+// implements no optional interface itself: Unwrap lets http.ResponseController+// reach the real writer, so flushing and hijacking keep working for streams.+type timedWriter struct {+ http.ResponseWriter+ route string+ start time.Time+ code int+ ttfbOK bool+}++func (t *timedWriter) Unwrap() http.ResponseWriter { return t.ResponseWriter }++func (t *timedWriter) status() int {+ if t.code == 0 {+ return http.StatusOK+ }+ return t.code+}++func (t *timedWriter) observeTTFB() {+ if !t.ttfbOK {+ t.ttfbOK = true+ requestTTFBSeconds.WithLabelValues(t.route).Observe(time.Since(t.start).Seconds())+ }+}++func (t *timedWriter) WriteHeader(code int) {+ if t.code == 0 {+ t.code = code+ }+ t.observeTTFB()+ t.ResponseWriter.WriteHeader(code)+}++func (t *timedWriter) Write(b []byte) (int, error) {+ t.observeTTFB()+ return t.ResponseWriter.Write(b)+}++// Flush is kept because a streaming handler that type-asserts http.Flusher+// directly, rather than going through http.ResponseController, would otherwise+// stop finding one and buffer the whole stream.+func (t *timedWriter) Flush() {+ t.observeTTFB()+ if f, ok := t.ResponseWriter.(http.Flusher); ok {+ f.Flush()+ }+}diff --git a/internal/node/sidecar_metrics_test.go b/internal/node/sidecar_metrics_test.gonew file mode 100644index 00000000..a9feafe0--- /dev/null+++ b/internal/node/sidecar_metrics_test.go@@ -0,0 +1,93 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "net/http"+ "net/http/httptest"+ "testing"+)++func TestClassifyRouteIsAClosedVocabulary(t *testing.T) {+ // A route label must not vary with anything an off-node caller chooses,+ // or a peer could grow this node's metric space by inventing names.+ cases := map[string]string{+ "/healthz": "health",+ "/metrics": "health",+ "/sam/service/discover": "service-registry",+ "/sam/service/register": "service-registry",+ "/sam/12D3KooWpeer/mcp/calc/run": "egress",+ "/v1/models": "models",+ "/v1/chat/completions": "completions",+ "/v1/completions": "completions",+ "/": "mcp",+ "/mcp": "mcp",+ }+ for path, want := range cases {+ if got := classifyRoute(path); got != want {+ t.Errorf("classifyRoute(%q) = %q, want %q", path, got, want)+ }+ }+}++func TestObserveRequestsKeepsStreamingUsable(t *testing.T) {+ // The wrapper sits in front of MCP sessions and completions, both of which+ // stream. If it hides Flusher, a stream buffers until the handler returns+ // and every streaming consumer silently breaks.+ var (+ sawFlusher bool+ sawController bool+ )+ handler := observeRequests(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ _, sawFlusher = w.(http.Flusher)+ if err := http.NewResponseController(w).Flush(); err == nil {+ sawController = true+ }+ _, _ = w.Write([]byte("chunk"))+ }))++ rec := httptest.NewRecorder()+ handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/chat/completions", nil))++ if !sawFlusher {+ t.Error("handler could not type-assert http.Flusher through the wrapper")+ }+ if !sawController {+ t.Error("http.ResponseController could not reach the underlying writer")+ }+ if got := rec.Body.String(); got != "chunk" {+ t.Errorf("body = %q, want %q", got, "chunk")+ }+}++func TestObserveRequestsRecordsTheHandlerStatus(t *testing.T) {+ handler := observeRequests(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusTeapot)+ }))++ rec := httptest.NewRecorder()+ handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/models", nil))++ if rec.Code != http.StatusTeapot {+ t.Errorf("status = %d, want %d", rec.Code, http.StatusTeapot)+ }+}++func TestTimedWriterReportsOKWhenTheHandlerNeverSetAStatus(t *testing.T) {+ w := &timedWriter{ResponseWriter: httptest.NewRecorder(), route: "mcp"}+ if got := w.status(); got != http.StatusOK {+ t.Errorf("status = %d, want %d", got, http.StatusOK)+ }+}diff --git a/internal/sambox/dial.go b/internal/sambox/dial.goindex 161cbce5..8c77ea00 100644--- a/internal/sambox/dial.go+++ b/internal/sambox/dial.go@@ -19,6 +19,7 @@ import ("errors""fmt""net"+ "sync/atomic""syscall""time")@@ -61,14 +62,27 @@ func (d *AgentDialer) DialDestination(ctx context.Context, _ *Credentials, dst Dreturn nil, errors.New("sambox: AgentDialer requires a Router")}+ start := time.Now()+route, err := d.Router.Route(dst)if err != nil {+ recordFlow(routeUnresolved, 0, err)return nil, err}ctx, cancel := context.WithTimeout(ctx, dialTimeout)defer cancel()+ conn, err := d.dialRoute(ctx, route, dst)+ recordFlow(route.Kind.String(), time.Since(start), err)+ if err != nil {+ return nil, err+ }+ flowsActive.Inc()+ return &countedConn{Conn: conn}, nil+}++func (d *AgentDialer) dialRoute(ctx context.Context, route Route, dst Destination) (net.Conn, error) {switch route.Kind {case RouteMeshEntrypoint:return d.dialMeshEntrypoint()@@ -81,6 +95,33 @@ func (d *AgentDialer) DialDestination(ctx context.Context, _ *Credentials, dst D}}+// countedConn keeps the active-flow gauge honest. Both relay directions close+// their side, so the decrement has to happen exactly once.+type countedConn struct {+ net.Conn+ closed atomic.Bool+}++func (c *countedConn) Close() error {+ if c.closed.CompareAndSwap(false, true) {+ flowsActive.Dec()+ }+ return c.Conn.Close()+}++// CloseWrite keeps the half-close the relay depends on reachable through the+// wrapper. Advertising it unconditionally would be a trap: the relay falls back+// to a full close for connections that cannot half-close, and a wrapper that+// claims the capability without delivering it leaves the peer's copy blocked+// forever. So when the wrapped connection has no half-close, do what the relay+// would have done.+func (c *countedConn) CloseWrite() error {+ if cw, ok := c.Conn.(interface{ CloseWrite() error }); ok {+ return cw.CloseWrite()+ }+ return c.Close()+}+func (d *AgentDialer) dial(ctx context.Context, network, address string) (net.Conn, error) {if address == "" {return nil, fmt.Errorf("sambox: no %s address configured", network)diff --git a/internal/sambox/metrics.go b/internal/sambox/metrics.gonew file mode 100644index 00000000..7a07d6eb--- /dev/null+++ b/internal/sambox/metrics.go@@ -0,0 +1,90 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package sambox++import (+ "errors"+ "time"++ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"+)++// The boundary is the one place that sees every flow an agent opens, so it is+// the only honest place to measure what the boundary costs and what it refused.+// Labels stay closed vocabularies: a destination name is agent-controlled, and+// putting it in a label would let a sandbox grow the metric space without bound.++// flowSetupBuckets resolve from a sidecar hop on a Unix socket (tens of+// microseconds) up to a mesh dial that crosses the DHT (seconds). The default+// buckets start at 5ms, which is above the median this measures.+var flowSetupBuckets = []float64{+ 0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,+}++var (+ flowsTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "sam_box_flows_total",+ Help: "Flows a sandbox asked the boundary to open, by route class and outcome",+ },+ []string{"route", "outcome"},+ )++ flowSetupSeconds = promauto.NewHistogramVec(+ prometheus.HistogramOpts{+ Name: "sam_box_flow_setup_seconds",+ Help: "Time from an admitted CONNECT to a usable destination connection",+ Buckets: flowSetupBuckets,+ },+ []string{"route"},+ )++ flowsActive = promauto.NewGauge(+ prometheus.GaugeOpts{+ Name: "sam_box_flows_active",+ Help: "Flows currently relaying through the boundary",+ },+ )+)++// routeUnresolved labels a flow the router refused before it could be+// classified, so a denial is never miscounted against a real route.+const routeUnresolved = "unresolved"++// outcomeFor maps a dial result onto the closed vocabulary the counters use.+func outcomeFor(err error) string {+ switch {+ case err == nil:+ return "allowed"+ case errors.Is(err, ErrNotAllowed):+ return "denied"+ case errors.Is(err, ErrHostUnreachable):+ return "unreachable"+ case errors.Is(err, ErrConnectionRefused):+ return "refused"+ default:+ return "error"+ }+}++// recordFlow accounts one attempt to open a destination. setup is only+// meaningful when the attempt succeeded, so it is only observed then.+func recordFlow(route string, setup time.Duration, err error) {+ flowsTotal.WithLabelValues(route, outcomeFor(err)).Inc()+ if err == nil {+ flowSetupSeconds.WithLabelValues(route).Observe(setup.Seconds())+ }+}diff --git a/internal/sambox/metrics_server.go b/internal/sambox/metrics_server.gonew file mode 100644index 00000000..75264679--- /dev/null+++ b/internal/sambox/metrics_server.go@@ -0,0 +1,60 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package sambox++import (+ "context"+ "errors"+ "net"+ "net/http"+ "time"++ "github.com/prometheus/client_golang/prometheus/promhttp"+)++// ServeMetrics exposes this boundary's counters on addr until ctx ends.+//+// It is off unless an operator asks for it. The boundary sits between a+// sandbox and the mesh, so any listener it opens is one more thing reachable+// from wherever addr is bound; an experiment wants the numbers, a production+// sandbox usually does not. Nothing here is authenticated, which is why the+// caller has to name the address rather than get one by default.+func ServeMetrics(ctx context.Context, addr string) (*http.Server, error) {+ if addr == "" {+ return nil, errors.New("sambox: no metrics address configured")+ }++ listener, err := net.Listen("tcp", addr)+ if err != nil {+ return nil, err+ }++ mux := http.NewServeMux()+ mux.Handle("/metrics", promhttp.Handler())+ server := &http.Server{+ Handler: mux,+ ReadHeaderTimeout: 10 * time.Second,+ }++ go func() {+ <-ctx.Done()+ _ = server.Close()+ }()+ go func() {+ _ = server.Serve(listener)+ }()++ return server, nil+}diff --git a/internal/sambox/metrics_test.go b/internal/sambox/metrics_test.gonew file mode 100644index 00000000..93ce7a76--- /dev/null+++ b/internal/sambox/metrics_test.go@@ -0,0 +1,145 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package sambox++import (+ "errors"+ "fmt"+ "io"+ "net"+ "testing"+ "time"++ "github.com/prometheus/client_golang/prometheus"+ dto "github.com/prometheus/client_model/go"+)++func TestOutcomeForNamesEveryDenialReasonSeparately(t *testing.T) {+ // A denial and an unreachable host mean opposite things about a+ // deployment: one is policy working, the other is the mesh failing.+ // Collapsing them would make the counters unusable as evidence.+ cases := []struct {+ err error+ want string+ }{+ {nil, "allowed"},+ {fmt.Errorf("wrapped: %w", ErrNotAllowed), "denied"},+ {fmt.Errorf("wrapped: %w", ErrHostUnreachable), "unreachable"},+ {fmt.Errorf("wrapped: %w", ErrConnectionRefused), "refused"},+ {errors.New("something else"), "error"},+ }+ for _, tc := range cases {+ if got := outcomeFor(tc.err); got != tc.want {+ t.Errorf("outcomeFor(%v) = %q, want %q", tc.err, got, tc.want)+ }+ }+}++func TestRouteKindStringsAreStableLabels(t *testing.T) {+ // These strings are metric label values, so renaming one silently breaks+ // every dashboard and every recorded experiment that used it.+ cases := map[RouteKind]string{+ RouteMeshEntrypoint: "mesh-entrypoint",+ RouteMeshService: "mesh-service",+ RouteExternal: "external",+ }+ for kind, want := range cases {+ if got := kind.String(); got != want {+ t.Errorf("RouteKind(%d).String() = %q, want %q", int(kind), got, want)+ }+ }+}++func TestCountedConnPropagatesHalfClose(t *testing.T) {+ // The relay half-closes to signal EOF upstream. If the wrapper swallows+ // CloseWrite, a peer waiting on EOF hangs until a timeout instead.+ client, server := net.Pipe()+ defer func() { _ = client.Close() }()+ defer func() { _ = server.Close() }()++ var closed bool+ c := &countedConn{Conn: halfCloser{Conn: client, onCloseWrite: func() { closed = true }}}+ if err := c.CloseWrite(); err != nil {+ t.Fatalf("CloseWrite: %v", err)+ }+ if !closed {+ t.Error("CloseWrite did not reach the underlying connection")+ }+}++func TestCountedConnFallsBackToCloseWhenItCannotHalfClose(t *testing.T) {+ // The relay closes a connection outright when it cannot half-close, and+ // that full close is what unblocks the opposite copy. A wrapper that+ // advertises CloseWrite without delivering one defeats that fallback and+ // hangs the relay forever, which is a deadlock no metric test would show.+ client, server := net.Pipe()+ defer func() { _ = server.Close() }()++ c := &countedConn{Conn: client} // net.Pipe cannot half-close++ // The deadline goes on before the close, both because a closed pipe will+ // not accept one and so the pre-fix behaviour reports a blocked read+ // rather than hanging the package for the whole test timeout.+ if err := client.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {+ t.Fatalf("SetReadDeadline: %v", err)+ }+ if err := c.CloseWrite(); err != nil {+ t.Fatalf("CloseWrite: %v", err)+ }++ if _, err := client.Read(make([]byte, 1)); !errors.Is(err, io.ErrClosedPipe) {+ t.Errorf("read after CloseWrite = %v, want the connection closed", err)+ }+}++func TestCountedConnDecrementsOnceOnRepeatedClose(t *testing.T) {+ // Both relay directions close their side, so a naive decrement would run+ // twice and drive the active-flow gauge negative.+ client, server := net.Pipe()+ defer func() { _ = server.Close() }()++ c := &countedConn{Conn: client}+ flowsActive.Set(0)+ flowsActive.Inc()++ _ = c.Close()+ _ = c.Close()++ if got := gaugeValue(t, flowsActive); got != 0 {+ t.Errorf("flowsActive = %v after two closes, want 0", got)+ }+}++// gaugeValue reads a gauge without pulling in the prometheus test helpers,+// which would add a module for one assertion.+func gaugeValue(t *testing.T, g prometheus.Gauge) float64 {+ t.Helper()+ var m dto.Metric+ if err := g.Write(&m); err != nil {+ t.Fatalf("read gauge: %v", err)+ }+ return m.GetGauge().GetValue()+}++// halfCloser adds CloseWrite to a connection that lacks one.+type halfCloser struct {+ net.Conn+ onCloseWrite func()+}++func (h halfCloser) CloseWrite() error {+ h.onCloseWrite()+ return nil+}