Skip to content

Agent Mesh

sambox,node: measure what the boundary costs

Antonio Ojea24d4a46750a8
diff --git a/cmd/sam-box/main.go b/cmd/sam-box/main.go
index c62e5736..44f613a0 100644
--- a/cmd/sam-box/main.go
+++ b/cmd/sam-box/main.go
@@ -47,6 +47,7 @@ func main() {
issuer string
audience string
insecure bool
+ metricsAddr string
logLevel string
)
@@ -92,6 +93,13 @@ func main() {
_ = os.Remove(sandboxSocket)
}()
+ if metricsAddr != "" {
+ if _, err := sambox.ServeMetrics(cmd.Context(), metricsAddr); err != nil {
+ return fmt.Errorf("serve metrics: %w", err)
+ }
+ logger.Infof("Serving metrics on http://%s/metrics", metricsAddr)
+ }
+
server := &sambox.SOCKS5Server{
Dialer: &sambox.AgentDialer{
Router: &sambox.Router{Egress: egress},
@@ -124,6 +132,7 @@ func main() {
runCmd.Flags().StringVar(&issuer, "credential-issuer", "", "Issuer whose credentials attest an agent's identity, e.g. a cluster's service-account issuer; required with --bundle")
runCmd.Flags().StringVar(&audience, "credential-audience", "", "Audience an agent's credential must be scoped to; required with --bundle")
runCmd.Flags().BoolVar(&insecure, "insecure-unverified-bundle", false, "Trust the bundle's declared identity without a credential to back it, letting whoever can write the file decide which agent this sandbox is")
+ runCmd.Flags().StringVar(&metricsAddr, "metrics-addr", "", "Serve unauthenticated Prometheus metrics on this address, e.g. 127.0.0.1:9600; off by default")
runCmd.Flags().StringVar(&logLevel, "log-level", "info", "Log level (debug, info, warn, error)")
for _, required := range []string{"socket", "sidecar-socket"} {
if err := runCmd.MarkFlagRequired(required); err != nil {
diff --git a/internal/node/sidecar.go b/internal/node/sidecar.go
index 39e6b2c5..ef3fee4f 100644
--- a/internal/node/sidecar.go
+++ b/internal/node/sidecar.go
@@ -83,7 +83,7 @@ func StartSidecarServer(node *SamNode, addr, socketPath, token, certFile, keyFil
mux.Handle("/", withAuth(token, true, withMeshConnection(node, mcpHandler)))
server := &http.Server{
- Handler: mux,
+ Handler: observeRequests(mux),
// Bound header-read time only: bodies/responses can legitimately stream
// (MCP sessions, inference completions), so no ReadTimeout/WriteTimeout.
ReadHeaderTimeout: 10 * time.Second,
diff --git a/internal/node/sidecar_metrics.go b/internal/node/sidecar_metrics.go
new file mode 100644
index 00000000..58767451
--- /dev/null
+++ b/internal/node/sidecar_metrics.go
@@ -0,0 +1,152 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "net/http"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// Half of what the sidecar serves streams: an MCP session and a completion
+// both hold the response open for as long as the work lasts. Total handler
+// time therefore measures the workload, not the mesh. Time to first byte is
+// the number that reflects what the mesh added, so both are recorded and the
+// distinction is kept in the metric names rather than left to the reader.
+
+var requestBuckets = []float64{
+ 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30,
+}
+
+var (
+ requestTTFBSeconds = promauto.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Name: "sam_node_request_ttfb_seconds",
+ Help: "Time from accepting a sidecar request to its first response byte",
+ Buckets: requestBuckets,
+ },
+ []string{"route"},
+ )
+
+ requestDurationSeconds = promauto.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Name: "sam_node_request_duration_seconds",
+ Help: "Time a sidecar request occupied its handler, including streaming",
+ Buckets: requestBuckets,
+ },
+ []string{"route", "code"},
+ )
+
+ requestsInFlight = promauto.NewGauge(
+ prometheus.GaugeOpts{
+ Name: "sam_node_requests_in_flight",
+ Help: "Sidecar requests currently being served",
+ },
+ )
+)
+
+// classifyRoute reduces a path to a closed vocabulary. Paths carry peer ids and
+// service names chosen off-node, so the raw path can never become a label.
+func classifyRoute(path string) string {
+ switch {
+ case path == "/healthz", path == "/readyz", path == "/metrics":
+ return "health"
+ case strings.HasPrefix(path, "/sam/service/"):
+ return "service-registry"
+ case strings.HasPrefix(path, "/sam/"):
+ return "egress"
+ case path == "/v1/models":
+ return "models"
+ case path == "/v1/chat/completions", path == "/v1/completions":
+ return "completions"
+ default:
+ return "mcp"
+ }
+}
+
+// observeRequests times every sidecar request. It wraps the whole mux so a
+// route added later is measured without anyone remembering to instrument it.
+func observeRequests(next http.Handler) http.Handler {
+ return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ route := classifyRoute(r.URL.Path)
+ if route == "health" {
+ next.ServeHTTP(w, r)
+ return
+ }
+
+ start := time.Now()
+ requestsInFlight.Inc()
+ defer requestsInFlight.Dec()
+
+ rec := &timedWriter{ResponseWriter: w, route: route, start: start}
+ next.ServeHTTP(rec, r)
+
+ requestDurationSeconds.WithLabelValues(route, strconv.Itoa(rec.status())).Observe(time.Since(start).Seconds())
+ })
+}
+
+// timedWriter records when the first byte reached the client. It deliberately
+// implements no optional interface itself: Unwrap lets http.ResponseController
+// reach the real writer, so flushing and hijacking keep working for streams.
+type timedWriter struct {
+ http.ResponseWriter
+ route string
+ start time.Time
+ code int
+ ttfbOK bool
+}
+
+func (t *timedWriter) Unwrap() http.ResponseWriter { return t.ResponseWriter }
+
+func (t *timedWriter) status() int {
+ if t.code == 0 {
+ return http.StatusOK
+ }
+ return t.code
+}
+
+func (t *timedWriter) observeTTFB() {
+ if !t.ttfbOK {
+ t.ttfbOK = true
+ requestTTFBSeconds.WithLabelValues(t.route).Observe(time.Since(t.start).Seconds())
+ }
+}
+
+func (t *timedWriter) WriteHeader(code int) {
+ if t.code == 0 {
+ t.code = code
+ }
+ t.observeTTFB()
+ t.ResponseWriter.WriteHeader(code)
+}
+
+func (t *timedWriter) Write(b []byte) (int, error) {
+ t.observeTTFB()
+ return t.ResponseWriter.Write(b)
+}
+
+// Flush is kept because a streaming handler that type-asserts http.Flusher
+// directly, rather than going through http.ResponseController, would otherwise
+// stop finding one and buffer the whole stream.
+func (t *timedWriter) Flush() {
+ t.observeTTFB()
+ if f, ok := t.ResponseWriter.(http.Flusher); ok {
+ f.Flush()
+ }
+}
diff --git a/internal/node/sidecar_metrics_test.go b/internal/node/sidecar_metrics_test.go
new file mode 100644
index 00000000..a9feafe0
--- /dev/null
+++ b/internal/node/sidecar_metrics_test.go
@@ -0,0 +1,93 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "net/http"
+ "net/http/httptest"
+ "testing"
+)
+
+func TestClassifyRouteIsAClosedVocabulary(t *testing.T) {
+ // A route label must not vary with anything an off-node caller chooses,
+ // or a peer could grow this node's metric space by inventing names.
+ cases := map[string]string{
+ "/healthz": "health",
+ "/metrics": "health",
+ "/sam/service/discover": "service-registry",
+ "/sam/service/register": "service-registry",
+ "/sam/12D3KooWpeer/mcp/calc/run": "egress",
+ "/v1/models": "models",
+ "/v1/chat/completions": "completions",
+ "/v1/completions": "completions",
+ "/": "mcp",
+ "/mcp": "mcp",
+ }
+ for path, want := range cases {
+ if got := classifyRoute(path); got != want {
+ t.Errorf("classifyRoute(%q) = %q, want %q", path, got, want)
+ }
+ }
+}
+
+func TestObserveRequestsKeepsStreamingUsable(t *testing.T) {
+ // The wrapper sits in front of MCP sessions and completions, both of which
+ // stream. If it hides Flusher, a stream buffers until the handler returns
+ // and every streaming consumer silently breaks.
+ var (
+ sawFlusher bool
+ sawController bool
+ )
+ handler := observeRequests(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ _, sawFlusher = w.(http.Flusher)
+ if err := http.NewResponseController(w).Flush(); err == nil {
+ sawController = true
+ }
+ _, _ = w.Write([]byte("chunk"))
+ }))
+
+ rec := httptest.NewRecorder()
+ handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/chat/completions", nil))
+
+ if !sawFlusher {
+ t.Error("handler could not type-assert http.Flusher through the wrapper")
+ }
+ if !sawController {
+ t.Error("http.ResponseController could not reach the underlying writer")
+ }
+ if got := rec.Body.String(); got != "chunk" {
+ t.Errorf("body = %q, want %q", got, "chunk")
+ }
+}
+
+func TestObserveRequestsRecordsTheHandlerStatus(t *testing.T) {
+ handler := observeRequests(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusTeapot)
+ }))
+
+ rec := httptest.NewRecorder()
+ handler.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/models", nil))
+
+ if rec.Code != http.StatusTeapot {
+ t.Errorf("status = %d, want %d", rec.Code, http.StatusTeapot)
+ }
+}
+
+func TestTimedWriterReportsOKWhenTheHandlerNeverSetAStatus(t *testing.T) {
+ w := &timedWriter{ResponseWriter: httptest.NewRecorder(), route: "mcp"}
+ if got := w.status(); got != http.StatusOK {
+ t.Errorf("status = %d, want %d", got, http.StatusOK)
+ }
+}
diff --git a/internal/sambox/dial.go b/internal/sambox/dial.go
index 161cbce5..8c77ea00 100644
--- a/internal/sambox/dial.go
+++ b/internal/sambox/dial.go
@@ -19,6 +19,7 @@ import (
"errors"
"fmt"
"net"
+ "sync/atomic"
"syscall"
"time"
)
@@ -61,14 +62,27 @@ func (d *AgentDialer) DialDestination(ctx context.Context, _ *Credentials, dst D
return nil, errors.New("sambox: AgentDialer requires a Router")
}
+ start := time.Now()
+
route, err := d.Router.Route(dst)
if err != nil {
+ recordFlow(routeUnresolved, 0, err)
return nil, err
}
ctx, cancel := context.WithTimeout(ctx, dialTimeout)
defer cancel()
+ conn, err := d.dialRoute(ctx, route, dst)
+ recordFlow(route.Kind.String(), time.Since(start), err)
+ if err != nil {
+ return nil, err
+ }
+ flowsActive.Inc()
+ return &countedConn{Conn: conn}, nil
+}
+
+func (d *AgentDialer) dialRoute(ctx context.Context, route Route, dst Destination) (net.Conn, error) {
switch route.Kind {
case RouteMeshEntrypoint:
return d.dialMeshEntrypoint()
@@ -81,6 +95,33 @@ func (d *AgentDialer) DialDestination(ctx context.Context, _ *Credentials, dst D
}
}
+// countedConn keeps the active-flow gauge honest. Both relay directions close
+// their side, so the decrement has to happen exactly once.
+type countedConn struct {
+ net.Conn
+ closed atomic.Bool
+}
+
+func (c *countedConn) Close() error {
+ if c.closed.CompareAndSwap(false, true) {
+ flowsActive.Dec()
+ }
+ return c.Conn.Close()
+}
+
+// CloseWrite keeps the half-close the relay depends on reachable through the
+// wrapper. Advertising it unconditionally would be a trap: the relay falls back
+// to a full close for connections that cannot half-close, and a wrapper that
+// claims the capability without delivering it leaves the peer's copy blocked
+// forever. So when the wrapped connection has no half-close, do what the relay
+// would have done.
+func (c *countedConn) CloseWrite() error {
+ if cw, ok := c.Conn.(interface{ CloseWrite() error }); ok {
+ return cw.CloseWrite()
+ }
+ return c.Close()
+}
+
func (d *AgentDialer) dial(ctx context.Context, network, address string) (net.Conn, error) {
if address == "" {
return nil, fmt.Errorf("sambox: no %s address configured", network)
diff --git a/internal/sambox/metrics.go b/internal/sambox/metrics.go
new file mode 100644
index 00000000..7a07d6eb
--- /dev/null
+++ b/internal/sambox/metrics.go
@@ -0,0 +1,90 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sambox
+
+import (
+ "errors"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// The boundary is the one place that sees every flow an agent opens, so it is
+// the only honest place to measure what the boundary costs and what it refused.
+// Labels stay closed vocabularies: a destination name is agent-controlled, and
+// putting it in a label would let a sandbox grow the metric space without bound.
+
+// flowSetupBuckets resolve from a sidecar hop on a Unix socket (tens of
+// microseconds) up to a mesh dial that crosses the DHT (seconds). The default
+// buckets start at 5ms, which is above the median this measures.
+var flowSetupBuckets = []float64{
+ 0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10,
+}
+
+var (
+ flowsTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "sam_box_flows_total",
+ Help: "Flows a sandbox asked the boundary to open, by route class and outcome",
+ },
+ []string{"route", "outcome"},
+ )
+
+ flowSetupSeconds = promauto.NewHistogramVec(
+ prometheus.HistogramOpts{
+ Name: "sam_box_flow_setup_seconds",
+ Help: "Time from an admitted CONNECT to a usable destination connection",
+ Buckets: flowSetupBuckets,
+ },
+ []string{"route"},
+ )
+
+ flowsActive = promauto.NewGauge(
+ prometheus.GaugeOpts{
+ Name: "sam_box_flows_active",
+ Help: "Flows currently relaying through the boundary",
+ },
+ )
+)
+
+// routeUnresolved labels a flow the router refused before it could be
+// classified, so a denial is never miscounted against a real route.
+const routeUnresolved = "unresolved"
+
+// outcomeFor maps a dial result onto the closed vocabulary the counters use.
+func outcomeFor(err error) string {
+ switch {
+ case err == nil:
+ return "allowed"
+ case errors.Is(err, ErrNotAllowed):
+ return "denied"
+ case errors.Is(err, ErrHostUnreachable):
+ return "unreachable"
+ case errors.Is(err, ErrConnectionRefused):
+ return "refused"
+ default:
+ return "error"
+ }
+}
+
+// recordFlow accounts one attempt to open a destination. setup is only
+// meaningful when the attempt succeeded, so it is only observed then.
+func recordFlow(route string, setup time.Duration, err error) {
+ flowsTotal.WithLabelValues(route, outcomeFor(err)).Inc()
+ if err == nil {
+ flowSetupSeconds.WithLabelValues(route).Observe(setup.Seconds())
+ }
+}
diff --git a/internal/sambox/metrics_server.go b/internal/sambox/metrics_server.go
new file mode 100644
index 00000000..75264679
--- /dev/null
+++ b/internal/sambox/metrics_server.go
@@ -0,0 +1,60 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sambox
+
+import (
+ "context"
+ "errors"
+ "net"
+ "net/http"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus/promhttp"
+)
+
+// ServeMetrics exposes this boundary's counters on addr until ctx ends.
+//
+// It is off unless an operator asks for it. The boundary sits between a
+// sandbox and the mesh, so any listener it opens is one more thing reachable
+// from wherever addr is bound; an experiment wants the numbers, a production
+// sandbox usually does not. Nothing here is authenticated, which is why the
+// caller has to name the address rather than get one by default.
+func ServeMetrics(ctx context.Context, addr string) (*http.Server, error) {
+ if addr == "" {
+ return nil, errors.New("sambox: no metrics address configured")
+ }
+
+ listener, err := net.Listen("tcp", addr)
+ if err != nil {
+ return nil, err
+ }
+
+ mux := http.NewServeMux()
+ mux.Handle("/metrics", promhttp.Handler())
+ server := &http.Server{
+ Handler: mux,
+ ReadHeaderTimeout: 10 * time.Second,
+ }
+
+ go func() {
+ <-ctx.Done()
+ _ = server.Close()
+ }()
+ go func() {
+ _ = server.Serve(listener)
+ }()
+
+ return server, nil
+}
diff --git a/internal/sambox/metrics_test.go b/internal/sambox/metrics_test.go
new file mode 100644
index 00000000..93ce7a76
--- /dev/null
+++ b/internal/sambox/metrics_test.go
@@ -0,0 +1,145 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sambox
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "testing"
+ "time"
+
+ "github.com/prometheus/client_golang/prometheus"
+ dto "github.com/prometheus/client_model/go"
+)
+
+func TestOutcomeForNamesEveryDenialReasonSeparately(t *testing.T) {
+ // A denial and an unreachable host mean opposite things about a
+ // deployment: one is policy working, the other is the mesh failing.
+ // Collapsing them would make the counters unusable as evidence.
+ cases := []struct {
+ err error
+ want string
+ }{
+ {nil, "allowed"},
+ {fmt.Errorf("wrapped: %w", ErrNotAllowed), "denied"},
+ {fmt.Errorf("wrapped: %w", ErrHostUnreachable), "unreachable"},
+ {fmt.Errorf("wrapped: %w", ErrConnectionRefused), "refused"},
+ {errors.New("something else"), "error"},
+ }
+ for _, tc := range cases {
+ if got := outcomeFor(tc.err); got != tc.want {
+ t.Errorf("outcomeFor(%v) = %q, want %q", tc.err, got, tc.want)
+ }
+ }
+}
+
+func TestRouteKindStringsAreStableLabels(t *testing.T) {
+ // These strings are metric label values, so renaming one silently breaks
+ // every dashboard and every recorded experiment that used it.
+ cases := map[RouteKind]string{
+ RouteMeshEntrypoint: "mesh-entrypoint",
+ RouteMeshService: "mesh-service",
+ RouteExternal: "external",
+ }
+ for kind, want := range cases {
+ if got := kind.String(); got != want {
+ t.Errorf("RouteKind(%d).String() = %q, want %q", int(kind), got, want)
+ }
+ }
+}
+
+func TestCountedConnPropagatesHalfClose(t *testing.T) {
+ // The relay half-closes to signal EOF upstream. If the wrapper swallows
+ // CloseWrite, a peer waiting on EOF hangs until a timeout instead.
+ client, server := net.Pipe()
+ defer func() { _ = client.Close() }()
+ defer func() { _ = server.Close() }()
+
+ var closed bool
+ c := &countedConn{Conn: halfCloser{Conn: client, onCloseWrite: func() { closed = true }}}
+ if err := c.CloseWrite(); err != nil {
+ t.Fatalf("CloseWrite: %v", err)
+ }
+ if !closed {
+ t.Error("CloseWrite did not reach the underlying connection")
+ }
+}
+
+func TestCountedConnFallsBackToCloseWhenItCannotHalfClose(t *testing.T) {
+ // The relay closes a connection outright when it cannot half-close, and
+ // that full close is what unblocks the opposite copy. A wrapper that
+ // advertises CloseWrite without delivering one defeats that fallback and
+ // hangs the relay forever, which is a deadlock no metric test would show.
+ client, server := net.Pipe()
+ defer func() { _ = server.Close() }()
+
+ c := &countedConn{Conn: client} // net.Pipe cannot half-close
+
+ // The deadline goes on before the close, both because a closed pipe will
+ // not accept one and so the pre-fix behaviour reports a blocked read
+ // rather than hanging the package for the whole test timeout.
+ if err := client.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil {
+ t.Fatalf("SetReadDeadline: %v", err)
+ }
+ if err := c.CloseWrite(); err != nil {
+ t.Fatalf("CloseWrite: %v", err)
+ }
+
+ if _, err := client.Read(make([]byte, 1)); !errors.Is(err, io.ErrClosedPipe) {
+ t.Errorf("read after CloseWrite = %v, want the connection closed", err)
+ }
+}
+
+func TestCountedConnDecrementsOnceOnRepeatedClose(t *testing.T) {
+ // Both relay directions close their side, so a naive decrement would run
+ // twice and drive the active-flow gauge negative.
+ client, server := net.Pipe()
+ defer func() { _ = server.Close() }()
+
+ c := &countedConn{Conn: client}
+ flowsActive.Set(0)
+ flowsActive.Inc()
+
+ _ = c.Close()
+ _ = c.Close()
+
+ if got := gaugeValue(t, flowsActive); got != 0 {
+ t.Errorf("flowsActive = %v after two closes, want 0", got)
+ }
+}
+
+// gaugeValue reads a gauge without pulling in the prometheus test helpers,
+// which would add a module for one assertion.
+func gaugeValue(t *testing.T, g prometheus.Gauge) float64 {
+ t.Helper()
+ var m dto.Metric
+ if err := g.Write(&m); err != nil {
+ t.Fatalf("read gauge: %v", err)
+ }
+ return m.GetGauge().GetValue()
+}
+
+// halfCloser adds CloseWrite to a connection that lacks one.
+type halfCloser struct {
+ net.Conn
+ onCloseWrite func()
+}
+
+func (h halfCloser) CloseWrite() error {
+ h.onCloseWrite()
+ return nil
+}