Skip to content

Agent Mesh

bench: a workload that answers the same question twice

Antonio Ojea00346e95ecc2
diff --git a/Makefile b/Makefile
index 82dfce66..59b44b46 100644
--- a/Makefile
+++ b/Makefile
@@ -35,6 +35,7 @@ build: $(OUT_DIR)/libinterceptor.so
go build -v -o "$(OUT_DIR)/mcp-client" ./cmd/mcp-client
go build -v -o "$(OUT_DIR)/nano-init" ./cmd/nano-init
go build -v -o "$(OUT_DIR)/sam-box" ./cmd/sam-box
+ go build -v -o "$(OUT_DIR)/sam-bench" ./cmd/sam-bench
go build -v -o "$(OUT_DIR)/sam-console" ./cmd/sam-console
diff --git a/cmd/sam-bench/main.go b/cmd/sam-bench/main.go
new file mode 100644
index 00000000..3e730cf0
--- /dev/null
+++ b/cmd/sam-bench/main.go
@@ -0,0 +1,214 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Command sam-bench measures what an agent experiences when it reaches the
+// mesh through a sandbox boundary.
+//
+// It issues a fixed workload rather than an interesting one. The chaos agent
+// in this repo is driven by a real model and is the right tool for asking
+// whether the mesh survives contact with an autonomous caller; it is the wrong
+// tool for asking how long something takes, because it never asks twice for
+// the same thing. This asks for exactly the same thing every time, so two runs
+// differ only where the mesh differs.
+//
+// One invocation is one observation: it records the workload, the latencies it
+// saw, and what the processes involved reported about themselves before and
+// after, into a single JSON document meant to be kept.
+package main
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "os"
+ "os/signal"
+ "strings"
+ "syscall"
+ "time"
+
+ "github.com/spf13/cobra"
+
+ "github.com/google/sam/internal/bench"
+)
+
+// observation is one complete, self-describing measurement.
+type observation struct {
+ // Labels record the conditions the operator varied, such as how many
+ // agents were running. Nothing infers them, because nothing can.
+ Labels map[string]string `json:"labels,omitempty"`
+
+ Started time.Time `json:"started"`
+ Report *bench.Report `json:"report"`
+
+ // Before and after are keyed by scrape source. Counters are cumulative,
+ // so a report quotes the difference, and keeping both ends means the
+ // difference can be recomputed rather than trusted.
+ Before map[string]map[string]float64 `json:"metrics_before,omitempty"`
+ After map[string]map[string]float64 `json:"metrics_after,omitempty"`
+}
+
+func main() {
+ var (
+ socket string
+ target string
+ method string
+ body string
+ headers []string
+ requests int
+ concurrency int
+ warmup int
+ newFlow bool
+ timeout time.Duration
+ scrape []string
+ labels []string
+ out string
+ )
+
+ rootCmd := &cobra.Command{
+ Use: "sam-bench",
+ Short: "Measure the mesh from where an agent stands",
+ Long: "Issues a fixed workload through a sandbox boundary and records what it cost,\n" +
+ "alongside what the processes involved reported about themselves.",
+ SilenceUsage: true,
+ RunE: func(cmd *cobra.Command, _ []string) error {
+ header, err := parseHeaders(headers)
+ if err != nil {
+ return err
+ }
+ tags, err := parseLabels(labels)
+ if err != nil {
+ return err
+ }
+
+ obs := observation{Labels: tags, Started: time.Now()}
+
+ obs.Before, err = scrapeAll(cmd.Context(), scrape)
+ if err != nil {
+ return err
+ }
+
+ obs.Report, err = bench.Run(cmd.Context(), bench.Options{
+ Socket: socket,
+ Target: target,
+ Method: method,
+ Body: []byte(body),
+ Header: header,
+ Requests: requests,
+ Concurrency: concurrency,
+ Warmup: warmup,
+ NewFlowPerRequest: newFlow,
+ Timeout: timeout,
+ })
+ if err != nil {
+ return err
+ }
+
+ obs.After, err = scrapeAll(cmd.Context(), scrape)
+ if err != nil {
+ return err
+ }
+
+ return write(out, obs)
+ },
+ }
+
+ flags := rootCmd.Flags()
+ flags.StringVar(&socket, "socket", "", "Sandbox boundary SOCKS5 socket to measure through; omit to measure the same workload without a boundary, which is the baseline")
+ flags.StringVar(&target, "target", "", "URL to request, e.g. http://mesh.sam.alt/v1/models (required)")
+ flags.StringVar(&method, "method", "GET", "HTTP method")
+ flags.StringVar(&body, "body", "", "Request body")
+ flags.StringArrayVar(&headers, "header", nil, "Request header as Name: value, repeatable")
+ flags.IntVar(&requests, "requests", 100, "Requests to issue after warmup")
+ flags.IntVar(&concurrency, "concurrency", 1, "Requests in flight at once")
+ flags.IntVar(&warmup, "warmup", 10, "Requests issued before measurement starts, reported separately")
+ flags.BoolVar(&newFlow, "new-flow-per-request", false, "Open a fresh boundary flow per request, measuring admission rather than transfer")
+ flags.DurationVar(&timeout, "timeout", 30*time.Second, "Bound on a single request")
+ flags.StringArrayVar(&scrape, "scrape", nil, "Metrics endpoint to record before and after the run, repeatable")
+ flags.StringArrayVar(&labels, "label", nil, "Condition to record with the observation as name=value, repeatable")
+ flags.StringVar(&out, "out", "", "File to write the observation to; default stdout")
+ if err := rootCmd.MarkFlagRequired("target"); err != nil {
+ panic(err)
+ }
+
+ ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+ defer stop()
+
+ if err := rootCmd.ExecuteContext(ctx); err != nil {
+ fmt.Fprintf(os.Stderr, "sam-bench: %v\n", err)
+ os.Exit(1)
+ }
+}
+
+// scrapeAll records every endpoint, refusing to continue if one is missing:
+// an observation with a hole in it is worse than no observation, because it
+// still looks like data.
+func scrapeAll(ctx context.Context, urls []string) (map[string]map[string]float64, error) {
+ if len(urls) == 0 {
+ return nil, nil
+ }
+
+ out := make(map[string]map[string]float64, len(urls))
+ for _, url := range urls {
+ snap, err := bench.Scrape(ctx, url)
+ if err != nil {
+ return nil, err
+ }
+ out[url] = snap.Flatten()
+ }
+ return out, nil
+}
+
+func parseHeaders(raw []string) (map[string][]string, error) {
+ if len(raw) == 0 {
+ return nil, nil
+ }
+ header := map[string][]string{}
+ for _, h := range raw {
+ name, value, found := strings.Cut(h, ":")
+ if !found {
+ return nil, fmt.Errorf("header %q is not Name: value", h)
+ }
+ header[strings.TrimSpace(name)] = append(header[strings.TrimSpace(name)], strings.TrimSpace(value))
+ }
+ return header, nil
+}
+
+func parseLabels(raw []string) (map[string]string, error) {
+ if len(raw) == 0 {
+ return nil, nil
+ }
+ labels := map[string]string{}
+ for _, l := range raw {
+ name, value, found := strings.Cut(l, "=")
+ if !found {
+ return nil, fmt.Errorf("label %q is not name=value", l)
+ }
+ labels[strings.TrimSpace(name)] = strings.TrimSpace(value)
+ }
+ return labels, nil
+}
+
+func write(path string, obs observation) error {
+ encoded, err := json.MarshalIndent(obs, "", " ")
+ if err != nil {
+ return err
+ }
+ encoded = append(encoded, '\n')
+
+ if path == "" {
+ _, err = os.Stdout.Write(encoded)
+ return err
+ }
+ return os.WriteFile(path, encoded, 0o600)
+}
diff --git a/internal/bench/bench.go b/internal/bench/bench.go
new file mode 100644
index 00000000..37db1407
--- /dev/null
+++ b/internal/bench/bench.go
@@ -0,0 +1,383 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package bench drives a reproducible agent workload against the mesh.
+//
+// The chaos agent that ships with the repo is the realism story: a real model
+// deciding what to call next. That makes it useless as an instrument, because
+// two runs never issue the same requests and a difference between them cannot
+// be attributed to the thing under test. This package is the opposite trade:
+// it does exactly what it is told, so a difference between two runs is a
+// difference in the mesh.
+//
+// It reaches the mesh the way an agent does, through the sandbox boundary's
+// SOCKS5 socket, so what it measures is what an agent would experience rather
+// than what an operator with host access would.
+package bench
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "net"
+ "net/http"
+ "sort"
+ "sync"
+ "time"
+
+ "golang.org/x/net/proxy"
+)
+
+// Options describe one measurement run. The zero value is not useful: at
+// minimum a Target and a positive Requests count are required.
+type Options struct {
+ // Socket is the sandbox boundary's SOCKS5 Unix socket. Empty measures the
+ // same workload without the boundary, which is the baseline the mesh
+ // numbers are only meaningful against.
+ Socket string
+
+ // Target is the URL to request, e.g. http://mesh.sam.alt/v1/models.
+ Target string
+
+ // Method and Body are the request to issue. An empty Method means GET.
+ Method string
+ Body []byte
+
+ // Header carries anything the target needs, such as a content type.
+ Header http.Header
+
+ // Requests is how many to issue in total, after warmup.
+ Requests int
+
+ // Concurrency is how many are in flight at once.
+ Concurrency int
+
+ // Warmup requests are issued and discarded before measurement starts, so
+ // first-call costs (DHT lookup, provider discovery, connection setup) do
+ // not contaminate the steady-state distribution. They are reported
+ // separately rather than thrown away.
+ Warmup int
+
+ // NewFlowPerRequest opens a fresh boundary flow for every request instead
+ // of reusing connections. This separates the cost of admitting a flow from
+ // the cost of carrying one, which are different questions.
+ NewFlowPerRequest bool
+
+ // Timeout bounds a single request.
+ Timeout time.Duration
+}
+
+// Sample is one completed request.
+type Sample struct {
+ // TTFB is the time until the response headers were available, which is
+ // what an agent waiting on a first token actually experiences.
+ TTFB time.Duration
+
+ // Total additionally includes reading the body to completion.
+ Total time.Duration
+
+ // Status is the HTTP status, or 0 if the request never got one.
+ Status int
+
+ // Err is set when the request did not complete.
+ Err error
+}
+
+// Report is the outcome of a run, in a shape that serialises to JSON for
+// later analysis without needing the raw samples.
+type Report struct {
+ Target string `json:"target"`
+ ThroughBoundary bool `json:"through_boundary"`
+ NewFlowPerRequest bool `json:"new_flow_per_request"`
+ Concurrency int `json:"concurrency"`
+ Requests int `json:"requests"`
+ Warmup int `json:"warmup"`
+ Succeeded int `json:"succeeded"`
+ Failed int `json:"failed"`
+ Elapsed float64 `json:"elapsed_seconds"`
+ Throughput float64 `json:"requests_per_second"`
+ TTFB Distribution `json:"ttfb_ms"`
+ Total Distribution `json:"total_ms"`
+ WarmupTTFB *Distribution `json:"warmup_ttfb_ms,omitempty"`
+
+ // Errors counts failures by their message, so a run that mostly failed
+ // cannot be mistaken for a fast one.
+ Errors map[string]int `json:"errors,omitempty"`
+}
+
+// Distribution summarises a set of latencies in milliseconds. Percentiles are
+// computed from the full sorted sample rather than from bucketed counts, so
+// they are exact for the run rather than an interpolation.
+type Distribution struct {
+ Count int `json:"count"`
+ Min float64 `json:"min"`
+ Mean float64 `json:"mean"`
+ P50 float64 `json:"p50"`
+ P90 float64 `json:"p90"`
+ P95 float64 `json:"p95"`
+ P99 float64 `json:"p99"`
+ Max float64 `json:"max"`
+ // StdDev is reported so a reader can see whether a mean is describing
+ // anything, rather than averaging a bimodal distribution into fiction.
+ StdDev float64 `json:"stddev"`
+}
+
+// Run issues the workload and returns its report.
+func Run(ctx context.Context, opts Options) (*Report, error) {
+ if opts.Target == "" {
+ return nil, errors.New("bench: no target")
+ }
+ if opts.Requests <= 0 {
+ return nil, errors.New("bench: no requests to issue")
+ }
+ if opts.Concurrency <= 0 {
+ opts.Concurrency = 1
+ }
+ if opts.Timeout <= 0 {
+ opts.Timeout = 30 * time.Second
+ }
+
+ client, err := newClient(opts)
+ if err != nil {
+ return nil, err
+ }
+ defer client.CloseIdleConnections()
+
+ report := &Report{
+ Target: opts.Target,
+ ThroughBoundary: opts.Socket != "",
+ NewFlowPerRequest: opts.NewFlowPerRequest,
+ Concurrency: opts.Concurrency,
+ Requests: opts.Requests,
+ Warmup: opts.Warmup,
+ Errors: map[string]int{},
+ }
+
+ if opts.Warmup > 0 {
+ warm := issue(ctx, client, opts, opts.Warmup)
+ d := summarise(ttfbOf(warm))
+ report.WarmupTTFB = &d
+ }
+
+ start := time.Now()
+ samples := issue(ctx, client, opts, opts.Requests)
+ report.Elapsed = time.Since(start).Seconds()
+ if report.Elapsed > 0 {
+ report.Throughput = float64(len(samples)) / report.Elapsed
+ }
+
+ var ok []Sample
+ for _, s := range samples {
+ if s.Err != nil {
+ report.Failed++
+ report.Errors[s.Err.Error()]++
+ continue
+ }
+ if s.Status >= 400 {
+ report.Failed++
+ report.Errors[fmt.Sprintf("HTTP %d", s.Status)]++
+ continue
+ }
+ report.Succeeded++
+ ok = append(ok, s)
+ }
+
+ report.TTFB = summarise(ttfbOf(ok))
+ report.Total = summarise(totalOf(ok))
+ return report, nil
+}
+
+// issue runs n requests at the configured concurrency.
+func issue(ctx context.Context, client *http.Client, opts Options, n int) []Sample {
+ var (
+ wg sync.WaitGroup
+ mu sync.Mutex
+ samples = make([]Sample, 0, n)
+ work = make(chan struct{})
+ )
+
+ for range opts.Concurrency {
+ wg.Add(1)
+ go func() {
+ defer wg.Done()
+ for range work {
+ s := once(ctx, client, opts)
+ mu.Lock()
+ samples = append(samples, s)
+ mu.Unlock()
+ }
+ }()
+ }
+
+ for range n {
+ select {
+ case work <- struct{}{}:
+ case <-ctx.Done():
+ close(work)
+ wg.Wait()
+ return samples
+ }
+ }
+ close(work)
+ wg.Wait()
+ return samples
+}
+
+// once issues a single request and times it.
+func once(ctx context.Context, client *http.Client, opts Options) Sample {
+ ctx, cancel := context.WithTimeout(ctx, opts.Timeout)
+ defer cancel()
+
+ method := opts.Method
+ if method == "" {
+ method = http.MethodGet
+ }
+
+ var body io.Reader
+ if len(opts.Body) > 0 {
+ body = bytes.NewReader(opts.Body)
+ }
+
+ req, err := http.NewRequestWithContext(ctx, method, opts.Target, body)
+ if err != nil {
+ return Sample{Err: err}
+ }
+ for k, values := range opts.Header {
+ for _, v := range values {
+ req.Header.Add(k, v)
+ }
+ }
+
+ start := time.Now()
+ resp, err := client.Do(req)
+ if err != nil {
+ return Sample{Err: err}
+ }
+ ttfb := time.Since(start)
+
+ // The body has to be drained for the connection to be reusable, and
+ // draining is part of what a caller waits for, so it is timed too.
+ _, copyErr := io.Copy(io.Discard, resp.Body)
+ closeErr := resp.Body.Close()
+ total := time.Since(start)
+
+ if copyErr != nil {
+ return Sample{Err: copyErr, Status: resp.StatusCode}
+ }
+ if closeErr != nil {
+ return Sample{Err: closeErr, Status: resp.StatusCode}
+ }
+ return Sample{TTFB: ttfb, Total: total, Status: resp.StatusCode}
+}
+
+// newClient builds the transport the workload runs over. Through a boundary it
+// dials the SOCKS5 socket, which is the only path a sandboxed agent has.
+func newClient(opts Options) (*http.Client, error) {
+ transport := &http.Transport{
+ MaxIdleConnsPerHost: opts.Concurrency,
+ DisableKeepAlives: opts.NewFlowPerRequest,
+ }
+
+ if opts.Socket != "" {
+ dialer, err := proxy.SOCKS5("unix", opts.Socket, nil, proxy.Direct)
+ if err != nil {
+ return nil, fmt.Errorf("bench: dial boundary %s: %w", opts.Socket, err)
+ }
+ contextDialer, ok := dialer.(proxy.ContextDialer)
+ if !ok {
+ return nil, errors.New("bench: SOCKS5 dialer does not support contexts")
+ }
+ transport.DialContext = contextDialer.DialContext
+ } else {
+ transport.DialContext = (&net.Dialer{}).DialContext
+ }
+
+ return &http.Client{Transport: transport}, nil
+}
+
+func ttfbOf(samples []Sample) []time.Duration {
+ out := make([]time.Duration, 0, len(samples))
+ for _, s := range samples {
+ if s.Err == nil {
+ out = append(out, s.TTFB)
+ }
+ }
+ return out
+}
+
+func totalOf(samples []Sample) []time.Duration {
+ out := make([]time.Duration, 0, len(samples))
+ for _, s := range samples {
+ if s.Err == nil {
+ out = append(out, s.Total)
+ }
+ }
+ return out
+}
+
+// summarise reduces latencies to a distribution in milliseconds.
+func summarise(values []time.Duration) Distribution {
+ if len(values) == 0 {
+ return Distribution{}
+ }
+
+ ms := make([]float64, len(values))
+ for i, v := range values {
+ ms[i] = float64(v.Nanoseconds()) / 1e6
+ }
+ sort.Float64s(ms)
+
+ var sum float64
+ for _, v := range ms {
+ sum += v
+ }
+ mean := sum / float64(len(ms))
+
+ var variance float64
+ for _, v := range ms {
+ variance += (v - mean) * (v - mean)
+ }
+ variance /= float64(len(ms))
+
+ return Distribution{
+ Count: len(ms),
+ Min: ms[0],
+ Mean: mean,
+ P50: percentile(ms, 50),
+ P90: percentile(ms, 90),
+ P95: percentile(ms, 95),
+ P99: percentile(ms, 99),
+ Max: ms[len(ms)-1],
+ StdDev: math.Sqrt(variance),
+ }
+}
+
+// percentile returns the p-th percentile of sorted values using the
+// nearest-rank method, which never invents a value that was not observed.
+func percentile(sorted []float64, p float64) float64 {
+ if len(sorted) == 0 {
+ return 0
+ }
+ rank := int(math.Ceil(p / 100 * float64(len(sorted))))
+ if rank < 1 {
+ rank = 1
+ }
+ if rank > len(sorted) {
+ rank = len(sorted)
+ }
+ return sorted[rank-1]
+}
diff --git a/internal/bench/bench_test.go b/internal/bench/bench_test.go
new file mode 100644
index 00000000..e902492c
--- /dev/null
+++ b/internal/bench/bench_test.go
@@ -0,0 +1,161 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bench
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "sync/atomic"
+ "testing"
+ "time"
+)
+
+func TestPercentileUsesNearestRank(t *testing.T) {
+ // Every number this package reports rests on this function. Interpolating
+ // percentiles would report a latency that was never observed, which is
+ // exactly the kind of number a reader would take literally.
+ sorted := []float64{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
+ cases := []struct {
+ p float64
+ want float64
+ }{
+ {50, 5},
+ {90, 9},
+ {95, 10},
+ {99, 10},
+ {100, 10},
+ }
+ for _, tc := range cases {
+ if got := percentile(sorted, tc.p); got != tc.want {
+ t.Errorf("percentile(p%v) = %v, want %v", tc.p, got, tc.want)
+ }
+ }
+}
+
+func TestPercentileHandlesDegenerateInput(t *testing.T) {
+ if got := percentile(nil, 50); got != 0 {
+ t.Errorf("percentile of nothing = %v, want 0", got)
+ }
+ if got := percentile([]float64{42}, 99); got != 42 {
+ t.Errorf("percentile of one sample = %v, want 42", got)
+ }
+}
+
+func TestSummariseReportsTheShapeNotJustTheMean(t *testing.T) {
+ // A mean alone cannot tell a steady mesh from one that stalls on one
+ // request in twenty, and the tail is the whole point of the measurement.
+ values := make([]time.Duration, 0, 100)
+ for i := range 99 {
+ _ = i
+ values = append(values, time.Millisecond)
+ }
+ values = append(values, time.Second)
+
+ d := summarise(values)
+ if d.Count != 100 {
+ t.Errorf("Count = %d, want 100", d.Count)
+ }
+ if d.P50 != 1 {
+ t.Errorf("P50 = %v ms, want 1", d.P50)
+ }
+ if d.Max != 1000 {
+ t.Errorf("Max = %v ms, want 1000", d.Max)
+ }
+ if d.StdDev <= 0 {
+ t.Errorf("StdDev = %v, want a positive spread for a bimodal sample", d.StdDev)
+ }
+}
+
+func TestSummariseOfNothingIsEmptyNotAPanic(t *testing.T) {
+ if got := summarise(nil); got.Count != 0 {
+ t.Errorf("Count = %d, want 0", got.Count)
+ }
+}
+
+func TestRunIssuesExactlyTheRequestedLoad(t *testing.T) {
+ // A load generator that quietly issues a different number of requests
+ // than it reports would corrupt every throughput figure derived from it.
+ var served atomic.Int64
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ served.Add(1)
+ _, _ = w.Write([]byte("ok"))
+ }))
+ defer server.Close()
+
+ report, err := Run(context.Background(), Options{
+ Target: server.URL,
+ Requests: 50,
+ Warmup: 10,
+ Concurrency: 5,
+ })
+ if err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+
+ if got := served.Load(); got != 60 {
+ t.Errorf("server saw %d requests, want 60 (50 measured + 10 warmup)", got)
+ }
+ if report.Succeeded != 50 {
+ t.Errorf("Succeeded = %d, want 50", report.Succeeded)
+ }
+ if report.Failed != 0 {
+ t.Errorf("Failed = %d, want 0: %v", report.Failed, report.Errors)
+ }
+ if report.WarmupTTFB == nil || report.WarmupTTFB.Count != 10 {
+ t.Errorf("warmup was not reported separately: %+v", report.WarmupTTFB)
+ }
+ if report.TTFB.Count != 50 {
+ t.Errorf("TTFB.Count = %d, want 50", report.TTFB.Count)
+ }
+}
+
+func TestRunCountsErrorStatusesAsFailures(t *testing.T) {
+ // A mesh that answers "denied" very quickly must never look like a mesh
+ // that answers correctly very quickly.
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusForbidden)
+ }))
+ defer server.Close()
+
+ report, err := Run(context.Background(), Options{Target: server.URL, Requests: 5, Concurrency: 1})
+ if err != nil {
+ t.Fatalf("Run: %v", err)
+ }
+
+ if report.Succeeded != 0 {
+ t.Errorf("Succeeded = %d, want 0", report.Succeeded)
+ }
+ if report.Failed != 5 {
+ t.Errorf("Failed = %d, want 5", report.Failed)
+ }
+ if report.Errors["HTTP 403"] != 5 {
+ t.Errorf("Errors = %v, want 5 recorded as HTTP 403", report.Errors)
+ }
+ if report.TTFB.Count != 0 {
+ t.Errorf("failed requests leaked into the latency distribution: %+v", report.TTFB)
+ }
+}
+
+func TestRunRejectsAWorkloadItCannotMeasure(t *testing.T) {
+ for _, opts := range []Options{
+ {Requests: 1},
+ {Target: "http://example.invalid"},
+ } {
+ if _, err := Run(context.Background(), opts); err == nil {
+ t.Errorf("Run(%+v) accepted an unmeasurable workload", opts)
+ }
+ }
+}
diff --git a/internal/bench/scrape.go b/internal/bench/scrape.go
new file mode 100644
index 00000000..259a7a1d
--- /dev/null
+++ b/internal/bench/scrape.go
@@ -0,0 +1,250 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bench
+
+import (
+ "bufio"
+ "context"
+ "fmt"
+ "net/http"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// A run's client-side latencies say what an agent experienced; the servers'
+// own counters say what they did to produce it, including the flows they
+// refused, which never appear as a latency at all. A report needs both, so
+// this reads the exposition format directly.
+//
+// It is deliberately a few lines rather than the upstream parser: that would
+// promote a transitive dependency to a direct one for the sake of splitting
+// on a brace.
+
+// Series is one exported sample: a metric name, its labels, and its value.
+type Series struct {
+ Name string
+ Labels map[string]string
+ Value float64
+}
+
+// Snapshot is everything one process exported at one instant.
+type Snapshot struct {
+ Source string `json:"source"`
+ Taken time.Time `json:"taken"`
+ Series []Series `json:"-"`
+}
+
+// Value returns the first sample of name whose labels include match, and
+// whether one existed. A missing series and a zero one mean different things:
+// a counter that was never incremented is absent from the exposition entirely.
+func (s *Snapshot) Value(name string, match map[string]string) (float64, bool) {
+ for _, series := range s.Series {
+ if series.Name != name {
+ continue
+ }
+ found := true
+ for k, v := range match {
+ if series.Labels[k] != v {
+ found = false
+ break
+ }
+ }
+ if found {
+ return series.Value, true
+ }
+ }
+ return 0, false
+}
+
+// Sum adds every sample of name, ignoring labels. It is how a counter split
+// across label sets, such as flows by route, becomes one total.
+func (s *Snapshot) Sum(name string) float64 {
+ var total float64
+ for _, series := range s.Series {
+ if series.Name == name {
+ total += series.Value
+ }
+ }
+ return total
+}
+
+// Flatten renders the snapshot as canonical "name{k=v,...}" keys, which is
+// what gets recorded in a result file. Keeping every series rather than a
+// chosen few means a question nobody thought to ask before the run can still
+// be answered from the record afterwards.
+func (s *Snapshot) Flatten() map[string]float64 {
+ out := make(map[string]float64, len(s.Series))
+ for _, series := range s.Series {
+ out[series.Key()] = series.Value
+ }
+ return out
+}
+
+// Key renders a series as a stable, sorted identifier.
+func (s Series) Key() string {
+ if len(s.Labels) == 0 {
+ return s.Name
+ }
+
+ keys := make([]string, 0, len(s.Labels))
+ for k := range s.Labels {
+ keys = append(keys, k)
+ }
+ sort.Strings(keys)
+
+ var b strings.Builder
+ b.WriteString(s.Name)
+ b.WriteByte('{')
+ for i, k := range keys {
+ if i > 0 {
+ b.WriteByte(',')
+ }
+ b.WriteString(k)
+ b.WriteByte('=')
+ b.WriteString(s.Labels[k])
+ }
+ b.WriteByte('}')
+ return b.String()
+}
+
+// Scrape reads one process's metrics endpoint.
+func Scrape(ctx context.Context, url string) (*Snapshot, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil, fmt.Errorf("scrape %s: %w", url, err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+
+ if resp.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("scrape %s: HTTP %d", url, resp.StatusCode)
+ }
+
+ series, err := parseExposition(resp.Body)
+ if err != nil {
+ return nil, fmt.Errorf("scrape %s: %w", url, err)
+ }
+ return &Snapshot{Source: url, Taken: time.Now(), Series: series}, nil
+}
+
+// parseExposition reads the Prometheus text format, ignoring HELP and TYPE.
+func parseExposition(r interface{ Read([]byte) (int, error) }) ([]Series, error) {
+ var out []Series
+
+ scanner := bufio.NewScanner(r)
+ scanner.Buffer(make([]byte, 0, 64*1024), 4*1024*1024)
+ for scanner.Scan() {
+ line := strings.TrimSpace(scanner.Text())
+ if line == "" || strings.HasPrefix(line, "#") {
+ continue
+ }
+
+ // A sample is "name[{labels}] value [timestamp]". Splitting on the
+ // last space would break on a label value containing one, so the
+ // name and labels are taken from the front instead.
+ name, rest, ok := splitSample(line)
+ if !ok {
+ continue
+ }
+
+ fields := strings.Fields(rest)
+ if len(fields) == 0 {
+ continue
+ }
+ value, err := strconv.ParseFloat(fields[0], 64)
+ if err != nil {
+ // NaN and +Inf are legal values that carry no information here.
+ continue
+ }
+
+ labels, err := parseLabels(line)
+ if err != nil {
+ return nil, err
+ }
+ out = append(out, Series{Name: name, Labels: labels, Value: value})
+ }
+ return out, scanner.Err()
+}
+
+// splitSample separates the metric name from everything after its labels.
+func splitSample(line string) (name, rest string, ok bool) {
+ if open := strings.IndexByte(line, '{'); open >= 0 {
+ close := strings.LastIndexByte(line, '}')
+ if close < open {
+ return "", "", false
+ }
+ return line[:open], line[close+1:], true
+ }
+ name, rest, found := strings.Cut(line, " ")
+ return name, rest, found
+}
+
+// parseLabels reads the label set of a sample, if it has one.
+func parseLabels(line string) (map[string]string, error) {
+ open := strings.IndexByte(line, '{')
+ if open < 0 {
+ return nil, nil
+ }
+ close := strings.LastIndexByte(line, '}')
+ if close < open {
+ return nil, fmt.Errorf("unterminated label set: %q", line)
+ }
+
+ labels := map[string]string{}
+ for _, pair := range splitLabelPairs(line[open+1 : close]) {
+ key, value, found := strings.Cut(pair, "=")
+ if !found {
+ continue
+ }
+ unquoted, err := strconv.Unquote(strings.TrimSpace(value))
+ if err != nil {
+ unquoted = strings.Trim(strings.TrimSpace(value), `"`)
+ }
+ labels[strings.TrimSpace(key)] = unquoted
+ }
+ return labels, nil
+}
+
+// splitLabelPairs splits on commas that are not inside a quoted value.
+func splitLabelPairs(s string) []string {
+ var (
+ pairs []string
+ start int
+ quoted bool
+ )
+ for i := 0; i < len(s); i++ {
+ switch s[i] {
+ case '\\':
+ i++
+ case '"':
+ quoted = !quoted
+ case ',':
+ if !quoted {
+ pairs = append(pairs, s[start:i])
+ start = i + 1
+ }
+ }
+ }
+ if start < len(s) {
+ pairs = append(pairs, s[start:])
+ }
+ return pairs
+}
diff --git a/internal/bench/scrape_test.go b/internal/bench/scrape_test.go
new file mode 100644
index 00000000..d69f5ac7
--- /dev/null
+++ b/internal/bench/scrape_test.go
@@ -0,0 +1,109 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package bench
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "strings"
+ "testing"
+)
+
+func TestParseExpositionReadsLabelledSamples(t *testing.T) {
+ // The numbers a report quotes come out of this parser, so it has to be
+ // right about the cases the real endpoints actually emit.
+ const exposition = `# HELP sam_box_flows_total Flows a sandbox asked to open
+# TYPE sam_box_flows_total counter
+sam_box_flows_total{route="mesh-service",outcome="allowed"} 42
+sam_box_flows_total{route="unresolved",outcome="denied"} 7
+process_resident_memory_bytes 1.8874368e+07
+sam_box_flow_setup_seconds_bucket{route="mesh-service",le="0.001"} 12
+`
+
+ series, err := parseExposition(strings.NewReader(exposition))
+ if err != nil {
+ t.Fatalf("parseExposition: %v", err)
+ }
+ snap := &Snapshot{Series: series}
+
+ if got, ok := snap.Value("sam_box_flows_total", map[string]string{"outcome": "denied"}); !ok || got != 7 {
+ t.Errorf("denied flows = %v (found %v), want 7", got, ok)
+ }
+ if got := snap.Sum("sam_box_flows_total"); got != 49 {
+ t.Errorf("total flows = %v, want 49", got)
+ }
+ if got, ok := snap.Value("process_resident_memory_bytes", nil); !ok || got != 18874368 {
+ t.Errorf("rss = %v (found %v), want 18874368", got, ok)
+ }
+ if got, ok := snap.Value("sam_box_flow_setup_seconds_bucket", map[string]string{"le": "0.001"}); !ok || got != 12 {
+ t.Errorf("bucket = %v (found %v), want 12", got, ok)
+ }
+}
+
+func TestValueDistinguishesAbsentFromZero(t *testing.T) {
+ // A counter that was never incremented is not exported at all. Reporting
+ // it as zero would turn "this never happened" into "this happened zero
+ // times", and only one of those is evidence.
+ snap := &Snapshot{Series: []Series{{Name: "present", Value: 0}}}
+
+ if _, ok := snap.Value("present", nil); !ok {
+ t.Error("an exported zero was reported as absent")
+ }
+ if _, ok := snap.Value("absent", nil); ok {
+ t.Error("a series that was never exported was reported as present")
+ }
+}
+
+func TestParseExpositionKeepsCommasInsideLabelValues(t *testing.T) {
+ // Error reasons and model names can contain commas. Splitting naively
+ // would silently corrupt the label set rather than fail loudly.
+ series, err := parseExposition(strings.NewReader(`m{a="x,y",b="z"} 1` + "\n"))
+ if err != nil {
+ t.Fatalf("parseExposition: %v", err)
+ }
+ if len(series) != 1 {
+ t.Fatalf("got %d series, want 1", len(series))
+ }
+ if got := series[0].Labels["a"]; got != "x,y" {
+ t.Errorf("label a = %q, want %q", got, "x,y")
+ }
+ if got := series[0].Labels["b"]; got != "z" {
+ t.Errorf("label b = %q, want %q", got, "z")
+ }
+}
+
+func TestScrapeReadsALiveEndpoint(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ _, _ = w.Write([]byte("sam_node_requests_in_flight 3\n"))
+ }))
+ defer server.Close()
+
+ snap, err := Scrape(context.Background(), server.URL)
+ if err != nil {
+ t.Fatalf("Scrape: %v", err)
+ }
+ if got, ok := snap.Value("sam_node_requests_in_flight", nil); !ok || got != 3 {
+ t.Errorf("in flight = %v (found %v), want 3", got, ok)
+ }
+}
+
+func TestScrapeReportsAnUnreachableEndpoint(t *testing.T) {
+ // A silently empty snapshot would show up in a report as a process that
+ // used no memory and served no requests.
+ if _, err := Scrape(context.Background(), "http://127.0.0.1:1/metrics"); err == nil {
+ t.Error("Scrape of an unreachable endpoint returned no error")
+ }
+}