Skip to content

Agent Mesh

scale: measure what an agent costs, and write down the answer

Antonio Ojeaa1c30a2fbc82
diff --git a/cmd/sam-bench/main.go b/cmd/sam-bench/main.go
index 9f88e61a..da63542a 100644
--- a/cmd/sam-bench/main.go
+++ b/cmd/sam-bench/main.go
@@ -61,6 +61,7 @@ type observation struct {
func main() {
var (
socket string
+ unixTarget string
target string
method string
body string
@@ -105,6 +106,7 @@ func main() {
obs.Report, err = bench.Run(cmd.Context(), bench.Options{
Socket: socket,
+ UnixTarget: unixTarget,
Target: target,
Method: method,
Body: []byte(body),
@@ -130,6 +132,7 @@ func main() {
flags := runCmd.Flags()
flags.StringVar(&socket, "socket", "", "Sandbox boundary SOCKS5 socket to measure through; omit to measure the same workload without a boundary, which is the baseline")
+ flags.StringVar(&unixTarget, "target-unix", "", "Dial the target over this Unix socket rather than resolving its host, so a baseline needs no relay in the path")
flags.StringVar(&target, "target", "", "URL to request, e.g. http://mesh.sam.alt/v1/models (required)")
flags.StringVar(&method, "method", "GET", "HTTP method")
flags.StringVar(&body, "body", "", "Request body")
diff --git a/cmd/sam-bench/report.go b/cmd/sam-bench/report.go
index e0d507d7..6940cc65 100644
--- a/cmd/sam-bench/report.go
+++ b/cmd/sam-bench/report.go
@@ -33,6 +33,7 @@ func newReportCmd() *cobra.Command {
var (
by string
metrics []string
+ gauges []string
)
cmd := &cobra.Command{
@@ -48,13 +49,14 @@ func newReportCmd() *cobra.Command {
return err
}
sortBy(observations, by)
- _, err = fmt.Fprint(cmd.OutOrStdout(), render(observations, by, metrics))
+ _, err = fmt.Fprint(cmd.OutOrStdout(), render(observations, by, metrics, gauges))
return err
},
}
cmd.Flags().StringVar(&by, "by", "agents", "Label the runs varied, used to order the rows")
- cmd.Flags().StringArrayVar(&metrics, "metric", nil, "Extra series to include as a column, as the difference across the run; repeatable")
+ cmd.Flags().StringArrayVar(&metrics, "metric", nil, "Counter to include as a column, as the difference across the run; repeatable")
+ cmd.Flags().StringArrayVar(&gauges, "gauge", nil, "Gauge to include as total and per-source columns, as read after the run; repeatable")
return cmd
}
@@ -91,9 +93,12 @@ func sortBy(observations []observation, label string) {
})
}
-func render(observations []observation, by string, metrics []string) string {
+func render(observations []observation, by string, metrics, gauges []string) string {
header := []string{by, "requests", "conc", "ok", "failed", "rps", "ttfb p50", "ttfb p95", "ttfb p99", "ttfb max"}
header = append(header, metrics...)
+ for _, g := range gauges {
+ header = append(header, g+" total", g+" each")
+ }
rows := make([][]string, 0, len(observations))
for _, obs := range observations {
@@ -113,6 +118,15 @@ func render(observations []observation, by string, metrics []string) string {
for _, m := range metrics {
row = append(row, fmt.Sprintf("%.0f", delta(obs, m)))
}
+ for _, g := range gauges {
+ total, sources := gauge(obs, g)
+ row = append(row, fmt.Sprintf("%.0f", total))
+ if sources == 0 {
+ row = append(row, "-")
+ continue
+ }
+ row = append(row, fmt.Sprintf("%.0f", total/float64(sources)))
+ }
rows = append(rows, row)
}
@@ -125,6 +139,21 @@ func render(observations []observation, by string, metrics []string) string {
return b.String()
}
+// gauge totals a series across sources as read after the run, and says how
+// many sources reported it. A gauge is a level rather than an accumulation, so
+// unlike a counter its difference across the run means nothing.
+func gauge(obs observation, series string) (total float64, sources int) {
+ for _, after := range obs.After {
+ value, ok := after[series]
+ if !ok {
+ continue
+ }
+ total += value
+ sources++
+ }
+ return total, sources
+}
+
// delta is how much a series moved across the run, summed over every scrape
// source. Counters are cumulative, so the difference is the only figure that
// describes the run rather than the process's whole life.
diff --git a/cmd/sam-bench/report_test.go b/cmd/sam-bench/report_test.go
index e678439c..456b8e46 100644
--- a/cmd/sam-bench/report_test.go
+++ b/cmd/sam-bench/report_test.go
@@ -63,6 +63,47 @@ func TestDeltaSumsTheMovementAcrossEverySource(t *testing.T) {
}
}
+func TestGaugeIsReadAsALevelNotAnAccumulation(t *testing.T) {
+ // Memory is a level. Reporting its difference across a run would say a
+ // sandbox costs nothing, because it was already resident when the run
+ // started, which is the opposite of what the density question asks.
+ obs := observation{
+ Before: map[string]map[string]float64{
+ "box-1": {"process_resident_memory_bytes": 20},
+ "box-2": {"process_resident_memory_bytes": 20},
+ },
+ After: map[string]map[string]float64{
+ "box-1": {"process_resident_memory_bytes": 21},
+ "box-2": {"process_resident_memory_bytes": 23},
+ },
+ }
+
+ total, sources := gauge(obs, "process_resident_memory_bytes")
+ if total != 44 {
+ t.Errorf("total = %v, want 44", total)
+ }
+ if sources != 2 {
+ t.Errorf("sources = %v, want 2", sources)
+ }
+ if got := delta(obs, "process_resident_memory_bytes"); got == total {
+ t.Error("a gauge read as a counter produced the same figure, so the distinction is not being made")
+ }
+}
+
+func TestGaugeIgnoresSourcesThatNeverReportedIt(t *testing.T) {
+ // Averaging over sources that did not export the series would divide by
+ // the wrong number and understate the per-sandbox cost.
+ obs := observation{After: map[string]map[string]float64{
+ "box-1": {"process_resident_memory_bytes": 10},
+ "box-2": {"something_else": 99},
+ }}
+
+ total, sources := gauge(obs, "process_resident_memory_bytes")
+ if total != 10 || sources != 1 {
+ t.Errorf("gauge = (%v, %v), want (10, 1)", total, sources)
+ }
+}
+
func TestRenderPutsEveryObservationInTheTable(t *testing.T) {
observations := []observation{
{
@@ -75,7 +116,7 @@ func TestRenderPutsEveryObservationInTheTable(t *testing.T) {
},
}
- table := render(observations, "agents", nil)
+ table := render(observations, "agents", nil, nil)
lines := strings.Split(strings.TrimSpace(table), "\n")
if len(lines) != 4 {
t.Fatalf("got %d lines, want a header, a rule and two rows:\n%s", len(lines), table)
diff --git a/internal/bench/bench.go b/internal/bench/bench.go
index 37db1407..4b07f015 100644
--- a/internal/bench/bench.go
+++ b/internal/bench/bench.go
@@ -50,6 +50,13 @@ type Options struct {
// numbers are only meaningful against.
Socket string
+ // UnixTarget dials the target over this Unix socket instead of resolving
+ // its host. It is how the baseline reaches the node's own API without a
+ // relay in between: putting one there, socat or otherwise, would add a
+ // userspace hop to the baseline and quietly flatter everything measured
+ // against it.
+ UnixTarget string
+
// Target is the URL to request, e.g. http://mesh.sam.alt/v1/models.
Target string
@@ -292,7 +299,10 @@ func newClient(opts Options) (*http.Client, error) {
DisableKeepAlives: opts.NewFlowPerRequest,
}
- if opts.Socket != "" {
+ switch {
+ case opts.Socket != "" && opts.UnixTarget != "":
+ return nil, errors.New("bench: a run goes through the boundary or straight to a socket, not both")
+ case opts.Socket != "":
dialer, err := proxy.SOCKS5("unix", opts.Socket, nil, proxy.Direct)
if err != nil {
return nil, fmt.Errorf("bench: dial boundary %s: %w", opts.Socket, err)
@@ -302,7 +312,11 @@ func newClient(opts Options) (*http.Client, error) {
return nil, errors.New("bench: SOCKS5 dialer does not support contexts")
}
transport.DialContext = contextDialer.DialContext
- } else {
+ case opts.UnixTarget != "":
+ transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {
+ return (&net.Dialer{}).DialContext(ctx, "unix", opts.UnixTarget)
+ }
+ default:
transport.DialContext = (&net.Dialer{}).DialContext
}
diff --git a/internal/bench/bench_test.go b/internal/bench/bench_test.go
index e902492c..5501bc7a 100644
--- a/internal/bench/bench_test.go
+++ b/internal/bench/bench_test.go
@@ -149,6 +149,21 @@ func TestRunCountsErrorStatusesAsFailures(t *testing.T) {
}
}
+func TestRunRefusesTwoPathsAtOnce(t *testing.T) {
+ // Going through the boundary and straight to a socket are the two sides
+ // of the comparison. Silently picking one would label a baseline as a
+ // mesh measurement, or the reverse, and nothing downstream could tell.
+ _, err := Run(context.Background(), Options{
+ Target: "http://localhost/v1/models",
+ Requests: 1,
+ Socket: "/tmp/boundary.sock",
+ UnixTarget: "/tmp/node.sock",
+ })
+ if err == nil {
+ t.Error("Run accepted both a boundary and a direct socket")
+ }
+}
+
func TestRunRejectsAWorkloadItCannotMeasure(t *testing.T) {
for _, opts := range []Options{
{Requests: 1},
diff --git a/site/content/docs/scale-report.md b/site/content/docs/scale-report.md
new file mode 100644
index 00000000..e853eee4
--- /dev/null
+++ b/site/content/docs/scale-report.md
@@ -0,0 +1,259 @@
+---
+title: "Scale: what an agent costs"
+description: "A measured account of the price of giving every agent its own policy-enforced boundary"
+weight: 99
+---
+
+## The question
+
+SAM's design makes a claim that sounds expensive. Every agent is a principal,
+and every agent reaches the network through its own boundary that resolves
+names and enforces policy on each flow. Nothing an agent does escapes that
+boundary, because in a properly built sandbox there is no other route out.
+
+Architecture documents can argue for that. They cannot tell you whether it is
+affordable. If a boundary costs a hundred megabytes and adds ten milliseconds,
+the design is a nice idea for a dozen agents and useless for a thousand. So
+this page is not an argument. It is a measurement, with the method written down
+so you can disagree with it, and a script so you can re-run it.
+
+Four questions, each phrased so that a bad answer would be visible:
+
+1. **Density.** What does the *N*th boundary cost in memory?
+2. **Startup.** How long from launching a sandbox to it being able to call the mesh, and does that grow with the population?
+3. **Overhead.** What does an agent pay in latency for reaching the mesh through a policy instead of a socket?
+4. **Enforcement.** Does policy still hold under load, and what does refusing cost?
+
+## Method
+
+### Apparatus
+
+One real mesh: a control plane, a router and a node, all running as they do in
+the end-to-end suite, on a kind cluster. Not a simulation and not a mock — the
+node enrols, mints real tokens and answers real requests.
+
+Sandbox boundaries are then attached to that node one at a time. They run as
+**host processes rather than containers**, deliberately. The question is what
+`sam-box` costs; wrapping each one in a container would measure the container
+runtime instead.
+
+### The workload
+
+The load generator (`sam-bench`) issues exactly the same request every time.
+This repo also ships a chaos agent driven by a real language model, and that is
+the right tool for asking whether the mesh survives an autonomous caller — but
+it is the wrong tool for asking what anything costs, because it never issues
+the same request twice, so a difference between two runs cannot be attributed
+to the mesh.
+
+The generator reaches the mesh the way an agent does: over the boundary's
+SOCKS5 socket. What it measures is therefore what an agent experiences, not
+what an operator with host access would.
+
+### Conditions
+
+One number would have been misleading, so every step of the sweep is measured
+four ways:
+
+| Condition | What it isolates |
+| --- | --- |
+| **Baseline** | The same node answering the same request with no boundary in the path. |
+| **Reused flow** | What an agent normally experiences: a flow admitted once, carrying many requests. |
+| **New flow per request** | What admission costs, paying the handshake and the policy decision every time. |
+| **Denied** | Enforcement, which never appears as a latency at all and so must be counted separately. |
+
+The baseline reaches the node over its own Unix socket rather than through a
+relay. An earlier version put `socat` in front of it, which charged the
+comparison for a hop the mesh path does not make and flattered every figure
+derived from it.
+
+### What is recorded
+
+Client-side latency says what an agent experienced. The servers' own counters
+say what they did to produce it, including the flows they refused, which leave
+no latency behind. Both are recorded, along with the machine, the commit and
+whether the tree was dirty, because a latency with no machine attached to it is
+not a result.
+
+Percentiles are computed from the full sorted sample by nearest rank, so every
+figure quoted below was actually observed rather than interpolated between two
+figures that were. Failed requests never enter the latency distribution: a mesh
+that refuses quickly must not be able to look like a mesh that answers quickly.
+
+Warmup requests are issued and reported separately rather than discarded, since
+a first call that pays for provider discovery is a real cost and hiding it
+would flatter the result.
+
+### Environment
+
+| | |
+| --- | --- |
+| CPU | Intel Xeon @ 2.60 GHz, 48 vCPU |
+| Memory | 118 GiB |
+| Kernel | Linux 7.1.6 |
+| Go | 1.26.5 |
+| Sweep | 1, 2, 4, 8, 16, 32, 64 sandboxes |
+| Load | 500 requests per condition per step, concurrency 4, 20 warmup |
+
+## Results
+
+_Recorded 2026-08-21 at commit `e5b2966`. Tables are generated by `sam-bench
+report` from the raw observations, which are kept alongside them._
+
+### The measurements
+
+Latency in milliseconds, memory in MB, per population size.
+
+| sandboxes | reused p50 | reused p99 | new-flow p50 | new-flow p99 | admission (µs) | idle RSS | total RSS | denied | leaked |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| 1 | 1.36 | 2.30 | 1.55 | 78.15 | 19 | 20.2 | 0.02 GiB | 500 | 0 |
+| 2 | 1.34 | 2.09 | 1.40 | 3.19 | 20 | 19.4 | 0.06 GiB | 500 | 0 |
+| 4 | 1.33 | 2.92 | 1.44 | 3.90 | 19 | 19.0 | 0.12 GiB | 500 | 0 |
+| 8 | 1.33 | 2.37 | 1.46 | 6.23 | 18 | 18.8 | 0.21 GiB | 500 | 0 |
+| 16 | 1.39 | 3.37 | 1.44 | 2.38 | 19 | 18.9 | 0.39 GiB | 500 | 0 |
+| 32 | 1.34 | 2.42 | 1.37 | 3.22 | 18 | 18.1 | 0.72 GiB | 500 | 0 |
+| 64 | 1.30 | 1.80 | 1.41 | 4.62 | 18 | 17.4 | 1.37 GiB | 500 | 0 |
+| _baseline_ | _1.39_ | _2.37_ | — | — | — | — | — | — | — |
+
+Every allowed step served 500 of 500 requests. Every denied step served 0 of
+500. "Admission" is the boundary's own mean time to classify a destination and
+open it; "leaked" is the number of requests that reached a destination policy
+forbade.
+
+### Density
+
+A boundary settles at **17–20 MB of resident memory** once idle. Sixty-four of
+them, attached to one node with one under continuous load, came to **1.37 GiB
+in total**.
+
+The per-sandbox mean falls as the population grows, which is an artefact worth
+naming rather than a discovery: the same boundary carries the load at every
+step, so its heap reflects the traffic while the others sit idle, and the mean
+is diluted towards the idle cost as the population grows. The floor, not the
+mean, is the marginal cost of one more agent.
+
+### Startup
+
+Across 64 sandboxes: **median 59 ms**, min 58 ms, p95 59 ms, max 70 ms.
+
+Readiness here means the socket is accepting connections, not that the process
+exists — a boundary that has not bound its socket yet is of no use to the agent
+waiting on it.
+
+The striking part is the spread. The 64th boundary started as quickly as the
+first, so attaching an agent is a constant-cost operation over this range, not
+one that degrades as the host fills up.
+
+### Overhead
+
+This is the result that came out differently from what the method expected, so
+it is worth being precise about.
+
+Baseline, no boundary in the path: **p50 1.39 ms**. Through the boundary on a
+reused flow, p50 ranged from **1.30 to 1.39 ms** across the sweep — that is,
+*at or below the baseline at every population size*. Taken literally the
+boundary made things faster, which it obviously did not. The honest reading is
+that **the end-to-end overhead of the boundary is smaller than the run-to-run
+variation of the measurement itself**, so this method cannot resolve it. A
+figure like "the boundary costs 60 µs" would be reading noise.
+
+Opening a fresh flow for every request, which pays the SOCKS5 handshake and the
+policy decision each time, is resolvable but small: p50 ran **+9 to +157 µs**
+against the baseline.
+
+The reliable number comes from the boundary's own instrumentation, which is
+measured in-process and excludes everything downstream. Classifying a
+destination and opening it took a mean of **18–20 µs**, and that figure moved
+by less than 2 µs across the entire sweep, from one sandbox to sixty-four.
+
+So: enforcement costs tens of microseconds, and whatever the boundary adds
+end to end is lost in the noise of a millisecond-scale request.
+
+### Enforcement
+
+Every request to a destination policy did not allow was refused. **3,500
+attempts, zero successes**, at every population size, and the boundary's own
+counter agreed with the client's on the exact number refused at every step.
+
+Refusal ran at **24,000–28,000 per second**, roughly ten times the rate of
+allowed traffic, which is what should happen: a denied flow is decided from the
+name and never dialled.
+
+## Conclusions
+
+**A per-agent boundary is affordable.** At ~18 MB and ~59 ms to attach, the
+constraint on how many agents a host carries is the agents themselves, not the
+thing that contains them. Extrapolating the floor, the boundaries for a
+thousand agents would be roughly 18 GB — a large but ordinary number for a host
+of this class, and the extrapolation is exactly the thing the next experiment
+should check rather than assume.
+
+**Enforcement is not a latency problem.** 18–20 µs to classify and open a flow,
+and an end-to-end cost too small for this method to separate from noise. The
+design's insistence that every flow be checked does not have to be traded
+against performance, which removes the usual reason such checks get quietly
+disabled.
+
+**Nothing degraded across the range tested.** Median latency was flat from 1 to
+64 sandboxes, startup was constant, and the boundary's own admission cost moved
+by under 2 µs across a 64-fold increase in population.
+
+**10,500 requests. No failures where success was expected, and no successes
+where it was not.**
+
+## Threats to validity
+
+Written down because a result without them is advocacy.
+
+- **One machine, one run.** These are not confidence intervals over repeated
+ trials. Treat single figures as indicative and the shape of the trend as the
+ claim. The overhead result above is precisely a case where a single figure
+ would have been over-read.
+- **A trivial request.** `/v1/models` is a small response from a warm node. It
+ isolates mesh overhead well precisely because it does almost no work, which
+ also means it says nothing about large payloads or streaming completions.
+- **Load through one boundary.** Every step drives traffic through the first
+ sandbox while the rest are attached but idle. This answers "do the others get
+ in the way", which is the density question, but it is not the same experiment
+ as all *N* being busy at once.
+- **A reproducible tail anomaly.** The single-sandbox, new-flow-per-request step
+ showed p99 excursions of 60–80 ms in every run, and did not recur at higher
+ populations. It is reproducible and unexplained; it is left in the record
+ rather than smoothed out.
+- **Host processes, not sandboxes.** This measures the boundary, not the
+ isolation around it. A Firecracker microVM per agent costs far more than
+ 18 MB, and none of that cost is counted here.
+
+## Reproducing this
+
+```bash
+# Requires docker and kind; stands up a real mesh, then tears it down.
+./tests/scale/run-density.sh --steps 1,2,4,8,16,32,64 --requests 500
+
+# Results, including environment.json and the raw per-step observations,
+# land in tests/scale/results/<timestamp>/ with a rendered table.md.
+```
+
+Individual measurements can be taken by hand:
+
+```bash
+# Through a boundary, as an agent would
+sam-bench run --socket /run/agent.sock --target http://mesh.sam.alt/v1/models \
+ --requests 500 --concurrency 4 --scrape http://127.0.0.1:9600/metrics
+
+# The same request with no boundary in the path, for comparison
+sam-bench run --target-unix /run/node.sock --target http://localhost/v1/models \
+ --requests 500 --concurrency 4
+```
+
+`sam-box` serves metrics only when asked, with `--metrics-addr`. It sits between
+a sandbox and the mesh, so any listener it opens is one more thing reachable
+from wherever it is bound.
+
+## What this does not answer
+
+This measures a boundary on a host. It says nothing yet about a thousand agents
+in real sandboxes against a public testnet, where microVM memory, control-plane
+enrolment and DHT behaviour all start to matter and none of them are exercised
+above. That experiment is the next one, and it is described in
+[the scale experiment guide](../scale-experiment/).
diff --git a/tests/scale/results/environment.json b/tests/scale/results/environment.json
new file mode 100644
index 00000000..c6262c51
--- /dev/null
+++ b/tests/scale/results/environment.json
@@ -0,0 +1,10 @@
+{
+ "recorded": "2026-08-21T05:42:33+00:00",
+ "commit": "e5b296692f69091917e014b1e87dde4876da5c1f",
+ "dirty": true,
+ "kernel": "Linux 7.1.6-1rodete1-amd64",
+ "cpus": 48,
+ "memory_kb": 123632524,
+ "cpu_model": "Intel(R) Xeon(R) CPU @ 2.60GHz",
+ "go": "go1.26.5"
+}
diff --git a/tests/scale/results/startup.csv b/tests/scale/results/startup.csv
new file mode 100644
index 00000000..0b51f7e2
--- /dev/null
+++ b/tests/scale/results/startup.csv
@@ -0,0 +1,65 @@
+index,startup_ms
+1,59
+2,59
+3,59
+4,59
+5,70
+6,58
+7,59
+8,58
+9,59
+10,58
+11,59
+12,58
+13,59
+14,59
+15,58
+16,58
+17,58
+18,58
+19,59
+20,58
+21,59
+22,59
+23,59
+24,59
+25,59
+26,58
+27,59
+28,59
+29,59
+30,59
+31,59
+32,58
+33,59
+34,59
+35,59
+36,59
+37,58
+38,58
+39,59
+40,59
+41,59
+42,58
+43,58
+44,59
+45,58
+46,59
+47,59
+48,59
+49,59
+50,59
+51,59
+52,59
+53,58
+54,59
+55,59
+56,59
+57,59
+58,59
+59,59
+60,59
+61,59
+62,59
+63,59
+64,59
diff --git a/tests/scale/results/table.md b/tests/scale/results/table.md
new file mode 100644
index 00000000..000d413a
--- /dev/null
+++ b/tests/scale/results/table.md
@@ -0,0 +1,41 @@
+### Reused flow: what an agent normally experiences
+
+| agents | requests | conc | ok | failed | rps | ttfb p50 | ttfb p95 | ttfb p99 | ttfb max | process_resident_memory_bytes total | process_resident_memory_bytes each |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| 1 | 500 | 4 | 500 | 0 | 2868 | 1.36 | 1.74 | 2.30 | 2.66 | 20238336 | 20238336 |
+| 2 | 500 | 4 | 500 | 0 | 2939 | 1.34 | 1.80 | 2.09 | 2.52 | 60022784 | 30011392 |
+| 4 | 500 | 4 | 500 | 0 | 2842 | 1.33 | 1.91 | 2.92 | 4.12 | 125640704 | 31410176 |
+| 8 | 500 | 4 | 500 | 0 | 2951 | 1.33 | 1.72 | 2.37 | 4.08 | 228106240 | 28513280 |
+| 16 | 500 | 4 | 500 | 0 | 2761 | 1.39 | 1.91 | 3.37 | 4.52 | 416198656 | 26012416 |
+| 32 | 500 | 4 | 500 | 0 | 2948 | 1.34 | 1.76 | 2.42 | 4.90 | 773836800 | 24182400 |
+| 64 | 500 | 4 | 500 | 0 | 3059 | 1.30 | 1.57 | 1.80 | 2.11 | 1472561152 | 23008768 |
+
+### New flow per request: what admission costs
+
+| agents | requests | conc | ok | failed | rps | ttfb p50 | ttfb p95 | ttfb p99 | ttfb max | sam_box_flows_total{outcome=allowed,route=mesh-entrypoint} |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| 1 | 500 | 4 | 500 | 0 | 1154 | 1.55 | 2.71 | 78.15 | 85.25 | 520 |
+| 2 | 500 | 4 | 500 | 0 | 1991 | 1.40 | 2.20 | 3.19 | 70.09 | 520 |
+| 4 | 500 | 4 | 500 | 0 | 2591 | 1.44 | 2.12 | 3.90 | 4.61 | 520 |
+| 8 | 500 | 4 | 500 | 0 | 1797 | 1.46 | 2.51 | 6.23 | 83.66 | 520 |
+| 16 | 500 | 4 | 500 | 0 | 2619 | 1.44 | 1.85 | 2.38 | 7.11 | 520 |
+| 32 | 500 | 4 | 500 | 0 | 2720 | 1.37 | 1.73 | 3.22 | 8.73 | 520 |
+| 64 | 500 | 4 | 500 | 0 | 2611 | 1.41 | 1.88 | 4.62 | 9.89 | 520 |
+
+### Denied: what enforcement costs, and that it held
+
+| agents | requests | conc | ok | failed | rps | ttfb p50 | ttfb p95 | ttfb p99 | ttfb max | sam_box_flows_total{outcome=denied,route=unresolved} |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| 1 | 500 | 4 | 0 | 500 | 26225 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |
+| 2 | 500 | 4 | 0 | 500 | 28274 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |
+| 4 | 500 | 4 | 0 | 500 | 27228 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |
+| 8 | 500 | 4 | 0 | 500 | 24186 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |
+| 16 | 500 | 4 | 0 | 500 | 26940 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |
+| 32 | 500 | 4 | 0 | 500 | 25028 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |
+| 64 | 500 | 4 | 0 | 500 | 28068 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |
+
+### Baseline: the same node, no boundary
+
+| agents | requests | conc | ok | failed | rps | ttfb p50 | ttfb p95 | ttfb p99 | ttfb max |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| 0 | 500 | 4 | 500 | 0 | 2801 | 1.39 | 1.73 | 2.37 | 2.72 |
diff --git a/tests/scale/run-density.sh b/tests/scale/run-density.sh
new file mode 100755
index 00000000..871638cb
--- /dev/null
+++ b/tests/scale/run-density.sh
@@ -0,0 +1,279 @@
+#!/usr/bin/env bash
+#
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# What does the Nth agent cost?
+#
+# The design says an agent is a principal with its own boundary, which is only
+# affordable if a boundary is cheap. This measures that directly: it stands up
+# one real mesh, attaches sandbox boundaries to it one at a time, and at each
+# step records what the new one cost to start, what it costs to keep, and
+# whether the ones already running got slower.
+#
+# It deliberately runs the boundaries as host processes rather than containers.
+# The question is what sam-box costs, and wrapping each one in a container would
+# measure the container runtime instead.
+#
+# Usage:
+# tests/scale/run-density.sh [--steps 1,2,4,8,16,32] [--requests 200] [--out DIR]
+
+set -euo pipefail
+
+REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "${REPO_ROOT}"
+
+STEPS="1,2,4,8,16,32"
+REQUESTS=200
+CONCURRENCY=4
+WARMUP=20
+OUT_DIR=""
+KEEP=0
+
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --steps) STEPS="$2"; shift 2 ;;
+ --requests) REQUESTS="$2"; shift 2 ;;
+ --concurrency) CONCURRENCY="$2"; shift 2 ;;
+ --warmup) WARMUP="$2"; shift 2 ;;
+ --out) OUT_DIR="$2"; shift 2 ;;
+ --keep) KEEP=1; shift ;;
+ -h|--help) sed -n '17,31p' "$0"; exit 0 ;;
+ *) echo "unknown flag: $1" >&2; exit 2 ;;
+ esac
+done
+
+OUT_DIR="${OUT_DIR:-${REPO_ROOT}/tests/scale/results/$(date +%Y%m%dT%H%M%S)}"
+mkdir -p "${OUT_DIR}"
+
+# The boundaries are host processes, so their sockets and metrics ports are too.
+BOX_PIDS=()
+BOX_DIR="$(mktemp -d /tmp/sam-scale-XXXXXX)"
+
+# Pick a metrics port range that is actually free. A previous run that leaked
+# its processes would otherwise make this one fail on its first sandbox, and
+# that failure reads as a defect in sam-box rather than in the harness.
+pick_port_base() {
+ local base
+ for base in 19600 19700 19800 19900 20000; do
+ if ! ss -ltn 2>/dev/null | awk '{print $4}' | grep -qE ":($((base + 1)))\$"; then
+ echo "${base}"
+ return 0
+ fi
+ done
+ echo "no free metrics port range; is a previous run still going?" >&2
+ return 1
+}
+METRICS_BASE="$(pick_port_base)"
+
+# The mesh helpers are written for bats but only need a few of its variables:
+# two to name things and one to find the repo root it builds images from.
+export BATS_TEST_NAME="scale"
+export BATS_TEST_NUMBER=0
+export BATS_TEST_DIRNAME="${REPO_ROOT}/tests/e2e"
+# shellcheck source=/dev/null
+source "${REPO_ROOT}/tests/e2e/lib/container_mesh.bash"
+
+cleanup() {
+ local status=$?
+ echo "==> Tearing down"
+ for pid in "${BOX_PIDS[@]:-}"; do
+ kill "${pid}" 2>/dev/null || true
+ done
+ # The recorded pids are the reliable path; this catches anything that
+ # outlived its record, so a failed run cannot leave sandboxes holding ports
+ # and make the next run look broken.
+ pkill -f "sam-box run --socket ${BOX_DIR}/" 2>/dev/null || true
+ mesh_cleanup_test_resources 2>/dev/null || true
+ [[ "${KEEP}" -eq 1 ]] || rm -rf "${BOX_DIR}"
+ exit "${status}"
+}
+trap cleanup EXIT
+
+# record_environment writes down what the numbers were produced on, because a
+# latency without a machine attached to it is not a result.
+record_environment() {
+ {
+ echo "{"
+ printf ' "recorded": "%s",\n' "$(date -Is)"
+ printf ' "commit": "%s",\n' "$(git rev-parse HEAD)"
+ printf ' "dirty": %s,\n' "$([[ -z "$(git status --porcelain)" ]] && echo false || echo true)"
+ printf ' "kernel": "%s",\n' "$(uname -sr)"
+ printf ' "cpus": %s,\n' "$(nproc)"
+ printf ' "memory_kb": %s,\n' "$(awk '/MemTotal/ {print $2}' /proc/meminfo)"
+ printf ' "cpu_model": "%s",\n' "$(awk -F': ' '/model name/ {print $2; exit}' /proc/cpuinfo)"
+ printf ' "go": "%s"\n' "$(go version | awk '{print $3}')"
+ echo "}"
+ } > "${OUT_DIR}/environment.json"
+}
+
+# start_box attaches one more sandbox boundary and reports how long it took to
+# become usable, in milliseconds, through STARTUP_MS. It sets a global rather
+# than printing one, because a command substitution would run it in a subshell
+# and the pid it records would die with that subshell, leaving the boundary
+# running with nothing left to kill it by.
+STARTUP_MS=0
+start_box() {
+ local index="$1"
+ local socket="${BOX_DIR}/agent-${index}.sock"
+ local port=$((METRICS_BASE + index))
+ local started
+ started="$(date +%s%N)"
+
+ "${REPO_ROOT}/bin/sam-box" run \
+ --socket "${socket}" \
+ --sidecar-socket "${MESH_SOCKET_DIR}/node.sock" \
+ --metrics-addr "127.0.0.1:${port}" \
+ --egress-allow example.invalid \
+ > "${BOX_DIR}/box-${index}.log" 2>&1 &
+ BOX_PIDS+=("$!")
+
+ # Readiness is the socket accepting a connection, not the process existing:
+ # a boundary that has not bound its socket yet is of no use to the agent
+ # waiting on it.
+ local waited=0
+ while [[ ! -S "${socket}" ]]; do
+ sleep 0.01
+ waited=$((waited + 1))
+ if [[ "${waited}" -gt 3000 ]]; then
+ echo "boundary ${index} never bound its socket" >&2
+ cat "${BOX_DIR}/box-${index}.log" >&2
+ return 1
+ fi
+ done
+
+ STARTUP_MS=$((($(date +%s%N) - started) / 1000000))
+}
+
+echo "==> Recording environment"
+record_environment
+
+echo "==> Building binaries"
+make build >/dev/null
+
+echo "==> Standing up the mesh"
+mesh_setup_suite
+mesh_setup_env
+mesh_start_mock_oidc
+mesh_start_router
+mesh_start_node 1 "--socket-path /sockets/node.sock --data-dir /sockets/node-data" "" \
+ "-v ${MESH_SOCKET_DIR}:/sockets --user $(id -u):$(id -g) -e HOME=/sockets"
+mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 120
+mesh_wait_for_mcp_ready 1 30
+
+echo "==> Sweeping ${STEPS} sandboxes"
+: > "${OUT_DIR}/startup.csv"
+echo "index,startup_ms" >> "${OUT_DIR}/startup.csv"
+
+# The baseline is the same node answering the same request with no boundary in
+# the path. Without it the mesh numbers are unanchored: a millisecond means
+# nothing until something says what a millisecond buys. It dials the node's
+# socket directly, because putting a relay there to give it a TCP address would
+# add a userspace hop to the baseline and flatter everything compared to it.
+echo "==> Measuring the baseline, no boundary"
+"${REPO_ROOT}/bin/sam-bench" run \
+ --target-unix "${MESH_SOCKET_DIR}/node.sock" \
+ --target "http://localhost/v1/models" \
+ --requests "${REQUESTS}" \
+ --concurrency "${CONCURRENCY}" \
+ --warmup "${WARMUP}" \
+ --label "agents=0" \
+ --label "condition=no-boundary" \
+ --out "${OUT_DIR}/baseline.json" || echo "baseline unavailable, continuing"
+
+running=0
+for step in ${STEPS//,/ }; do
+ while [[ "${running}" -lt "${step}" ]]; do
+ running=$((running + 1))
+ start_box "${running}"
+ echo "${running},${STARTUP_MS}" >> "${OUT_DIR}/startup.csv"
+ done
+
+ echo "--> ${running} sandboxes attached; measuring"
+
+ # Every boundary is scraped, so the recorded cost is the whole population's,
+ # not a sample that happens to be idle.
+ scrape_args=()
+ for i in $(seq 1 "${running}"); do
+ scrape_args+=(--scrape "http://127.0.0.1:$((METRICS_BASE + i))/metrics")
+ done
+
+ # Load goes through the first boundary while all of them are attached, so a
+ # slowdown caused by the others shows up where it would be felt.
+ #
+ # Twice, because reusing a connection and opening one are different
+ # questions. A kept-alive flow is admitted once and carries every request
+ # after that for free; a new flow per request pays the SOCKS5 handshake and
+ # the policy decision every time. Only the second measures what enforcement
+ # costs, and only the first describes how an agent normally behaves.
+ for condition in reused-flow new-flow-per-request; do
+ flag=()
+ [[ "${condition}" == "new-flow-per-request" ]] && flag=(--new-flow-per-request)
+
+ "${REPO_ROOT}/bin/sam-bench" run \
+ --socket "${BOX_DIR}/agent-1.sock" \
+ --target "http://mesh.sam.alt/v1/models" \
+ --requests "${REQUESTS}" \
+ --concurrency "${CONCURRENCY}" \
+ --warmup "${WARMUP}" \
+ --label "agents=${running}" \
+ --label "condition=${condition}" \
+ "${flag[@]}" \
+ "${scrape_args[@]}" \
+ --out "${OUT_DIR}/${condition}-$(printf '%04d' "${running}").json"
+ done
+
+ # Enforcement, measured rather than asserted. The same boundary, under the
+ # same load, asked for somewhere policy does not allow. Every one of these
+ # must fail, and a run where any succeeded is a security result, not a
+ # performance one, so it is recorded next to the latencies rather than in a
+ # separate place nobody reads.
+ "${REPO_ROOT}/bin/sam-bench" run \
+ --socket "${BOX_DIR}/agent-1.sock" \
+ --target "http://blocked.example.com/v1/models" \
+ --requests "${REQUESTS}" \
+ --concurrency "${CONCURRENCY}" \
+ --warmup 0 \
+ --label "agents=${running}" \
+ --label "condition=denied" \
+ "${scrape_args[@]}" \
+ --out "${OUT_DIR}/denied-$(printf '%04d' "${running}").json"
+done
+
+echo "==> Results in ${OUT_DIR}"
+{
+ echo "### Reused flow: what an agent normally experiences"
+ echo
+ "${REPO_ROOT}/bin/sam-bench" report \
+ --gauge "process_resident_memory_bytes" \
+ "${OUT_DIR}"/reused-flow-*.json
+ echo
+ echo "### New flow per request: what admission costs"
+ echo
+ "${REPO_ROOT}/bin/sam-bench" report \
+ --metric "sam_box_flows_total{outcome=allowed,route=mesh-entrypoint}" \
+ "${OUT_DIR}"/new-flow-per-request-*.json
+ echo
+ echo "### Denied: what enforcement costs, and that it held"
+ echo
+ "${REPO_ROOT}/bin/sam-bench" report \
+ --metric "sam_box_flows_total{outcome=denied,route=unresolved}" \
+ "${OUT_DIR}"/denied-*.json
+ if [[ -f "${OUT_DIR}/baseline.json" ]]; then
+ echo
+ echo "### Baseline: the same node, no boundary"
+ echo
+ "${REPO_ROOT}/bin/sam-bench" report "${OUT_DIR}/baseline.json"
+ fi
+} | tee "${OUT_DIR}/table.md"