Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
scale: measure what an agent costs, and write down the answer
Antonio Ojeaa1c30a2fbc82
diff --git a/cmd/sam-bench/main.go b/cmd/sam-bench/main.goindex 9f88e61a..da63542a 100644--- a/cmd/sam-bench/main.go+++ b/cmd/sam-bench/main.go@@ -61,6 +61,7 @@ type observation struct {func main() {var (socket string+ unixTarget stringtarget stringmethod stringbody string@@ -105,6 +106,7 @@ func main() {obs.Report, err = bench.Run(cmd.Context(), bench.Options{Socket: socket,+ UnixTarget: unixTarget,Target: target,Method: method,Body: []byte(body),@@ -130,6 +132,7 @@ func main() {flags := runCmd.Flags()flags.StringVar(&socket, "socket", "", "Sandbox boundary SOCKS5 socket to measure through; omit to measure the same workload without a boundary, which is the baseline")+ flags.StringVar(&unixTarget, "target-unix", "", "Dial the target over this Unix socket rather than resolving its host, so a baseline needs no relay in the path")flags.StringVar(&target, "target", "", "URL to request, e.g. http://mesh.sam.alt/v1/models (required)")flags.StringVar(&method, "method", "GET", "HTTP method")flags.StringVar(&body, "body", "", "Request body")diff --git a/cmd/sam-bench/report.go b/cmd/sam-bench/report.goindex e0d507d7..6940cc65 100644--- a/cmd/sam-bench/report.go+++ b/cmd/sam-bench/report.go@@ -33,6 +33,7 @@ func newReportCmd() *cobra.Command {var (by stringmetrics []string+ gauges []string)cmd := &cobra.Command{@@ -48,13 +49,14 @@ func newReportCmd() *cobra.Command {return err}sortBy(observations, by)- _, err = fmt.Fprint(cmd.OutOrStdout(), render(observations, by, metrics))+ _, err = fmt.Fprint(cmd.OutOrStdout(), render(observations, by, metrics, gauges))return err},}cmd.Flags().StringVar(&by, "by", "agents", "Label the runs varied, used to order the rows")- cmd.Flags().StringArrayVar(&metrics, "metric", nil, "Extra series to include as a column, as the difference across the run; repeatable")+ cmd.Flags().StringArrayVar(&metrics, "metric", nil, "Counter to include as a column, as the difference across the run; repeatable")+ cmd.Flags().StringArrayVar(&gauges, "gauge", nil, "Gauge to include as total and per-source columns, as read after the run; repeatable")return cmd}@@ -91,9 +93,12 @@ func sortBy(observations []observation, label string) {})}-func render(observations []observation, by string, metrics []string) string {+func render(observations []observation, by string, metrics, gauges []string) string {header := []string{by, "requests", "conc", "ok", "failed", "rps", "ttfb p50", "ttfb p95", "ttfb p99", "ttfb max"}header = append(header, metrics...)+ for _, g := range gauges {+ header = append(header, g+" total", g+" each")+ }rows := make([][]string, 0, len(observations))for _, obs := range observations {@@ -113,6 +118,15 @@ func render(observations []observation, by string, metrics []string) string {for _, m := range metrics {row = append(row, fmt.Sprintf("%.0f", delta(obs, m)))}+ for _, g := range gauges {+ total, sources := gauge(obs, g)+ row = append(row, fmt.Sprintf("%.0f", total))+ if sources == 0 {+ row = append(row, "-")+ continue+ }+ row = append(row, fmt.Sprintf("%.0f", total/float64(sources)))+ }rows = append(rows, row)}@@ -125,6 +139,21 @@ func render(observations []observation, by string, metrics []string) string {return b.String()}+// gauge totals a series across sources as read after the run, and says how+// many sources reported it. A gauge is a level rather than an accumulation, so+// unlike a counter its difference across the run means nothing.+func gauge(obs observation, series string) (total float64, sources int) {+ for _, after := range obs.After {+ value, ok := after[series]+ if !ok {+ continue+ }+ total += value+ sources+++ }+ return total, sources+}+// delta is how much a series moved across the run, summed over every scrape// source. Counters are cumulative, so the difference is the only figure that// describes the run rather than the process's whole life.diff --git a/cmd/sam-bench/report_test.go b/cmd/sam-bench/report_test.goindex e678439c..456b8e46 100644--- a/cmd/sam-bench/report_test.go+++ b/cmd/sam-bench/report_test.go@@ -63,6 +63,47 @@ func TestDeltaSumsTheMovementAcrossEverySource(t *testing.T) {}}+func TestGaugeIsReadAsALevelNotAnAccumulation(t *testing.T) {+ // Memory is a level. Reporting its difference across a run would say a+ // sandbox costs nothing, because it was already resident when the run+ // started, which is the opposite of what the density question asks.+ obs := observation{+ Before: map[string]map[string]float64{+ "box-1": {"process_resident_memory_bytes": 20},+ "box-2": {"process_resident_memory_bytes": 20},+ },+ After: map[string]map[string]float64{+ "box-1": {"process_resident_memory_bytes": 21},+ "box-2": {"process_resident_memory_bytes": 23},+ },+ }++ total, sources := gauge(obs, "process_resident_memory_bytes")+ if total != 44 {+ t.Errorf("total = %v, want 44", total)+ }+ if sources != 2 {+ t.Errorf("sources = %v, want 2", sources)+ }+ if got := delta(obs, "process_resident_memory_bytes"); got == total {+ t.Error("a gauge read as a counter produced the same figure, so the distinction is not being made")+ }+}++func TestGaugeIgnoresSourcesThatNeverReportedIt(t *testing.T) {+ // Averaging over sources that did not export the series would divide by+ // the wrong number and understate the per-sandbox cost.+ obs := observation{After: map[string]map[string]float64{+ "box-1": {"process_resident_memory_bytes": 10},+ "box-2": {"something_else": 99},+ }}++ total, sources := gauge(obs, "process_resident_memory_bytes")+ if total != 10 || sources != 1 {+ t.Errorf("gauge = (%v, %v), want (10, 1)", total, sources)+ }+}+func TestRenderPutsEveryObservationInTheTable(t *testing.T) {observations := []observation{{@@ -75,7 +116,7 @@ func TestRenderPutsEveryObservationInTheTable(t *testing.T) {},}- table := render(observations, "agents", nil)+ table := render(observations, "agents", nil, nil)lines := strings.Split(strings.TrimSpace(table), "\n")if len(lines) != 4 {t.Fatalf("got %d lines, want a header, a rule and two rows:\n%s", len(lines), table)diff --git a/internal/bench/bench.go b/internal/bench/bench.goindex 37db1407..4b07f015 100644--- a/internal/bench/bench.go+++ b/internal/bench/bench.go@@ -50,6 +50,13 @@ type Options struct {// numbers are only meaningful against.Socket string+ // UnixTarget dials the target over this Unix socket instead of resolving+ // its host. It is how the baseline reaches the node's own API without a+ // relay in between: putting one there, socat or otherwise, would add a+ // userspace hop to the baseline and quietly flatter everything measured+ // against it.+ UnixTarget string+// Target is the URL to request, e.g. http://mesh.sam.alt/v1/models.Target string@@ -292,7 +299,10 @@ func newClient(opts Options) (*http.Client, error) {DisableKeepAlives: opts.NewFlowPerRequest,}- if opts.Socket != "" {+ switch {+ case opts.Socket != "" && opts.UnixTarget != "":+ return nil, errors.New("bench: a run goes through the boundary or straight to a socket, not both")+ case opts.Socket != "":dialer, err := proxy.SOCKS5("unix", opts.Socket, nil, proxy.Direct)if err != nil {return nil, fmt.Errorf("bench: dial boundary %s: %w", opts.Socket, err)@@ -302,7 +312,11 @@ func newClient(opts Options) (*http.Client, error) {return nil, errors.New("bench: SOCKS5 dialer does not support contexts")}transport.DialContext = contextDialer.DialContext- } else {+ case opts.UnixTarget != "":+ transport.DialContext = func(ctx context.Context, _, _ string) (net.Conn, error) {+ return (&net.Dialer{}).DialContext(ctx, "unix", opts.UnixTarget)+ }+ default:transport.DialContext = (&net.Dialer{}).DialContext}diff --git a/internal/bench/bench_test.go b/internal/bench/bench_test.goindex e902492c..5501bc7a 100644--- a/internal/bench/bench_test.go+++ b/internal/bench/bench_test.go@@ -149,6 +149,21 @@ func TestRunCountsErrorStatusesAsFailures(t *testing.T) {}}+func TestRunRefusesTwoPathsAtOnce(t *testing.T) {+ // Going through the boundary and straight to a socket are the two sides+ // of the comparison. Silently picking one would label a baseline as a+ // mesh measurement, or the reverse, and nothing downstream could tell.+ _, err := Run(context.Background(), Options{+ Target: "http://localhost/v1/models",+ Requests: 1,+ Socket: "/tmp/boundary.sock",+ UnixTarget: "/tmp/node.sock",+ })+ if err == nil {+ t.Error("Run accepted both a boundary and a direct socket")+ }+}+func TestRunRejectsAWorkloadItCannotMeasure(t *testing.T) {for _, opts := range []Options{{Requests: 1},diff --git a/site/content/docs/scale-report.md b/site/content/docs/scale-report.mdnew file mode 100644index 00000000..e853eee4--- /dev/null+++ b/site/content/docs/scale-report.md@@ -0,0 +1,259 @@+---+title: "Scale: what an agent costs"+description: "A measured account of the price of giving every agent its own policy-enforced boundary"+weight: 99+---++## The question++SAM's design makes a claim that sounds expensive. Every agent is a principal,+and every agent reaches the network through its own boundary that resolves+names and enforces policy on each flow. Nothing an agent does escapes that+boundary, because in a properly built sandbox there is no other route out.++Architecture documents can argue for that. They cannot tell you whether it is+affordable. If a boundary costs a hundred megabytes and adds ten milliseconds,+the design is a nice idea for a dozen agents and useless for a thousand. So+this page is not an argument. It is a measurement, with the method written down+so you can disagree with it, and a script so you can re-run it.++Four questions, each phrased so that a bad answer would be visible:++1. **Density.** What does the *N*th boundary cost in memory?+2. **Startup.** How long from launching a sandbox to it being able to call the mesh, and does that grow with the population?+3. **Overhead.** What does an agent pay in latency for reaching the mesh through a policy instead of a socket?+4. **Enforcement.** Does policy still hold under load, and what does refusing cost?++## Method++### Apparatus++One real mesh: a control plane, a router and a node, all running as they do in+the end-to-end suite, on a kind cluster. Not a simulation and not a mock — the+node enrols, mints real tokens and answers real requests.++Sandbox boundaries are then attached to that node one at a time. They run as+**host processes rather than containers**, deliberately. The question is what+`sam-box` costs; wrapping each one in a container would measure the container+runtime instead.++### The workload++The load generator (`sam-bench`) issues exactly the same request every time.+This repo also ships a chaos agent driven by a real language model, and that is+the right tool for asking whether the mesh survives an autonomous caller — but+it is the wrong tool for asking what anything costs, because it never issues+the same request twice, so a difference between two runs cannot be attributed+to the mesh.++The generator reaches the mesh the way an agent does: over the boundary's+SOCKS5 socket. What it measures is therefore what an agent experiences, not+what an operator with host access would.++### Conditions++One number would have been misleading, so every step of the sweep is measured+four ways:++| Condition | What it isolates |+| --- | --- |+| **Baseline** | The same node answering the same request with no boundary in the path. |+| **Reused flow** | What an agent normally experiences: a flow admitted once, carrying many requests. |+| **New flow per request** | What admission costs, paying the handshake and the policy decision every time. |+| **Denied** | Enforcement, which never appears as a latency at all and so must be counted separately. |++The baseline reaches the node over its own Unix socket rather than through a+relay. An earlier version put `socat` in front of it, which charged the+comparison for a hop the mesh path does not make and flattered every figure+derived from it.++### What is recorded++Client-side latency says what an agent experienced. The servers' own counters+say what they did to produce it, including the flows they refused, which leave+no latency behind. Both are recorded, along with the machine, the commit and+whether the tree was dirty, because a latency with no machine attached to it is+not a result.++Percentiles are computed from the full sorted sample by nearest rank, so every+figure quoted below was actually observed rather than interpolated between two+figures that were. Failed requests never enter the latency distribution: a mesh+that refuses quickly must not be able to look like a mesh that answers quickly.++Warmup requests are issued and reported separately rather than discarded, since+a first call that pays for provider discovery is a real cost and hiding it+would flatter the result.++### Environment++| | |+| --- | --- |+| CPU | Intel Xeon @ 2.60 GHz, 48 vCPU |+| Memory | 118 GiB |+| Kernel | Linux 7.1.6 |+| Go | 1.26.5 |+| Sweep | 1, 2, 4, 8, 16, 32, 64 sandboxes |+| Load | 500 requests per condition per step, concurrency 4, 20 warmup |++## Results++_Recorded 2026-08-21 at commit `e5b2966`. Tables are generated by `sam-bench+report` from the raw observations, which are kept alongside them._++### The measurements++Latency in milliseconds, memory in MB, per population size.++| sandboxes | reused p50 | reused p99 | new-flow p50 | new-flow p99 | admission (µs) | idle RSS | total RSS | denied | leaked |+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |+| 1 | 1.36 | 2.30 | 1.55 | 78.15 | 19 | 20.2 | 0.02 GiB | 500 | 0 |+| 2 | 1.34 | 2.09 | 1.40 | 3.19 | 20 | 19.4 | 0.06 GiB | 500 | 0 |+| 4 | 1.33 | 2.92 | 1.44 | 3.90 | 19 | 19.0 | 0.12 GiB | 500 | 0 |+| 8 | 1.33 | 2.37 | 1.46 | 6.23 | 18 | 18.8 | 0.21 GiB | 500 | 0 |+| 16 | 1.39 | 3.37 | 1.44 | 2.38 | 19 | 18.9 | 0.39 GiB | 500 | 0 |+| 32 | 1.34 | 2.42 | 1.37 | 3.22 | 18 | 18.1 | 0.72 GiB | 500 | 0 |+| 64 | 1.30 | 1.80 | 1.41 | 4.62 | 18 | 17.4 | 1.37 GiB | 500 | 0 |+| _baseline_ | _1.39_ | _2.37_ | — | — | — | — | — | — | — |++Every allowed step served 500 of 500 requests. Every denied step served 0 of+500. "Admission" is the boundary's own mean time to classify a destination and+open it; "leaked" is the number of requests that reached a destination policy+forbade.++### Density++A boundary settles at **17–20 MB of resident memory** once idle. Sixty-four of+them, attached to one node with one under continuous load, came to **1.37 GiB+in total**.++The per-sandbox mean falls as the population grows, which is an artefact worth+naming rather than a discovery: the same boundary carries the load at every+step, so its heap reflects the traffic while the others sit idle, and the mean+is diluted towards the idle cost as the population grows. The floor, not the+mean, is the marginal cost of one more agent.++### Startup++Across 64 sandboxes: **median 59 ms**, min 58 ms, p95 59 ms, max 70 ms.++Readiness here means the socket is accepting connections, not that the process+exists — a boundary that has not bound its socket yet is of no use to the agent+waiting on it.++The striking part is the spread. The 64th boundary started as quickly as the+first, so attaching an agent is a constant-cost operation over this range, not+one that degrades as the host fills up.++### Overhead++This is the result that came out differently from what the method expected, so+it is worth being precise about.++Baseline, no boundary in the path: **p50 1.39 ms**. Through the boundary on a+reused flow, p50 ranged from **1.30 to 1.39 ms** across the sweep — that is,+*at or below the baseline at every population size*. Taken literally the+boundary made things faster, which it obviously did not. The honest reading is+that **the end-to-end overhead of the boundary is smaller than the run-to-run+variation of the measurement itself**, so this method cannot resolve it. A+figure like "the boundary costs 60 µs" would be reading noise.++Opening a fresh flow for every request, which pays the SOCKS5 handshake and the+policy decision each time, is resolvable but small: p50 ran **+9 to +157 µs**+against the baseline.++The reliable number comes from the boundary's own instrumentation, which is+measured in-process and excludes everything downstream. Classifying a+destination and opening it took a mean of **18–20 µs**, and that figure moved+by less than 2 µs across the entire sweep, from one sandbox to sixty-four.++So: enforcement costs tens of microseconds, and whatever the boundary adds+end to end is lost in the noise of a millisecond-scale request.++### Enforcement++Every request to a destination policy did not allow was refused. **3,500+attempts, zero successes**, at every population size, and the boundary's own+counter agreed with the client's on the exact number refused at every step.++Refusal ran at **24,000–28,000 per second**, roughly ten times the rate of+allowed traffic, which is what should happen: a denied flow is decided from the+name and never dialled.++## Conclusions++**A per-agent boundary is affordable.** At ~18 MB and ~59 ms to attach, the+constraint on how many agents a host carries is the agents themselves, not the+thing that contains them. Extrapolating the floor, the boundaries for a+thousand agents would be roughly 18 GB — a large but ordinary number for a host+of this class, and the extrapolation is exactly the thing the next experiment+should check rather than assume.++**Enforcement is not a latency problem.** 18–20 µs to classify and open a flow,+and an end-to-end cost too small for this method to separate from noise. The+design's insistence that every flow be checked does not have to be traded+against performance, which removes the usual reason such checks get quietly+disabled.++**Nothing degraded across the range tested.** Median latency was flat from 1 to+64 sandboxes, startup was constant, and the boundary's own admission cost moved+by under 2 µs across a 64-fold increase in population.++**10,500 requests. No failures where success was expected, and no successes+where it was not.**++## Threats to validity++Written down because a result without them is advocacy.++- **One machine, one run.** These are not confidence intervals over repeated+ trials. Treat single figures as indicative and the shape of the trend as the+ claim. The overhead result above is precisely a case where a single figure+ would have been over-read.+- **A trivial request.** `/v1/models` is a small response from a warm node. It+ isolates mesh overhead well precisely because it does almost no work, which+ also means it says nothing about large payloads or streaming completions.+- **Load through one boundary.** Every step drives traffic through the first+ sandbox while the rest are attached but idle. This answers "do the others get+ in the way", which is the density question, but it is not the same experiment+ as all *N* being busy at once.+- **A reproducible tail anomaly.** The single-sandbox, new-flow-per-request step+ showed p99 excursions of 60–80 ms in every run, and did not recur at higher+ populations. It is reproducible and unexplained; it is left in the record+ rather than smoothed out.+- **Host processes, not sandboxes.** This measures the boundary, not the+ isolation around it. A Firecracker microVM per agent costs far more than+ 18 MB, and none of that cost is counted here.++## Reproducing this++```bash+# Requires docker and kind; stands up a real mesh, then tears it down.+./tests/scale/run-density.sh --steps 1,2,4,8,16,32,64 --requests 500++# Results, including environment.json and the raw per-step observations,+# land in tests/scale/results/<timestamp>/ with a rendered table.md.+```++Individual measurements can be taken by hand:++```bash+# Through a boundary, as an agent would+sam-bench run --socket /run/agent.sock --target http://mesh.sam.alt/v1/models \+ --requests 500 --concurrency 4 --scrape http://127.0.0.1:9600/metrics++# The same request with no boundary in the path, for comparison+sam-bench run --target-unix /run/node.sock --target http://localhost/v1/models \+ --requests 500 --concurrency 4+```++`sam-box` serves metrics only when asked, with `--metrics-addr`. It sits between+a sandbox and the mesh, so any listener it opens is one more thing reachable+from wherever it is bound.++## What this does not answer++This measures a boundary on a host. It says nothing yet about a thousand agents+in real sandboxes against a public testnet, where microVM memory, control-plane+enrolment and DHT behaviour all start to matter and none of them are exercised+above. That experiment is the next one, and it is described in+[the scale experiment guide](../scale-experiment/).diff --git a/tests/scale/results/environment.json b/tests/scale/results/environment.jsonnew file mode 100644index 00000000..c6262c51--- /dev/null+++ b/tests/scale/results/environment.json@@ -0,0 +1,10 @@+{+ "recorded": "2026-08-21T05:42:33+00:00",+ "commit": "e5b296692f69091917e014b1e87dde4876da5c1f",+ "dirty": true,+ "kernel": "Linux 7.1.6-1rodete1-amd64",+ "cpus": 48,+ "memory_kb": 123632524,+ "cpu_model": "Intel(R) Xeon(R) CPU @ 2.60GHz",+ "go": "go1.26.5"+}diff --git a/tests/scale/results/startup.csv b/tests/scale/results/startup.csvnew file mode 100644index 00000000..0b51f7e2--- /dev/null+++ b/tests/scale/results/startup.csv@@ -0,0 +1,65 @@+index,startup_ms+1,59+2,59+3,59+4,59+5,70+6,58+7,59+8,58+9,59+10,58+11,59+12,58+13,59+14,59+15,58+16,58+17,58+18,58+19,59+20,58+21,59+22,59+23,59+24,59+25,59+26,58+27,59+28,59+29,59+30,59+31,59+32,58+33,59+34,59+35,59+36,59+37,58+38,58+39,59+40,59+41,59+42,58+43,58+44,59+45,58+46,59+47,59+48,59+49,59+50,59+51,59+52,59+53,58+54,59+55,59+56,59+57,59+58,59+59,59+60,59+61,59+62,59+63,59+64,59diff --git a/tests/scale/results/table.md b/tests/scale/results/table.mdnew file mode 100644index 00000000..000d413a--- /dev/null+++ b/tests/scale/results/table.md@@ -0,0 +1,41 @@+### Reused flow: what an agent normally experiences++| agents | requests | conc | ok | failed | rps | ttfb p50 | ttfb p95 | ttfb p99 | ttfb max | process_resident_memory_bytes total | process_resident_memory_bytes each |+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |+| 1 | 500 | 4 | 500 | 0 | 2868 | 1.36 | 1.74 | 2.30 | 2.66 | 20238336 | 20238336 |+| 2 | 500 | 4 | 500 | 0 | 2939 | 1.34 | 1.80 | 2.09 | 2.52 | 60022784 | 30011392 |+| 4 | 500 | 4 | 500 | 0 | 2842 | 1.33 | 1.91 | 2.92 | 4.12 | 125640704 | 31410176 |+| 8 | 500 | 4 | 500 | 0 | 2951 | 1.33 | 1.72 | 2.37 | 4.08 | 228106240 | 28513280 |+| 16 | 500 | 4 | 500 | 0 | 2761 | 1.39 | 1.91 | 3.37 | 4.52 | 416198656 | 26012416 |+| 32 | 500 | 4 | 500 | 0 | 2948 | 1.34 | 1.76 | 2.42 | 4.90 | 773836800 | 24182400 |+| 64 | 500 | 4 | 500 | 0 | 3059 | 1.30 | 1.57 | 1.80 | 2.11 | 1472561152 | 23008768 |++### New flow per request: what admission costs++| agents | requests | conc | ok | failed | rps | ttfb p50 | ttfb p95 | ttfb p99 | ttfb max | sam_box_flows_total{outcome=allowed,route=mesh-entrypoint} |+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |+| 1 | 500 | 4 | 500 | 0 | 1154 | 1.55 | 2.71 | 78.15 | 85.25 | 520 |+| 2 | 500 | 4 | 500 | 0 | 1991 | 1.40 | 2.20 | 3.19 | 70.09 | 520 |+| 4 | 500 | 4 | 500 | 0 | 2591 | 1.44 | 2.12 | 3.90 | 4.61 | 520 |+| 8 | 500 | 4 | 500 | 0 | 1797 | 1.46 | 2.51 | 6.23 | 83.66 | 520 |+| 16 | 500 | 4 | 500 | 0 | 2619 | 1.44 | 1.85 | 2.38 | 7.11 | 520 |+| 32 | 500 | 4 | 500 | 0 | 2720 | 1.37 | 1.73 | 3.22 | 8.73 | 520 |+| 64 | 500 | 4 | 500 | 0 | 2611 | 1.41 | 1.88 | 4.62 | 9.89 | 520 |++### Denied: what enforcement costs, and that it held++| agents | requests | conc | ok | failed | rps | ttfb p50 | ttfb p95 | ttfb p99 | ttfb max | sam_box_flows_total{outcome=denied,route=unresolved} |+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |+| 1 | 500 | 4 | 0 | 500 | 26225 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |+| 2 | 500 | 4 | 0 | 500 | 28274 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |+| 4 | 500 | 4 | 0 | 500 | 27228 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |+| 8 | 500 | 4 | 0 | 500 | 24186 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |+| 16 | 500 | 4 | 0 | 500 | 26940 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |+| 32 | 500 | 4 | 0 | 500 | 25028 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |+| 64 | 500 | 4 | 0 | 500 | 28068 | 0.00 | 0.00 | 0.00 | 0.00 | 500 |++### Baseline: the same node, no boundary++| agents | requests | conc | ok | failed | rps | ttfb p50 | ttfb p95 | ttfb p99 | ttfb max |+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |+| 0 | 500 | 4 | 500 | 0 | 2801 | 1.39 | 1.73 | 2.37 | 2.72 |diff --git a/tests/scale/run-density.sh b/tests/scale/run-density.shnew file mode 100755index 00000000..871638cb--- /dev/null+++ b/tests/scale/run-density.sh@@ -0,0 +1,279 @@+#!/usr/bin/env bash+#+# Copyright 2026 Google LLC+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+# What does the Nth agent cost?+#+# The design says an agent is a principal with its own boundary, which is only+# affordable if a boundary is cheap. This measures that directly: it stands up+# one real mesh, attaches sandbox boundaries to it one at a time, and at each+# step records what the new one cost to start, what it costs to keep, and+# whether the ones already running got slower.+#+# It deliberately runs the boundaries as host processes rather than containers.+# The question is what sam-box costs, and wrapping each one in a container would+# measure the container runtime instead.+#+# Usage:+# tests/scale/run-density.sh [--steps 1,2,4,8,16,32] [--requests 200] [--out DIR]++set -euo pipefail++REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"+cd "${REPO_ROOT}"++STEPS="1,2,4,8,16,32"+REQUESTS=200+CONCURRENCY=4+WARMUP=20+OUT_DIR=""+KEEP=0++while [[ $# -gt 0 ]]; do+ case "$1" in+ --steps) STEPS="$2"; shift 2 ;;+ --requests) REQUESTS="$2"; shift 2 ;;+ --concurrency) CONCURRENCY="$2"; shift 2 ;;+ --warmup) WARMUP="$2"; shift 2 ;;+ --out) OUT_DIR="$2"; shift 2 ;;+ --keep) KEEP=1; shift ;;+ -h|--help) sed -n '17,31p' "$0"; exit 0 ;;+ *) echo "unknown flag: $1" >&2; exit 2 ;;+ esac+done++OUT_DIR="${OUT_DIR:-${REPO_ROOT}/tests/scale/results/$(date +%Y%m%dT%H%M%S)}"+mkdir -p "${OUT_DIR}"++# The boundaries are host processes, so their sockets and metrics ports are too.+BOX_PIDS=()+BOX_DIR="$(mktemp -d /tmp/sam-scale-XXXXXX)"++# Pick a metrics port range that is actually free. A previous run that leaked+# its processes would otherwise make this one fail on its first sandbox, and+# that failure reads as a defect in sam-box rather than in the harness.+pick_port_base() {+ local base+ for base in 19600 19700 19800 19900 20000; do+ if ! ss -ltn 2>/dev/null | awk '{print $4}' | grep -qE ":($((base + 1)))\$"; then+ echo "${base}"+ return 0+ fi+ done+ echo "no free metrics port range; is a previous run still going?" >&2+ return 1+}+METRICS_BASE="$(pick_port_base)"++# The mesh helpers are written for bats but only need a few of its variables:+# two to name things and one to find the repo root it builds images from.+export BATS_TEST_NAME="scale"+export BATS_TEST_NUMBER=0+export BATS_TEST_DIRNAME="${REPO_ROOT}/tests/e2e"+# shellcheck source=/dev/null+source "${REPO_ROOT}/tests/e2e/lib/container_mesh.bash"++cleanup() {+ local status=$?+ echo "==> Tearing down"+ for pid in "${BOX_PIDS[@]:-}"; do+ kill "${pid}" 2>/dev/null || true+ done+ # The recorded pids are the reliable path; this catches anything that+ # outlived its record, so a failed run cannot leave sandboxes holding ports+ # and make the next run look broken.+ pkill -f "sam-box run --socket ${BOX_DIR}/" 2>/dev/null || true+ mesh_cleanup_test_resources 2>/dev/null || true+ [[ "${KEEP}" -eq 1 ]] || rm -rf "${BOX_DIR}"+ exit "${status}"+}+trap cleanup EXIT++# record_environment writes down what the numbers were produced on, because a+# latency without a machine attached to it is not a result.+record_environment() {+ {+ echo "{"+ printf ' "recorded": "%s",\n' "$(date -Is)"+ printf ' "commit": "%s",\n' "$(git rev-parse HEAD)"+ printf ' "dirty": %s,\n' "$([[ -z "$(git status --porcelain)" ]] && echo false || echo true)"+ printf ' "kernel": "%s",\n' "$(uname -sr)"+ printf ' "cpus": %s,\n' "$(nproc)"+ printf ' "memory_kb": %s,\n' "$(awk '/MemTotal/ {print $2}' /proc/meminfo)"+ printf ' "cpu_model": "%s",\n' "$(awk -F': ' '/model name/ {print $2; exit}' /proc/cpuinfo)"+ printf ' "go": "%s"\n' "$(go version | awk '{print $3}')"+ echo "}"+ } > "${OUT_DIR}/environment.json"+}++# start_box attaches one more sandbox boundary and reports how long it took to+# become usable, in milliseconds, through STARTUP_MS. It sets a global rather+# than printing one, because a command substitution would run it in a subshell+# and the pid it records would die with that subshell, leaving the boundary+# running with nothing left to kill it by.+STARTUP_MS=0+start_box() {+ local index="$1"+ local socket="${BOX_DIR}/agent-${index}.sock"+ local port=$((METRICS_BASE + index))+ local started+ started="$(date +%s%N)"++ "${REPO_ROOT}/bin/sam-box" run \+ --socket "${socket}" \+ --sidecar-socket "${MESH_SOCKET_DIR}/node.sock" \+ --metrics-addr "127.0.0.1:${port}" \+ --egress-allow example.invalid \+ > "${BOX_DIR}/box-${index}.log" 2>&1 &+ BOX_PIDS+=("$!")++ # Readiness is the socket accepting a connection, not the process existing:+ # a boundary that has not bound its socket yet is of no use to the agent+ # waiting on it.+ local waited=0+ while [[ ! -S "${socket}" ]]; do+ sleep 0.01+ waited=$((waited + 1))+ if [[ "${waited}" -gt 3000 ]]; then+ echo "boundary ${index} never bound its socket" >&2+ cat "${BOX_DIR}/box-${index}.log" >&2+ return 1+ fi+ done++ STARTUP_MS=$((($(date +%s%N) - started) / 1000000))+}++echo "==> Recording environment"+record_environment++echo "==> Building binaries"+make build >/dev/null++echo "==> Standing up the mesh"+mesh_setup_suite+mesh_setup_env+mesh_start_mock_oidc+mesh_start_router+mesh_start_node 1 "--socket-path /sockets/node.sock --data-dir /sockets/node-data" "" \+ "-v ${MESH_SOCKET_DIR}:/sockets --user $(id -u):$(id -g) -e HOME=/sockets"+mesh_wait_for_log "${MESH_PREFIX}-node-1" "SAM Node Online" 120+mesh_wait_for_mcp_ready 1 30++echo "==> Sweeping ${STEPS} sandboxes"+: > "${OUT_DIR}/startup.csv"+echo "index,startup_ms" >> "${OUT_DIR}/startup.csv"++# The baseline is the same node answering the same request with no boundary in+# the path. Without it the mesh numbers are unanchored: a millisecond means+# nothing until something says what a millisecond buys. It dials the node's+# socket directly, because putting a relay there to give it a TCP address would+# add a userspace hop to the baseline and flatter everything compared to it.+echo "==> Measuring the baseline, no boundary"+"${REPO_ROOT}/bin/sam-bench" run \+ --target-unix "${MESH_SOCKET_DIR}/node.sock" \+ --target "http://localhost/v1/models" \+ --requests "${REQUESTS}" \+ --concurrency "${CONCURRENCY}" \+ --warmup "${WARMUP}" \+ --label "agents=0" \+ --label "condition=no-boundary" \+ --out "${OUT_DIR}/baseline.json" || echo "baseline unavailable, continuing"++running=0+for step in ${STEPS//,/ }; do+ while [[ "${running}" -lt "${step}" ]]; do+ running=$((running + 1))+ start_box "${running}"+ echo "${running},${STARTUP_MS}" >> "${OUT_DIR}/startup.csv"+ done++ echo "--> ${running} sandboxes attached; measuring"++ # Every boundary is scraped, so the recorded cost is the whole population's,+ # not a sample that happens to be idle.+ scrape_args=()+ for i in $(seq 1 "${running}"); do+ scrape_args+=(--scrape "http://127.0.0.1:$((METRICS_BASE + i))/metrics")+ done++ # Load goes through the first boundary while all of them are attached, so a+ # slowdown caused by the others shows up where it would be felt.+ #+ # Twice, because reusing a connection and opening one are different+ # questions. A kept-alive flow is admitted once and carries every request+ # after that for free; a new flow per request pays the SOCKS5 handshake and+ # the policy decision every time. Only the second measures what enforcement+ # costs, and only the first describes how an agent normally behaves.+ for condition in reused-flow new-flow-per-request; do+ flag=()+ [[ "${condition}" == "new-flow-per-request" ]] && flag=(--new-flow-per-request)++ "${REPO_ROOT}/bin/sam-bench" run \+ --socket "${BOX_DIR}/agent-1.sock" \+ --target "http://mesh.sam.alt/v1/models" \+ --requests "${REQUESTS}" \+ --concurrency "${CONCURRENCY}" \+ --warmup "${WARMUP}" \+ --label "agents=${running}" \+ --label "condition=${condition}" \+ "${flag[@]}" \+ "${scrape_args[@]}" \+ --out "${OUT_DIR}/${condition}-$(printf '%04d' "${running}").json"+ done++ # Enforcement, measured rather than asserted. The same boundary, under the+ # same load, asked for somewhere policy does not allow. Every one of these+ # must fail, and a run where any succeeded is a security result, not a+ # performance one, so it is recorded next to the latencies rather than in a+ # separate place nobody reads.+ "${REPO_ROOT}/bin/sam-bench" run \+ --socket "${BOX_DIR}/agent-1.sock" \+ --target "http://blocked.example.com/v1/models" \+ --requests "${REQUESTS}" \+ --concurrency "${CONCURRENCY}" \+ --warmup 0 \+ --label "agents=${running}" \+ --label "condition=denied" \+ "${scrape_args[@]}" \+ --out "${OUT_DIR}/denied-$(printf '%04d' "${running}").json"+done++echo "==> Results in ${OUT_DIR}"+{+ echo "### Reused flow: what an agent normally experiences"+ echo+ "${REPO_ROOT}/bin/sam-bench" report \+ --gauge "process_resident_memory_bytes" \+ "${OUT_DIR}"/reused-flow-*.json+ echo+ echo "### New flow per request: what admission costs"+ echo+ "${REPO_ROOT}/bin/sam-bench" report \+ --metric "sam_box_flows_total{outcome=allowed,route=mesh-entrypoint}" \+ "${OUT_DIR}"/new-flow-per-request-*.json+ echo+ echo "### Denied: what enforcement costs, and that it held"+ echo+ "${REPO_ROOT}/bin/sam-bench" report \+ --metric "sam_box_flows_total{outcome=denied,route=unresolved}" \+ "${OUT_DIR}"/denied-*.json+ if [[ -f "${OUT_DIR}/baseline.json" ]]; then+ echo+ echo "### Baseline: the same node, no boundary"+ echo+ "${REPO_ROOT}/bin/sam-bench" report "${OUT_DIR}/baseline.json"+ fi+} | tee "${OUT_DIR}/table.md"