Skip to content

Agent Mesh

bench: turn observations into a table

Antonio Ojea460e67f0359c
diff --git a/cmd/sam-bench/main.go b/cmd/sam-bench/main.go
index 3e730cf0..9f88e61a 100644
--- a/cmd/sam-bench/main.go
+++ b/cmd/sam-bench/main.go
@@ -78,6 +78,11 @@ func main() {
rootCmd := &cobra.Command{
Use: "sam-bench",
Short: "Measure the mesh from where an agent stands",
+ }
+
+ runCmd := &cobra.Command{
+ Use: "run",
+ Short: "Issue a fixed workload and record what it cost",
Long: "Issues a fixed workload through a sandbox boundary and records what it cost,\n" +
"alongside what the processes involved reported about themselves.",
SilenceUsage: true,
@@ -123,7 +128,7 @@ func main() {
},
}
- flags := rootCmd.Flags()
+ flags := runCmd.Flags()
flags.StringVar(&socket, "socket", "", "Sandbox boundary SOCKS5 socket to measure through; omit to measure the same workload without a boundary, which is the baseline")
flags.StringVar(&target, "target", "", "URL to request, e.g. http://mesh.sam.alt/v1/models (required)")
flags.StringVar(&method, "method", "GET", "HTTP method")
@@ -137,9 +142,10 @@ func main() {
flags.StringArrayVar(&scrape, "scrape", nil, "Metrics endpoint to record before and after the run, repeatable")
flags.StringArrayVar(&labels, "label", nil, "Condition to record with the observation as name=value, repeatable")
flags.StringVar(&out, "out", "", "File to write the observation to; default stdout")
- if err := rootCmd.MarkFlagRequired("target"); err != nil {
+ if err := runCmd.MarkFlagRequired("target"); err != nil {
panic(err)
}
+ rootCmd.AddCommand(runCmd, newReportCmd())
ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
defer stop()
diff --git a/cmd/sam-bench/report.go b/cmd/sam-bench/report.go
new file mode 100644
index 00000000..e0d507d7
--- /dev/null
+++ b/cmd/sam-bench/report.go
@@ -0,0 +1,139 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "sort"
+ "strconv"
+ "strings"
+
+ "github.com/spf13/cobra"
+)
+
+// Reading a directory of observations by eye is how a result gets quoted
+// wrongly. This turns them into one table, ordered by the condition that was
+// varied, so a trend is visible rather than asserted.
+
+func newReportCmd() *cobra.Command {
+ var (
+ by string
+ metrics []string
+ )
+
+ cmd := &cobra.Command{
+ Use: "report <observation.json>...",
+ Short: "Turn recorded observations into a table",
+ Long: "Reads observations written by `sam-bench run` and renders them as a Markdown\n" +
+ "table, ordered by the label that was varied across the runs.",
+ Args: cobra.MinimumNArgs(1),
+ SilenceUsage: true,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ observations, err := load(args)
+ if err != nil {
+ return err
+ }
+ sortBy(observations, by)
+ _, err = fmt.Fprint(cmd.OutOrStdout(), render(observations, by, metrics))
+ return err
+ },
+ }
+
+ cmd.Flags().StringVar(&by, "by", "agents", "Label the runs varied, used to order the rows")
+ cmd.Flags().StringArrayVar(&metrics, "metric", nil, "Extra series to include as a column, as the difference across the run; repeatable")
+ return cmd
+}
+
+func load(paths []string) ([]observation, error) {
+ out := make([]observation, 0, len(paths))
+ for _, path := range paths {
+ raw, err := os.ReadFile(path) // #nosec G304 -- the operator names their own result files
+ if err != nil {
+ return nil, err
+ }
+ var obs observation
+ if err := json.Unmarshal(raw, &obs); err != nil {
+ return nil, fmt.Errorf("%s: %w", path, err)
+ }
+ if obs.Report == nil {
+ return nil, fmt.Errorf("%s: no report in observation", path)
+ }
+ out = append(out, obs)
+ }
+ return out, nil
+}
+
+// sortBy orders rows by the varied label, numerically where it is a number so
+// that 8 agents comes before 64 rather than after it.
+func sortBy(observations []observation, label string) {
+ sort.SliceStable(observations, func(i, j int) bool {
+ a, b := observations[i].Labels[label], observations[j].Labels[label]
+ na, aerr := strconv.ParseFloat(a, 64)
+ nb, berr := strconv.ParseFloat(b, 64)
+ if aerr == nil && berr == nil {
+ return na < nb
+ }
+ return a < b
+ })
+}
+
+func render(observations []observation, by string, metrics []string) string {
+ header := []string{by, "requests", "conc", "ok", "failed", "rps", "ttfb p50", "ttfb p95", "ttfb p99", "ttfb max"}
+ header = append(header, metrics...)
+
+ rows := make([][]string, 0, len(observations))
+ for _, obs := range observations {
+ r := obs.Report
+ row := []string{
+ obs.Labels[by],
+ strconv.Itoa(r.Requests),
+ strconv.Itoa(r.Concurrency),
+ strconv.Itoa(r.Succeeded),
+ strconv.Itoa(r.Failed),
+ fmt.Sprintf("%.0f", r.Throughput),
+ ms(r.TTFB.P50),
+ ms(r.TTFB.P95),
+ ms(r.TTFB.P99),
+ ms(r.TTFB.Max),
+ }
+ for _, m := range metrics {
+ row = append(row, fmt.Sprintf("%.0f", delta(obs, m)))
+ }
+ rows = append(rows, row)
+ }
+
+ var b strings.Builder
+ b.WriteString("| " + strings.Join(header, " | ") + " |\n")
+ b.WriteString("|" + strings.Repeat(" --- |", len(header)) + "\n")
+ for _, row := range rows {
+ b.WriteString("| " + strings.Join(row, " | ") + " |\n")
+ }
+ return b.String()
+}
+
+// delta is how much a series moved across the run, summed over every scrape
+// source. Counters are cumulative, so the difference is the only figure that
+// describes the run rather than the process's whole life.
+func delta(obs observation, series string) float64 {
+ var total float64
+ for source, after := range obs.After {
+ total += after[series] - obs.Before[source][series]
+ }
+ return total
+}
+
+func ms(v float64) string { return strconv.FormatFloat(v, 'f', 2, 64) }
diff --git a/cmd/sam-bench/report_test.go b/cmd/sam-bench/report_test.go
new file mode 100644
index 00000000..e678439c
--- /dev/null
+++ b/cmd/sam-bench/report_test.go
@@ -0,0 +1,89 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/google/sam/internal/bench"
+)
+
+func TestSortByOrdersNumericLabelsNumerically(t *testing.T) {
+ // Lexical ordering would put 128 agents between 1 and 16, and a trend
+ // read off a table in that order would be nonsense.
+ observations := []observation{
+ {Labels: map[string]string{"agents": "128"}},
+ {Labels: map[string]string{"agents": "1"}},
+ {Labels: map[string]string{"agents": "16"}},
+ {Labels: map[string]string{"agents": "2"}},
+ }
+ sortBy(observations, "agents")
+
+ var got []string
+ for _, o := range observations {
+ got = append(got, o.Labels["agents"])
+ }
+ want := []string{"1", "2", "16", "128"}
+ for i := range want {
+ if got[i] != want[i] {
+ t.Fatalf("order = %v, want %v", got, want)
+ }
+ }
+}
+
+func TestDeltaSumsTheMovementAcrossEverySource(t *testing.T) {
+ // Counters are cumulative and there is one per sandbox, so a run's flow
+ // count is the movement summed over all of them, not any single value.
+ obs := observation{
+ Before: map[string]map[string]float64{
+ "box-1": {"sam_box_flows_total{outcome=allowed}": 10},
+ "box-2": {"sam_box_flows_total{outcome=allowed}": 5},
+ },
+ After: map[string]map[string]float64{
+ "box-1": {"sam_box_flows_total{outcome=allowed}": 60},
+ "box-2": {"sam_box_flows_total{outcome=allowed}": 45},
+ },
+ }
+
+ if got := delta(obs, "sam_box_flows_total{outcome=allowed}"); got != 90 {
+ t.Errorf("delta = %v, want 90 (50 + 40)", got)
+ }
+}
+
+func TestRenderPutsEveryObservationInTheTable(t *testing.T) {
+ observations := []observation{
+ {
+ Labels: map[string]string{"agents": "1"},
+ Report: &bench.Report{Requests: 100, Concurrency: 4, Succeeded: 100, Throughput: 250},
+ },
+ {
+ Labels: map[string]string{"agents": "64"},
+ Report: &bench.Report{Requests: 100, Concurrency: 4, Succeeded: 99, Failed: 1, Throughput: 120},
+ },
+ }
+
+ table := render(observations, "agents", nil)
+ lines := strings.Split(strings.TrimSpace(table), "\n")
+ if len(lines) != 4 {
+ t.Fatalf("got %d lines, want a header, a rule and two rows:\n%s", len(lines), table)
+ }
+ if !strings.Contains(lines[0], "agents") {
+ t.Errorf("header does not name the varied label: %q", lines[0])
+ }
+ if !strings.Contains(lines[3], "64") {
+ t.Errorf("last row is not the 64-agent run: %q", lines[3])
+ }
+}