Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
bench: turn observations into a table
Antonio Ojea460e67f0359c
diff --git a/cmd/sam-bench/main.go b/cmd/sam-bench/main.goindex 3e730cf0..9f88e61a 100644--- a/cmd/sam-bench/main.go+++ b/cmd/sam-bench/main.go@@ -78,6 +78,11 @@ func main() {rootCmd := &cobra.Command{Use: "sam-bench",Short: "Measure the mesh from where an agent stands",+ }++ runCmd := &cobra.Command{+ Use: "run",+ Short: "Issue a fixed workload and record what it cost",Long: "Issues a fixed workload through a sandbox boundary and records what it cost,\n" +"alongside what the processes involved reported about themselves.",SilenceUsage: true,@@ -123,7 +128,7 @@ func main() {},}- flags := rootCmd.Flags()+ flags := runCmd.Flags()flags.StringVar(&socket, "socket", "", "Sandbox boundary SOCKS5 socket to measure through; omit to measure the same workload without a boundary, which is the baseline")flags.StringVar(&target, "target", "", "URL to request, e.g. http://mesh.sam.alt/v1/models (required)")flags.StringVar(&method, "method", "GET", "HTTP method")@@ -137,9 +142,10 @@ func main() {flags.StringArrayVar(&scrape, "scrape", nil, "Metrics endpoint to record before and after the run, repeatable")flags.StringArrayVar(&labels, "label", nil, "Condition to record with the observation as name=value, repeatable")flags.StringVar(&out, "out", "", "File to write the observation to; default stdout")- if err := rootCmd.MarkFlagRequired("target"); err != nil {+ if err := runCmd.MarkFlagRequired("target"); err != nil {panic(err)}+ rootCmd.AddCommand(runCmd, newReportCmd())ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)defer stop()diff --git a/cmd/sam-bench/report.go b/cmd/sam-bench/report.gonew file mode 100644index 00000000..e0d507d7--- /dev/null+++ b/cmd/sam-bench/report.go@@ -0,0 +1,139 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package main++import (+ "encoding/json"+ "fmt"+ "os"+ "sort"+ "strconv"+ "strings"++ "github.com/spf13/cobra"+)++// Reading a directory of observations by eye is how a result gets quoted+// wrongly. This turns them into one table, ordered by the condition that was+// varied, so a trend is visible rather than asserted.++func newReportCmd() *cobra.Command {+ var (+ by string+ metrics []string+ )++ cmd := &cobra.Command{+ Use: "report <observation.json>...",+ Short: "Turn recorded observations into a table",+ Long: "Reads observations written by `sam-bench run` and renders them as a Markdown\n" ++ "table, ordered by the label that was varied across the runs.",+ Args: cobra.MinimumNArgs(1),+ SilenceUsage: true,+ RunE: func(cmd *cobra.Command, args []string) error {+ observations, err := load(args)+ if err != nil {+ return err+ }+ sortBy(observations, by)+ _, err = fmt.Fprint(cmd.OutOrStdout(), render(observations, by, metrics))+ return err+ },+ }++ cmd.Flags().StringVar(&by, "by", "agents", "Label the runs varied, used to order the rows")+ cmd.Flags().StringArrayVar(&metrics, "metric", nil, "Extra series to include as a column, as the difference across the run; repeatable")+ return cmd+}++func load(paths []string) ([]observation, error) {+ out := make([]observation, 0, len(paths))+ for _, path := range paths {+ raw, err := os.ReadFile(path) // #nosec G304 -- the operator names their own result files+ if err != nil {+ return nil, err+ }+ var obs observation+ if err := json.Unmarshal(raw, &obs); err != nil {+ return nil, fmt.Errorf("%s: %w", path, err)+ }+ if obs.Report == nil {+ return nil, fmt.Errorf("%s: no report in observation", path)+ }+ out = append(out, obs)+ }+ return out, nil+}++// sortBy orders rows by the varied label, numerically where it is a number so+// that 8 agents comes before 64 rather than after it.+func sortBy(observations []observation, label string) {+ sort.SliceStable(observations, func(i, j int) bool {+ a, b := observations[i].Labels[label], observations[j].Labels[label]+ na, aerr := strconv.ParseFloat(a, 64)+ nb, berr := strconv.ParseFloat(b, 64)+ if aerr == nil && berr == nil {+ return na < nb+ }+ return a < b+ })+}++func render(observations []observation, by string, metrics []string) string {+ header := []string{by, "requests", "conc", "ok", "failed", "rps", "ttfb p50", "ttfb p95", "ttfb p99", "ttfb max"}+ header = append(header, metrics...)++ rows := make([][]string, 0, len(observations))+ for _, obs := range observations {+ r := obs.Report+ row := []string{+ obs.Labels[by],+ strconv.Itoa(r.Requests),+ strconv.Itoa(r.Concurrency),+ strconv.Itoa(r.Succeeded),+ strconv.Itoa(r.Failed),+ fmt.Sprintf("%.0f", r.Throughput),+ ms(r.TTFB.P50),+ ms(r.TTFB.P95),+ ms(r.TTFB.P99),+ ms(r.TTFB.Max),+ }+ for _, m := range metrics {+ row = append(row, fmt.Sprintf("%.0f", delta(obs, m)))+ }+ rows = append(rows, row)+ }++ var b strings.Builder+ b.WriteString("| " + strings.Join(header, " | ") + " |\n")+ b.WriteString("|" + strings.Repeat(" --- |", len(header)) + "\n")+ for _, row := range rows {+ b.WriteString("| " + strings.Join(row, " | ") + " |\n")+ }+ return b.String()+}++// delta is how much a series moved across the run, summed over every scrape+// source. Counters are cumulative, so the difference is the only figure that+// describes the run rather than the process's whole life.+func delta(obs observation, series string) float64 {+ var total float64+ for source, after := range obs.After {+ total += after[series] - obs.Before[source][series]+ }+ return total+}++func ms(v float64) string { return strconv.FormatFloat(v, 'f', 2, 64) }diff --git a/cmd/sam-bench/report_test.go b/cmd/sam-bench/report_test.gonew file mode 100644index 00000000..e678439c--- /dev/null+++ b/cmd/sam-bench/report_test.go@@ -0,0 +1,89 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package main++import (+ "strings"+ "testing"++ "github.com/google/sam/internal/bench"+)++func TestSortByOrdersNumericLabelsNumerically(t *testing.T) {+ // Lexical ordering would put 128 agents between 1 and 16, and a trend+ // read off a table in that order would be nonsense.+ observations := []observation{+ {Labels: map[string]string{"agents": "128"}},+ {Labels: map[string]string{"agents": "1"}},+ {Labels: map[string]string{"agents": "16"}},+ {Labels: map[string]string{"agents": "2"}},+ }+ sortBy(observations, "agents")++ var got []string+ for _, o := range observations {+ got = append(got, o.Labels["agents"])+ }+ want := []string{"1", "2", "16", "128"}+ for i := range want {+ if got[i] != want[i] {+ t.Fatalf("order = %v, want %v", got, want)+ }+ }+}++func TestDeltaSumsTheMovementAcrossEverySource(t *testing.T) {+ // Counters are cumulative and there is one per sandbox, so a run's flow+ // count is the movement summed over all of them, not any single value.+ obs := observation{+ Before: map[string]map[string]float64{+ "box-1": {"sam_box_flows_total{outcome=allowed}": 10},+ "box-2": {"sam_box_flows_total{outcome=allowed}": 5},+ },+ After: map[string]map[string]float64{+ "box-1": {"sam_box_flows_total{outcome=allowed}": 60},+ "box-2": {"sam_box_flows_total{outcome=allowed}": 45},+ },+ }++ if got := delta(obs, "sam_box_flows_total{outcome=allowed}"); got != 90 {+ t.Errorf("delta = %v, want 90 (50 + 40)", got)+ }+}++func TestRenderPutsEveryObservationInTheTable(t *testing.T) {+ observations := []observation{+ {+ Labels: map[string]string{"agents": "1"},+ Report: &bench.Report{Requests: 100, Concurrency: 4, Succeeded: 100, Throughput: 250},+ },+ {+ Labels: map[string]string{"agents": "64"},+ Report: &bench.Report{Requests: 100, Concurrency: 4, Succeeded: 99, Failed: 1, Throughput: 120},+ },+ }++ table := render(observations, "agents", nil)+ lines := strings.Split(strings.TrimSpace(table), "\n")+ if len(lines) != 4 {+ t.Fatalf("got %d lines, want a header, a rule and two rows:\n%s", len(lines), table)+ }+ if !strings.Contains(lines[0], "agents") {+ t.Errorf("header does not name the varied label: %q", lines[0])+ }+ if !strings.Contains(lines[3], "64") {+ t.Errorf("last row is not the 64-agent run: %q", lines[3])+ }+}