Skip to content

Agent Mesh

node: count the agents a node is serving

Antonio Ojead95881fe406a
diff --git a/internal/node/agent.go b/internal/node/agent.go
index f24a6de8..7cf2b757 100644
--- a/internal/node/agent.go
+++ b/internal/node/agent.go
@@ -65,7 +65,9 @@ func agentFromLocalGateway(r *http.Request) string {
if !fromLocalSocket(r) {
return ""
}
- return agentClaim(r.Header.Get(api.HeaderSamAgent))
+ agentID := agentClaim(r.Header.Get(api.HeaderSamAgent))
+ recordAgentSeen(agentID)
+ return agentID
}
// agentClaim validates an agent identifier arriving from elsewhere, returning
diff --git a/internal/node/agent_metrics.go b/internal/node/agent_metrics.go
new file mode 100644
index 00000000..62e44376
--- /dev/null
+++ b/internal/node/agent_metrics.go
@@ -0,0 +1,83 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "sync"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// How many agents a node is actually serving is the one number a mesh of
+// agents cannot be described without, and until now nothing counted it. A node
+// knew how many peers it had, which is a statement about hosts, not about the
+// principals running on them.
+//
+// It is a count and not a label. Putting the agent identifier on a metric
+// would be the obvious way to answer the same question, and it would put a
+// thousand label values on a thousand series the first time anyone ran this at
+// the scale it is for.
+
+// maxTrackedAgents bounds the set behind the gauge. The identifiers come from
+// the local gateway, which is trusted, so this is not defending against an
+// attacker so much as against a bug: a gateway generating a fresh identity per
+// request would otherwise grow this map until the node died, and a memory leak
+// in the thing measuring the experiment is a bad way to end one.
+const maxTrackedAgents = 100_000
+
+var (
+ agentsSeen = promauto.NewGauge(
+ prometheus.GaugeOpts{
+ Name: "sam_node_agents_seen",
+ Help: "Distinct agents this node has served for a local gateway",
+ },
+ )
+
+ agentsUntrackedTotal = promauto.NewCounter(
+ prometheus.CounterOpts{
+ Name: "sam_node_agents_untracked_total",
+ Help: "Agent claims not counted because the tracking limit was reached",
+ },
+ )
+)
+
+var seenAgents = struct {
+ sync.Mutex
+ ids map[string]struct{}
+}{ids: make(map[string]struct{})}
+
+// recordAgentSeen counts an agent the first time this node serves it.
+func recordAgentSeen(agentID string) {
+ if agentID == "" {
+ return
+ }
+
+ seenAgents.Lock()
+ defer seenAgents.Unlock()
+
+ if _, known := seenAgents.ids[agentID]; known {
+ return
+ }
+ if len(seenAgents.ids) >= maxTrackedAgents {
+ // Counted rather than silently dropped: a gauge that stops moving
+ // looks like a mesh that stopped growing.
+ agentsUntrackedTotal.Inc()
+ return
+ }
+
+ seenAgents.ids[agentID] = struct{}{}
+ agentsSeen.Set(float64(len(seenAgents.ids)))
+}
diff --git a/internal/node/agent_metrics_test.go b/internal/node/agent_metrics_test.go
new file mode 100644
index 00000000..5dea4705
--- /dev/null
+++ b/internal/node/agent_metrics_test.go
@@ -0,0 +1,104 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "strconv"
+ "testing"
+
+ "github.com/prometheus/client_golang/prometheus"
+ dto "github.com/prometheus/client_model/go"
+)
+
+func resetSeenAgents() {
+ seenAgents.Lock()
+ defer seenAgents.Unlock()
+ seenAgents.ids = make(map[string]struct{})
+ agentsSeen.Set(0)
+}
+
+func TestAgentsSeenCountsEachAgentOnce(t *testing.T) {
+ // The gauge answers "how many agents is this node serving". Counting a
+ // busy agent repeatedly would answer "how many requests arrived", which
+ // is a different question that other metrics already answer.
+ resetSeenAgents()
+
+ for range 5 {
+ recordAgentSeen("reviewer-7.prod.acme.example")
+ }
+ recordAgentSeen("planner-2.prod.acme.example")
+
+ if got := gaugeValue(t, agentsSeen); got != 2 {
+ t.Errorf("agents seen = %v, want 2", got)
+ }
+}
+
+func TestAgentsSeenIgnoresAnAbsentClaim(t *testing.T) {
+ // Unattributed requests are normal: a node's own housekeeping carries no
+ // agent. Counting the empty string would invent an agent that never ran.
+ resetSeenAgents()
+
+ recordAgentSeen("")
+
+ if got := gaugeValue(t, agentsSeen); got != 0 {
+ t.Errorf("agents seen = %v, want 0", got)
+ }
+}
+
+func TestAgentsSeenStopsGrowingAtTheLimit(t *testing.T) {
+ // A gateway minting a fresh identity per request would otherwise grow this
+ // map until the node died, which is a poor way for an experiment to end.
+ resetSeenAgents()
+
+ seenAgents.Lock()
+ for i := range maxTrackedAgents {
+ seenAgents.ids[strconv.Itoa(i)] = struct{}{}
+ }
+ seenAgents.Unlock()
+
+ before := counterValue(t, agentsUntrackedTotal)
+ recordAgentSeen("one-too-many.prod.acme.example")
+
+ if got := counterValue(t, agentsUntrackedTotal); got != before+1 {
+ t.Errorf("untracked total = %v, want %v: the limit was hit silently", got, before+1)
+ }
+
+ seenAgents.Lock()
+ overLimit := len(seenAgents.ids) > maxTrackedAgents
+ seenAgents.Unlock()
+ if overLimit {
+ t.Error("the tracking set grew past its limit")
+ }
+
+ resetSeenAgents()
+}
+
+func gaugeValue(t *testing.T, g prometheus.Gauge) float64 {
+ t.Helper()
+ var m dto.Metric
+ if err := g.Write(&m); err != nil {
+ t.Fatalf("read gauge: %v", err)
+ }
+ return m.GetGauge().GetValue()
+}
+
+func counterValue(t *testing.T, c prometheus.Counter) float64 {
+ t.Helper()
+ var m dto.Metric
+ if err := c.Write(&m); err != nil {
+ t.Fatalf("read counter: %v", err)
+ }
+ return m.GetCounter().GetValue()
+}