Skip to content

Agent Mesh

controlplane, router: say why a member credential is refused

Antonio Ojeaa8f6520cf78b
diff --git a/internal/controlplane/member_auth_test.go b/internal/controlplane/member_auth_test.go
new file mode 100644
index 00000000..6bba0942
--- /dev/null
+++ b/internal/controlplane/member_auth_test.go
@@ -0,0 +1,156 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package controlplane
+
+import (
+ "context"
+ "crypto/ed25519"
+ "crypto/rand"
+ "encoding/base64"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/google/agentmesh/api"
+ "github.com/google/agentmesh/internal/identity"
+ "github.com/libp2p/go-libp2p/core/crypto"
+ "github.com/libp2p/go-libp2p/core/peer"
+)
+
+// A member credential that is presented and not accepted is counted under
+// the reason it was refused for, and one that is accepted, or absent, is
+// not counted at all.
+func TestAdmittedNodeCountsRefusalsByReason(t *testing.T) {
+ issuer, _ := startCustomMockOIDC(t)
+ srv, store, baseURL := setupTestServer(t, issuer)
+ defer func() {
+ _ = srv.Close()
+ _ = store.Close()
+ }()
+ const adminToken = "super-secret-admin-token"
+ srv.config.AdminToken = adminToken
+ srv.config.AutoApproveEnrollment = true
+ ctx := context.Background()
+ if err := store.SaveMeshPolicy(ctx, []*api.PolicyRole{{Name: api.RoleNode}}, nil); err != nil {
+ t.Fatal(err)
+ }
+
+ enroll := func(t *testing.T) (*api.BootstrapEnrollResponse, peer.ID) {
+ t.Helper()
+ token := createAdminBootstrapToken(t, baseURL, adminToken, api.RoleNode, 1)
+ priv, pub, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pID, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ pubBytes, err := crypto.MarshalPublicKey(pub)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out := bootstrapEnroll(t, baseURL, token, priv, pID, pubBytes, api.RoleNode, nil)
+ if out.Status != api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {
+ t.Fatalf("enroll: status %v (%s)", out.Status, out.ErrorMessage)
+ }
+ return out, pID
+ }
+ request := func(bearer string) *http.Request {
+ r := httptest.NewRequest(http.MethodGet, "/policies", nil)
+ if bearer != "" {
+ r.Header.Set("Authorization", "Bearer "+bearer)
+ }
+ return r
+ }
+ encode := func(b []byte) string { return base64.StdEncoding.EncodeToString(b) }
+
+ admitted, admittedID := enroll(t)
+ banned, bannedID := enroll(t)
+ if err := store.SetNodeBanned(ctx, bannedID.String(), true); err != nil {
+ t.Fatal(err)
+ }
+ expired, expiredID := enroll(t)
+ expiredRecord, err := store.GetNode(ctx, expiredID.String())
+ if err != nil {
+ t.Fatal(err)
+ }
+ expiredRecord.ExpiresAt = time.Now().Add(-time.Minute)
+ if err := store.EnrollNode(ctx, expiredRecord); err != nil {
+ t.Fatal(err)
+ }
+ // A credential this control plane signed for a node its store has no
+ // record of: what every enrolled node holds after the store is reset.
+ signingKey, _, err := store.GetCurrentKey(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, strangerID := newTestKey(t)
+ stranger, err := identity.MintBootstrapBiscuitToken(signingKey, strangerID, api.RoleNode, time.Now().Add(time.Hour), nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ // A credential signed by a key this control plane never had.
+ _, otherKey, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ forged, err := identity.MintBootstrapBiscuitToken(otherKey, admittedID, api.RoleNode, time.Now().Add(time.Hour), nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ reasons := []string{refusalMalformed, refusalInvalidBiscuit, refusalUnknownNode, refusalBanned, refusalSessionExpired}
+ cases := []struct {
+ name string
+ bearer string
+ admit bool
+ reason string
+ }{
+ {name: "no credential", bearer: "", admit: false, reason: ""},
+ {name: "admitted node", bearer: encode(admitted.BiscuitToken), admit: true, reason: ""},
+ {name: "not base64", bearer: "not*base64", admit: false, reason: refusalMalformed},
+ {name: "not a biscuit", bearer: encode([]byte("not a biscuit")), admit: false, reason: refusalInvalidBiscuit},
+ {name: "signed by another control plane", bearer: encode(forged), admit: false, reason: refusalInvalidBiscuit},
+ {name: "node unknown to the store", bearer: encode(stranger), admit: false, reason: refusalUnknownNode},
+ {name: "banned node", bearer: encode(banned.BiscuitToken), admit: false, reason: refusalBanned},
+ {name: "session expired", bearer: encode(expired.BiscuitToken), admit: false, reason: refusalSessionExpired},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ before := make(map[string]float64, len(reasons))
+ for _, reason := range reasons {
+ before[reason] = counterValue(t, memberAuthRefusalsTotal, reason)
+ }
+ record, err := srv.admittedNode(request(tc.bearer))
+ if err != nil {
+ t.Fatalf("admittedNode: %v", err)
+ }
+ if (record != nil) != tc.admit {
+ t.Fatalf("admitted = %v, want %v", record != nil, tc.admit)
+ }
+ for _, reason := range reasons {
+ want := before[reason]
+ if reason == tc.reason {
+ want++
+ }
+ if got := counterValue(t, memberAuthRefusalsTotal, reason); got != want {
+ t.Errorf("refusals{reason=%q} = %v, want %v", reason, got, want)
+ }
+ }
+ })
+ }
+}
diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.go
index a3a6a68c..dad98e48 100644
--- a/internal/controlplane/metrics.go
+++ b/internal/controlplane/metrics.go
@@ -77,8 +77,38 @@ var (
Help: "Times the operator signing key file could not be reloaded; the keys already in use were kept",
},
)
+
+ // memberAuthRefusalsTotal counts the member credentials presented on a
+ // mesh endpoint that the control plane did not accept, by reason. A
+ // request with no credential is not counted: on /oauth/token that is
+ // the anonymous flow, and elsewhere the 401 alone says what to send.
+ memberAuthRefusalsTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "agentmesh_control_plane_member_auth_refusals_total",
+ Help: "Member credentials presented on mesh endpoints and refused, by reason (malformed, invalid_biscuit, unknown_node, banned, session_expired)",
+ },
+ []string{"reason"},
+ )
+)
+
+// Member credential refusal reasons. unknown_node is the one an operator
+// acts on: the credential verifies under this control plane's keys, but
+// the store has no enrollment record for the node, so the store was reset
+// or the node was removed.
+const (
+ refusalMalformed = "malformed"
+ refusalInvalidBiscuit = "invalid_biscuit"
+ refusalUnknownNode = "unknown_node"
+ refusalBanned = "banned"
+ refusalSessionExpired = "session_expired"
)
+func init() {
+ for _, reason := range []string{refusalMalformed, refusalInvalidBiscuit, refusalUnknownNode, refusalBanned, refusalSessionExpired} {
+ memberAuthRefusalsTotal.WithLabelValues(reason)
+ }
+}
+
// observeRoute counts and times requests to one registered mux pattern. The
// pattern, not the request path, is the label: paths carry peer ids and token
// ids chosen off-plane, so the raw path can never become a label.
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index 8006dcc5..d35fa1b0 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -1904,7 +1904,10 @@ func (s *Server) admittedNodeWithChallenge(w http.ResponseWriter, r *http.Reques
// admittedNode returns the enrolled, admitted node whose biscuit the request
// bears, or nil when the request carries no such credential. The error is a
// store that could not be read, which says nothing about the credential;
-// the caller answers it as an outage, not as a refusal.
+// the caller answers it as an outage, not as a refusal. A credential that
+// was presented and not accepted is counted and logged with its reason, so
+// that a member's 401 can be read from the control plane and not inferred
+// from the member.
func (s *Server) admittedNode(r *http.Request) (*storage.EnrolledNode, error) {
authHeader := r.Header.Get("Authorization")
if !strings.HasPrefix(authHeader, "Bearer ") {
@@ -1912,6 +1915,7 @@ func (s *Server) admittedNode(r *http.Request) (*storage.EnrolledNode, error) {
}
biscuitBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authHeader, "Bearer "))
if err != nil {
+ refuseMember(r, refusalMalformed, "", err)
return nil, nil
}
trustedKeys, err := s.store.GetAllValidPublicKeys(r.Context())
@@ -1920,18 +1924,44 @@ func (s *Server) admittedNode(r *http.Request) (*storage.EnrolledNode, error) {
}
peerID, err := identity.VerifyAndExtractPeerID(trustedKeys, biscuitBytes, s.config.BiscuitTimeout)
if err != nil {
+ refuseMember(r, refusalInvalidBiscuit, "", err)
return nil, nil
}
nodeRecord, err := s.store.GetNode(r.Context(), peerID.String())
if err != nil && !errors.Is(err, storage.ErrNotFound) {
return nil, fmt.Errorf("node %s: %w", peerID, err)
}
- if nodeRecord == nil || nodeRecord.CheckAdmission(time.Now()) != nil {
+ if nodeRecord == nil {
+ refuseMember(r, refusalUnknownNode, peerID.String(), errors.New("no enrollment record: the store was reset or the node was removed"))
+ return nil, nil
+ }
+ if err := nodeRecord.CheckAdmission(time.Now()); err != nil {
+ reason := refusalSessionExpired
+ if errors.Is(err, storage.ErrNodeBanned) {
+ reason = refusalBanned
+ }
+ refuseMember(r, reason, peerID.String(), err)
return nil, nil
}
return nodeRecord, nil
}
+// refuseMember records a member credential that was presented and not
+// accepted. A credential the keys do not verify or a session that lapsed
+// is the member's to renew; a node this store does not know, or has
+// banned, is the operator's to look at.
+func refuseMember(r *http.Request, reason, peerID string, err error) {
+ memberAuthRefusalsTotal.WithLabelValues(reason).Inc()
+ switch reason {
+ case refusalUnknownNode, refusalBanned:
+ logger.Warnw("Member credential refused", "reason", reason, "path", r.URL.Path, "peer_id", peerID, "error", err)
+ case refusalMalformed:
+ logger.Debugw("Member credential refused", "reason", reason, "path", r.URL.Path, "error", err)
+ default:
+ logger.Infow("Member credential refused", "reason", reason, "path", r.URL.Path, "peer_id", peerID, "error", err)
+ }
+}
+
// writeStoreUnavailable answers a mesh request the control plane could not
// authenticate because its own store did not answer: 503, so the member
// retries at its next sync, and a log line, so the outage is read from the
diff --git a/internal/router/goaway.go b/internal/router/goaway.go
index af6f1451..5f44793f 100644
--- a/internal/router/goaway.go
+++ b/internal/router/goaway.go
@@ -136,6 +136,7 @@ func (r *Router) drainMembers() {
}
members := r.members()
if len(members) == 0 {
+ logger.Info("[GoAway] Draining: no members attached, nothing to send away")
return
}
retryAfter := r.config.ShutdownLeaseTTL
diff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.md
index 603c55b1..334316cb 100644
--- a/site/content/docs/reference/control-plane.md
+++ b/site/content/docs/reference/control-plane.md
@@ -184,3 +184,15 @@ with `403`). The console uses these routes.
by role and state, active routers, mesh-connected peers as reported by
router leases, rows per store table, service reports held in memory,
per-route request counts and latencies, and the Go runtime.
+
+A member whose request on a mesh endpoint (`/policies`, `/egress`,
+`/revocations`, `/token/exchange`, `/sts/token`, ...) is answered 401 is
+counted in `agentmesh_control_plane_member_auth_refusals_total{reason}` and
+logged with the same reason: `malformed` (not a base64 credential),
+`invalid_biscuit` (not signed by a key this control plane trusts, or
+expired), `unknown_node` (the credential verifies but the store has no
+enrollment record for the node: the store was reset, or the node was
+removed), `banned`, `session_expired`. `unknown_node` is the one to look
+at after a control plane restart on an empty store: every node then
+refused is a node that needs a bootstrap token
+(`--bootstrap-token-path`) to enroll again on its own.