Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
controlplane, router: say why a member credential is refused
Antonio Ojeaa8f6520cf78b
diff --git a/internal/controlplane/member_auth_test.go b/internal/controlplane/member_auth_test.gonew file mode 100644index 00000000..6bba0942--- /dev/null+++ b/internal/controlplane/member_auth_test.go@@ -0,0 +1,156 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package controlplane++import (+ "context"+ "crypto/ed25519"+ "crypto/rand"+ "encoding/base64"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"++ "github.com/google/agentmesh/api"+ "github.com/google/agentmesh/internal/identity"+ "github.com/libp2p/go-libp2p/core/crypto"+ "github.com/libp2p/go-libp2p/core/peer"+)++// A member credential that is presented and not accepted is counted under+// the reason it was refused for, and one that is accepted, or absent, is+// not counted at all.+func TestAdmittedNodeCountsRefusalsByReason(t *testing.T) {+ issuer, _ := startCustomMockOIDC(t)+ srv, store, baseURL := setupTestServer(t, issuer)+ defer func() {+ _ = srv.Close()+ _ = store.Close()+ }()+ const adminToken = "super-secret-admin-token"+ srv.config.AdminToken = adminToken+ srv.config.AutoApproveEnrollment = true+ ctx := context.Background()+ if err := store.SaveMeshPolicy(ctx, []*api.PolicyRole{{Name: api.RoleNode}}, nil); err != nil {+ t.Fatal(err)+ }++ enroll := func(t *testing.T) (*api.BootstrapEnrollResponse, peer.ID) {+ t.Helper()+ token := createAdminBootstrapToken(t, baseURL, adminToken, api.RoleNode, 1)+ priv, pub, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ if err != nil {+ t.Fatal(err)+ }+ pID, err := peer.IDFromPrivateKey(priv)+ if err != nil {+ t.Fatal(err)+ }+ pubBytes, err := crypto.MarshalPublicKey(pub)+ if err != nil {+ t.Fatal(err)+ }+ out := bootstrapEnroll(t, baseURL, token, priv, pID, pubBytes, api.RoleNode, nil)+ if out.Status != api.EnrollmentStatus_ENROLLMENT_STATUS_APPROVED {+ t.Fatalf("enroll: status %v (%s)", out.Status, out.ErrorMessage)+ }+ return out, pID+ }+ request := func(bearer string) *http.Request {+ r := httptest.NewRequest(http.MethodGet, "/policies", nil)+ if bearer != "" {+ r.Header.Set("Authorization", "Bearer "+bearer)+ }+ return r+ }+ encode := func(b []byte) string { return base64.StdEncoding.EncodeToString(b) }++ admitted, admittedID := enroll(t)+ banned, bannedID := enroll(t)+ if err := store.SetNodeBanned(ctx, bannedID.String(), true); err != nil {+ t.Fatal(err)+ }+ expired, expiredID := enroll(t)+ expiredRecord, err := store.GetNode(ctx, expiredID.String())+ if err != nil {+ t.Fatal(err)+ }+ expiredRecord.ExpiresAt = time.Now().Add(-time.Minute)+ if err := store.EnrollNode(ctx, expiredRecord); err != nil {+ t.Fatal(err)+ }+ // A credential this control plane signed for a node its store has no+ // record of: what every enrolled node holds after the store is reset.+ signingKey, _, err := store.GetCurrentKey(ctx)+ if err != nil {+ t.Fatal(err)+ }+ _, strangerID := newTestKey(t)+ stranger, err := identity.MintBootstrapBiscuitToken(signingKey, strangerID, api.RoleNode, time.Now().Add(time.Hour), nil, nil)+ if err != nil {+ t.Fatal(err)+ }+ // A credential signed by a key this control plane never had.+ _, otherKey, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }+ forged, err := identity.MintBootstrapBiscuitToken(otherKey, admittedID, api.RoleNode, time.Now().Add(time.Hour), nil, nil)+ if err != nil {+ t.Fatal(err)+ }++ reasons := []string{refusalMalformed, refusalInvalidBiscuit, refusalUnknownNode, refusalBanned, refusalSessionExpired}+ cases := []struct {+ name string+ bearer string+ admit bool+ reason string+ }{+ {name: "no credential", bearer: "", admit: false, reason: ""},+ {name: "admitted node", bearer: encode(admitted.BiscuitToken), admit: true, reason: ""},+ {name: "not base64", bearer: "not*base64", admit: false, reason: refusalMalformed},+ {name: "not a biscuit", bearer: encode([]byte("not a biscuit")), admit: false, reason: refusalInvalidBiscuit},+ {name: "signed by another control plane", bearer: encode(forged), admit: false, reason: refusalInvalidBiscuit},+ {name: "node unknown to the store", bearer: encode(stranger), admit: false, reason: refusalUnknownNode},+ {name: "banned node", bearer: encode(banned.BiscuitToken), admit: false, reason: refusalBanned},+ {name: "session expired", bearer: encode(expired.BiscuitToken), admit: false, reason: refusalSessionExpired},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ before := make(map[string]float64, len(reasons))+ for _, reason := range reasons {+ before[reason] = counterValue(t, memberAuthRefusalsTotal, reason)+ }+ record, err := srv.admittedNode(request(tc.bearer))+ if err != nil {+ t.Fatalf("admittedNode: %v", err)+ }+ if (record != nil) != tc.admit {+ t.Fatalf("admitted = %v, want %v", record != nil, tc.admit)+ }+ for _, reason := range reasons {+ want := before[reason]+ if reason == tc.reason {+ want+++ }+ if got := counterValue(t, memberAuthRefusalsTotal, reason); got != want {+ t.Errorf("refusals{reason=%q} = %v, want %v", reason, got, want)+ }+ }+ })+ }+}diff --git a/internal/controlplane/metrics.go b/internal/controlplane/metrics.goindex a3a6a68c..dad98e48 100644--- a/internal/controlplane/metrics.go+++ b/internal/controlplane/metrics.go@@ -77,8 +77,38 @@ var (Help: "Times the operator signing key file could not be reloaded; the keys already in use were kept",},)++ // memberAuthRefusalsTotal counts the member credentials presented on a+ // mesh endpoint that the control plane did not accept, by reason. A+ // request with no credential is not counted: on /oauth/token that is+ // the anonymous flow, and elsewhere the 401 alone says what to send.+ memberAuthRefusalsTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Name: "agentmesh_control_plane_member_auth_refusals_total",+ Help: "Member credentials presented on mesh endpoints and refused, by reason (malformed, invalid_biscuit, unknown_node, banned, session_expired)",+ },+ []string{"reason"},+ )+)++// Member credential refusal reasons. unknown_node is the one an operator+// acts on: the credential verifies under this control plane's keys, but+// the store has no enrollment record for the node, so the store was reset+// or the node was removed.+const (+ refusalMalformed = "malformed"+ refusalInvalidBiscuit = "invalid_biscuit"+ refusalUnknownNode = "unknown_node"+ refusalBanned = "banned"+ refusalSessionExpired = "session_expired")+func init() {+ for _, reason := range []string{refusalMalformed, refusalInvalidBiscuit, refusalUnknownNode, refusalBanned, refusalSessionExpired} {+ memberAuthRefusalsTotal.WithLabelValues(reason)+ }+}+// observeRoute counts and times requests to one registered mux pattern. The// pattern, not the request path, is the label: paths carry peer ids and token// ids chosen off-plane, so the raw path can never become a label.diff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex 8006dcc5..d35fa1b0 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -1904,7 +1904,10 @@ func (s *Server) admittedNodeWithChallenge(w http.ResponseWriter, r *http.Reques// admittedNode returns the enrolled, admitted node whose biscuit the request// bears, or nil when the request carries no such credential. The error is a// store that could not be read, which says nothing about the credential;-// the caller answers it as an outage, not as a refusal.+// the caller answers it as an outage, not as a refusal. A credential that+// was presented and not accepted is counted and logged with its reason, so+// that a member's 401 can be read from the control plane and not inferred+// from the member.func (s *Server) admittedNode(r *http.Request) (*storage.EnrolledNode, error) {authHeader := r.Header.Get("Authorization")if !strings.HasPrefix(authHeader, "Bearer ") {@@ -1912,6 +1915,7 @@ func (s *Server) admittedNode(r *http.Request) (*storage.EnrolledNode, error) {}biscuitBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authHeader, "Bearer "))if err != nil {+ refuseMember(r, refusalMalformed, "", err)return nil, nil}trustedKeys, err := s.store.GetAllValidPublicKeys(r.Context())@@ -1920,18 +1924,44 @@ func (s *Server) admittedNode(r *http.Request) (*storage.EnrolledNode, error) {}peerID, err := identity.VerifyAndExtractPeerID(trustedKeys, biscuitBytes, s.config.BiscuitTimeout)if err != nil {+ refuseMember(r, refusalInvalidBiscuit, "", err)return nil, nil}nodeRecord, err := s.store.GetNode(r.Context(), peerID.String())if err != nil && !errors.Is(err, storage.ErrNotFound) {return nil, fmt.Errorf("node %s: %w", peerID, err)}- if nodeRecord == nil || nodeRecord.CheckAdmission(time.Now()) != nil {+ if nodeRecord == nil {+ refuseMember(r, refusalUnknownNode, peerID.String(), errors.New("no enrollment record: the store was reset or the node was removed"))+ return nil, nil+ }+ if err := nodeRecord.CheckAdmission(time.Now()); err != nil {+ reason := refusalSessionExpired+ if errors.Is(err, storage.ErrNodeBanned) {+ reason = refusalBanned+ }+ refuseMember(r, reason, peerID.String(), err)return nil, nil}return nodeRecord, nil}+// refuseMember records a member credential that was presented and not+// accepted. A credential the keys do not verify or a session that lapsed+// is the member's to renew; a node this store does not know, or has+// banned, is the operator's to look at.+func refuseMember(r *http.Request, reason, peerID string, err error) {+ memberAuthRefusalsTotal.WithLabelValues(reason).Inc()+ switch reason {+ case refusalUnknownNode, refusalBanned:+ logger.Warnw("Member credential refused", "reason", reason, "path", r.URL.Path, "peer_id", peerID, "error", err)+ case refusalMalformed:+ logger.Debugw("Member credential refused", "reason", reason, "path", r.URL.Path, "error", err)+ default:+ logger.Infow("Member credential refused", "reason", reason, "path", r.URL.Path, "peer_id", peerID, "error", err)+ }+}+// writeStoreUnavailable answers a mesh request the control plane could not// authenticate because its own store did not answer: 503, so the member// retries at its next sync, and a log line, so the outage is read from thediff --git a/internal/router/goaway.go b/internal/router/goaway.goindex af6f1451..5f44793f 100644--- a/internal/router/goaway.go+++ b/internal/router/goaway.go@@ -136,6 +136,7 @@ func (r *Router) drainMembers() {}members := r.members()if len(members) == 0 {+ logger.Info("[GoAway] Draining: no members attached, nothing to send away")return}retryAfter := r.config.ShutdownLeaseTTLdiff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.mdindex 603c55b1..334316cb 100644--- a/site/content/docs/reference/control-plane.md+++ b/site/content/docs/reference/control-plane.md@@ -184,3 +184,15 @@ with `403`). The console uses these routes.by role and state, active routers, mesh-connected peers as reported byrouter leases, rows per store table, service reports held in memory,per-route request counts and latencies, and the Go runtime.++A member whose request on a mesh endpoint (`/policies`, `/egress`,+`/revocations`, `/token/exchange`, `/sts/token`, ...) is answered 401 is+counted in `agentmesh_control_plane_member_auth_refusals_total{reason}` and+logged with the same reason: `malformed` (not a base64 credential),+`invalid_biscuit` (not signed by a key this control plane trusts, or+expired), `unknown_node` (the credential verifies but the store has no+enrollment record for the node: the store was reset, or the node was+removed), `banned`, `session_expired`. `unknown_node` is the one to look+at after a control plane restart on an empty store: every node then+refused is a node that needs a bootstrap token+(`--bootstrap-token-path`) to enroll again on its own.