Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
controlplane: a store that does not answer is 503 to a member, not 401
Antonio Ojea3361372ac328
diff --git a/internal/controlplane/catalog_test.go b/internal/controlplane/catalog_test.goindex 3b34b94c..c390a726 100644--- a/internal/controlplane/catalog_test.go+++ b/internal/controlplane/catalog_test.go@@ -19,14 +19,18 @@ import ("context""crypto/ed25519""encoding/base64"+ "errors""io""net/http"+ "path/filepath""strconv"+ "sync/atomic""testing""time""github.com/google/agentmesh/api""github.com/google/agentmesh/internal/identity"+ "github.com/google/agentmesh/internal/storage""github.com/libp2p/go-libp2p/core/crypto""github.com/libp2p/go-libp2p/core/peer""google.golang.org/protobuf/encoding/protojson"@@ -320,6 +324,74 @@ func TestHandleNodeCatalog_Admission(t *testing.T) {}}+// failingStore is a store whose reads fail on demand, the way a database+// that stopped answering looks to the control plane.+type failingStore struct {+ storage.Store+ fail atomic.Bool+}++func (f *failingStore) GetAllValidPublicKeys(ctx context.Context) ([]ed25519.PublicKey, error) {+ if f.fail.Load() {+ return nil, errors.New("store: connection refused")+ }+ return f.Store.GetAllValidPublicKeys(ctx)+}++func (f *failingStore) GetNode(ctx context.Context, peerID string) (*storage.EnrolledNode, error) {+ if f.fail.Load() {+ return nil, errors.New("store: connection refused")+ }+ return f.Store.GetNode(ctx, peerID)+}++// A store that does not answer is an outage, not a refusal: the member is+// told 503 and the control plane logs it, where before it was told 401, the+// answer a revoked credential gets, and nothing was logged. On the testnet+// a database stall produced hundreds of such 401s per step of the ladder.+func TestMemberAuthAnswersStoreFailureAs503(t *testing.T) {+ t.Parallel()++ dbPath := filepath.Join(t.TempDir(), "control-plane.db")+ real, err := storage.NewSQLStore("sqlite", dbPath)+ if err != nil {+ t.Fatal(err)+ }+ store := &failingStore{Store: real}+ srv, err := NewServer(Options{+ ListenAddr: "127.0.0.1:0", DriverName: "sqlite", DataSourceName: dbPath,+ AllowedAudiences: []string{"agentmesh-audience"}, BiscuitTimeout: 10 * time.Second,+ }, store)+ if err != nil {+ t.Fatal(err)+ }+ if err := srv.Start(); err != nil {+ t.Fatal(err)+ }+ defer func() {+ _ = srv.Close()+ _ = real.Close()+ }()+ cpURL := "http://" + srv.listener.Addr().String()++ ctx := context.Background()+ priv, biscuitBytes := enrollRefreshTestNode(t, ctx, real)+ body := catalogBody(t, &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "calc"})+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusNoContent {+ t.Fatalf("admitted node with a healthy store: got status %d, want %d", got, http.StatusNoContent)+ }++ store.fail.Store(true)+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusServiceUnavailable {+ t.Fatalf("admitted node with a failing store: got status %d, want %d, not a refusal", got, http.StatusServiceUnavailable)+ }++ store.fail.Store(false)+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusNoContent {+ t.Fatalf("admitted node once the store answers again: got status %d, want %d", got, http.StatusNoContent)+ }+}+// The cache is keyed by the canonical base58 form from the verified biscuit,// but the enrollment record's PeerID comes off the wire and may be any valid// encoding of the same peer (e.g. CIDv1 base32). The view lookup and thediff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex e04a2895..e7e617bb 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -1661,9 +1661,15 @@ func (s *Server) unservedEgress(ctx context.Context, policy *api.PolicyConfig) (// admittedNodeWithChallenge returns the enrolled, admitted node whose biscuit// the request bears after verifying the signed freshness challenge in// HeaderChallengeTimestamp and HeaderChallengeSignature against the node's-// stored public key. Every failure writes a uniform 401 Unauthorized.+// stored public key. A credential that does not verify writes a uniform 401+// Unauthorized; a store that could not be read writes 503, so that a member+// retries instead of taking a database outage for a revocation.func (s *Server) admittedNodeWithChallenge(w http.ResponseWriter, r *http.Request, challengeFn func(peerID string, ts int64) []byte) *storage.EnrolledNode {- nodeRecord := s.admittedNode(r)+ nodeRecord, err := s.admittedNode(r)+ if err != nil {+ writeStoreUnavailable(w, r, err)+ return nil+ }if nodeRecord == nil {writeChallengeError(w, "Unauthorized: node credential required")return nil@@ -1689,29 +1695,43 @@ func (s *Server) admittedNodeWithChallenge(w http.ResponseWriter, r *http.Reques}// admittedNode returns the enrolled, admitted node whose biscuit the request-// bears, or nil.-func (s *Server) admittedNode(r *http.Request) *storage.EnrolledNode {+// bears, or nil when the request carries no such credential. The error is a+// store that could not be read, which says nothing about the credential;+// the caller answers it as an outage, not as a refusal.+func (s *Server) admittedNode(r *http.Request) (*storage.EnrolledNode, error) {authHeader := r.Header.Get("Authorization")if !strings.HasPrefix(authHeader, "Bearer ") {- return nil+ return nil, nil}biscuitBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authHeader, "Bearer "))if err != nil {- return nil+ return nil, nil}trustedKeys, err := s.store.GetAllValidPublicKeys(r.Context())if err != nil {- return nil+ return nil, fmt.Errorf("signing keys: %w", err)}peerID, err := identity.VerifyAndExtractPeerID(trustedKeys, biscuitBytes, s.config.BiscuitTimeout)if err != nil {- return nil+ return nil, nil}nodeRecord, err := s.store.GetNode(r.Context(), peerID.String())- if err != nil || nodeRecord == nil || nodeRecord.CheckAdmission(time.Now()) != nil {- return nil+ if err != nil && !errors.Is(err, storage.ErrNotFound) {+ return nil, fmt.Errorf("node %s: %w", peerID, err)}- return nodeRecord+ if nodeRecord == nil || nodeRecord.CheckAdmission(time.Now()) != nil {+ return nil, nil+ }+ return nodeRecord, nil+}++// writeStoreUnavailable answers a mesh request the control plane could not+// authenticate because its own store did not answer: 503, so the member+// retries at its next sync, and a log line, so the outage is read from the+// control plane and not inferred from members reporting 401.+func writeStoreUnavailable(w http.ResponseWriter, r *http.Request, err error) {+ logger.Errorw("Store unavailable while authenticating a member", "path", r.URL.Path, "error", err)+ http.Error(w, "Service unavailable: control plane store did not answer", http.StatusServiceUnavailable)}// HandleEgress HTTP GET `/egress`: the egress destinations the requestingdiff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.goindex 2b8414c8..37ccaeb3 100644--- a/internal/controlplane/sts.go+++ b/internal/controlplane/sts.go@@ -743,7 +743,11 @@ func (s *Server) HandleTokenExchange(w http.ResponseWriter, r *http.Request) {http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)return}- nodeRecord := s.admittedNode(r)+ nodeRecord, err := s.admittedNode(r)+ if err != nil {+ writeStoreUnavailable(w, r, err)+ return+ }if nodeRecord == nil {writeChallengeError(w, "Unauthorized: node credential required")return@@ -944,7 +948,11 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) {http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)return}- nodeRecord := s.admittedNode(r)+ nodeRecord, err := s.admittedNode(r)+ if err != nil {+ writeStoreUnavailable(w, r, err)+ return+ }if nodeRecord == nil {writeChallengeError(w, "Unauthorized: node credential required")return@@ -1553,7 +1561,11 @@ func decodeBase64Biscuit(encoded string) ([]byte, error) {}func (s *Server) resolveDefaultActorPeer(ctx context.Context, r *http.Request) (peer.ID, error) {- if nodeRecord := s.admittedNode(r); nodeRecord != nil {+ nodeRecord, err := s.admittedNode(r)+ if err != nil {+ return "", err+ }+ if nodeRecord != nil {return peer.Decode(nodeRecord.PeerID)}_, pub, err := s.store.GetCurrentKey(ctx)@@ -1708,7 +1720,12 @@ func (s *Server) handleOAuthTokenExchangeGrant(w http.ResponseWriter, r *http.Re}}} else {- admitted := s.admittedNode(r)+ admitted, storeErr := s.admittedNode(r)+ if storeErr != nil {+ logger.Errorw("Store unavailable while authenticating a member", "path", r.URL.Path, "error", storeErr)+ writeOAuthError(w, http.StatusServiceUnavailable, "temporarily_unavailable", "control plane store did not answer")+ return+ }actorPeerID, err := s.resolveDefaultActorPeer(r.Context(), r)if err != nil {writeOAuthError(w, http.StatusBadRequest, "invalid_request", "Could not resolve actor peer_id: "+err.Error())