Skip to content

Agent Mesh

controlplane: a store that does not answer is 503 to a member, not 401

Antonio Ojea3361372ac328
diff --git a/internal/controlplane/catalog_test.go b/internal/controlplane/catalog_test.go
index 3b34b94c..c390a726 100644
--- a/internal/controlplane/catalog_test.go
+++ b/internal/controlplane/catalog_test.go
@@ -19,14 +19,18 @@ import (
"context"
"crypto/ed25519"
"encoding/base64"
+ "errors"
"io"
"net/http"
+ "path/filepath"
"strconv"
+ "sync/atomic"
"testing"
"time"
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/identity"
+ "github.com/google/agentmesh/internal/storage"
"github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/peer"
"google.golang.org/protobuf/encoding/protojson"
@@ -320,6 +324,74 @@ func TestHandleNodeCatalog_Admission(t *testing.T) {
}
}
+// failingStore is a store whose reads fail on demand, the way a database
+// that stopped answering looks to the control plane.
+type failingStore struct {
+ storage.Store
+ fail atomic.Bool
+}
+
+func (f *failingStore) GetAllValidPublicKeys(ctx context.Context) ([]ed25519.PublicKey, error) {
+ if f.fail.Load() {
+ return nil, errors.New("store: connection refused")
+ }
+ return f.Store.GetAllValidPublicKeys(ctx)
+}
+
+func (f *failingStore) GetNode(ctx context.Context, peerID string) (*storage.EnrolledNode, error) {
+ if f.fail.Load() {
+ return nil, errors.New("store: connection refused")
+ }
+ return f.Store.GetNode(ctx, peerID)
+}
+
+// A store that does not answer is an outage, not a refusal: the member is
+// told 503 and the control plane logs it, where before it was told 401, the
+// answer a revoked credential gets, and nothing was logged. On the testnet
+// a database stall produced hundreds of such 401s per step of the ladder.
+func TestMemberAuthAnswersStoreFailureAs503(t *testing.T) {
+ t.Parallel()
+
+ dbPath := filepath.Join(t.TempDir(), "control-plane.db")
+ real, err := storage.NewSQLStore("sqlite", dbPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ store := &failingStore{Store: real}
+ srv, err := NewServer(Options{
+ ListenAddr: "127.0.0.1:0", DriverName: "sqlite", DataSourceName: dbPath,
+ AllowedAudiences: []string{"agentmesh-audience"}, BiscuitTimeout: 10 * time.Second,
+ }, store)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := srv.Start(); err != nil {
+ t.Fatal(err)
+ }
+ defer func() {
+ _ = srv.Close()
+ _ = real.Close()
+ }()
+ cpURL := "http://" + srv.listener.Addr().String()
+
+ ctx := context.Background()
+ priv, biscuitBytes := enrollRefreshTestNode(t, ctx, real)
+ body := catalogBody(t, &api.ServiceInfo{Type: api.ServiceType_SERVICE_TYPE_MCP, Name: "calc"})
+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusNoContent {
+ t.Fatalf("admitted node with a healthy store: got status %d, want %d", got, http.StatusNoContent)
+ }
+
+ store.fail.Store(true)
+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusServiceUnavailable {
+ t.Fatalf("admitted node with a failing store: got status %d, want %d, not a refusal", got, http.StatusServiceUnavailable)
+ }
+
+ store.fail.Store(false)
+ if got := postCatalog(t, cpURL, bearer(biscuitBytes), priv, body); got != http.StatusNoContent {
+ t.Fatalf("admitted node once the store answers again: got status %d, want %d", got, http.StatusNoContent)
+ }
+}
+
// The cache is keyed by the canonical base58 form from the verified biscuit,
// but the enrollment record's PeerID comes off the wire and may be any valid
// encoding of the same peer (e.g. CIDv1 base32). The view lookup and the
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index e04a2895..e7e617bb 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -1661,9 +1661,15 @@ func (s *Server) unservedEgress(ctx context.Context, policy *api.PolicyConfig) (
// admittedNodeWithChallenge returns the enrolled, admitted node whose biscuit
// the request bears after verifying the signed freshness challenge in
// HeaderChallengeTimestamp and HeaderChallengeSignature against the node's
-// stored public key. Every failure writes a uniform 401 Unauthorized.
+// stored public key. A credential that does not verify writes a uniform 401
+// Unauthorized; a store that could not be read writes 503, so that a member
+// retries instead of taking a database outage for a revocation.
func (s *Server) admittedNodeWithChallenge(w http.ResponseWriter, r *http.Request, challengeFn func(peerID string, ts int64) []byte) *storage.EnrolledNode {
- nodeRecord := s.admittedNode(r)
+ nodeRecord, err := s.admittedNode(r)
+ if err != nil {
+ writeStoreUnavailable(w, r, err)
+ return nil
+ }
if nodeRecord == nil {
writeChallengeError(w, "Unauthorized: node credential required")
return nil
@@ -1689,29 +1695,43 @@ func (s *Server) admittedNodeWithChallenge(w http.ResponseWriter, r *http.Reques
}
// admittedNode returns the enrolled, admitted node whose biscuit the request
-// bears, or nil.
-func (s *Server) admittedNode(r *http.Request) *storage.EnrolledNode {
+// bears, or nil when the request carries no such credential. The error is a
+// store that could not be read, which says nothing about the credential;
+// the caller answers it as an outage, not as a refusal.
+func (s *Server) admittedNode(r *http.Request) (*storage.EnrolledNode, error) {
authHeader := r.Header.Get("Authorization")
if !strings.HasPrefix(authHeader, "Bearer ") {
- return nil
+ return nil, nil
}
biscuitBytes, err := base64.StdEncoding.DecodeString(strings.TrimPrefix(authHeader, "Bearer "))
if err != nil {
- return nil
+ return nil, nil
}
trustedKeys, err := s.store.GetAllValidPublicKeys(r.Context())
if err != nil {
- return nil
+ return nil, fmt.Errorf("signing keys: %w", err)
}
peerID, err := identity.VerifyAndExtractPeerID(trustedKeys, biscuitBytes, s.config.BiscuitTimeout)
if err != nil {
- return nil
+ return nil, nil
}
nodeRecord, err := s.store.GetNode(r.Context(), peerID.String())
- if err != nil || nodeRecord == nil || nodeRecord.CheckAdmission(time.Now()) != nil {
- return nil
+ if err != nil && !errors.Is(err, storage.ErrNotFound) {
+ return nil, fmt.Errorf("node %s: %w", peerID, err)
}
- return nodeRecord
+ if nodeRecord == nil || nodeRecord.CheckAdmission(time.Now()) != nil {
+ return nil, nil
+ }
+ return nodeRecord, nil
+}
+
+// writeStoreUnavailable answers a mesh request the control plane could not
+// authenticate because its own store did not answer: 503, so the member
+// retries at its next sync, and a log line, so the outage is read from the
+// control plane and not inferred from members reporting 401.
+func writeStoreUnavailable(w http.ResponseWriter, r *http.Request, err error) {
+ logger.Errorw("Store unavailable while authenticating a member", "path", r.URL.Path, "error", err)
+ http.Error(w, "Service unavailable: control plane store did not answer", http.StatusServiceUnavailable)
}
// HandleEgress HTTP GET `/egress`: the egress destinations the requesting
diff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.go
index 2b8414c8..37ccaeb3 100644
--- a/internal/controlplane/sts.go
+++ b/internal/controlplane/sts.go
@@ -743,7 +743,11 @@ func (s *Server) HandleTokenExchange(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
- nodeRecord := s.admittedNode(r)
+ nodeRecord, err := s.admittedNode(r)
+ if err != nil {
+ writeStoreUnavailable(w, r, err)
+ return
+ }
if nodeRecord == nil {
writeChallengeError(w, "Unauthorized: node credential required")
return
@@ -944,7 +948,11 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) {
http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
return
}
- nodeRecord := s.admittedNode(r)
+ nodeRecord, err := s.admittedNode(r)
+ if err != nil {
+ writeStoreUnavailable(w, r, err)
+ return
+ }
if nodeRecord == nil {
writeChallengeError(w, "Unauthorized: node credential required")
return
@@ -1553,7 +1561,11 @@ func decodeBase64Biscuit(encoded string) ([]byte, error) {
}
func (s *Server) resolveDefaultActorPeer(ctx context.Context, r *http.Request) (peer.ID, error) {
- if nodeRecord := s.admittedNode(r); nodeRecord != nil {
+ nodeRecord, err := s.admittedNode(r)
+ if err != nil {
+ return "", err
+ }
+ if nodeRecord != nil {
return peer.Decode(nodeRecord.PeerID)
}
_, pub, err := s.store.GetCurrentKey(ctx)
@@ -1708,7 +1720,12 @@ func (s *Server) handleOAuthTokenExchangeGrant(w http.ResponseWriter, r *http.Re
}
}
} else {
- admitted := s.admittedNode(r)
+ admitted, storeErr := s.admittedNode(r)
+ if storeErr != nil {
+ logger.Errorw("Store unavailable while authenticating a member", "path", r.URL.Path, "error", storeErr)
+ writeOAuthError(w, http.StatusServiceUnavailable, "temporarily_unavailable", "control plane store did not answer")
+ return
+ }
actorPeerID, err := s.resolveDefaultActorPeer(r.Context(), r)
if err != nil {
writeOAuthError(w, http.StatusBadRequest, "invalid_request", "Could not resolve actor peer_id: "+err.Error())