Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
fix biscuit timeout flakes
Antonio Ojea3580129e6c16
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.goindex f7b8aefb..770f9ffe 100644--- a/internal/controlplane/server.go+++ b/internal/controlplane/server.go@@ -845,7 +845,7 @@ func (s *Server) HandleRouterLease(w http.ResponseWriter, r *http.Request) {}// Enforce role("router") or role("bootstrap") inside the biscuit- authorizer, err := b.Authorizer(verifyingKey)+ authorizer, err := b.Authorizer(verifyingKey, identity.AuthorizerOptions(s.config.BiscuitTimeout)...)if err != nil {http.Error(w, "Internal authorizer error", http.StatusInternalServerError)returndiff --git a/internal/identity/biscuit.go b/internal/identity/biscuit.goindex 48cab2ff..2b2de055 100644--- a/internal/identity/biscuit.go+++ b/internal/identity/biscuit.go@@ -31,6 +31,22 @@ import ("github.com/libp2p/go-libp2p/core/peer")+// DefaultAuthorizerTimeout bounds Datalog evaluation when no timeout is configured.+// biscuit-go defaults to 2ms of wall-clock time, of which a single authorization of+// a realistic token already spends ~0.14ms (~1.1ms under -race), so ordinary+// scheduling noise turns into a spurious denial. The amount of work is bounded by+// the fact and iteration limits; this deadline only caps how long the caller waits.+const DefaultAuthorizerTimeout = 1 * time.Second++// AuthorizerOptions returns the authorizer options enforcing a Datalog evaluation+// budget. A non-positive timeout falls back to DefaultAuthorizerTimeout.+func AuthorizerOptions(timeout time.Duration) []biscuit.AuthorizerOption {+ if timeout <= 0 {+ timeout = DefaultAuthorizerTimeout+ }+ return []biscuit.AuthorizerOption{biscuit.WithWorldOptions(datalog.WithMaxDuration(timeout))}+}+// MintBiscuitToken generates a signed Biscuit token for a peer with policy rules based on JWT claims.// labels are control-plane-attested key=value claims (canonical, pre-validated); empty means no claims.func MintBiscuitToken(signingKey ed25519.PrivateKey, claims jwt.MapClaims, token *oidc.IDToken, remotePeer peer.ID, biscuitExpiry time.Time, roles []string, policyRoles []*api.PolicyRole, labels map[string]string) ([]byte, []string, error) {@@ -225,10 +241,7 @@ func VerifyBiscuitAndGetKey(biscuitData []byte, expectedPeer peer.ID, trustedPubreturn nil, nil, fmt.Errorf("malformed biscuit: %w", err)}- var authOpts []biscuit.AuthorizerOption- if timeout > 0 {- authOpts = append(authOpts, biscuit.WithWorldOptions(datalog.WithMaxDuration(timeout)))- }+ authOpts := AuthorizerOptions(timeout)var lastErr errorvar authorized bool@@ -356,10 +369,7 @@ func VerifyAndExtractPeerID(trustedPublicKeys []ed25519.PublicKey, biscuitData [return "", fmt.Errorf("malformed biscuit: %w", err)}- var authOpts []biscuit.AuthorizerOption- if timeout > 0 {- authOpts = append(authOpts, biscuit.WithWorldOptions(datalog.WithMaxDuration(timeout)))- }+ authOpts := AuthorizerOptions(timeout)var authorizer biscuit.Authorizervar verified bool@@ -420,13 +430,13 @@ func VerifyAndExtractPeerID(trustedPublicKeys []ed25519.PublicKey, biscuitData [// VerifyBiscuitRole checks that the biscuit is signed by the control plane's public key// and contains the specified role fact.-func VerifyBiscuitRole(biscuitData []byte, controlPlanePubKey ed25519.PublicKey, expectedRole string) error {+func VerifyBiscuitRole(biscuitData []byte, controlPlanePubKey ed25519.PublicKey, expectedRole string, timeout time.Duration) error {b, err := biscuit.Unmarshal(biscuitData)if err != nil {return fmt.Errorf("malformed biscuit: %w", err)}- authorizer, err := b.Authorizer(controlPlanePubKey)+ authorizer, err := b.Authorizer(controlPlanePubKey, AuthorizerOptions(timeout)...)if err != nil {return fmt.Errorf("failed to create authorizer: %w", err)}diff --git a/internal/identity/biscuit_test.go b/internal/identity/biscuit_test.goindex caf89bd7..fc98ea0d 100644--- a/internal/identity/biscuit_test.go+++ b/internal/identity/biscuit_test.go@@ -17,6 +17,7 @@ package identityimport ("crypto/ed25519""crypto/rand"+ "errors""sync""testing""time"@@ -574,3 +575,36 @@ func TestVerifyAndExtractPeerID_MultipleTrustedKeys(t *testing.T) {t.Errorf("expected peer ID %s, got %s", dummyPeer, extractedPeer)}}++func TestVerifyBiscuitRole_TimeoutIsHonored(t *testing.T) {+ pub, priv, err := ed25519.GenerateKey(rand.Reader)+ if err != nil {+ t.Fatal(err)+ }++ privNode, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)+ if err != nil {+ t.Fatal(err)+ }+ dummyPeer, err := peer.IDFromPrivateKey(privNode)+ if err != nil {+ t.Fatal(err)+ }++ biscuitData, err := MintBootstrapBiscuitToken(priv, dummyPeer, api.RoleRouter, time.Now().Add(1*time.Hour), nil, nil)+ if err != nil {+ t.Fatalf("MintBootstrapBiscuitToken failed: %v", err)+ }++ // A zero timeout must fall back to the generous default, not to the 2ms+ // biscuit-go default which spuriously fails under load.+ if err := VerifyBiscuitRole(biscuitData, pub, api.RoleRouter, 0); err != nil {+ t.Errorf("VerifyBiscuitRole with zero timeout failed: %v", err)+ }++ // An explicit timeout must reach the Datalog world.+ err = VerifyBiscuitRole(biscuitData, pub, api.RoleRouter, time.Nanosecond)+ if !errors.Is(err, datalog.ErrWorldRunLimitTimeout) {+ t.Errorf("expected datalog timeout error, got %v", err)+ }+}diff --git a/internal/node/enroll.go b/internal/node/enroll.goindex e5cddc6b..963c12d9 100644--- a/internal/node/enroll.go+++ b/internal/node/enroll.go@@ -136,7 +136,7 @@ func (n *SamNode) processEnrollResponse(resp *http.Response) (*api.EnrollRespons}if n.config.RequiredRole != "" {- if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, ed25519.PublicKey(enrollResp.ControlPlanePublicKey), n.config.RequiredRole); err != nil {+ if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, ed25519.PublicKey(enrollResp.ControlPlanePublicKey), n.config.RequiredRole, n.BiscuitTimeout); err != nil {return nil, fmt.Errorf("enrolled biscuit token lacks required role %q: %w", n.config.RequiredRole, err)}}@@ -317,7 +317,7 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, breturn fmt.Errorf("received invalid control plane public key size: %d bytes", len(enrollResp.ControlPlanePublicKey))}- if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, ed25519.PublicKey(enrollResp.ControlPlanePublicKey), n.config.RequiredRole); err != nil {+ if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, ed25519.PublicKey(enrollResp.ControlPlanePublicKey), n.config.RequiredRole, n.BiscuitTimeout); err != nil {return fmt.Errorf("enrolled biscuit token lacks required role %q: %w", n.config.RequiredRole, err)}diff --git a/internal/node/labels_gate.go b/internal/node/labels_gate.goindex 4d02c343..f85e9b0c 100644--- a/internal/node/labels_gate.go+++ b/internal/node/labels_gate.go@@ -21,8 +21,6 @@ import ("sort""time"- "github.com/biscuit-auth/biscuit-go/v2"- "github.com/biscuit-auth/biscuit-go/v2/datalog""github.com/google/sam/api""github.com/google/sam/internal/identity""github.com/libp2p/go-libp2p/core/peer"@@ -106,11 +104,7 @@ func (n *SamNode) checkPeerLabels(providerBiscuit []byte, peerID peer.ID, requirreturn fmt.Errorf("provider %s biscuit verification failed: %w", peerID, err)}- var authOpts []biscuit.AuthorizerOption- if n.BiscuitTimeout > 0 {- authOpts = append(authOpts, biscuit.WithWorldOptions(datalog.WithMaxDuration(n.BiscuitTimeout)))- }- authorizer, err := b.Authorizer(key, authOpts...)+ authorizer, err := b.Authorizer(key, identity.AuthorizerOptions(n.BiscuitTimeout)...)if err != nil {return fmt.Errorf("provider %s authorizer instantiation failed: %w", peerID, err)}diff --git a/internal/node/middleware.go b/internal/node/middleware.goindex 29a9a3e0..c9c6ac85 100644--- a/internal/node/middleware.go+++ b/internal/node/middleware.go@@ -21,8 +21,8 @@ import ("sync/atomic""github.com/biscuit-auth/biscuit-go/v2"- "github.com/biscuit-auth/biscuit-go/v2/datalog""github.com/google/sam/api"+ "github.com/google/sam/internal/identity""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer""github.com/libp2p/go-msgio"@@ -194,11 +194,7 @@ func (n *SamNode) Authorize(rawToken []byte, req RequestContext, pubKey ed25519.return fmt.Errorf("invalid biscuit: %w", err)}- var authOpts []biscuit.AuthorizerOption- if n.BiscuitTimeout > 0 {- authOpts = append(authOpts, biscuit.WithWorldOptions(datalog.WithMaxDuration(n.BiscuitTimeout)))- }- authorizer, err := b.Authorizer(pubKey, authOpts...)+ authorizer, err := b.Authorizer(pubKey, identity.AuthorizerOptions(n.BiscuitTimeout)...)if err != nil {return err}@@ -340,10 +336,7 @@ func (n *SamNode) injectIdentityFacts(authorizer biscuit.Authorizer, pubKey ed25copy(keys, n.trustedKeys)n.keysMu.RUnlock()- var authOpts []biscuit.AuthorizerOption- if n.BiscuitTimeout > 0 {- authOpts = append(authOpts, biscuit.WithWorldOptions(datalog.WithMaxDuration(n.BiscuitTimeout)))- }+ authOpts := identity.AuthorizerOptions(n.BiscuitTimeout)var auth biscuit.Authorizervar authErr errordiff --git a/internal/node/node.go b/internal/node/node.goindex fe26b24a..cf850958 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -36,7 +36,6 @@ import ("time""github.com/biscuit-auth/biscuit-go/v2"- "github.com/biscuit-auth/biscuit-go/v2/datalog""github.com/google/sam/api""github.com/google/sam/internal/identity"samdiscovery "github.com/google/sam/internal/node/discovery"@@ -277,7 +276,7 @@ func (n *SamNode) Start(ctx context.Context) error {if biscuitBytes := n.GetIdentity(); len(biscuitBytes) > 0 {controlPlanePubKeyBytes, _, err := n.Store.LoadMeshConfig()if err == nil && len(controlPlanePubKeyBytes) > 0 {- if err := identity.VerifyBiscuitRole(biscuitBytes, ed25519.PublicKey(controlPlanePubKeyBytes), n.config.RequiredRole); err != nil {+ if err := identity.VerifyBiscuitRole(biscuitBytes, ed25519.PublicKey(controlPlanePubKeyBytes), n.config.RequiredRole, n.BiscuitTimeout); err != nil {return fmt.Errorf("loaded identity fails role requirement %q: %w", n.config.RequiredRole, err)}}@@ -829,7 +828,7 @@ func (n *SamNode) performRouterAuthHandshake(s network.Stream, biscuitBytes []by}// Enforce role("router") inside the biscuit- authorizer, err := b.Authorizer(trustedKeys[0])+ authorizer, err := b.Authorizer(trustedKeys[0], identity.AuthorizerOptions(n.BiscuitTimeout)...)if err != nil {return false, fmt.Errorf("authorizer instantiation failed: %w", err)}@@ -1435,9 +1434,7 @@ func (n *SamNode) verifyBiscuit(biscuitData []byte, remotePeer peer.ID) (*biscuiif len(tk.Key) != ed25519.PublicKeySize {continue}- authorizer, err := b.Authorizer(tk.Key, biscuit.WithWorldOptions(- datalog.WithMaxDuration(5*time.Second),- ))+ authorizer, err := b.Authorizer(tk.Key, identity.AuthorizerOptions(n.BiscuitTimeout)...)if err != nil {lastErr = fmt.Errorf("authorizer error: %w", err)continuediff --git a/internal/router/config.go b/internal/router/config.goindex ce18dbc2..fc3de6b4 100644--- a/internal/router/config.go+++ b/internal/router/config.go@@ -19,6 +19,7 @@ import ("time""github.com/google/sam/api"+ "github.com/google/sam/internal/identity")// Options holds configuration details for the sam-router.@@ -69,7 +70,7 @@ func (o *Options) Default() {o.KeysDBPath = "router.key"}if o.BiscuitTimeout <= 0 {- o.BiscuitTimeout = 100 * time.Millisecond+ o.BiscuitTimeout = identity.DefaultAuthorizerTimeout}if o.RequiredRole == "" {o.RequiredRole = api.RoleRouterdiff --git a/internal/router/router.go b/internal/router/router.goindex a9dfbe75..ded95979 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -326,7 +326,7 @@ func (r *Router) enroll(peerID peer.ID) error {r.trustedPublicKeys = []ed25519.PublicKey{enrollResp.ControlPlanePublicKey}r.keysMu.Unlock()- if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, enrollResp.ControlPlanePublicKey, r.config.RequiredRole); err != nil {+ if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, enrollResp.ControlPlanePublicKey, r.config.RequiredRole, r.config.BiscuitTimeout); err != nil {return fmt.Errorf("enrolled biscuit token lacks required role %q: %w", r.config.RequiredRole, err)}@@ -448,7 +448,7 @@ func (r *Router) enrollBootstrap(peerID peer.ID) error {r.trustedPublicKeys = []ed25519.PublicKey{enrollResp.ControlPlanePublicKey}r.keysMu.Unlock()- if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, enrollResp.ControlPlanePublicKey, r.config.RequiredRole); err != nil {+ if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, enrollResp.ControlPlanePublicKey, r.config.RequiredRole, r.config.BiscuitTimeout); err != nil {return fmt.Errorf("enrolled biscuit token lacks required role %q: %w", r.config.RequiredRole, err)}@@ -939,7 +939,7 @@ func (r *Router) performMutualAuth(s network.Stream) error {}// Enforce required role inside the biscuit- authorizer, err := b.Authorizer(trustedKeys[0])+ authorizer, err := b.Authorizer(trustedKeys[0], identity.AuthorizerOptions(r.config.BiscuitTimeout)...)if err != nil {return fmt.Errorf("authorizer instantiation failed: %w", err)}diff --git a/internal/router/router_test.go b/internal/router/router_test.goindex 6f4a66a7..d80b6d35 100644--- a/internal/router/router_test.go+++ b/internal/router/router_test.go@@ -794,3 +794,11 @@ func TestRouterConnectionManagerWatermarks(t *testing.T) {t.Fatalf("expected default watermarks 1000/4000, got Low: %d, High: %d", defaultOpts.LowWaterMark, defaultOpts.HighWaterMark)}}++func TestRouterDefaultBiscuitTimeout(t *testing.T) {+ opts := Options{}+ opts.Default()+ if opts.BiscuitTimeout != identity.DefaultAuthorizerTimeout {+ t.Fatalf("expected default biscuit timeout %v, got %v", identity.DefaultAuthorizerTimeout, opts.BiscuitTimeout)+ }+}diff --git a/tests/integration/login_test.go b/tests/integration/login_test.goindex 154a5d05..4b0734a7 100644--- a/tests/integration/login_test.go+++ b/tests/integration/login_test.go@@ -198,7 +198,6 @@ func TestSamNodeJoinOfflineAccessSavesRefreshToken(t *testing.T) {}}-// TestSamNodeRunJoinNonInteractive covers the safe fallback: since the test// harness gives the child process no TTY (stdin defaults to /dev/null),// "run --join" must not attempt (and block on) an interactive OIDC login; it@@ -405,4 +404,3 @@ func normalizeControlPlaneURLForTest(url string) string {}return strings.TrimSuffix(url, "/")}-