Skip to content

Agent Mesh

fix biscuit timeout flakes

Antonio Ojea3580129e6c16
diff --git a/internal/controlplane/server.go b/internal/controlplane/server.go
index f7b8aefb..770f9ffe 100644
--- a/internal/controlplane/server.go
+++ b/internal/controlplane/server.go
@@ -845,7 +845,7 @@ func (s *Server) HandleRouterLease(w http.ResponseWriter, r *http.Request) {
}
// Enforce role("router") or role("bootstrap") inside the biscuit
- authorizer, err := b.Authorizer(verifyingKey)
+ authorizer, err := b.Authorizer(verifyingKey, identity.AuthorizerOptions(s.config.BiscuitTimeout)...)
if err != nil {
http.Error(w, "Internal authorizer error", http.StatusInternalServerError)
return
diff --git a/internal/identity/biscuit.go b/internal/identity/biscuit.go
index 48cab2ff..2b2de055 100644
--- a/internal/identity/biscuit.go
+++ b/internal/identity/biscuit.go
@@ -31,6 +31,22 @@ import (
"github.com/libp2p/go-libp2p/core/peer"
)
+// DefaultAuthorizerTimeout bounds Datalog evaluation when no timeout is configured.
+// biscuit-go defaults to 2ms of wall-clock time, of which a single authorization of
+// a realistic token already spends ~0.14ms (~1.1ms under -race), so ordinary
+// scheduling noise turns into a spurious denial. The amount of work is bounded by
+// the fact and iteration limits; this deadline only caps how long the caller waits.
+const DefaultAuthorizerTimeout = 1 * time.Second
+
+// AuthorizerOptions returns the authorizer options enforcing a Datalog evaluation
+// budget. A non-positive timeout falls back to DefaultAuthorizerTimeout.
+func AuthorizerOptions(timeout time.Duration) []biscuit.AuthorizerOption {
+ if timeout <= 0 {
+ timeout = DefaultAuthorizerTimeout
+ }
+ return []biscuit.AuthorizerOption{biscuit.WithWorldOptions(datalog.WithMaxDuration(timeout))}
+}
+
// MintBiscuitToken generates a signed Biscuit token for a peer with policy rules based on JWT claims.
// labels are control-plane-attested key=value claims (canonical, pre-validated); empty means no claims.
func MintBiscuitToken(signingKey ed25519.PrivateKey, claims jwt.MapClaims, token *oidc.IDToken, remotePeer peer.ID, biscuitExpiry time.Time, roles []string, policyRoles []*api.PolicyRole, labels map[string]string) ([]byte, []string, error) {
@@ -225,10 +241,7 @@ func VerifyBiscuitAndGetKey(biscuitData []byte, expectedPeer peer.ID, trustedPub
return nil, nil, fmt.Errorf("malformed biscuit: %w", err)
}
- var authOpts []biscuit.AuthorizerOption
- if timeout > 0 {
- authOpts = append(authOpts, biscuit.WithWorldOptions(datalog.WithMaxDuration(timeout)))
- }
+ authOpts := AuthorizerOptions(timeout)
var lastErr error
var authorized bool
@@ -356,10 +369,7 @@ func VerifyAndExtractPeerID(trustedPublicKeys []ed25519.PublicKey, biscuitData [
return "", fmt.Errorf("malformed biscuit: %w", err)
}
- var authOpts []biscuit.AuthorizerOption
- if timeout > 0 {
- authOpts = append(authOpts, biscuit.WithWorldOptions(datalog.WithMaxDuration(timeout)))
- }
+ authOpts := AuthorizerOptions(timeout)
var authorizer biscuit.Authorizer
var verified bool
@@ -420,13 +430,13 @@ func VerifyAndExtractPeerID(trustedPublicKeys []ed25519.PublicKey, biscuitData [
// VerifyBiscuitRole checks that the biscuit is signed by the control plane's public key
// and contains the specified role fact.
-func VerifyBiscuitRole(biscuitData []byte, controlPlanePubKey ed25519.PublicKey, expectedRole string) error {
+func VerifyBiscuitRole(biscuitData []byte, controlPlanePubKey ed25519.PublicKey, expectedRole string, timeout time.Duration) error {
b, err := biscuit.Unmarshal(biscuitData)
if err != nil {
return fmt.Errorf("malformed biscuit: %w", err)
}
- authorizer, err := b.Authorizer(controlPlanePubKey)
+ authorizer, err := b.Authorizer(controlPlanePubKey, AuthorizerOptions(timeout)...)
if err != nil {
return fmt.Errorf("failed to create authorizer: %w", err)
}
diff --git a/internal/identity/biscuit_test.go b/internal/identity/biscuit_test.go
index caf89bd7..fc98ea0d 100644
--- a/internal/identity/biscuit_test.go
+++ b/internal/identity/biscuit_test.go
@@ -17,6 +17,7 @@ package identity
import (
"crypto/ed25519"
"crypto/rand"
+ "errors"
"sync"
"testing"
"time"
@@ -574,3 +575,36 @@ func TestVerifyAndExtractPeerID_MultipleTrustedKeys(t *testing.T) {
t.Errorf("expected peer ID %s, got %s", dummyPeer, extractedPeer)
}
}
+
+func TestVerifyBiscuitRole_TimeoutIsHonored(t *testing.T) {
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ privNode, _, err := crypto.GenerateKeyPair(crypto.Ed25519, -1)
+ if err != nil {
+ t.Fatal(err)
+ }
+ dummyPeer, err := peer.IDFromPrivateKey(privNode)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ biscuitData, err := MintBootstrapBiscuitToken(priv, dummyPeer, api.RoleRouter, time.Now().Add(1*time.Hour), nil, nil)
+ if err != nil {
+ t.Fatalf("MintBootstrapBiscuitToken failed: %v", err)
+ }
+
+ // A zero timeout must fall back to the generous default, not to the 2ms
+ // biscuit-go default which spuriously fails under load.
+ if err := VerifyBiscuitRole(biscuitData, pub, api.RoleRouter, 0); err != nil {
+ t.Errorf("VerifyBiscuitRole with zero timeout failed: %v", err)
+ }
+
+ // An explicit timeout must reach the Datalog world.
+ err = VerifyBiscuitRole(biscuitData, pub, api.RoleRouter, time.Nanosecond)
+ if !errors.Is(err, datalog.ErrWorldRunLimitTimeout) {
+ t.Errorf("expected datalog timeout error, got %v", err)
+ }
+}
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index e5cddc6b..963c12d9 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -136,7 +136,7 @@ func (n *SamNode) processEnrollResponse(resp *http.Response) (*api.EnrollRespons
}
if n.config.RequiredRole != "" {
- if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, ed25519.PublicKey(enrollResp.ControlPlanePublicKey), n.config.RequiredRole); err != nil {
+ if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, ed25519.PublicKey(enrollResp.ControlPlanePublicKey), n.config.RequiredRole, n.BiscuitTimeout); err != nil {
return nil, fmt.Errorf("enrolled biscuit token lacks required role %q: %w", n.config.RequiredRole, err)
}
}
@@ -317,7 +317,7 @@ func (n *SamNode) EnrollBootstrap(ctx context.Context, controlPlaneURL string, b
return fmt.Errorf("received invalid control plane public key size: %d bytes", len(enrollResp.ControlPlanePublicKey))
}
- if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, ed25519.PublicKey(enrollResp.ControlPlanePublicKey), n.config.RequiredRole); err != nil {
+ if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, ed25519.PublicKey(enrollResp.ControlPlanePublicKey), n.config.RequiredRole, n.BiscuitTimeout); err != nil {
return fmt.Errorf("enrolled biscuit token lacks required role %q: %w", n.config.RequiredRole, err)
}
diff --git a/internal/node/labels_gate.go b/internal/node/labels_gate.go
index 4d02c343..f85e9b0c 100644
--- a/internal/node/labels_gate.go
+++ b/internal/node/labels_gate.go
@@ -21,8 +21,6 @@ import (
"sort"
"time"
- "github.com/biscuit-auth/biscuit-go/v2"
- "github.com/biscuit-auth/biscuit-go/v2/datalog"
"github.com/google/sam/api"
"github.com/google/sam/internal/identity"
"github.com/libp2p/go-libp2p/core/peer"
@@ -106,11 +104,7 @@ func (n *SamNode) checkPeerLabels(providerBiscuit []byte, peerID peer.ID, requir
return fmt.Errorf("provider %s biscuit verification failed: %w", peerID, err)
}
- var authOpts []biscuit.AuthorizerOption
- if n.BiscuitTimeout > 0 {
- authOpts = append(authOpts, biscuit.WithWorldOptions(datalog.WithMaxDuration(n.BiscuitTimeout)))
- }
- authorizer, err := b.Authorizer(key, authOpts...)
+ authorizer, err := b.Authorizer(key, identity.AuthorizerOptions(n.BiscuitTimeout)...)
if err != nil {
return fmt.Errorf("provider %s authorizer instantiation failed: %w", peerID, err)
}
diff --git a/internal/node/middleware.go b/internal/node/middleware.go
index 29a9a3e0..c9c6ac85 100644
--- a/internal/node/middleware.go
+++ b/internal/node/middleware.go
@@ -21,8 +21,8 @@ import (
"sync/atomic"
"github.com/biscuit-auth/biscuit-go/v2"
- "github.com/biscuit-auth/biscuit-go/v2/datalog"
"github.com/google/sam/api"
+ "github.com/google/sam/internal/identity"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/libp2p/go-msgio"
@@ -194,11 +194,7 @@ func (n *SamNode) Authorize(rawToken []byte, req RequestContext, pubKey ed25519.
return fmt.Errorf("invalid biscuit: %w", err)
}
- var authOpts []biscuit.AuthorizerOption
- if n.BiscuitTimeout > 0 {
- authOpts = append(authOpts, biscuit.WithWorldOptions(datalog.WithMaxDuration(n.BiscuitTimeout)))
- }
- authorizer, err := b.Authorizer(pubKey, authOpts...)
+ authorizer, err := b.Authorizer(pubKey, identity.AuthorizerOptions(n.BiscuitTimeout)...)
if err != nil {
return err
}
@@ -340,10 +336,7 @@ func (n *SamNode) injectIdentityFacts(authorizer biscuit.Authorizer, pubKey ed25
copy(keys, n.trustedKeys)
n.keysMu.RUnlock()
- var authOpts []biscuit.AuthorizerOption
- if n.BiscuitTimeout > 0 {
- authOpts = append(authOpts, biscuit.WithWorldOptions(datalog.WithMaxDuration(n.BiscuitTimeout)))
- }
+ authOpts := identity.AuthorizerOptions(n.BiscuitTimeout)
var auth biscuit.Authorizer
var authErr error
diff --git a/internal/node/node.go b/internal/node/node.go
index fe26b24a..cf850958 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -36,7 +36,6 @@ import (
"time"
"github.com/biscuit-auth/biscuit-go/v2"
- "github.com/biscuit-auth/biscuit-go/v2/datalog"
"github.com/google/sam/api"
"github.com/google/sam/internal/identity"
samdiscovery "github.com/google/sam/internal/node/discovery"
@@ -277,7 +276,7 @@ func (n *SamNode) Start(ctx context.Context) error {
if biscuitBytes := n.GetIdentity(); len(biscuitBytes) > 0 {
controlPlanePubKeyBytes, _, err := n.Store.LoadMeshConfig()
if err == nil && len(controlPlanePubKeyBytes) > 0 {
- if err := identity.VerifyBiscuitRole(biscuitBytes, ed25519.PublicKey(controlPlanePubKeyBytes), n.config.RequiredRole); err != nil {
+ if err := identity.VerifyBiscuitRole(biscuitBytes, ed25519.PublicKey(controlPlanePubKeyBytes), n.config.RequiredRole, n.BiscuitTimeout); err != nil {
return fmt.Errorf("loaded identity fails role requirement %q: %w", n.config.RequiredRole, err)
}
}
@@ -829,7 +828,7 @@ func (n *SamNode) performRouterAuthHandshake(s network.Stream, biscuitBytes []by
}
// Enforce role("router") inside the biscuit
- authorizer, err := b.Authorizer(trustedKeys[0])
+ authorizer, err := b.Authorizer(trustedKeys[0], identity.AuthorizerOptions(n.BiscuitTimeout)...)
if err != nil {
return false, fmt.Errorf("authorizer instantiation failed: %w", err)
}
@@ -1435,9 +1434,7 @@ func (n *SamNode) verifyBiscuit(biscuitData []byte, remotePeer peer.ID) (*biscui
if len(tk.Key) != ed25519.PublicKeySize {
continue
}
- authorizer, err := b.Authorizer(tk.Key, biscuit.WithWorldOptions(
- datalog.WithMaxDuration(5*time.Second),
- ))
+ authorizer, err := b.Authorizer(tk.Key, identity.AuthorizerOptions(n.BiscuitTimeout)...)
if err != nil {
lastErr = fmt.Errorf("authorizer error: %w", err)
continue
diff --git a/internal/router/config.go b/internal/router/config.go
index ce18dbc2..fc3de6b4 100644
--- a/internal/router/config.go
+++ b/internal/router/config.go
@@ -19,6 +19,7 @@ import (
"time"
"github.com/google/sam/api"
+ "github.com/google/sam/internal/identity"
)
// Options holds configuration details for the sam-router.
@@ -69,7 +70,7 @@ func (o *Options) Default() {
o.KeysDBPath = "router.key"
}
if o.BiscuitTimeout <= 0 {
- o.BiscuitTimeout = 100 * time.Millisecond
+ o.BiscuitTimeout = identity.DefaultAuthorizerTimeout
}
if o.RequiredRole == "" {
o.RequiredRole = api.RoleRouter
diff --git a/internal/router/router.go b/internal/router/router.go
index a9dfbe75..ded95979 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -326,7 +326,7 @@ func (r *Router) enroll(peerID peer.ID) error {
r.trustedPublicKeys = []ed25519.PublicKey{enrollResp.ControlPlanePublicKey}
r.keysMu.Unlock()
- if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, enrollResp.ControlPlanePublicKey, r.config.RequiredRole); err != nil {
+ if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, enrollResp.ControlPlanePublicKey, r.config.RequiredRole, r.config.BiscuitTimeout); err != nil {
return fmt.Errorf("enrolled biscuit token lacks required role %q: %w", r.config.RequiredRole, err)
}
@@ -448,7 +448,7 @@ func (r *Router) enrollBootstrap(peerID peer.ID) error {
r.trustedPublicKeys = []ed25519.PublicKey{enrollResp.ControlPlanePublicKey}
r.keysMu.Unlock()
- if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, enrollResp.ControlPlanePublicKey, r.config.RequiredRole); err != nil {
+ if err := identity.VerifyBiscuitRole(enrollResp.BiscuitToken, enrollResp.ControlPlanePublicKey, r.config.RequiredRole, r.config.BiscuitTimeout); err != nil {
return fmt.Errorf("enrolled biscuit token lacks required role %q: %w", r.config.RequiredRole, err)
}
@@ -939,7 +939,7 @@ func (r *Router) performMutualAuth(s network.Stream) error {
}
// Enforce required role inside the biscuit
- authorizer, err := b.Authorizer(trustedKeys[0])
+ authorizer, err := b.Authorizer(trustedKeys[0], identity.AuthorizerOptions(r.config.BiscuitTimeout)...)
if err != nil {
return fmt.Errorf("authorizer instantiation failed: %w", err)
}
diff --git a/internal/router/router_test.go b/internal/router/router_test.go
index 6f4a66a7..d80b6d35 100644
--- a/internal/router/router_test.go
+++ b/internal/router/router_test.go
@@ -794,3 +794,11 @@ func TestRouterConnectionManagerWatermarks(t *testing.T) {
t.Fatalf("expected default watermarks 1000/4000, got Low: %d, High: %d", defaultOpts.LowWaterMark, defaultOpts.HighWaterMark)
}
}
+
+func TestRouterDefaultBiscuitTimeout(t *testing.T) {
+ opts := Options{}
+ opts.Default()
+ if opts.BiscuitTimeout != identity.DefaultAuthorizerTimeout {
+ t.Fatalf("expected default biscuit timeout %v, got %v", identity.DefaultAuthorizerTimeout, opts.BiscuitTimeout)
+ }
+}
diff --git a/tests/integration/login_test.go b/tests/integration/login_test.go
index 154a5d05..4b0734a7 100644
--- a/tests/integration/login_test.go
+++ b/tests/integration/login_test.go
@@ -198,7 +198,6 @@ func TestSamNodeJoinOfflineAccessSavesRefreshToken(t *testing.T) {
}
}
-
// TestSamNodeRunJoinNonInteractive covers the safe fallback: since the test
// harness gives the child process no TTY (stdin defaults to /dev/null),
// "run --join" must not attempt (and block on) an interactive OIDC login; it
@@ -405,4 +404,3 @@ func normalizeControlPlaneURLForTest(url string) string {
}
return strings.TrimSuffix(url, "/")
}
-