Skip to content

Agent Mesh

identity, node, router: bound pre-authentication work (audit M23, M6)

Antonio Ojea0f2ceba31f6c
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index 8c63e131..13848a5c 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -622,8 +622,6 @@ func main() {
// Start renewal loop
meshNode.StartRenewalLoop(ctx, oidcIssuerFlag, clientIDFlag, clientSecretFlag, jwtPathFlag)
- meshNode.Host.SetStreamHandler(api.AuthProtocolID, meshNode.HandleAuthHandshake)
-
// Start Sidecar API Server (multiplexed with MCP)
sidecarSrv, err := node.StartSidecarServer(meshNode, bindAddrFlag, resolveSocketPath(cmd), apiTokenFlag, tlsCertFlag, tlsKeyFlag, tlsCAFlag)
if err != nil {
diff --git a/internal/identity/appended_blocks_test.go b/internal/identity/appended_blocks_test.go
new file mode 100644
index 00000000..bc3f653e
--- /dev/null
+++ b/internal/identity/appended_blocks_test.go
@@ -0,0 +1,146 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package identity
+
+import (
+ "crypto/ed25519"
+ "crypto/rand"
+ "errors"
+ "fmt"
+ "runtime"
+ "testing"
+ "time"
+
+ "github.com/biscuit-auth/biscuit-go/v2"
+ "github.com/biscuit-auth/biscuit-go/v2/parser"
+ "github.com/google/sam/api"
+)
+
+// appendJoinBomb attenuates token with a block any holder can add offline: a
+// few hundred facts and a 3-way self-join over them. biscuit-go runs a rule
+// to completion before it checks the deadline, so evaluating this block
+// pins a core for the whole budget and leaks the worker goroutine.
+func appendJoinBomb(t *testing.T, token *biscuit.Biscuit, facts int) []byte {
+ t.Helper()
+ block := token.CreateBlock()
+ for i := 0; i < facts; i++ {
+ f, err := parser.FromStringFact(fmt.Sprintf(`f(%d)`, i))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := block.AddFact(f); err != nil {
+ t.Fatal(err)
+ }
+ }
+ rule, err := parser.FromStringRule(`g($a, $b, $c) <- f($a), f($b), f($c)`)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := block.AddRule(rule); err != nil {
+ t.Fatal(err)
+ }
+ bombed, err := token.Append(rand.Reader, block.Build())
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, err := bombed.Serialize()
+ if err != nil {
+ t.Fatal(err)
+ }
+ return data
+}
+
+// Every inbound verify path must refuse a token with appended blocks before
+// building an authorizer: rejection has to cost microseconds regardless of
+// the block's contents, and must not leave a Datalog worker running.
+func TestInboundVerifyRejectsAppendedBlocksWithoutEvaluatingThem(t *testing.T) {
+ pub, priv, err := ed25519.GenerateKey(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ peerID := newTestPeer(t)
+ keys := []ed25519.PublicKey{pub}
+
+ clean, err := MintBootstrapBiscuitToken(priv, peerID, api.RoleNode, time.Now().Add(time.Hour), nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ token, err := biscuit.Unmarshal(clean)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if token.BlockCount() != 0 {
+ t.Fatalf("freshly minted token reports %d appended blocks, want 0", token.BlockCount())
+ }
+ bombed := appendJoinBomb(t, token, 200)
+
+ // The budget is what an attacker would pin per call; the assertion is that
+ // rejection does not come anywhere near it.
+ const budget = 2 * time.Second
+ const fastEnough = budget / 4
+
+ paths := map[string]func([]byte) error{
+ "VerifyBiscuit": func(d []byte) error {
+ _, err := VerifyBiscuit(d, peerID, keys, budget)
+ return err
+ },
+ "VerifyBiscuitAndGetKey": func(d []byte) error {
+ _, _, err := VerifyBiscuitAndGetKey(d, peerID, keys, budget)
+ return err
+ },
+ "VerifyBiscuitAndGetExpiry": func(d []byte) error {
+ _, err := VerifyBiscuitAndGetExpiry(d, peerID, keys, budget)
+ return err
+ },
+ "VerifyAndExtractPeerID": func(d []byte) error {
+ _, err := VerifyAndExtractPeerID(keys, d, budget)
+ return err
+ },
+ "VerifyExpiredAndExtractPeerID": func(d []byte) error {
+ _, err := VerifyExpiredAndExtractPeerID(keys, d, budget)
+ return err
+ },
+ "VerifyBiscuitRole": func(d []byte) error {
+ return VerifyBiscuitRole(d, pub, api.RoleNode, budget)
+ },
+ }
+
+ for name, verify := range paths {
+ t.Run(name, func(t *testing.T) {
+ if err := verify(clean); err != nil {
+ t.Fatalf("clean token rejected: %v", err)
+ }
+
+ before := runtime.NumGoroutine()
+ start := time.Now()
+ err := verify(bombed)
+ took := time.Since(start)
+ if err == nil {
+ t.Fatal("token with an appended block accepted")
+ }
+ if !errors.Is(err, ErrAppendedBlocks) {
+ t.Errorf("err = %v, want ErrAppendedBlocks", err)
+ }
+ if took > fastEnough {
+ t.Errorf("rejection took %v; the block was evaluated", took)
+ }
+ // Give a leaked worker a moment to show up before counting.
+ time.Sleep(20 * time.Millisecond)
+ if after := runtime.NumGoroutine(); after > before {
+ t.Errorf("goroutines grew %d -> %d; a Datalog worker was left running", before, after)
+ }
+ })
+ }
+}
diff --git a/internal/identity/biscuit.go b/internal/identity/biscuit.go
index 5ea57d39..a7acf6bc 100644
--- a/internal/identity/biscuit.go
+++ b/internal/identity/biscuit.go
@@ -34,17 +34,56 @@ import (
// DefaultAuthorizerTimeout bounds Datalog evaluation when no timeout is configured.
// biscuit-go defaults to 2ms of wall-clock time, of which a single authorization of
// a realistic token already spends ~0.14ms (~1.1ms under -race), so ordinary
-// scheduling noise turns into a spurious denial. The amount of work is bounded by
-// the fact and iteration limits; this deadline only caps how long the caller waits.
+// scheduling noise turns into a spurious denial.
const DefaultAuthorizerTimeout = 1 * time.Second
+// Datalog world limits, biscuit-go's defaults made explicit. They bound the
+// number of derived facts and fixpoint iterations, but neither preempts a
+// single rule: one self-join over a few hundred facts runs to completion
+// whatever these say, and on timeout biscuit-go's worker goroutine keeps
+// computing. The only bound on attacker-authored rules is therefore that SAM
+// never evaluates any: see UnmarshalInbound.
+const (
+ maxDatalogFacts = 1000
+ maxDatalogIterations = 100
+)
+
// AuthorizerOptions returns the authorizer options enforcing a Datalog evaluation
// budget. A non-positive timeout falls back to DefaultAuthorizerTimeout.
func AuthorizerOptions(timeout time.Duration) []biscuit.AuthorizerOption {
if timeout <= 0 {
timeout = DefaultAuthorizerTimeout
}
- return []biscuit.AuthorizerOption{biscuit.WithWorldOptions(datalog.WithMaxDuration(timeout))}
+ return []biscuit.AuthorizerOption{biscuit.WithWorldOptions(
+ datalog.WithMaxDuration(timeout),
+ datalog.WithMaxFacts(maxDatalogFacts),
+ datalog.WithMaxIterations(maxDatalogIterations),
+ )}
+}
+
+// ErrAppendedBlocks is returned for a token that carries attenuation blocks.
+var ErrAppendedBlocks = errors.New("biscuit carries appended blocks; SAM tokens are authority-block only")
+
+// UnmarshalInbound parses a token received from a peer or a client and
+// refuses one with appended blocks.
+//
+// Appending needs no root key, so appended blocks are the one place a token
+// holder can put Datalog of their own. SAM reads nothing from them: facts
+// there are invisible to the authorizer and RequireAuthorityBinding ignores
+// them. What they can still do is cost CPU: a block with a self-join rule
+// over a few hundred facts pins a core for the whole evaluation budget on
+// every verifier that evaluates it, and leaks the worker goroutine (see the
+// limits above). The control plane never mints such blocks, so a token that
+// has any is not one SAM issued in its current form.
+func UnmarshalInbound(biscuitData []byte) (*biscuit.Biscuit, error) {
+ b, err := biscuit.Unmarshal(biscuitData)
+ if err != nil {
+ return nil, fmt.Errorf("malformed biscuit: %w", err)
+ }
+ if n := b.BlockCount(); n > 0 {
+ return nil, fmt.Errorf("%w (%d)", ErrAppendedBlocks, n)
+ }
+ return b, nil
}
// EnforceExpiration injects the current time and the expiration check into an
@@ -343,9 +382,9 @@ func VerifyBiscuitAndGetExpiry(biscuitData []byte, expectedPeer peer.ID, trusted
}
func verifyBiscuit(biscuitData []byte, expectedPeer peer.ID, trustedPublicKeys []ed25519.PublicKey, timeout time.Duration) (*biscuit.Biscuit, ed25519.PublicKey, time.Time, error) {
- b, err := biscuit.Unmarshal(biscuitData)
+ b, err := UnmarshalInbound(biscuitData)
if err != nil {
- return nil, nil, time.Time{}, fmt.Errorf("malformed biscuit: %w", err)
+ return nil, nil, time.Time{}, err
}
authOpts := AuthorizerOptions(timeout)
@@ -470,9 +509,9 @@ func VerifyAndExtractPeerID(trustedPublicKeys []ed25519.PublicKey, biscuitData [
}
func extractPeerID(trustedPublicKeys []ed25519.PublicKey, biscuitData []byte, timeout time.Duration, enforceExpiry bool) (peer.ID, error) {
- b, err := biscuit.Unmarshal(biscuitData)
+ b, err := UnmarshalInbound(biscuitData)
if err != nil {
- return "", fmt.Errorf("malformed biscuit: %w", err)
+ return "", err
}
authOpts := AuthorizerOptions(timeout)
@@ -545,9 +584,9 @@ func extractPeerID(trustedPublicKeys []ed25519.PublicKey, biscuitData []byte, ti
// should trigger a refresh rather than refuse to boot. Do not use it to admit a token
// received from a peer.
func VerifyBiscuitRole(biscuitData []byte, controlPlanePubKey ed25519.PublicKey, expectedRole string, timeout time.Duration) error {
- b, err := biscuit.Unmarshal(biscuitData)
+ b, err := UnmarshalInbound(biscuitData)
if err != nil {
- return fmt.Errorf("malformed biscuit: %w", err)
+ return err
}
return RequireRole(b, controlPlanePubKey, expectedRole, timeout)
}
diff --git a/internal/identity/peer_binding_test.go b/internal/identity/peer_binding_test.go
index f4afc7e8..ca879dbb 100644
--- a/internal/identity/peer_binding_test.go
+++ b/internal/identity/peer_binding_test.go
@@ -17,6 +17,7 @@ package identity
import (
"crypto/ed25519"
"crypto/rand"
+ "errors"
"testing"
"time"
@@ -127,10 +128,13 @@ func TestPeerBindingRejectsAppendedBlock(t *testing.T) {
if _, err := VerifyBiscuit(forged, attacker, keys, time.Second); err == nil {
t.Error("VerifyBiscuit admitted a token re-bound by an appended block")
}
- // Attenuation must stay usable for the peer the authority block names,
- // otherwise this fix would break delegation instead of the bypass.
- if _, err := VerifyBiscuit(forged, victim, keys, time.Second); err != nil {
- t.Errorf("VerifyBiscuit rejected an attenuated token held by its real owner: %v", err)
+ // Nor is the token accepted from its real owner: appended blocks are the
+ // one place a holder can put Datalog of their own, and even a check there
+ // is a rule biscuit-go runs to completion with no deadline, so SAM does
+ // not evaluate any (ErrAppendedBlocks). RequireAuthorityBinding above is
+ // the defence in depth behind that gate.
+ if _, err := VerifyBiscuit(forged, victim, keys, time.Second); !errors.Is(err, ErrAppendedBlocks) {
+ t.Errorf("VerifyBiscuit on an attenuated token: err = %v, want ErrAppendedBlocks", err)
}
}
diff --git a/internal/node/middleware.go b/internal/node/middleware.go
index c29daff3..435953c1 100644
--- a/internal/node/middleware.go
+++ b/internal/node/middleware.go
@@ -19,6 +19,7 @@ import (
"fmt"
"runtime/debug"
"sync/atomic"
+ "time"
"github.com/biscuit-auth/biscuit-go/v2"
"github.com/google/sam/api"
@@ -113,6 +114,12 @@ func (n *SamNode) WithBiscuitAuth(next func(network.Stream, RequestContext)) net
return
}
+ // The peer is not authorized yet: it gets the handshake budget to
+ // produce its frame, not an open-ended hold on this goroutine.
+ if err := ts.SetReadDeadline(time.Now().Add(authHandshakeTimeout)); err != nil {
+ logger.Debugf("[Auth] Failed to set auth frame deadline for %s: %v", remotePeer, err)
+ }
+
// Read AuthFrame
reader := msgio.NewVarintReaderSize(ts, 1024*64)
msg, err := reader.ReadMsg()
@@ -157,6 +164,12 @@ func (n *SamNode) WithBiscuitAuth(next func(network.Stream, RequestContext)) net
return
}
+ // Authorized: the session itself is long-lived, so the pre-auth
+ // deadline comes off.
+ if err := ts.SetReadDeadline(time.Time{}); err != nil {
+ logger.Debugf("[Auth] Failed to clear auth frame deadline for %s: %v", remotePeer, err)
+ }
+
next(ts, reqCtx)
}
}
@@ -203,7 +216,7 @@ func (n *SamNode) Authorize(rawToken []byte, req RequestContext, pubKey ed25519.
if len(pubKey) != ed25519.PublicKeySize {
return fmt.Errorf("invalid public key size: %d", len(pubKey))
}
- b, err := biscuit.Unmarshal(rawToken)
+ b, err := identity.UnmarshalInbound(rawToken)
if err != nil {
return fmt.Errorf("invalid biscuit: %w", err)
}
diff --git a/internal/node/middleware_test.go b/internal/node/middleware_test.go
index 7d9d231d..374cdd40 100644
--- a/internal/node/middleware_test.go
+++ b/internal/node/middleware_test.go
@@ -31,6 +31,7 @@ import (
"github.com/biscuit-auth/biscuit-go/v2/parser"
"github.com/google/sam/api"
"github.com/google/sam/internal/identity"
+ "github.com/google/sam/internal/ratelimit"
lru "github.com/hashicorp/golang-lru/v2"
golog "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p/core/crypto"
@@ -657,7 +658,7 @@ func TestRevocation(t *testing.T) {
}
cache, _ := lru.New[string, int64](10000)
- rl, _ := NewPeerRateLimiter(100)
+ rl, _ := ratelimit.NewPeerRateLimiter(100)
node := &SamNode{
trustedKeys: []TrustedKey{{Key: pub, ReceivedAt: time.Now()}},
revokedPeers: cache,
@@ -746,7 +747,7 @@ func TestWithBiscuitAuth_MutualBiscuit(t *testing.T) {
t.Fatal(err)
}
- rl, _ := NewPeerRateLimiter(100)
+ rl, _ := ratelimit.NewPeerRateLimiter(100)
node := &SamNode{
trustedKeys: []TrustedKey{{Key: pub, ReceivedAt: time.Now()}},
rateLimiter: rl,
diff --git a/internal/node/node.go b/internal/node/node.go
index 7e671d04..7a336e60 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -39,6 +39,7 @@ import (
"github.com/google/sam/api"
"github.com/google/sam/internal/identity"
samdiscovery "github.com/google/sam/internal/node/discovery"
+ "github.com/google/sam/internal/ratelimit"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/ipfs/go-cid"
golog "github.com/ipfs/go-log/v2"
@@ -160,11 +161,15 @@ type SamNode struct {
keysMu sync.RWMutex
MeshPolicyRules []biscuit.Rule
MeshPolicyMu sync.RWMutex
- rateLimiter *PeerRateLimiter
- services *ServiceRegistry
- BoundHTTPAddr string
- BoundSocketPath string
- AllowLoopback bool
+ rateLimiter *ratelimit.PeerRateLimiter
+ // handshakeLimiter bounds /sam/auth attempts per peer separately from
+ // rateLimiter, so a peer's authenticated traffic cannot starve its own
+ // re-authentication and vice versa.
+ handshakeLimiter *ratelimit.PeerRateLimiter
+ services *ServiceRegistry
+ BoundHTTPAddr string
+ BoundSocketPath string
+ AllowLoopback bool
authSuccess chan struct{}
authOnce sync.Once
@@ -291,10 +296,14 @@ func NewSamNode(cfg Options) (*SamNode, error) {
}
var err error
- node.rateLimiter, err = NewPeerRateLimiter(RateLimiterSize)
+ node.rateLimiter, err = ratelimit.NewPeerRateLimiter(RateLimiterSize)
if err != nil {
return nil, fmt.Errorf("failed to create rate limiter: %w", err)
}
+ node.handshakeLimiter, err = ratelimit.NewPeerRateLimiter(RateLimiterSize)
+ if err != nil {
+ return nil, fmt.Errorf("failed to create handshake rate limiter: %w", err)
+ }
node.revokedPeers, err = lru.New[string, int64](RevocationCacheSize)
if err != nil {
return nil, fmt.Errorf("failed to create revocation cache: %w", err)
@@ -1547,6 +1556,13 @@ func (n *SamNode) getTrustedPublicKeys() []ed25519.PublicKey {
return keys
}
+// authHandshakeTimeout bounds how long an unauthenticated peer may hold a
+// /sam/auth stream open: it has to send its frame and read the reply within
+// it. Any internet peer can open these streams, so without a deadline each
+// one is a goroutine held for as long as the peer likes. A var so tests can
+// shorten it.
+var authHandshakeTimeout = 10 * time.Second
+
// HandleAuthHandshake is the core libp2p stream handler for /sam/auth/1.0.0.
// This is the "Admission Office" of the mesh node.
func (n *SamNode) HandleAuthHandshake(s network.Stream) {
@@ -1564,6 +1580,15 @@ func (n *SamNode) HandleAuthHandshake(s network.Stream) {
}
}
+ if n.handshakeLimiter != nil && !n.handshakeLimiter.Allow(remotePeer.String()) {
+ logger.Warnf("[AuthN] Handshake rate limit exceeded for %s", remotePeer)
+ _ = s.Reset()
+ return
+ }
+ if err := s.SetDeadline(time.Now().Add(authHandshakeTimeout)); err != nil {
+ logger.Debugf("[AuthN] Failed to set handshake deadline for %s: %v", remotePeer, err)
+ }
+
reader := msgio.NewVarintReaderSize(s, 1024*64)
msg, err := reader.ReadMsg()
if err != nil {
diff --git a/internal/node/node_test.go b/internal/node/node_test.go
index aa6c0430..5459f536 100644
--- a/internal/node/node_test.go
+++ b/internal/node/node_test.go
@@ -26,6 +26,7 @@ import (
"github.com/biscuit-auth/biscuit-go/v2"
"github.com/google/sam/api"
"github.com/google/sam/internal/identity"
+ "github.com/google/sam/internal/ratelimit"
lru "github.com/hashicorp/golang-lru/v2"
"github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p/core/crypto"
@@ -428,6 +429,115 @@ func TestHandleAuthHandshake(t *testing.T) {
}
}
+// Any internet peer can open /sam/auth streams. An idle one must be closed on
+// the node's schedule, not the peer's, and a peer opening them in a tight loop
+// must be cut off before the node verifies every frame it sends.
+func TestHandleAuthHandshakeBoundsUnauthenticatedPeers(t *testing.T) {
+ pub, priv, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+
+ serverHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = serverHost.Close() }()
+ clientHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = clientHost.Close() }()
+
+ limiter, err := ratelimit.NewPeerRateLimiter(16)
+ if err != nil {
+ t.Fatal(err)
+ }
+ node := &SamNode{
+ trustedKeys: []TrustedKey{{Key: pub, ReceivedAt: time.Now()}},
+ BiscuitTimeout: time.Second,
+ handshakeLimiter: limiter,
+ }
+ serverHost.SetStreamHandler(api.AuthProtocolID, node.HandleAuthHandshake)
+ if err := clientHost.Connect(ctx, peer.AddrInfo{ID: serverHost.ID(), Addrs: serverHost.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+
+ t.Run("idle stream is closed by the deadline", func(t *testing.T) {
+ old := authHandshakeTimeout
+ authHandshakeTimeout = 200 * time.Millisecond
+ t.Cleanup(func() { authHandshakeTimeout = old })
+
+ s, err := clientHost.NewStream(ctx, serverHost.ID(), api.AuthProtocolID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = s.Close() }()
+ // Send nothing. The node must give up on us, which the client sees as
+ // its read failing; without the deadline this read blocks forever.
+ if err := s.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
+ t.Fatal(err)
+ }
+ start := time.Now()
+ _, err = msgio.NewVarintReaderSize(s, 1024*64).ReadMsg()
+ if err == nil {
+ t.Fatal("idle handshake stream answered")
+ }
+ if took := time.Since(start); took > 4*time.Second {
+ t.Fatalf("idle stream was held for %v; the node did not apply its own deadline", took)
+ }
+ })
+
+ t.Run("tight loop is rate limited", func(t *testing.T) {
+ builder := biscuit.NewBuilder(priv)
+ for _, f := range []biscuit.Fact{
+ {Predicate: biscuit.Predicate{Name: api.FactNode, IDs: []biscuit.Term{biscuit.String(clientHost.ID().String())}}},
+ {Predicate: biscuit.Predicate{Name: api.FactExpiration, IDs: []biscuit.Term{biscuit.Date(time.Now().Add(time.Hour))}}},
+ } {
+ if err := builder.AddAuthorityFact(f); err != nil {
+ t.Fatal(err)
+ }
+ }
+ b, err := builder.Build()
+ if err != nil {
+ t.Fatal(err)
+ }
+ token, err := b.Serialize()
+ if err != nil {
+ t.Fatal(err)
+ }
+ frame, err := proto.Marshal(&api.AuthFrame{Biscuit: token})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ attempts := 3 * ratelimit.PeerBurst
+ answered := 0
+ for i := 0; i < attempts; i++ {
+ s, err := clientHost.NewStream(ctx, serverHost.ID(), api.AuthProtocolID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = s.SetDeadline(time.Now().Add(5 * time.Second))
+ if err := msgio.NewVarintWriter(s).WriteMsg(frame); err == nil {
+ if _, err := msgio.NewVarintReaderSize(s, 1024*64).ReadMsg(); err == nil {
+ answered++
+ }
+ }
+ _ = s.Close()
+ }
+ // The token is valid, so every answer is a success: the only reason
+ // some go unanswered is the limiter.
+ if answered < ratelimit.PeerBurst {
+ t.Errorf("only %d of the first %d handshakes answered; limiter is too strict", answered, ratelimit.PeerBurst)
+ }
+ if answered == attempts {
+ t.Errorf("all %d back-to-back handshakes were verified; no per-peer limit applied", attempts)
+ }
+ })
+}
+
// TestPerformRouterAuthHandshakeRequiresRouterRole: a router whose biscuit
// verifies but carries no role("router") is a fatal auth failure, so the node
// gives up on it instead of retrying.
diff --git a/internal/node/rate_limiter.go b/internal/ratelimit/peer.go
similarity index 92%
rename from internal/node/rate_limiter.go
rename to internal/ratelimit/peer.go
index 8689fe9f..d438797b 100644
--- a/internal/node/rate_limiter.go
+++ b/internal/ratelimit/peer.go
@@ -12,7 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package node
+// Package ratelimit bounds how often an individual peer may make a node or
+// router do pre-authentication work.
+package ratelimit
import (
"sync"
diff --git a/internal/node/rate_limiter_test.go b/internal/ratelimit/peer_test.go
similarity index 99%
rename from internal/node/rate_limiter_test.go
rename to internal/ratelimit/peer_test.go
index a28fe91e..d8a2971d 100644
--- a/internal/node/rate_limiter_test.go
+++ b/internal/ratelimit/peer_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package node
+package ratelimit
import (
"testing"
diff --git a/internal/router/router.go b/internal/router/router.go
index 00eeb140..74a4d34a 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -37,6 +37,7 @@ import (
"github.com/google/sam/api"
"github.com/google/sam/internal/identity"
+ "github.com/google/sam/internal/ratelimit"
golog "github.com/ipfs/go-log/v2"
"github.com/libp2p/go-libp2p"
dht "github.com/libp2p/go-libp2p-kad-dht"
@@ -111,6 +112,9 @@ type Router struct {
EventTopic *pubsub.Topic
authenticatedPeers sync.Map
bannedPeers sync.Map
+ // handshakeLimiter bounds /sam/auth attempts per peer; any internet peer
+ // can open those streams.
+ handshakeLimiter *ratelimit.PeerRateLimiter
// Keys & Identity
biscuitToken []byte
@@ -137,10 +141,17 @@ func NewRouter(ctx context.Context, config Options) (*Router, error) {
ctx, cancel := context.WithCancel(ctx)
+ handshakeLimiter, err := ratelimit.NewPeerRateLimiter(handshakeLimiterSize)
+ if err != nil {
+ cancel()
+ return nil, fmt.Errorf("failed to create handshake rate limiter: %w", err)
+ }
+
return &Router{
- config: config,
- ctx: ctx,
- cancel: cancel,
+ config: config,
+ ctx: ctx,
+ cancel: cancel,
+ handshakeLimiter: handshakeLimiter,
}, nil
}
@@ -1058,6 +1069,15 @@ func recoverStreamHandler(name string, next network.StreamHandler) network.Strea
}
}
+// authHandshakeTimeout bounds how long an unauthenticated peer may hold a
+// /sam/auth stream: it has to send its frame and read the reply within it.
+// A var so tests can shorten it.
+var authHandshakeTimeout = 10 * time.Second
+
+// handshakeLimiterSize is how many distinct peers' handshake budgets are
+// tracked at once (LRU beyond that).
+const handshakeLimiterSize = 4096
+
// HandleAuthHandshake processes incoming auth connections.
// It is part of mutual auth:
// 1. Receives client's Biscuit.
@@ -1073,6 +1093,15 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
return
}
+ if r.handshakeLimiter != nil && !r.handshakeLimiter.Allow(remotePeer.String()) {
+ logger.Warnf("[AuthN] Handshake rate limit exceeded for %s", remotePeer)
+ _ = s.Reset()
+ return
+ }
+ if err := s.SetDeadline(time.Now().Add(authHandshakeTimeout)); err != nil {
+ logger.Debugf("[AuthN] Failed to set handshake deadline for %s: %v", remotePeer, err)
+ }
+
reader := msgio.NewVarintReaderSize(s, 1024*64)
msg, err := reader.ReadMsg()
if err != nil {
diff --git a/internal/router/router_test.go b/internal/router/router_test.go
index e3a134f7..eec83e02 100644
--- a/internal/router/router_test.go
+++ b/internal/router/router_test.go
@@ -36,6 +36,7 @@ import (
"github.com/google/sam/api"
"github.com/google/sam/internal/controlplane"
"github.com/google/sam/internal/identity"
+ "github.com/google/sam/internal/ratelimit"
"github.com/google/sam/internal/storage"
"github.com/libp2p/go-libp2p"
"github.com/libp2p/go-libp2p/core/crypto"
@@ -909,6 +910,95 @@ func TestPerformMutualAuth(t *testing.T) {
}
}
+// The router's /sam/auth handler is reachable by any internet peer. An idle
+// stream must be closed on the router's schedule, and a peer handshaking in a
+// tight loop must be cut off before every frame is verified.
+func TestHandleAuthHandshakeBoundsUnauthenticatedPeers(t *testing.T) {
+ cpPub, cpPriv, err := ed25519.GenerateKey(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+
+ serverHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = serverHost.Close() }()
+ clientHost, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = clientHost.Close() }()
+
+ r, err := NewRouter(ctx, Options{BiscuitTimeout: time.Second, RequiredRole: api.RoleRouter})
+ if err != nil {
+ t.Fatal(err)
+ }
+ r.Host = serverHost
+ r.biscuitToken = []byte("router-biscuit")
+ r.trustedPublicKeys = []ed25519.PublicKey{cpPub}
+ serverHost.SetStreamHandler(api.AuthProtocolID, r.HandleAuthHandshake)
+ if err := clientHost.Connect(ctx, peer.AddrInfo{ID: serverHost.ID(), Addrs: serverHost.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+
+ t.Run("idle stream is closed by the deadline", func(t *testing.T) {
+ old := authHandshakeTimeout
+ authHandshakeTimeout = 200 * time.Millisecond
+ t.Cleanup(func() { authHandshakeTimeout = old })
+
+ s, err := clientHost.NewStream(ctx, serverHost.ID(), api.AuthProtocolID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = s.Close() }()
+ if err := s.SetReadDeadline(time.Now().Add(5 * time.Second)); err != nil {
+ t.Fatal(err)
+ }
+ start := time.Now()
+ if _, err := msgio.NewVarintReaderSize(s, 1024*64).ReadMsg(); err == nil {
+ t.Fatal("idle handshake stream answered")
+ }
+ if took := time.Since(start); took > 4*time.Second {
+ t.Fatalf("idle stream was held for %v; the router did not apply its own deadline", took)
+ }
+ })
+
+ t.Run("tight loop is rate limited", func(t *testing.T) {
+ token, err := identity.MintBootstrapBiscuitToken(cpPriv, clientHost.ID(), api.RoleNode, time.Now().Add(time.Hour), nil, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ frame, err := proto.Marshal(&api.AuthFrame{Biscuit: token})
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ attempts := 3 * ratelimit.PeerBurst
+ answered := 0
+ for i := 0; i < attempts; i++ {
+ s, err := clientHost.NewStream(ctx, serverHost.ID(), api.AuthProtocolID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _ = s.SetDeadline(time.Now().Add(5 * time.Second))
+ if err := msgio.NewVarintWriter(s).WriteMsg(frame); err == nil {
+ if _, err := msgio.NewVarintReaderSize(s, 1024*64).ReadMsg(); err == nil {
+ answered++
+ }
+ }
+ _ = s.Close()
+ }
+ if answered < ratelimit.PeerBurst {
+ t.Errorf("only %d of the first %d handshakes answered; limiter is too strict", answered, ratelimit.PeerBurst)
+ }
+ if answered == attempts {
+ t.Errorf("all %d back-to-back handshakes were verified; no per-peer limit applied", attempts)
+ }
+ })
+}
+
// reconcileBannedPeers replaces the blocklist with the control plane's ban
// set rather than merging into it. The removal half is the point: the control
// plane can unban a peer and there is no event for that, so a peer that drops