Skip to content

Agent Mesh

node,router: reconcile peer bans against the control plane's ban set

HosniBelfekia4ce1c313e43
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index 8962adc0..111eae8d 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -350,7 +350,7 @@ func main() {
var controlPlanePubKey ed25519.PublicKey
var routerAddrs []multiaddr.Multiaddr
- storedPubKey, syncedAddrs, err := node.SyncMeshConfig(context.Background(), store)
+ storedPubKey, syncedAddrs, bannedPeerIDs, err := node.SyncMeshConfig(context.Background(), store)
if err != nil {
logger.Warnf("Failed to sync mesh config: %v", err)
}
@@ -475,6 +475,7 @@ func main() {
ControlPlanePubKey: controlPlanePubKey,
RouterAddrs: routerAddrs,
Store: store,
+ BannedPeerIDs: bannedPeerIDs,
MeshID: meshFlag,
DiscoveryInterval: discoveryIntervalFlag,
ListenAddrs: listenAddrs,
@@ -542,6 +543,7 @@ func main() {
PrivKey: priv,
RouterAddrs: initRouterAddrs,
Store: store,
+ BannedPeerIDs: bannedPeerIDs,
MeshID: meshFlag,
DiscoveryInterval: discoveryIntervalFlag,
ListenAddrs: listenAddrs,
@@ -599,11 +601,12 @@ func main() {
}
enrollCancel()
- storedPubKey, newRouterAddrs, err := node.SyncMeshConfig(context.Background(), store)
+ storedPubKey, newRouterAddrs, postEnrollBannedPeerIDs, err := node.SyncMeshConfig(context.Background(), store)
if err != nil {
logger.Warnf("Failed to sync mesh config post-enrollment: %v", err)
}
controlPlanePubKey = storedPubKey
+ bannedPeerIDs = postEnrollBannedPeerIDs
logger.Debugf("listenAddrs: %v, allowLoopback: %v", listenAddrs, allowLoopbackFlag)
meshNode, err = node.NewSamNode(node.Options{
@@ -611,6 +614,7 @@ func main() {
ControlPlanePubKey: controlPlanePubKey,
RouterAddrs: newRouterAddrs,
Store: store,
+ BannedPeerIDs: bannedPeerIDs,
MeshID: meshFlag,
DiscoveryInterval: discoveryIntervalFlag,
ListenAddrs: listenAddrs,
diff --git a/internal/node/controlplane.go b/internal/node/controlplane.go
index e6a3c37d..de40ed4a 100644
--- a/internal/node/controlplane.go
+++ b/internal/node/controlplane.go
@@ -134,17 +134,26 @@ func mergeTrustedKeys(existing []TrustedKey, fetched []ed25519.PublicKey, now ti
// SyncMeshConfig loads the mesh configuration from the store, attempts to refresh it
// via HTTP from the control plane, and updates the store if successful.
-// It returns the control plane public key and the latest multiaddresses.
-func SyncMeshConfig(ctx context.Context, s *Store) ([]byte, []multiaddr.Multiaddr, error) {
+// It returns the control plane public key, the latest multiaddresses, and the
+// control plane's current ban set.
+//
+// The ban set is deliberately not persisted. MeshEvent_BANNED is published once
+// and gossip has no replay, so a node that restarted or was offline has to be
+// told again; /info is that catch-up, and reading it fresh each start is also
+// what makes an unban take effect. Nil means the control plane was not reached,
+// which is not the same as "nothing is banned": callers must not treat it as an
+// instruction to clear anything.
+func SyncMeshConfig(ctx context.Context, s *Store) ([]byte, []multiaddr.Multiaddr, []string, error) {
pubKey, storedAddrsStr, err := s.LoadMeshConfig()
if err != nil {
- return nil, nil, fmt.Errorf("failed to load mesh config from store: %w", err)
+ return nil, nil, nil, fmt.Errorf("failed to load mesh config from store: %w", err)
}
controlPlaneURL, err := s.LoadControlPlaneURL()
if err != nil {
- return nil, nil, fmt.Errorf("failed to load control plane URL from store: %w", err)
+ return nil, nil, nil, fmt.Errorf("failed to load control plane URL from store: %w", err)
}
+ var bannedPeerIDs []string
var routerAddrs []multiaddr.Multiaddr
// Parse stored addresses
@@ -163,7 +172,7 @@ func SyncMeshConfig(ctx context.Context, s *Store) ([]byte, []multiaddr.Multiadd
info, err := FetchControlPlaneInfo(ctx, controlPlaneURL)
if err != nil {
logger.Warnf("Failed to fetch updated addresses via HTTP (using cached): %v", err)
- } else if len(info.RouterAddresses) > 0 {
+ } else if bannedPeerIDs = info.GetBannedPeerIds(); len(info.RouterAddresses) > 0 {
logger.Infof("Discovered latest router addresses: %v", info.RouterAddresses)
var newRouterAddrs []multiaddr.Multiaddr
for _, addrStr := range info.RouterAddresses {
@@ -199,7 +208,7 @@ func SyncMeshConfig(ctx context.Context, s *Store) ([]byte, []multiaddr.Multiadd
}
}
- return pubKey, routerAddrs, nil
+ return pubKey, routerAddrs, bannedPeerIDs, nil
}
// FetchMeshPolicy retrieves the latest mesh policy from the control plane's /policies endpoint using a biscuit token.
diff --git a/internal/node/controlplane_test.go b/internal/node/controlplane_test.go
index 04b6cc08..e47508e3 100644
--- a/internal/node/controlplane_test.go
+++ b/internal/node/controlplane_test.go
@@ -162,10 +162,13 @@ func TestSyncMeshConfig(t *testing.T) {
defer store.Close() //nolint:errcheck
// Initial store is empty, so SyncMeshConfig should just return empty
- pubKey, addrs, err := SyncMeshConfig(context.Background(), store)
+ pubKey, addrs, bannedPeerIDs, err := SyncMeshConfig(context.Background(), store)
if err != nil {
t.Fatalf("SyncMeshConfig failed: %v", err)
}
+ if len(bannedPeerIDs) != 0 {
+ t.Errorf("Expected no banned peers for empty store, got %v", bannedPeerIDs)
+ }
if len(pubKey) != 0 || len(addrs) != 0 {
t.Errorf("Expected empty result for empty store, got pubKey=%v, addrs=%v", pubKey, addrs)
}
@@ -180,7 +183,7 @@ func TestSyncMeshConfig(t *testing.T) {
}
// Call SyncMeshConfig, it should fetch new addrs from server
- pubKey, addrs, err = SyncMeshConfig(context.Background(), store)
+ pubKey, addrs, _, err = SyncMeshConfig(context.Background(), store)
if err != nil {
t.Fatalf("SyncMeshConfig failed: %v", err)
}
diff --git a/internal/node/node.go b/internal/node/node.go
index f6fc054f..dfd99550 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -299,6 +299,16 @@ func NewSamNode(cfg Options) (*SamNode, error) {
if err != nil {
return nil, fmt.Errorf("failed to create revocation cache: %w", err)
}
+ // Seed from the control plane's ban set so the gater enforces existing
+ // bans from the first connection, instead of waiting for an event that
+ // was already published while this node was down.
+ for _, id := range cfg.BannedPeerIDs {
+ if _, err := peer.Decode(id); err != nil {
+ logger.Warnf("Ignoring undecodable banned peer ID %q from the control plane: %v", id, err)
+ continue
+ }
+ node.revokedPeers.Add(id, time.Now().Unix())
+ }
node.peerLabelGate, err = lru.New[string, time.Time](labelGateCacheSize)
if err != nil {
return nil, fmt.Errorf("failed to create label gate cache: %w", err)
@@ -1276,6 +1286,9 @@ func (n *SamNode) handleBannedEvent(event *api.MeshEvent) {
n.revokedPeers.Add(event.PeerId, event.Timestamp)
}
// Drop any prior admission, otherwise the relay ACL keeps honouring it.
+ // The cache entry above is not written to disk: a restarted node picks the
+ // ban back up from the control plane's ban set in /info (see
+ // SyncMeshConfig), which is also how an unban reaches it.
if p, err := peer.Decode(event.PeerId); err == nil {
n.authPeers.Delete(p)
if n.Host != nil {
diff --git a/internal/node/options.go b/internal/node/options.go
index f70529a6..98f3b146 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -40,13 +40,19 @@ type Options struct {
ControlPlanePubKey ed25519.PublicKey
RouterAddrs []multiaddr.Multiaddr
Store *Store
- MeshID string
- DiscoveryInterval string
- ListenAddrs []string
- EnableRelay bool
- NodeConfig *NodeConfigComplete
- KeyGracePeriod time.Duration
- AllowLoopback bool
+
+ // BannedPeerIDs seeds the revocation cache from the control plane's ban
+ // set (see SyncMeshConfig). Without it a restarted node would enforce no
+ // ban until the next MeshEvent_BANNED, which for an existing ban never
+ // comes.
+ BannedPeerIDs []string
+ MeshID string
+ DiscoveryInterval string
+ ListenAddrs []string
+ EnableRelay bool
+ NodeConfig *NodeConfigComplete
+ KeyGracePeriod time.Duration
+ AllowLoopback bool
// AnnouncePrivateAddrs controls whether RFC1918/ULA addresses are published
// to the mesh. Nil means true: private meshes reach each other over exactly
// those addresses. Set false on nodes that are only reachable via routers or
diff --git a/internal/router/router.go b/internal/router/router.go
index 3f733d5b..bce3d493 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -721,7 +721,10 @@ func (r *Router) listenForControlPlaneEvents(ctx context.Context) {
if bannedPeer, err := peer.Decode(event.PeerId); err == nil {
logger.Infof("[Router Event] Received BANNED event for peer %s, evicting from authenticated peers and adding to blocklist", bannedPeer)
r.authenticatedPeers.Delete(bannedPeer)
- r.bannedPeers.Store(bannedPeer, true)
+ // The timestamp is what stops an /info answer that was
+ // already in flight from undoing this (see
+ // reconcileBannedPeers).
+ r.bannedPeers.Store(bannedPeer, time.Now())
if r.Host != nil {
_ = r.Host.Network().ClosePeer(bannedPeer)
}
@@ -853,6 +856,67 @@ func (r *Router) renewLease() {
}
}
+// reconcileBannedPeers replaces the local blocklist with the control plane's
+// ban set. It is a replacement rather than a merge: the control plane can
+// unban a peer, and there is no event for that, so a peer that has dropped off
+// the list has to drop out of the blocklist too.
+//
+// fetchedAt is when the request that produced peerIDs was issued, and it is
+// what makes the replacement safe. A ban is written to the control plane's
+// database before MeshEvent_BANNED is published, so a response *issued after*
+// the ban is guaranteed to carry it, and its absence really does mean
+// unbanned. A response issued before the ban cannot say anything about it, so
+// bans recorded at or after fetchedAt are left alone -- otherwise an /info
+// still in flight when an event arrived would immediately undo it, which is a
+// race the router hits on startup, where the first poll overlaps live gossip.
+//
+// The boundary is inclusive because the answer has to be strictly newer than
+// the ban to speak to it. A clock coarse enough to time the fetch and the ban
+// in the same tick -- Windows readily does -- otherwise reads a simultaneous
+// ban as the older of the two and drops it.
+func (r *Router) reconcileBannedPeers(peerIDs []string, fetchedAt time.Time) {
+ banned := make(map[peer.ID]struct{}, len(peerIDs))
+ for _, id := range peerIDs {
+ p, err := peer.Decode(id)
+ if err != nil {
+ logger.Warnf("[Federation] Ignoring undecodable banned peer ID %q: %v", id, err)
+ continue
+ }
+ banned[p] = struct{}{}
+ }
+
+ // Drop bans the control plane no longer holds, except any that landed at
+ // or after the moment this answer was asked for.
+ r.bannedPeers.Range(func(key, value any) bool {
+ p, ok := key.(peer.ID)
+ if !ok {
+ return true
+ }
+ if _, still := banned[p]; still {
+ return true
+ }
+ if bannedAt, ok := value.(time.Time); ok && !bannedAt.Before(fetchedAt) {
+ return true
+ }
+ logger.Infof("[Federation] Peer %s is no longer banned, removing from blocklist", p)
+ r.bannedPeers.Delete(p)
+ return true
+ })
+
+ // Apply the ones it does, evicting anything already admitted.
+ for p := range banned {
+ if _, already := r.bannedPeers.Load(p); already {
+ continue
+ }
+ logger.Infof("[Federation] Peer %s is banned, adding to blocklist", p)
+ r.bannedPeers.Store(p, fetchedAt)
+ r.authenticatedPeers.Delete(p)
+ if r.Host != nil {
+ _ = r.Host.Network().ClosePeer(p)
+ }
+ }
+}
+
func (r *Router) runFederationLoop() {
defer r.wg.Done()
ticker := time.NewTicker(30 * time.Second)
@@ -872,6 +936,10 @@ func (r *Router) runFederationLoop() {
func (r *Router) connectBootstrapRouters() {
client := &http.Client{Timeout: 10 * time.Second}
+ // Taken before the request: anything banned after this point cannot be
+ // reflected in the answer, so reconciliation must not read its absence as
+ // an unban (see reconcileBannedPeers).
+ fetchedAt := time.Now()
resp, err := client.Get(r.config.ControlPlaneURL + "/info")
if err != nil {
logger.Errorf("[Federation] Failed to fetch router info: %v", err)
@@ -894,6 +962,10 @@ func (r *Router) connectBootstrapRouters() {
return
}
+ // /info carries the whole ban set, so this is also where a router that
+ // restarted or missed a MeshEvent_BANNED catches up.
+ r.reconcileBannedPeers(info.GetBannedPeerIds(), fetchedAt)
+
for _, addrStr := range info.RouterAddresses {
ma, err := multiaddr.NewMultiaddr(addrStr)
if err != nil {
diff --git a/internal/router/router_test.go b/internal/router/router_test.go
index 5798219d..71fe0628 100644
--- a/internal/router/router_test.go
+++ b/internal/router/router_test.go
@@ -873,3 +873,69 @@ func TestPerformMutualAuthAcceptsRotatedKey(t *testing.T) {
t.Fatal("peer not recorded as authenticated")
}
}
+
+// reconcileBannedPeers replaces the blocklist with the control plane's ban
+// set rather than merging into it. The removal half is the point: the control
+// plane can unban a peer and there is no event for that, so a peer that drops
+// off /info has to drop out of the blocklist too.
+func TestReconcileBannedPeers(t *testing.T) {
+ newPeer := func(t *testing.T) peer.ID {
+ t.Helper()
+ priv, _, err := crypto.GenerateEd25519Key(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ id, err := peer.IDFromPrivateKey(priv)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return id
+ }
+
+ stillBanned := newPeer(t)
+ unbanned := newPeer(t)
+ newlyBanned := newPeer(t)
+
+ r := &Router{}
+ // Prior state, as a running router would hold it.
+ past := time.Now().Add(-time.Minute)
+ r.bannedPeers.Store(stillBanned, past)
+ r.bannedPeers.Store(unbanned, past)
+ r.authenticatedPeers.Store(newlyBanned, true)
+
+ fetchedAt := time.Now()
+ r.reconcileBannedPeers([]string{stillBanned.String(), newlyBanned.String()}, fetchedAt)
+
+ if _, banned := r.bannedPeers.Load(stillBanned); !banned {
+ t.Error("a peer still in the ban set must stay banned")
+ }
+ if _, banned := r.bannedPeers.Load(newlyBanned); !banned {
+ t.Error("a peer newly in the ban set must become banned")
+ }
+ if _, banned := r.bannedPeers.Load(unbanned); banned {
+ t.Error("a peer no longer in the ban set must be unbanned: /info is the only signal an unban has")
+ }
+ if _, admitted := r.authenticatedPeers.Load(newlyBanned); admitted {
+ t.Error("banning a peer must drop any prior admission")
+ }
+
+ // An undecodable entry must be skipped, not abort the reconciliation.
+ r.reconcileBannedPeers([]string{"not-a-peer-id", stillBanned.String()}, time.Now())
+ if _, banned := r.bannedPeers.Load(stillBanned); !banned {
+ t.Error("a malformed entry must not discard the valid ones")
+ }
+
+ // A ban recorded in the same tick as the fetch must survive. The answer
+ // was issued no later than the ban, so its silence cannot mean unbanned,
+ // and a clock too coarse to separate the two -- Windows readily is -- must
+ // not be what decides it. This is the startup race, and a strict
+ // comparison here let it through: TestRouterGossipSubBannedEvent failed
+ // roughly one run in ten with the ban deleted at the instant it arrived.
+ simultaneous := newPeer(t)
+ tick := time.Now()
+ r.bannedPeers.Store(simultaneous, tick)
+ r.reconcileBannedPeers(nil, tick)
+ if _, banned := r.bannedPeers.Load(simultaneous); !banned {
+ t.Error("a ban recorded at the fetch instant must survive: the answer is not newer than the ban, so it cannot report it unbanned")
+ }
+}
diff --git a/mobile/sam-node-ffi/ffi/ffi.go b/mobile/sam-node-ffi/ffi/ffi.go
index 478d1e6c..ea26a847 100644
--- a/mobile/sam-node-ffi/ffi/ffi.go
+++ b/mobile/sam-node-ffi/ffi/ffi.go
@@ -144,7 +144,7 @@ func StartNode(configJSON string) error {
var routerAddrs []multiaddr.Multiaddr
// Sync config from stored/synced configuration
- storedPubKey, syncedAddrs, err := node.SyncMeshConfig(context.Background(), store)
+ storedPubKey, syncedAddrs, bannedPeerIDs, err := node.SyncMeshConfig(context.Background(), store)
if err == nil && len(storedPubKey) > 0 {
controlPlanePubKey = storedPubKey
routerAddrs = syncedAddrs
@@ -191,6 +191,7 @@ func StartNode(configJSON string) error {
ControlPlanePubKey: controlPlanePubKey,
RouterAddrs: routerAddrs,
Store: store,
+ BannedPeerIDs: bannedPeerIDs,
MeshID: meshID,
DiscoveryInterval: discoveryInterval,
ListenAddrs: listenAddrs,
@@ -374,7 +375,7 @@ func EnrollNode(dataDir string, controlPlaneURL string, jwt string, allowLoopbac
return fmt.Errorf("failed to save control plane URL: %w", err)
}
- _, _, err = node.SyncMeshConfig(enrollCtx, store)
+ _, _, _, err = node.SyncMeshConfig(enrollCtx, store)
if err != nil {
return fmt.Errorf("failed to sync mesh config post-enrollment: %w", err)
}