Skip to content

Agent Mesh

node: dial the routers at once, be ready on the first, and redial one that drops

Antonio Ojeafce09966e39f
diff --git a/internal/node/node.go b/internal/node/node.go
index d924de78..41698101 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -29,6 +29,7 @@ import (
"io"
"net"
"net/http"
+ "slices"
"strconv"
"strings"
"sync"
@@ -161,14 +162,16 @@ func (n *SamNode) isAdmitted(p peer.ID) bool {
}
type SamNode struct {
- config Options
- Host host.Host
- DHT *dht.IpfsDHT
- PubSub *pubsub.PubSub
- Discovery *samdiscovery.Discovery
- Store *Store
- RouterPeerID peer.ID
- authenticatedRouters map[peer.ID]bool
+ config Options
+ Host host.Host
+ DHT *dht.IpfsDHT
+ PubSub *pubsub.PubSub
+ Discovery *samdiscovery.Discovery
+ Store *Store
+ RouterPeerID peer.ID
+ authenticatedRouters map[peer.ID]bool
+ // redialing holds the routers a redialRouter goroutine is working on.
+ redialing map[peer.ID]bool
peerLastEventTime map[string]int64
mu sync.Mutex
nodeConfig *NodeConfigComplete
@@ -231,20 +234,11 @@ func (n *SamNode) getTokenSource() TokenSource {
// UpdateRelays updates the current relays used by AutoRelay.
func (n *SamNode) UpdateRelays(addrs []multiaddr.Multiaddr) {
+ newRelays := routerPeers(context.Background(), addrs)
n.mu.Lock()
defer n.mu.Unlock()
- var newRelays []peer.AddrInfo
- for _, addr := range addrs {
- resolvedAddrs, err := resolveAddrIfNeeded(context.Background(), addr)
- if err != nil {
- resolvedAddrs = []multiaddr.Multiaddr{addr}
- }
- for _, resolved := range resolvedAddrs {
- if addrInfo, err := peer.AddrInfoFromP2pAddr(resolved); err == nil && addrInfo.ID != "" {
- newRelays = append(newRelays, *addrInfo)
- n.Host.Peerstore().AddAddrs(addrInfo.ID, addrInfo.Addrs, peerstore.PermanentAddrTTL)
- }
- }
+ for _, relay := range newRelays {
+ n.Host.Peerstore().AddAddrs(relay.ID, relay.Addrs, peerstore.PermanentAddrTTL)
}
n.currentRelays = newRelays
logger.Infof("[Relay] Updated current relays for AutoRelay: %v", newRelays)
@@ -358,6 +352,46 @@ func resolveAddrIfNeeded(ctx context.Context, addr multiaddr.Multiaddr) ([]multi
return madns.DefaultResolver.Resolve(ctx, stripP2pFromDnsaddr(addr))
}
+// routerPeers resolves router addresses and groups them by peer, in first-seen
+// order. A router is advertised under one multiaddr per interface, and a
+// dnsaddr entry resolves to every router behind the name, so the same peer
+// arrives several times; it is one peer with all of its addresses, dialed
+// once and handshaken once. An address that cannot be resolved is kept as
+// written for libp2p to resolve at dial time.
+func routerPeers(ctx context.Context, addrs []multiaddr.Multiaddr) []peer.AddrInfo {
+ var order []peer.ID
+ byPeer := map[peer.ID]*peer.AddrInfo{}
+ for _, addr := range addrs {
+ resolved, err := resolveAddrIfNeeded(ctx, addr)
+ if err != nil {
+ resolved = []multiaddr.Multiaddr{addr}
+ }
+ for _, r := range resolved {
+ info, err := peer.AddrInfoFromP2pAddr(r)
+ if err != nil || info.ID == "" {
+ logger.Warnf("Failed to parse router addr %s: %v", r, err)
+ continue
+ }
+ known, seen := byPeer[info.ID]
+ if !seen {
+ byPeer[info.ID] = info
+ order = append(order, info.ID)
+ continue
+ }
+ for _, a := range info.Addrs {
+ if !slices.ContainsFunc(known.Addrs, a.Equal) {
+ known.Addrs = append(known.Addrs, a)
+ }
+ }
+ }
+ }
+ out := make([]peer.AddrInfo, 0, len(order))
+ for _, id := range order {
+ out = append(out, *byPeer[id])
+ }
+ return out
+}
+
// NewSamNode creates a new Agent instance secured with the 4-layer pipeline.
// NewSamNode initializes options and structures without starting background tasks or network interfaces.
func NewSamNode(cfg Options) (*SamNode, error) {
@@ -498,23 +532,11 @@ func (n *SamNode) Start(ctx context.Context) error {
// Layer 2: Attach the Bouncer (Gater)
gater := &nodeConnGate{node: n}
- // Convert router multiaddrs to peer.AddrInfo to use as static relays
- var staticRelays []peer.AddrInfo
- for _, addr := range n.config.RouterAddrs {
- resolvedAddrs, err := resolveAddrIfNeeded(ctx, addr)
- if err != nil {
- resolvedAddrs = []multiaddr.Multiaddr{addr}
- }
- for _, resolved := range resolvedAddrs {
- if addrInfo, err := peer.AddrInfoFromP2pAddr(resolved); err == nil && addrInfo.ID != "" {
- staticRelays = append(staticRelays, *addrInfo)
- if n.RouterPeerID == "" {
- n.RouterPeerID = addrInfo.ID
- }
- } else {
- logger.Warnf("Failed to parse static relay addr %s: %v", resolved, err)
- }
- }
+ // The routers, one entry per peer with every address it is known by;
+ // they double as the static relays.
+ staticRelays := routerPeers(ctx, n.config.RouterAddrs)
+ if n.RouterPeerID == "" && len(staticRelays) > 0 {
+ n.RouterPeerID = staticRelays[0].ID
}
logger.Infof("Configured %d static relays: %v", len(staticRelays), staticRelays)
@@ -618,6 +640,7 @@ func (n *SamNode) Start(ctx context.Context) error {
n.mu.Unlock()
if wasAuthenticated {
logger.Warnf("[AuthN] Router %s disconnected; clearing authenticated session so the next check re-handshakes", remotePeer)
+ n.redialRouter(ctx, remotePeer)
}
},
})
@@ -658,21 +681,8 @@ func (n *SamNode) Start(ctx context.Context) error {
n.services.reprovideNow = n.triggerReprovide
n.applyPendingEgress(ctx)
- var authenticated bool
- var fatalAuthErr error
-
- for _, addr := range n.config.RouterAddrs {
- if err := n.ConnectAndAuthWithRouter(ctx, addr); err != nil {
- logger.Warnf("[AuthN] Failed to bootstrap and auth with router %s: %v", addr, err)
- if errors.Is(err, ErrFatalAuth) {
- fatalAuthErr = err
- }
- } else {
- authenticated = true
- }
- }
-
- if len(n.config.RouterAddrs) > 0 && !authenticated {
+ authenticated, fatalAuthErr := n.authWithRouters(ctx, staticRelays)
+ if len(staticRelays) > 0 && !authenticated {
if fatalAuthErr != nil {
return fmt.Errorf("fatal auth failure: %w", fatalAuthErr)
}
@@ -980,73 +990,184 @@ dhtLoop:
return nil
}
-func (n *SamNode) ConnectAndAuthWithRouter(ctx context.Context, addr multiaddr.Multiaddr) error {
- resolvedAddrs, err := resolveAddrIfNeeded(ctx, addr)
+// authWithRouters dials every router at once and returns as soon as one has
+// admitted this node, or once all have refused. A router that is down costs
+// its dial timeout; paid up front and one router after another, that was
+// every member's cold-join cost for as long as the control plane still
+// listed a router that had gone. The dials still in flight finish in the
+// background and record their sessions as they land; the connection monitor
+// re-handshakes whatever drops later.
+func (n *SamNode) authWithRouters(ctx context.Context, routers []peer.AddrInfo) (authenticated bool, fatal error) {
+ if len(routers) == 0 {
+ return false, nil
+ }
+ biscuitBytes, err := n.loadIdentityForAuth()
if err != nil {
- resolvedAddrs = []multiaddr.Multiaddr{addr}
+ return false, err
+ }
+ results := n.dialRouters(ctx, routers, biscuitBytes)
+ for range routers {
+ err := <-results
+ if err == nil {
+ return true, nil
+ }
+ if errors.Is(err, ErrFatalAuth) {
+ fatal = err
+ }
+ }
+ return false, fatal
+}
+
+// dialRouters runs authRouter against every router at once and delivers one
+// outcome per router on the returned channel, in completion order. The
+// channel is buffered for all of them, so a caller may stop reading early.
+func (n *SamNode) dialRouters(ctx context.Context, routers []peer.AddrInfo, biscuitBytes []byte) <-chan error {
+ results := make(chan error, len(routers))
+ for _, router := range routers {
+ go func() {
+ err := n.authRouter(ctx, router, biscuitBytes)
+ if err != nil {
+ logger.Warnf("[AuthN] Failed to bootstrap and auth with router %s: %v", router.ID, err)
+ }
+ results <- err
+ }()
+ }
+ return results
+}
+
+// redialRouter re-handshakes a router whose session just dropped, in the
+// background, with a doubling delay between attempts. The connection monitor
+// only acts once no router is left and only on its own interval, so without
+// this a rollout that restarts the routers one after another, each faster
+// than that interval, takes every session a member has before any is back;
+// and a router that restarts alone is never re-handshaken at all, the
+// member keeping one router fewer after each rollout. One redial per router
+// runs at a time, and a router that stays away is left to the monitor.
+func (n *SamNode) redialRouter(ctx context.Context, router peer.ID) {
+ if n.config.RouterRedialDelay < 0 {
+ return
+ }
+ n.mu.Lock()
+ if n.redialing == nil {
+ n.redialing = map[peer.ID]bool{}
+ }
+ if n.redialing[router] {
+ n.mu.Unlock()
+ return
}
+ n.redialing[router] = true
+ n.mu.Unlock()
+
+ go func() {
+ defer func() {
+ n.mu.Lock()
+ delete(n.redialing, router)
+ n.mu.Unlock()
+ }()
+ biscuitBytes, err := n.loadIdentityForAuth()
+ if err != nil {
+ return
+ }
+ delay := n.config.RouterRedialDelay
+ for attempt := 1; attempt <= n.config.RouterRedialAttempts; attempt++ {
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(delay):
+ }
+ if n.isAuthenticatedAndConnected(router) {
+ return
+ }
+ addrs := n.Host.Peerstore().Addrs(router)
+ if len(addrs) == 0 {
+ logger.Warnf("[AuthN] Router %s dropped and no address for it is known; leaving it to the connection monitor", router)
+ return
+ }
+ err := n.authRouter(ctx, peer.AddrInfo{ID: router, Addrs: addrs}, biscuitBytes)
+ if err == nil {
+ logger.Infof("[AuthN] Router %s re-admitted this node on redial %d", router, attempt)
+ n.triggerReprovide()
+ return
+ }
+ if errors.Is(err, ErrFatalAuth) {
+ logger.Warnf("[AuthN] Router %s refused this node's credential on redial: %v", router, err)
+ return
+ }
+ logger.Debugf("[AuthN] Redial %d of router %s failed: %v", attempt, router, err)
+ delay *= 2
+ }
+ logger.Warnf("[AuthN] Router %s did not come back within %d redials; leaving it to the connection monitor", router, n.config.RouterRedialAttempts)
+ }()
+}
- // Load biscuit from store once before the loop
+func (n *SamNode) loadIdentityForAuth() ([]byte, error) {
biscuitBytes, err := n.Store.LoadIdentity()
if err != nil {
- return fmt.Errorf("%w: failed to load identity from store: %w", ErrFatalAuth, err)
+ return nil, fmt.Errorf("%w: failed to load identity from store: %w", ErrFatalAuth, err)
}
if len(biscuitBytes) == 0 {
- return fmt.Errorf("%w: no identity biscuit found in store", ErrFatalAuth)
+ return nil, fmt.Errorf("%w: no identity biscuit found in store", ErrFatalAuth)
+ }
+ return biscuitBytes, nil
+}
+
+// authRouter connects to one router over whichever of its addresses answers
+// and runs the handshake, unless a live authenticated session to it already
+// exists: Host.Connect is a no-op on an existing connection, but the
+// handshake is not, and repeating it trips the router's per-peer handshake
+// limiter. The dial is bounded by RouterConnectTimeout so a router that is
+// down does not hold its caller for long.
+func (n *SamNode) authRouter(ctx context.Context, router peer.AddrInfo, biscuitBytes []byte) error {
+ if n.isAuthenticatedAndConnected(router.ID) {
+ return nil
+ }
+ routerCtx, cancel := context.WithTimeout(ctx, n.config.RouterConnectTimeout)
+ defer cancel()
+ if err := n.Host.Connect(routerCtx, router); err != nil {
+ if strings.Contains(err.Error(), "peer id mismatch") {
+ return fmt.Errorf("%w: %w", ErrFatalAuth, err)
+ }
+ return fmt.Errorf("failed to connect to router %s at %v: %w", router.ID, router.Addrs, err)
+ }
+ if err := n.authenticateRouterSession(routerCtx, router.ID, biscuitBytes); err != nil {
+ return fmt.Errorf("router %s: %w", router.ID, err)
+ }
+ n.authOnce.Do(func() {
+ close(n.authSuccess)
+ })
+ return nil
+}
+
+// ConnectAndAuthWithRouter connects to and authenticates with every router
+// behind addr: one peer for a direct address, every router behind the name
+// for a dnsaddr. They are dialed at once and all are waited for; it succeeds
+// when at least one admitted this node.
+func (n *SamNode) ConnectAndAuthWithRouter(ctx context.Context, addr multiaddr.Multiaddr) error {
+ biscuitBytes, err := n.loadIdentityForAuth()
+ if err != nil {
+ return err
}
+ routers := routerPeers(ctx, []multiaddr.Multiaddr{addr})
var connected bool
var lastFatalErr error
var errs []error
-
- for _, resolved := range resolvedAddrs {
- addrInfo, err := peer.AddrInfoFromP2pAddr(resolved)
- if err != nil {
- errs = append(errs, fmt.Errorf("failed to get AddrInfo from multiaddr %s: %w", resolved, err))
- continue
- }
-
- // A router advertises one multiaddr per interface, all the same peer.
- // Host.Connect is a no-op on an existing connection, but the
- // handshake is not: repeating it per address trips the router's
- // per-peer handshake limiter and logs a failure for every extra
- // address. One live authenticated session per router is enough.
- if n.isAuthenticatedAndConnected(addrInfo.ID) {
+ results := n.dialRouters(ctx, routers, biscuitBytes)
+ for range routers {
+ err := <-results
+ if err == nil {
connected = true
continue
}
-
- // Create a per-replica timeout context to prevent blocking on offline replicas
- replicaCtx, cancel := context.WithTimeout(ctx, n.config.RouterConnectTimeout)
-
- if err := n.Host.Connect(replicaCtx, *addrInfo); err != nil {
- cancel()
- if strings.Contains(err.Error(), "peer id mismatch") {
- lastFatalErr = fmt.Errorf("%w: %w", ErrFatalAuth, err)
- }
- errs = append(errs, fmt.Errorf("failed to connect to router %s: %w", resolved, err))
- continue
- }
-
- err = n.authenticateRouterSession(replicaCtx, addrInfo.ID, biscuitBytes)
- cancel()
- if err != nil {
- errs = append(errs, fmt.Errorf("router %s: %w", resolved, err))
- if errors.Is(err, ErrFatalAuth) {
- lastFatalErr = err
- }
- continue
+ errs = append(errs, err)
+ if errors.Is(err, ErrFatalAuth) {
+ lastFatalErr = err
}
- connected = true
}
if connected {
- n.authOnce.Do(func() {
- close(n.authSuccess)
- })
return nil
}
-
if lastFatalErr != nil {
return lastFatalErr
}
diff --git a/internal/node/options.go b/internal/node/options.go
index 21ed650e..3148fd84 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -30,6 +30,11 @@ const (
DefaultDiscoveryInterval = "30s"
DefaultConfigFile = "sam-node.yaml"
DefaultRouterConnectTimeout = 5 * time.Second
+ // A router pod that restarts is back in well under a minute; the first
+ // redial waits a moment for it, and six doublings reach past where the
+ // connection monitor's own interval takes over.
+ DefaultRouterRedialDelay = 2 * time.Second
+ DefaultRouterRedialAttempts = 6
// DefaultSocketName is the local API socket the node creates in its data directory.
DefaultSocketName = "sam.sock"
)
@@ -60,6 +65,13 @@ type Options struct {
AutoRelayBackoff time.Duration
// RouterConnectTimeout bounds each router address's dial (connect + stream open).
RouterConnectTimeout time.Duration
+ // RouterRedialDelay is how long after a router session drops the node
+ // waits before dialing that router again; each further attempt waits
+ // twice as long, up to RouterRedialAttempts of them. Zero uses the
+ // default; negative disables the redial, leaving a dropped router to the
+ // connection monitor.
+ RouterRedialDelay time.Duration
+ RouterRedialAttempts int
// BiscuitTimeout bounds Datalog evaluation when verifying biscuit tokens.
BiscuitTimeout time.Duration
// DHT Options
@@ -134,6 +146,12 @@ func (o *Options) Default() {
if o.RouterConnectTimeout == 0 {
o.RouterConnectTimeout = DefaultRouterConnectTimeout
}
+ if o.RouterRedialDelay == 0 {
+ o.RouterRedialDelay = DefaultRouterRedialDelay
+ }
+ if o.RouterRedialAttempts <= 0 {
+ o.RouterRedialAttempts = DefaultRouterRedialAttempts
+ }
if o.BiscuitTimeout <= 0 {
o.BiscuitTimeout = identity.DefaultAuthorizerTimeout
}
diff --git a/internal/node/router_auth_dedupe_test.go b/internal/node/router_auth_dedupe_test.go
index 434d38aa..7b65ba5c 100644
--- a/internal/node/router_auth_dedupe_test.go
+++ b/internal/node/router_auth_dedupe_test.go
@@ -18,7 +18,9 @@ import (
"bytes"
"context"
"crypto/ed25519"
+ "fmt"
"io"
+ "net"
"net/http"
"net/http/httptest"
"sync/atomic"
@@ -28,6 +30,7 @@ import (
"github.com/biscuit-auth/biscuit-go/v2"
"github.com/google/sam/api"
"github.com/libp2p/go-libp2p"
+ "github.com/libp2p/go-libp2p/core/crypto"
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
@@ -117,7 +120,8 @@ func newFakeRouter(t *testing.T, listenAddrs ...string) *fakeRouter {
}
// startNode brings up a node enrolled against this router and authenticated.
-func (r *fakeRouter) startNode(t *testing.T, ctx context.Context, routerAddrs []multiaddr.Multiaddr) *SamNode {
+// Each tweak may adjust the options before the node is built.
+func (r *fakeRouter) startNode(t *testing.T, ctx context.Context, routerAddrs []multiaddr.Multiaddr, tweaks ...func(*Options)) *SamNode {
t.Helper()
store, err := NewStore(t.TempDir())
if err != nil {
@@ -133,14 +137,18 @@ func (r *fakeRouter) startNode(t *testing.T, ctx context.Context, routerAddrs []
t.Fatal(err)
}
- node, err := NewSamNode(Options{
+ opts := Options{
PrivKey: privKey,
Store: store,
ControlPlanePubKey: r.cpPub,
RouterAddrs: routerAddrs,
ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
AllowLoopback: true,
- })
+ }
+ for _, tweak := range tweaks {
+ tweak(&opts)
+ }
+ node, err := NewSamNode(opts)
if err != nil {
t.Fatal(err)
}
@@ -235,7 +243,10 @@ func TestRouterDisconnectClearsAuthenticatedSession(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
- node := r.startNode(t, ctx, r.routerAddrs)
+ // The redial would restore the session on its own; this is about the
+ // state in between, so it is off here and TestRouterDropIsRedialed
+ // covers the redial.
+ node := r.startNode(t, ctx, r.routerAddrs, func(o *Options) { o.RouterRedialDelay = -1 })
if !node.IsConnected() {
t.Fatal("node does not report an authenticated router connection after Start")
@@ -323,3 +334,154 @@ func TestRefreshEnrollmentReadmitsConnectedRouters(t *testing.T) {
}
waitForConnectedness(t, node, r.h.ID(), network.Connected)
}
+
+// The control plane keeps advertising a router for as long as its lease
+// lasts, so after a rollout every joiner is handed a router that is gone.
+// Start must not pay that router's dial timeout before the node is ready:
+// the live router admits the node and the dead one is still being dialed
+// when Start returns.
+func TestStartIsReadyOnceOneRouterAdmitsTheNode(t *testing.T) {
+ live := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+
+ // A listener that accepts and never speaks: the dial completes at the
+ // TCP level, the security handshake then waits out the timeout.
+ deadListener, err := net.Listen("tcp", "127.0.0.1:0")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = deadListener.Close() }()
+ go func() {
+ for {
+ c, err := deadListener.Accept()
+ if err != nil {
+ return
+ }
+ defer func() { _ = c.Close() }()
+ }
+ }()
+ _, deadPub, err := crypto.GenerateEd25519Key(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ deadID, err := peer.IDFromPublicKey(deadPub)
+ if err != nil {
+ t.Fatal(err)
+ }
+ deadAddr := multiaddr.StringCast(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", deadListener.Addr().(*net.TCPAddr).Port, deadID))
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ // The dead router first, where a sequential dial would have waited on it.
+ routerAddrs := append([]multiaddr.Multiaddr{deadAddr}, live.routerAddrs...)
+
+ const connectTimeout = 3 * time.Second
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = store.Close() })
+ privKey := GetOrGenerateKey(store)
+ pid, _ := peer.IDFromPrivateKey(privKey)
+ if err := store.SaveIdentity(live.mint(pid.String(), api.RoleNode)); err != nil {
+ t.Fatal(err)
+ }
+ node, err := NewSamNode(Options{
+ PrivKey: privKey,
+ Store: store,
+ ControlPlanePubKey: live.cpPub,
+ RouterAddrs: routerAddrs,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ AllowLoopback: true,
+ RouterConnectTimeout: connectTimeout,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = node.Teardown() })
+
+ started := time.Now()
+ if err := node.Start(ctx); err != nil {
+ t.Fatalf("Start: %v", err)
+ }
+ took := time.Since(started)
+ if !node.IsConnected() {
+ t.Fatal("Start returned without an authenticated router")
+ }
+ if took >= connectTimeout {
+ t.Fatalf("Start took %v, at least the dead router's %v dial timeout; the live router should have made the node ready", took, connectTimeout)
+ }
+ if got := live.handshakes.Load(); got != 1 {
+ t.Fatalf("live router saw %d handshakes, want 1", got)
+ }
+}
+
+// Three entries for one router, as a control plane hands out when a dnsaddr
+// resolves to every address behind the name: one peer, one dial, one
+// handshake, and one static relay.
+func TestRouterPeersGroupsAddressesByPeer(t *testing.T) {
+ r := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0", "/ip4/127.0.0.1/tcp/0")
+ addrs := append(append([]multiaddr.Multiaddr{}, r.routerAddrs...), r.routerAddrs[0])
+
+ peers := routerPeers(context.Background(), addrs)
+ if len(peers) != 1 {
+ t.Fatalf("routerPeers = %d peers for %d addresses of one router, want 1", len(peers), len(addrs))
+ }
+ if peers[0].ID != r.h.ID() {
+ t.Fatalf("peer = %s, want the router %s", peers[0].ID, r.h.ID())
+ }
+ if len(peers[0].Addrs) != 2 {
+ t.Fatalf("peer carries %d addresses, want its 2 distinct ones: %v", len(peers[0].Addrs), peers[0].Addrs)
+ }
+}
+
+// A router that restarts drops every member's session. The connection
+// monitor only acts once a member has no router left, and only on its own
+// interval, so a rollout that restarts the routers one after another took
+// every session before any came back, and a router restarting alone was
+// never re-handshaken. The member must redial a dropped router by itself.
+func TestRouterDropIsRedialed(t *testing.T) {
+ r := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ h, handshakes := r.h, r.handshakes
+
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ // The monitor is kept out of the way: the redial must do this alone.
+ node := r.startNode(t, ctx, r.routerAddrs, func(o *Options) {
+ o.RouterRedialDelay = 100 * time.Millisecond
+ o.MonitorBootstrap = time.Hour
+ })
+ if got := handshakes.Load(); got != 1 {
+ t.Fatalf("handshakes after Start = %d, want 1", got)
+ }
+
+ // The router side closes, as a restarting pod does.
+ if err := h.Network().ClosePeer(node.Host.ID()); err != nil {
+ t.Fatal(err)
+ }
+ waitForConnectedness(t, node, h.ID(), network.NotConnected)
+
+ deadline := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) && (!node.IsConnected() || handshakes.Load() != 2) {
+ time.Sleep(20 * time.Millisecond)
+ }
+ if got := handshakes.Load(); got != 2 {
+ t.Fatalf("handshakes after the drop = %d, want 2: the node must handshake the router again", got)
+ }
+ if !node.IsConnected() {
+ t.Fatal("node does not report an authenticated session after the redial")
+ }
+
+ // Dropping it again while the first redial has finished starts another;
+ // the session comes back a second time.
+ if err := h.Network().ClosePeer(node.Host.ID()); err != nil {
+ t.Fatal(err)
+ }
+ waitForConnectedness(t, node, h.ID(), network.NotConnected)
+ deadline = time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) && handshakes.Load() < 3 {
+ time.Sleep(20 * time.Millisecond)
+ }
+ if got := handshakes.Load(); got != 3 {
+ t.Fatalf("handshakes after the second drop = %d, want 3", got)
+ }
+}