Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: dial the routers at once, be ready on the first, and redial one that drops
Antonio Ojeafce09966e39f
diff --git a/internal/node/node.go b/internal/node/node.goindex d924de78..41698101 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -29,6 +29,7 @@ import ("io""net""net/http"+ "slices""strconv""strings""sync"@@ -161,14 +162,16 @@ func (n *SamNode) isAdmitted(p peer.ID) bool {}type SamNode struct {- config Options- Host host.Host- DHT *dht.IpfsDHT- PubSub *pubsub.PubSub- Discovery *samdiscovery.Discovery- Store *Store- RouterPeerID peer.ID- authenticatedRouters map[peer.ID]bool+ config Options+ Host host.Host+ DHT *dht.IpfsDHT+ PubSub *pubsub.PubSub+ Discovery *samdiscovery.Discovery+ Store *Store+ RouterPeerID peer.ID+ authenticatedRouters map[peer.ID]bool+ // redialing holds the routers a redialRouter goroutine is working on.+ redialing map[peer.ID]boolpeerLastEventTime map[string]int64mu sync.MutexnodeConfig *NodeConfigComplete@@ -231,20 +234,11 @@ func (n *SamNode) getTokenSource() TokenSource {// UpdateRelays updates the current relays used by AutoRelay.func (n *SamNode) UpdateRelays(addrs []multiaddr.Multiaddr) {+ newRelays := routerPeers(context.Background(), addrs)n.mu.Lock()defer n.mu.Unlock()- var newRelays []peer.AddrInfo- for _, addr := range addrs {- resolvedAddrs, err := resolveAddrIfNeeded(context.Background(), addr)- if err != nil {- resolvedAddrs = []multiaddr.Multiaddr{addr}- }- for _, resolved := range resolvedAddrs {- if addrInfo, err := peer.AddrInfoFromP2pAddr(resolved); err == nil && addrInfo.ID != "" {- newRelays = append(newRelays, *addrInfo)- n.Host.Peerstore().AddAddrs(addrInfo.ID, addrInfo.Addrs, peerstore.PermanentAddrTTL)- }- }+ for _, relay := range newRelays {+ n.Host.Peerstore().AddAddrs(relay.ID, relay.Addrs, peerstore.PermanentAddrTTL)}n.currentRelays = newRelayslogger.Infof("[Relay] Updated current relays for AutoRelay: %v", newRelays)@@ -358,6 +352,46 @@ func resolveAddrIfNeeded(ctx context.Context, addr multiaddr.Multiaddr) ([]multireturn madns.DefaultResolver.Resolve(ctx, stripP2pFromDnsaddr(addr))}+// routerPeers resolves router addresses and groups them by peer, in first-seen+// order. A router is advertised under one multiaddr per interface, and a+// dnsaddr entry resolves to every router behind the name, so the same peer+// arrives several times; it is one peer with all of its addresses, dialed+// once and handshaken once. An address that cannot be resolved is kept as+// written for libp2p to resolve at dial time.+func routerPeers(ctx context.Context, addrs []multiaddr.Multiaddr) []peer.AddrInfo {+ var order []peer.ID+ byPeer := map[peer.ID]*peer.AddrInfo{}+ for _, addr := range addrs {+ resolved, err := resolveAddrIfNeeded(ctx, addr)+ if err != nil {+ resolved = []multiaddr.Multiaddr{addr}+ }+ for _, r := range resolved {+ info, err := peer.AddrInfoFromP2pAddr(r)+ if err != nil || info.ID == "" {+ logger.Warnf("Failed to parse router addr %s: %v", r, err)+ continue+ }+ known, seen := byPeer[info.ID]+ if !seen {+ byPeer[info.ID] = info+ order = append(order, info.ID)+ continue+ }+ for _, a := range info.Addrs {+ if !slices.ContainsFunc(known.Addrs, a.Equal) {+ known.Addrs = append(known.Addrs, a)+ }+ }+ }+ }+ out := make([]peer.AddrInfo, 0, len(order))+ for _, id := range order {+ out = append(out, *byPeer[id])+ }+ return out+}+// NewSamNode creates a new Agent instance secured with the 4-layer pipeline.// NewSamNode initializes options and structures without starting background tasks or network interfaces.func NewSamNode(cfg Options) (*SamNode, error) {@@ -498,23 +532,11 @@ func (n *SamNode) Start(ctx context.Context) error {// Layer 2: Attach the Bouncer (Gater)gater := &nodeConnGate{node: n}- // Convert router multiaddrs to peer.AddrInfo to use as static relays- var staticRelays []peer.AddrInfo- for _, addr := range n.config.RouterAddrs {- resolvedAddrs, err := resolveAddrIfNeeded(ctx, addr)- if err != nil {- resolvedAddrs = []multiaddr.Multiaddr{addr}- }- for _, resolved := range resolvedAddrs {- if addrInfo, err := peer.AddrInfoFromP2pAddr(resolved); err == nil && addrInfo.ID != "" {- staticRelays = append(staticRelays, *addrInfo)- if n.RouterPeerID == "" {- n.RouterPeerID = addrInfo.ID- }- } else {- logger.Warnf("Failed to parse static relay addr %s: %v", resolved, err)- }- }+ // The routers, one entry per peer with every address it is known by;+ // they double as the static relays.+ staticRelays := routerPeers(ctx, n.config.RouterAddrs)+ if n.RouterPeerID == "" && len(staticRelays) > 0 {+ n.RouterPeerID = staticRelays[0].ID}logger.Infof("Configured %d static relays: %v", len(staticRelays), staticRelays)@@ -618,6 +640,7 @@ func (n *SamNode) Start(ctx context.Context) error {n.mu.Unlock()if wasAuthenticated {logger.Warnf("[AuthN] Router %s disconnected; clearing authenticated session so the next check re-handshakes", remotePeer)+ n.redialRouter(ctx, remotePeer)}},})@@ -658,21 +681,8 @@ func (n *SamNode) Start(ctx context.Context) error {n.services.reprovideNow = n.triggerReproviden.applyPendingEgress(ctx)- var authenticated bool- var fatalAuthErr error-- for _, addr := range n.config.RouterAddrs {- if err := n.ConnectAndAuthWithRouter(ctx, addr); err != nil {- logger.Warnf("[AuthN] Failed to bootstrap and auth with router %s: %v", addr, err)- if errors.Is(err, ErrFatalAuth) {- fatalAuthErr = err- }- } else {- authenticated = true- }- }-- if len(n.config.RouterAddrs) > 0 && !authenticated {+ authenticated, fatalAuthErr := n.authWithRouters(ctx, staticRelays)+ if len(staticRelays) > 0 && !authenticated {if fatalAuthErr != nil {return fmt.Errorf("fatal auth failure: %w", fatalAuthErr)}@@ -980,73 +990,184 @@ dhtLoop:return nil}-func (n *SamNode) ConnectAndAuthWithRouter(ctx context.Context, addr multiaddr.Multiaddr) error {- resolvedAddrs, err := resolveAddrIfNeeded(ctx, addr)+// authWithRouters dials every router at once and returns as soon as one has+// admitted this node, or once all have refused. A router that is down costs+// its dial timeout; paid up front and one router after another, that was+// every member's cold-join cost for as long as the control plane still+// listed a router that had gone. The dials still in flight finish in the+// background and record their sessions as they land; the connection monitor+// re-handshakes whatever drops later.+func (n *SamNode) authWithRouters(ctx context.Context, routers []peer.AddrInfo) (authenticated bool, fatal error) {+ if len(routers) == 0 {+ return false, nil+ }+ biscuitBytes, err := n.loadIdentityForAuth()if err != nil {- resolvedAddrs = []multiaddr.Multiaddr{addr}+ return false, err+ }+ results := n.dialRouters(ctx, routers, biscuitBytes)+ for range routers {+ err := <-results+ if err == nil {+ return true, nil+ }+ if errors.Is(err, ErrFatalAuth) {+ fatal = err+ }+ }+ return false, fatal+}++// dialRouters runs authRouter against every router at once and delivers one+// outcome per router on the returned channel, in completion order. The+// channel is buffered for all of them, so a caller may stop reading early.+func (n *SamNode) dialRouters(ctx context.Context, routers []peer.AddrInfo, biscuitBytes []byte) <-chan error {+ results := make(chan error, len(routers))+ for _, router := range routers {+ go func() {+ err := n.authRouter(ctx, router, biscuitBytes)+ if err != nil {+ logger.Warnf("[AuthN] Failed to bootstrap and auth with router %s: %v", router.ID, err)+ }+ results <- err+ }()+ }+ return results+}++// redialRouter re-handshakes a router whose session just dropped, in the+// background, with a doubling delay between attempts. The connection monitor+// only acts once no router is left and only on its own interval, so without+// this a rollout that restarts the routers one after another, each faster+// than that interval, takes every session a member has before any is back;+// and a router that restarts alone is never re-handshaken at all, the+// member keeping one router fewer after each rollout. One redial per router+// runs at a time, and a router that stays away is left to the monitor.+func (n *SamNode) redialRouter(ctx context.Context, router peer.ID) {+ if n.config.RouterRedialDelay < 0 {+ return+ }+ n.mu.Lock()+ if n.redialing == nil {+ n.redialing = map[peer.ID]bool{}+ }+ if n.redialing[router] {+ n.mu.Unlock()+ return}+ n.redialing[router] = true+ n.mu.Unlock()++ go func() {+ defer func() {+ n.mu.Lock()+ delete(n.redialing, router)+ n.mu.Unlock()+ }()+ biscuitBytes, err := n.loadIdentityForAuth()+ if err != nil {+ return+ }+ delay := n.config.RouterRedialDelay+ for attempt := 1; attempt <= n.config.RouterRedialAttempts; attempt++ {+ select {+ case <-ctx.Done():+ return+ case <-time.After(delay):+ }+ if n.isAuthenticatedAndConnected(router) {+ return+ }+ addrs := n.Host.Peerstore().Addrs(router)+ if len(addrs) == 0 {+ logger.Warnf("[AuthN] Router %s dropped and no address for it is known; leaving it to the connection monitor", router)+ return+ }+ err := n.authRouter(ctx, peer.AddrInfo{ID: router, Addrs: addrs}, biscuitBytes)+ if err == nil {+ logger.Infof("[AuthN] Router %s re-admitted this node on redial %d", router, attempt)+ n.triggerReprovide()+ return+ }+ if errors.Is(err, ErrFatalAuth) {+ logger.Warnf("[AuthN] Router %s refused this node's credential on redial: %v", router, err)+ return+ }+ logger.Debugf("[AuthN] Redial %d of router %s failed: %v", attempt, router, err)+ delay *= 2+ }+ logger.Warnf("[AuthN] Router %s did not come back within %d redials; leaving it to the connection monitor", router, n.config.RouterRedialAttempts)+ }()+}- // Load biscuit from store once before the loop+func (n *SamNode) loadIdentityForAuth() ([]byte, error) {biscuitBytes, err := n.Store.LoadIdentity()if err != nil {- return fmt.Errorf("%w: failed to load identity from store: %w", ErrFatalAuth, err)+ return nil, fmt.Errorf("%w: failed to load identity from store: %w", ErrFatalAuth, err)}if len(biscuitBytes) == 0 {- return fmt.Errorf("%w: no identity biscuit found in store", ErrFatalAuth)+ return nil, fmt.Errorf("%w: no identity biscuit found in store", ErrFatalAuth)+ }+ return biscuitBytes, nil+}++// authRouter connects to one router over whichever of its addresses answers+// and runs the handshake, unless a live authenticated session to it already+// exists: Host.Connect is a no-op on an existing connection, but the+// handshake is not, and repeating it trips the router's per-peer handshake+// limiter. The dial is bounded by RouterConnectTimeout so a router that is+// down does not hold its caller for long.+func (n *SamNode) authRouter(ctx context.Context, router peer.AddrInfo, biscuitBytes []byte) error {+ if n.isAuthenticatedAndConnected(router.ID) {+ return nil+ }+ routerCtx, cancel := context.WithTimeout(ctx, n.config.RouterConnectTimeout)+ defer cancel()+ if err := n.Host.Connect(routerCtx, router); err != nil {+ if strings.Contains(err.Error(), "peer id mismatch") {+ return fmt.Errorf("%w: %w", ErrFatalAuth, err)+ }+ return fmt.Errorf("failed to connect to router %s at %v: %w", router.ID, router.Addrs, err)+ }+ if err := n.authenticateRouterSession(routerCtx, router.ID, biscuitBytes); err != nil {+ return fmt.Errorf("router %s: %w", router.ID, err)+ }+ n.authOnce.Do(func() {+ close(n.authSuccess)+ })+ return nil+}++// ConnectAndAuthWithRouter connects to and authenticates with every router+// behind addr: one peer for a direct address, every router behind the name+// for a dnsaddr. They are dialed at once and all are waited for; it succeeds+// when at least one admitted this node.+func (n *SamNode) ConnectAndAuthWithRouter(ctx context.Context, addr multiaddr.Multiaddr) error {+ biscuitBytes, err := n.loadIdentityForAuth()+ if err != nil {+ return err}+ routers := routerPeers(ctx, []multiaddr.Multiaddr{addr})var connected boolvar lastFatalErr errorvar errs []error-- for _, resolved := range resolvedAddrs {- addrInfo, err := peer.AddrInfoFromP2pAddr(resolved)- if err != nil {- errs = append(errs, fmt.Errorf("failed to get AddrInfo from multiaddr %s: %w", resolved, err))- continue- }-- // A router advertises one multiaddr per interface, all the same peer.- // Host.Connect is a no-op on an existing connection, but the- // handshake is not: repeating it per address trips the router's- // per-peer handshake limiter and logs a failure for every extra- // address. One live authenticated session per router is enough.- if n.isAuthenticatedAndConnected(addrInfo.ID) {+ results := n.dialRouters(ctx, routers, biscuitBytes)+ for range routers {+ err := <-results+ if err == nil {connected = truecontinue}-- // Create a per-replica timeout context to prevent blocking on offline replicas- replicaCtx, cancel := context.WithTimeout(ctx, n.config.RouterConnectTimeout)-- if err := n.Host.Connect(replicaCtx, *addrInfo); err != nil {- cancel()- if strings.Contains(err.Error(), "peer id mismatch") {- lastFatalErr = fmt.Errorf("%w: %w", ErrFatalAuth, err)- }- errs = append(errs, fmt.Errorf("failed to connect to router %s: %w", resolved, err))- continue- }-- err = n.authenticateRouterSession(replicaCtx, addrInfo.ID, biscuitBytes)- cancel()- if err != nil {- errs = append(errs, fmt.Errorf("router %s: %w", resolved, err))- if errors.Is(err, ErrFatalAuth) {- lastFatalErr = err- }- continue+ errs = append(errs, err)+ if errors.Is(err, ErrFatalAuth) {+ lastFatalErr = err}- connected = true}if connected {- n.authOnce.Do(func() {- close(n.authSuccess)- })return nil}-if lastFatalErr != nil {return lastFatalErr}diff --git a/internal/node/options.go b/internal/node/options.goindex 21ed650e..3148fd84 100644--- a/internal/node/options.go+++ b/internal/node/options.go@@ -30,6 +30,11 @@ const (DefaultDiscoveryInterval = "30s"DefaultConfigFile = "sam-node.yaml"DefaultRouterConnectTimeout = 5 * time.Second+ // A router pod that restarts is back in well under a minute; the first+ // redial waits a moment for it, and six doublings reach past where the+ // connection monitor's own interval takes over.+ DefaultRouterRedialDelay = 2 * time.Second+ DefaultRouterRedialAttempts = 6// DefaultSocketName is the local API socket the node creates in its data directory.DefaultSocketName = "sam.sock")@@ -60,6 +65,13 @@ type Options struct {AutoRelayBackoff time.Duration// RouterConnectTimeout bounds each router address's dial (connect + stream open).RouterConnectTimeout time.Duration+ // RouterRedialDelay is how long after a router session drops the node+ // waits before dialing that router again; each further attempt waits+ // twice as long, up to RouterRedialAttempts of them. Zero uses the+ // default; negative disables the redial, leaving a dropped router to the+ // connection monitor.+ RouterRedialDelay time.Duration+ RouterRedialAttempts int// BiscuitTimeout bounds Datalog evaluation when verifying biscuit tokens.BiscuitTimeout time.Duration// DHT Options@@ -134,6 +146,12 @@ func (o *Options) Default() {if o.RouterConnectTimeout == 0 {o.RouterConnectTimeout = DefaultRouterConnectTimeout}+ if o.RouterRedialDelay == 0 {+ o.RouterRedialDelay = DefaultRouterRedialDelay+ }+ if o.RouterRedialAttempts <= 0 {+ o.RouterRedialAttempts = DefaultRouterRedialAttempts+ }if o.BiscuitTimeout <= 0 {o.BiscuitTimeout = identity.DefaultAuthorizerTimeout}diff --git a/internal/node/router_auth_dedupe_test.go b/internal/node/router_auth_dedupe_test.goindex 434d38aa..7b65ba5c 100644--- a/internal/node/router_auth_dedupe_test.go+++ b/internal/node/router_auth_dedupe_test.go@@ -18,7 +18,9 @@ import ("bytes""context""crypto/ed25519"+ "fmt""io"+ "net""net/http""net/http/httptest""sync/atomic"@@ -28,6 +30,7 @@ import ("github.com/biscuit-auth/biscuit-go/v2""github.com/google/sam/api""github.com/libp2p/go-libp2p"+ "github.com/libp2p/go-libp2p/core/crypto""github.com/libp2p/go-libp2p/core/host""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer"@@ -117,7 +120,8 @@ func newFakeRouter(t *testing.T, listenAddrs ...string) *fakeRouter {}// startNode brings up a node enrolled against this router and authenticated.-func (r *fakeRouter) startNode(t *testing.T, ctx context.Context, routerAddrs []multiaddr.Multiaddr) *SamNode {+// Each tweak may adjust the options before the node is built.+func (r *fakeRouter) startNode(t *testing.T, ctx context.Context, routerAddrs []multiaddr.Multiaddr, tweaks ...func(*Options)) *SamNode {t.Helper()store, err := NewStore(t.TempDir())if err != nil {@@ -133,14 +137,18 @@ func (r *fakeRouter) startNode(t *testing.T, ctx context.Context, routerAddrs []t.Fatal(err)}- node, err := NewSamNode(Options{+ opts := Options{PrivKey: privKey,Store: store,ControlPlanePubKey: r.cpPub,RouterAddrs: routerAddrs,ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},AllowLoopback: true,- })+ }+ for _, tweak := range tweaks {+ tweak(&opts)+ }+ node, err := NewSamNode(opts)if err != nil {t.Fatal(err)}@@ -235,7 +243,10 @@ func TestRouterDisconnectClearsAuthenticatedSession(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)defer cancel()- node := r.startNode(t, ctx, r.routerAddrs)+ // The redial would restore the session on its own; this is about the+ // state in between, so it is off here and TestRouterDropIsRedialed+ // covers the redial.+ node := r.startNode(t, ctx, r.routerAddrs, func(o *Options) { o.RouterRedialDelay = -1 })if !node.IsConnected() {t.Fatal("node does not report an authenticated router connection after Start")@@ -323,3 +334,154 @@ func TestRefreshEnrollmentReadmitsConnectedRouters(t *testing.T) {}waitForConnectedness(t, node, r.h.ID(), network.Connected)}++// The control plane keeps advertising a router for as long as its lease+// lasts, so after a rollout every joiner is handed a router that is gone.+// Start must not pay that router's dial timeout before the node is ready:+// the live router admits the node and the dead one is still being dialed+// when Start returns.+func TestStartIsReadyOnceOneRouterAdmitsTheNode(t *testing.T) {+ live := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")++ // A listener that accepts and never speaks: the dial completes at the+ // TCP level, the security handshake then waits out the timeout.+ deadListener, err := net.Listen("tcp", "127.0.0.1:0")+ if err != nil {+ t.Fatal(err)+ }+ defer func() { _ = deadListener.Close() }()+ go func() {+ for {+ c, err := deadListener.Accept()+ if err != nil {+ return+ }+ defer func() { _ = c.Close() }()+ }+ }()+ _, deadPub, err := crypto.GenerateEd25519Key(nil)+ if err != nil {+ t.Fatal(err)+ }+ deadID, err := peer.IDFromPublicKey(deadPub)+ if err != nil {+ t.Fatal(err)+ }+ deadAddr := multiaddr.StringCast(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d/p2p/%s", deadListener.Addr().(*net.TCPAddr).Port, deadID))++ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()+ // The dead router first, where a sequential dial would have waited on it.+ routerAddrs := append([]multiaddr.Multiaddr{deadAddr}, live.routerAddrs...)++ const connectTimeout = 3 * time.Second+ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = store.Close() })+ privKey := GetOrGenerateKey(store)+ pid, _ := peer.IDFromPrivateKey(privKey)+ if err := store.SaveIdentity(live.mint(pid.String(), api.RoleNode)); err != nil {+ t.Fatal(err)+ }+ node, err := NewSamNode(Options{+ PrivKey: privKey,+ Store: store,+ ControlPlanePubKey: live.cpPub,+ RouterAddrs: routerAddrs,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ AllowLoopback: true,+ RouterConnectTimeout: connectTimeout,+ })+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = node.Teardown() })++ started := time.Now()+ if err := node.Start(ctx); err != nil {+ t.Fatalf("Start: %v", err)+ }+ took := time.Since(started)+ if !node.IsConnected() {+ t.Fatal("Start returned without an authenticated router")+ }+ if took >= connectTimeout {+ t.Fatalf("Start took %v, at least the dead router's %v dial timeout; the live router should have made the node ready", took, connectTimeout)+ }+ if got := live.handshakes.Load(); got != 1 {+ t.Fatalf("live router saw %d handshakes, want 1", got)+ }+}++// Three entries for one router, as a control plane hands out when a dnsaddr+// resolves to every address behind the name: one peer, one dial, one+// handshake, and one static relay.+func TestRouterPeersGroupsAddressesByPeer(t *testing.T) {+ r := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0", "/ip4/127.0.0.1/tcp/0")+ addrs := append(append([]multiaddr.Multiaddr{}, r.routerAddrs...), r.routerAddrs[0])++ peers := routerPeers(context.Background(), addrs)+ if len(peers) != 1 {+ t.Fatalf("routerPeers = %d peers for %d addresses of one router, want 1", len(peers), len(addrs))+ }+ if peers[0].ID != r.h.ID() {+ t.Fatalf("peer = %s, want the router %s", peers[0].ID, r.h.ID())+ }+ if len(peers[0].Addrs) != 2 {+ t.Fatalf("peer carries %d addresses, want its 2 distinct ones: %v", len(peers[0].Addrs), peers[0].Addrs)+ }+}++// A router that restarts drops every member's session. The connection+// monitor only acts once a member has no router left, and only on its own+// interval, so a rollout that restarts the routers one after another took+// every session before any came back, and a router restarting alone was+// never re-handshaken. The member must redial a dropped router by itself.+func TestRouterDropIsRedialed(t *testing.T) {+ r := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")+ h, handshakes := r.h, r.handshakes++ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()+ // The monitor is kept out of the way: the redial must do this alone.+ node := r.startNode(t, ctx, r.routerAddrs, func(o *Options) {+ o.RouterRedialDelay = 100 * time.Millisecond+ o.MonitorBootstrap = time.Hour+ })+ if got := handshakes.Load(); got != 1 {+ t.Fatalf("handshakes after Start = %d, want 1", got)+ }++ // The router side closes, as a restarting pod does.+ if err := h.Network().ClosePeer(node.Host.ID()); err != nil {+ t.Fatal(err)+ }+ waitForConnectedness(t, node, h.ID(), network.NotConnected)++ deadline := time.Now().Add(5 * time.Second)+ for time.Now().Before(deadline) && (!node.IsConnected() || handshakes.Load() != 2) {+ time.Sleep(20 * time.Millisecond)+ }+ if got := handshakes.Load(); got != 2 {+ t.Fatalf("handshakes after the drop = %d, want 2: the node must handshake the router again", got)+ }+ if !node.IsConnected() {+ t.Fatal("node does not report an authenticated session after the redial")+ }++ // Dropping it again while the first redial has finished starts another;+ // the session comes back a second time.+ if err := h.Network().ClosePeer(node.Host.ID()); err != nil {+ t.Fatal(err)+ }+ waitForConnectedness(t, node, h.ID(), network.NotConnected)+ deadline = time.Now().Add(5 * time.Second)+ for time.Now().Before(deadline) && handshakes.Load() < 3 {+ time.Sleep(20 * time.Millisecond)+ }+ if got := handshakes.Load(); got != 3 {+ t.Fatalf("handshakes after the second drop = %d, want 3", got)+ }+}