Skip to content

Agent Mesh

node: redial a router that drops the node as it re-admits it

HosniBelfekif5be60912ecd
diff --git a/internal/node/node.go b/internal/node/node.go
index 6f9cd5c8..66442707 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -1116,63 +1116,97 @@ func (n *AgentMeshNode) redialRouter(ctx context.Context, router peer.ID) {
}
if n.redialing[router] {
n.mu.Unlock()
+ logger.Debugf("[AuthN] Router %s dropped while a redial of it is running; that redial checks the session before it ends", router)
return
}
n.redialing[router] = true
n.mu.Unlock()
go func() {
- defer func() {
- n.mu.Lock()
- delete(n.redialing, router)
- n.mu.Unlock()
- }()
- biscuitBytes, err := n.loadIdentityForAuth()
- if err != nil {
- logger.Errorf("[AuthN] Cannot redial router %s without an identity: %v", router, err)
+ if n.redialUntilUp(ctx, router) {
return
}
- delay := n.config.RouterRedialDelay
- for attempt := 1; attempt <= n.config.RouterRedialAttempts; attempt++ {
- select {
- case <-ctx.Done():
- return
- case <-time.After(delay):
- }
- if n.isAuthenticatedAndConnected(router) {
- return
- }
- if !n.isAttached(router) {
- return
- }
- addrs := n.Host.Peerstore().Addrs(router)
- if len(addrs) == 0 {
- logger.Warnf("[AuthN] Router %s dropped and no address for it is known; replacing it", router)
- n.detach(router)
- return
- }
- err := n.authRouter(ctx, peer.AddrInfo{ID: router, Addrs: addrs}, biscuitBytes)
- if err == nil {
+ n.mu.Lock()
+ delete(n.redialing, router)
+ n.mu.Unlock()
+ }()
+}
+
+// redialUntilUp runs the redial attempts of redialRouter and reports whether
+// it released the redial's in-flight mark. It releases it only on a live
+// session, through releaseRedialIfUp; on every other exit the mark is left
+// to the caller.
+func (n *AgentMeshNode) redialUntilUp(ctx context.Context, router peer.ID) bool {
+ biscuitBytes, err := n.loadIdentityForAuth()
+ if err != nil {
+ logger.Errorf("[AuthN] Cannot redial router %s without an identity: %v", router, err)
+ return false
+ }
+ delay := n.config.RouterRedialDelay
+ for attempt := 1; attempt <= n.config.RouterRedialAttempts; attempt++ {
+ select {
+ case <-ctx.Done():
+ return false
+ case <-time.After(delay):
+ }
+ if n.releaseRedialIfUp(router) {
+ logger.Debugf("[AuthN] Router %s session was restored while its redial waited", router)
+ return true
+ }
+ if !n.isAttached(router) {
+ return false
+ }
+ addrs := n.Host.Peerstore().Addrs(router)
+ if len(addrs) == 0 {
+ logger.Warnf("[AuthN] Router %s dropped and no address for it is known; replacing it", router)
+ n.detach(router)
+ return false
+ }
+ err := n.authRouter(ctx, peer.AddrInfo{ID: router, Addrs: addrs}, biscuitBytes)
+ if err == nil {
+ if n.releaseRedialIfUp(router) {
logger.Infof("[AuthN] Router %s re-admitted this node on redial %d", router, attempt)
// A router that restarted lost this node's records.
n.triggerReprovide()
- return
+ return true
}
+ // A drop since the handshake found this redial's mark and left
+ // the session to it, or came before the session was recorded.
+ logger.Warnf("[AuthN] Router %s admitted this node on redial %d and dropped it again", router, attempt)
+ } else {
if errors.Is(err, ErrFatalAuth) {
logger.Warnf("[AuthN] Router %s refused this node's credential on redial: %v", router, err)
n.detach(router)
n.shun(router, n.config.RouterShunDuration)
- return
+ return false
}
logger.Debugf("[AuthN] Redial %d of router %s failed: %v", attempt, router, err)
- delay *= 2
}
- // Given up: the monitor tops up from the other routers, and this one
- // is a candidate again once it is back on /info and the shun lapses.
- logger.Warnf("[AuthN] Router %s did not come back within %d redials; replacing it", router, n.config.RouterRedialAttempts)
- n.detach(router)
- n.shun(router, n.config.RouterShunDuration)
- }()
+ delay *= 2
+ }
+ // Given up: the monitor tops up from the other routers, and this one
+ // is a candidate again once it is back on /info and the shun lapses.
+ logger.Warnf("[AuthN] Router %s did not keep a session with this node within %d redials; replacing it", router, n.config.RouterRedialAttempts)
+ n.detach(router)
+ n.shun(router, n.config.RouterShunDuration)
+ return false
+}
+
+// releaseRedialIfUp ends a redial of router on a live session: it clears the
+// redial's in-flight mark only while the session is up, and reports whether
+// it did. It holds n.mu, as the disconnect handler does to clear the
+// session, so a drop either came first and is seen here, and the redial goes
+// on, or comes after, finds no mark and starts a redial of its own. The
+// connection is checked as well as the session flag, since a drop that
+// lands before the handshake records the session clears nothing.
+func (n *AgentMeshNode) releaseRedialIfUp(router peer.ID) bool {
+ n.mu.Lock()
+ defer n.mu.Unlock()
+ if !n.authenticatedRouters[router] || n.Host.Network().Connectedness(router) != network.Connected {
+ return false
+ }
+ delete(n.redialing, router)
+ return true
}
func (n *AgentMeshNode) loadIdentityForAuth() ([]byte, error) {
@@ -1299,7 +1333,19 @@ func (n *AgentMeshNode) authenticateRouterSession(ctx context.Context, router pe
}
_ = s.Close()
+ // A router that closes the connection right after admitting the node can
+ // do so before the session is recorded, and the disconnect handler then
+ // finds none to clear. Recorded anyway, it would outlive the connection,
+ // and the next connection to the router, which AutoRelay opens within
+ // milliseconds, would make it look live although the router never
+ // authenticated it. The swarm closes a connection before it notifies
+ // the handler, and the handler takes n.mu, so a close either came first
+ // and is seen here or comes after and clears what is recorded now.
n.mu.Lock()
+ if s.Conn().IsClosed() {
+ n.mu.Unlock()
+ return fmt.Errorf("router %s closed the connection as it admitted this node", router)
+ }
n.authenticatedRouters[router] = true
n.RouterPeerID = router
n.mu.Unlock()
diff --git a/internal/node/router_auth_dedupe_test.go b/internal/node/router_auth_dedupe_test.go
index 7957f5cd..499af48e 100644
--- a/internal/node/router_auth_dedupe_test.go
+++ b/internal/node/router_auth_dedupe_test.go
@@ -597,8 +597,9 @@ func TestRouterDropIsRedialed(t *testing.T) {
t.Fatal("node does not report an authenticated session after the redial")
}
- // Dropping it again while the first redial has finished starts another;
- // the session comes back a second time.
+ // Dropping it again once the session is back starts another redial,
+ // whether or not the first one has released its mark yet; the session
+ // comes back a second time.
dropFromRouter(t, h, node)
deadline = time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) && handshakes.Load() < 3 {
@@ -609,6 +610,114 @@ func TestRouterDropIsRedialed(t *testing.T) {
}
}
+// A router that admits the node and drops it again at once, as one that
+// restarts twice in a row or sends members away right after admitting them
+// does, must not cost the node that router. Each drop lands either before
+// the node records the session or after it, while the redial still holds
+// its mark; in both orders the redial must go on until a session holds.
+func TestRouterDropAfterReadmitIsRedialed(t *testing.T) {
+ const drops = 2 // fewer than RouterRedialAttempts
+ r := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ // The first handshake is Start's; the next drops admit the node and
+ // close its connection, and the one after them is kept.
+ var seen atomic.Int32
+ r.h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {
+ member := s.Conn().RemotePeer()
+ n := seen.Add(1)
+ r.admit(s)
+ if n > 1 && n <= drops+1 {
+ _ = r.h.Network().ClosePeer(member)
+ }
+ })
+
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ // The monitor is kept out of the way: the redial must do this alone.
+ node := r.startNode(t, ctx, r.routerAddrs, func(o *Options) {
+ o.RouterRedialDelay = 100 * time.Millisecond
+ o.MonitorBootstrap = time.Hour
+ })
+ if got := r.handshakes.Load(); got != 1 {
+ t.Fatalf("handshakes after Start = %d, want 1", got)
+ }
+
+ dropFromRouter(t, r.h, node)
+
+ const want = drops + 2 // Start, the dropped redials, the one that holds
+ deadline := time.Now().Add(10 * time.Second)
+ for time.Now().Before(deadline) && (!node.IsConnected() || r.handshakes.Load() != want) {
+ time.Sleep(20 * time.Millisecond)
+ }
+ if got := r.handshakes.Load(); got != want {
+ t.Fatalf("handshakes = %d, want %d: the node must redial a router that dropped it as it re-admitted it", got, want)
+ }
+ if !node.IsConnected() {
+ t.Fatal("node does not report an authenticated session after the redials")
+ }
+ if !node.isAttached(r.h.ID()) {
+ t.Fatal("router is no longer attached, want it kept: it admitted the node again")
+ }
+ node.mu.Lock()
+ _, shunned := node.shunned[r.h.ID()]
+ node.mu.Unlock()
+ if shunned {
+ t.Fatal("router is shunned, want it kept: it dropped the node fewer times than RouterRedialAttempts")
+ }
+}
+
+func TestReleaseRedialIfUp(t *testing.T) {
+ r := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ // No redial and no monitor: the test sets the session and the mark itself.
+ node := r.startNode(t, ctx, r.routerAddrs, func(o *Options) {
+ o.RouterRedialDelay = -1
+ o.MonitorBootstrap = time.Hour
+ })
+ router := r.h.ID()
+ // A peer the node has never been connected to.
+ unconnectedKey, _, err := crypto.GenerateEd25519Key(nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ unconnected, err := peer.IDFromPrivateKey(unconnectedKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, tc := range []struct {
+ name string
+ router peer.ID
+ session bool
+ want bool
+ }{
+ {name: "live session", router: router, session: true, want: true},
+ {name: "session cleared", router: router, session: false, want: false},
+ {name: "connection gone", router: unconnected, session: true, want: false},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ node.mu.Lock()
+ node.redialing = map[peer.ID]bool{tc.router: true}
+ if tc.session {
+ node.authenticatedRouters[tc.router] = true
+ } else {
+ delete(node.authenticatedRouters, tc.router)
+ }
+ node.mu.Unlock()
+
+ if got := node.releaseRedialIfUp(tc.router); got != tc.want {
+ t.Fatalf("releaseRedialIfUp = %v, want %v", got, tc.want)
+ }
+ node.mu.Lock()
+ marked := node.redialing[tc.router]
+ node.mu.Unlock()
+ if marked == tc.want {
+ t.Fatalf("redial mark held = %v after releaseRedialIfUp returned %v, want the mark cleared exactly when it returns true", marked, tc.want)
+ }
+ })
+ }
+}
+
// A peer's record names a circuit through a router the node's catalog does
// not list: the node read the catalog while that router was restarting, or
// the router's lease is not carried right now. The call must handshake with