Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: redial a router that drops the node as it re-admits it
HosniBelfekif5be60912ecd
diff --git a/internal/node/node.go b/internal/node/node.goindex 6f9cd5c8..66442707 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -1116,63 +1116,97 @@ func (n *AgentMeshNode) redialRouter(ctx context.Context, router peer.ID) {}if n.redialing[router] {n.mu.Unlock()+ logger.Debugf("[AuthN] Router %s dropped while a redial of it is running; that redial checks the session before it ends", router)return}n.redialing[router] = truen.mu.Unlock()go func() {- defer func() {- n.mu.Lock()- delete(n.redialing, router)- n.mu.Unlock()- }()- biscuitBytes, err := n.loadIdentityForAuth()- if err != nil {- logger.Errorf("[AuthN] Cannot redial router %s without an identity: %v", router, err)+ if n.redialUntilUp(ctx, router) {return}- delay := n.config.RouterRedialDelay- for attempt := 1; attempt <= n.config.RouterRedialAttempts; attempt++ {- select {- case <-ctx.Done():- return- case <-time.After(delay):- }- if n.isAuthenticatedAndConnected(router) {- return- }- if !n.isAttached(router) {- return- }- addrs := n.Host.Peerstore().Addrs(router)- if len(addrs) == 0 {- logger.Warnf("[AuthN] Router %s dropped and no address for it is known; replacing it", router)- n.detach(router)- return- }- err := n.authRouter(ctx, peer.AddrInfo{ID: router, Addrs: addrs}, biscuitBytes)- if err == nil {+ n.mu.Lock()+ delete(n.redialing, router)+ n.mu.Unlock()+ }()+}++// redialUntilUp runs the redial attempts of redialRouter and reports whether+// it released the redial's in-flight mark. It releases it only on a live+// session, through releaseRedialIfUp; on every other exit the mark is left+// to the caller.+func (n *AgentMeshNode) redialUntilUp(ctx context.Context, router peer.ID) bool {+ biscuitBytes, err := n.loadIdentityForAuth()+ if err != nil {+ logger.Errorf("[AuthN] Cannot redial router %s without an identity: %v", router, err)+ return false+ }+ delay := n.config.RouterRedialDelay+ for attempt := 1; attempt <= n.config.RouterRedialAttempts; attempt++ {+ select {+ case <-ctx.Done():+ return false+ case <-time.After(delay):+ }+ if n.releaseRedialIfUp(router) {+ logger.Debugf("[AuthN] Router %s session was restored while its redial waited", router)+ return true+ }+ if !n.isAttached(router) {+ return false+ }+ addrs := n.Host.Peerstore().Addrs(router)+ if len(addrs) == 0 {+ logger.Warnf("[AuthN] Router %s dropped and no address for it is known; replacing it", router)+ n.detach(router)+ return false+ }+ err := n.authRouter(ctx, peer.AddrInfo{ID: router, Addrs: addrs}, biscuitBytes)+ if err == nil {+ if n.releaseRedialIfUp(router) {logger.Infof("[AuthN] Router %s re-admitted this node on redial %d", router, attempt)// A router that restarted lost this node's records.n.triggerReprovide()- return+ return true}+ // A drop since the handshake found this redial's mark and left+ // the session to it, or came before the session was recorded.+ logger.Warnf("[AuthN] Router %s admitted this node on redial %d and dropped it again", router, attempt)+ } else {if errors.Is(err, ErrFatalAuth) {logger.Warnf("[AuthN] Router %s refused this node's credential on redial: %v", router, err)n.detach(router)n.shun(router, n.config.RouterShunDuration)- return+ return false}logger.Debugf("[AuthN] Redial %d of router %s failed: %v", attempt, router, err)- delay *= 2}- // Given up: the monitor tops up from the other routers, and this one- // is a candidate again once it is back on /info and the shun lapses.- logger.Warnf("[AuthN] Router %s did not come back within %d redials; replacing it", router, n.config.RouterRedialAttempts)- n.detach(router)- n.shun(router, n.config.RouterShunDuration)- }()+ delay *= 2+ }+ // Given up: the monitor tops up from the other routers, and this one+ // is a candidate again once it is back on /info and the shun lapses.+ logger.Warnf("[AuthN] Router %s did not keep a session with this node within %d redials; replacing it", router, n.config.RouterRedialAttempts)+ n.detach(router)+ n.shun(router, n.config.RouterShunDuration)+ return false+}++// releaseRedialIfUp ends a redial of router on a live session: it clears the+// redial's in-flight mark only while the session is up, and reports whether+// it did. It holds n.mu, as the disconnect handler does to clear the+// session, so a drop either came first and is seen here, and the redial goes+// on, or comes after, finds no mark and starts a redial of its own. The+// connection is checked as well as the session flag, since a drop that+// lands before the handshake records the session clears nothing.+func (n *AgentMeshNode) releaseRedialIfUp(router peer.ID) bool {+ n.mu.Lock()+ defer n.mu.Unlock()+ if !n.authenticatedRouters[router] || n.Host.Network().Connectedness(router) != network.Connected {+ return false+ }+ delete(n.redialing, router)+ return true}func (n *AgentMeshNode) loadIdentityForAuth() ([]byte, error) {@@ -1299,7 +1333,19 @@ func (n *AgentMeshNode) authenticateRouterSession(ctx context.Context, router pe}_ = s.Close()+ // A router that closes the connection right after admitting the node can+ // do so before the session is recorded, and the disconnect handler then+ // finds none to clear. Recorded anyway, it would outlive the connection,+ // and the next connection to the router, which AutoRelay opens within+ // milliseconds, would make it look live although the router never+ // authenticated it. The swarm closes a connection before it notifies+ // the handler, and the handler takes n.mu, so a close either came first+ // and is seen here or comes after and clears what is recorded now.n.mu.Lock()+ if s.Conn().IsClosed() {+ n.mu.Unlock()+ return fmt.Errorf("router %s closed the connection as it admitted this node", router)+ }n.authenticatedRouters[router] = truen.RouterPeerID = routern.mu.Unlock()diff --git a/internal/node/router_auth_dedupe_test.go b/internal/node/router_auth_dedupe_test.goindex 7957f5cd..499af48e 100644--- a/internal/node/router_auth_dedupe_test.go+++ b/internal/node/router_auth_dedupe_test.go@@ -597,8 +597,9 @@ func TestRouterDropIsRedialed(t *testing.T) {t.Fatal("node does not report an authenticated session after the redial")}- // Dropping it again while the first redial has finished starts another;- // the session comes back a second time.+ // Dropping it again once the session is back starts another redial,+ // whether or not the first one has released its mark yet; the session+ // comes back a second time.dropFromRouter(t, h, node)deadline = time.Now().Add(5 * time.Second)for time.Now().Before(deadline) && handshakes.Load() < 3 {@@ -609,6 +610,114 @@ func TestRouterDropIsRedialed(t *testing.T) {}}+// A router that admits the node and drops it again at once, as one that+// restarts twice in a row or sends members away right after admitting them+// does, must not cost the node that router. Each drop lands either before+// the node records the session or after it, while the redial still holds+// its mark; in both orders the redial must go on until a session holds.+func TestRouterDropAfterReadmitIsRedialed(t *testing.T) {+ const drops = 2 // fewer than RouterRedialAttempts+ r := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")+ // The first handshake is Start's; the next drops admit the node and+ // close its connection, and the one after them is kept.+ var seen atomic.Int32+ r.h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {+ member := s.Conn().RemotePeer()+ n := seen.Add(1)+ r.admit(s)+ if n > 1 && n <= drops+1 {+ _ = r.h.Network().ClosePeer(member)+ }+ })++ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()+ // The monitor is kept out of the way: the redial must do this alone.+ node := r.startNode(t, ctx, r.routerAddrs, func(o *Options) {+ o.RouterRedialDelay = 100 * time.Millisecond+ o.MonitorBootstrap = time.Hour+ })+ if got := r.handshakes.Load(); got != 1 {+ t.Fatalf("handshakes after Start = %d, want 1", got)+ }++ dropFromRouter(t, r.h, node)++ const want = drops + 2 // Start, the dropped redials, the one that holds+ deadline := time.Now().Add(10 * time.Second)+ for time.Now().Before(deadline) && (!node.IsConnected() || r.handshakes.Load() != want) {+ time.Sleep(20 * time.Millisecond)+ }+ if got := r.handshakes.Load(); got != want {+ t.Fatalf("handshakes = %d, want %d: the node must redial a router that dropped it as it re-admitted it", got, want)+ }+ if !node.IsConnected() {+ t.Fatal("node does not report an authenticated session after the redials")+ }+ if !node.isAttached(r.h.ID()) {+ t.Fatal("router is no longer attached, want it kept: it admitted the node again")+ }+ node.mu.Lock()+ _, shunned := node.shunned[r.h.ID()]+ node.mu.Unlock()+ if shunned {+ t.Fatal("router is shunned, want it kept: it dropped the node fewer times than RouterRedialAttempts")+ }+}++func TestReleaseRedialIfUp(t *testing.T) {+ r := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ // No redial and no monitor: the test sets the session and the mark itself.+ node := r.startNode(t, ctx, r.routerAddrs, func(o *Options) {+ o.RouterRedialDelay = -1+ o.MonitorBootstrap = time.Hour+ })+ router := r.h.ID()+ // A peer the node has never been connected to.+ unconnectedKey, _, err := crypto.GenerateEd25519Key(nil)+ if err != nil {+ t.Fatal(err)+ }+ unconnected, err := peer.IDFromPrivateKey(unconnectedKey)+ if err != nil {+ t.Fatal(err)+ }++ for _, tc := range []struct {+ name string+ router peer.ID+ session bool+ want bool+ }{+ {name: "live session", router: router, session: true, want: true},+ {name: "session cleared", router: router, session: false, want: false},+ {name: "connection gone", router: unconnected, session: true, want: false},+ } {+ t.Run(tc.name, func(t *testing.T) {+ node.mu.Lock()+ node.redialing = map[peer.ID]bool{tc.router: true}+ if tc.session {+ node.authenticatedRouters[tc.router] = true+ } else {+ delete(node.authenticatedRouters, tc.router)+ }+ node.mu.Unlock()++ if got := node.releaseRedialIfUp(tc.router); got != tc.want {+ t.Fatalf("releaseRedialIfUp = %v, want %v", got, tc.want)+ }+ node.mu.Lock()+ marked := node.redialing[tc.router]+ node.mu.Unlock()+ if marked == tc.want {+ t.Fatalf("redial mark held = %v after releaseRedialIfUp returned %v, want the mark cleared exactly when it returns true", marked, tc.want)+ }+ })+ }+}+// A peer's record names a circuit through a router the node's catalog does// not list: the node read the catalog while that router was restarting, or// the router's lease is not carried right now. The call must handshake with