Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: retry the routers that did not admit it in time
Antonio Ojeaf6a3e53a7d50
diff --git a/internal/node/enroll.go b/internal/node/enroll.goindex 9240bfcc..78c1700a 100644--- a/internal/node/enroll.go+++ b/internal/node/enroll.go@@ -229,24 +229,33 @@ func (n *AgentMeshNode) connectToRouters(ctx context.Context, addrs []string) erif len(addrs) == 0 {return fmt.Errorf("failed to connect and authenticate after HTTP enrollment: control plane returned no router addresses")}+ if err := n.joinRouters(ctx, addrs); err != nil {+ return fmt.Errorf("failed to connect and authenticate with any router after HTTP enrollment: %w", err)+ }+ logger.Info("Successfully enrolled via HTTP and stored identity and mesh config.")+ return nil+}- var lastAuthErr error+// joinRouters is the handshake that follows an enrollment: every router+// the control plane answered with is dialled at once and the node is in+// the mesh when one admits it, retried as at Start, since the routers are+// busiest exactly when a fleet enrolls.+func (n *AgentMeshNode) joinRouters(ctx context.Context, addrs []string) error {+ parsed := make([]multiaddr.Multiaddr, 0, len(addrs))for _, addrStr := range addrs {addr, err := multiaddr.NewMultiaddr(addrStr)if err != nil {logger.Warnf("Failed to parse router address from response: %v", err)continue}- if err := n.ConnectAndAuthWithRouter(ctx, addr); err != nil {- logger.Warnf("Failed to connect and auth with router after enrollment: %v", err)- lastAuthErr = err- } else {- logger.Info("Successfully enrolled via HTTP and stored identity and mesh config.")- return nil- }+ parsed = append(parsed, addr)}-- return fmt.Errorf("failed to connect and authenticate with any router after HTTP enrollment (last error: %v)", lastAuthErr)+ routers := routerPeers(ctx, parsed)+ if len(routers) == 0 {+ return fmt.Errorf("no router peer behind %v", addrs)+ }+ _, err := n.admitByRouters(ctx, routers)+ return err}// EnrollBootstrap enrolls the node with the control plane using a pre-shared bootstrap token.@@ -417,26 +426,8 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL strif len(enrollResp.RouterAddresses) == 0 {return fmt.Errorf("failed to connect and authenticate after bootstrap enrollment: control plane returned no router addresses")}-- var lastAuthErr error- var authed bool- for _, addrStr := range enrollResp.RouterAddresses {- addr, err := multiaddr.NewMultiaddr(addrStr)- if err != nil {- logger.Warnf("Failed to parse router address: %v", err)- continue- }- if err := n.ConnectAndAuthWithRouter(ctx, addr); err != nil {- logger.Warnf("Failed to connect and auth with router: %v", err)- lastAuthErr = err- } else {- authed = true- break- }- }-- if !authed {- return fmt.Errorf("failed to connect/auth with router after bootstrap enrollment (last error: %v)", lastAuthErr)+ if err := n.joinRouters(ctx, enrollResp.RouterAddresses); err != nil {+ return fmt.Errorf("failed to connect/auth with router after bootstrap enrollment: %w", err)}logger.Info("Successfully enrolled via Bootstrap token and joined mesh.")diff --git a/internal/node/node.go b/internal/node/node.goindex 345707bc..97728608 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -687,12 +687,9 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {n.services.reprovideNow = n.triggerReproviden.applyPendingEgress(ctx)- authenticated, fatalAuthErr := n.authWithRouters(ctx, staticRelays)- if len(staticRelays) > 0 && !authenticated {- if fatalAuthErr != nil {- return fmt.Errorf("fatal auth failure: %w", fatalAuthErr)- }- return fmt.Errorf("failed to authenticate with any router: all connection attempts failed")+ authenticated, err := n.admitByRouters(ctx, staticRelays)+ if err != nil {+ return err}if authenticated {@@ -996,6 +993,40 @@ dhtLoop:return nil}+// admitByRouters runs authWithRouters until a router admits this node, with+// RouterRedialDelay doubling between attempts, up to RouterAuthAttempts of+// them. A router that is up but slow, as every router is while a fleet+// joins through it, fails the first handshake on its deadline and admits+// the node on the next; without the retry a member that had just enrolled+// exited at that first deadline. A router that refuses the credential ends+// it at once, and no router at all is reported as before. authenticated is+// false with a nil error only when there are no routers to admit the node.+func (n *AgentMeshNode) admitByRouters(ctx context.Context, routers []peer.AddrInfo) (authenticated bool, err error) {+ if len(routers) == 0 {+ return false, nil+ }+ delay := n.config.RouterRedialDelay+ for attempt := 1; ; attempt++ {+ authenticated, fatal := n.authWithRouters(ctx, routers)+ if authenticated {+ return true, nil+ }+ if fatal != nil {+ return false, fmt.Errorf("fatal auth failure: %w", fatal)+ }+ if attempt >= n.config.RouterAuthAttempts || delay <= 0 {+ return false, errors.New("failed to authenticate with any router: all connection attempts failed")+ }+ logger.Warnf("[AuthN] No router admitted this node on attempt %d of %d; retrying in %s", attempt, n.config.RouterAuthAttempts, delay)+ select {+ case <-ctx.Done():+ return false, ctx.Err()+ case <-time.After(delay):+ }+ delay *= 2+ }+}+// authWithRouters dials every router at once and returns as soon as one has// admitted this node, or once all have refused. A router that is down costs// its dial timeout; paid up front and one router after another, that wasdiff --git a/internal/node/options.go b/internal/node/options.goindex e6032a64..91d76e5d 100644--- a/internal/node/options.go+++ b/internal/node/options.go@@ -35,6 +35,11 @@ const (// connection monitor's own interval takes over.DefaultRouterRedialDelay = 2 * time.SecondDefaultRouterRedialAttempts = 6+ // DefaultRouterAuthAttempts with DefaultRouterRedialDelay doubling+ // between them waits 14 s in all beyond the dials themselves: enough+ // for a router that is busy admitting a fleet, short enough that a+ // member with no router at all still reports it within a minute.+ DefaultRouterAuthAttempts = 4// DefaultSocketName is the local API socket the node creates in its data directory.DefaultSocketName = "agentmesh.sock")@@ -92,6 +97,10 @@ type Options struct {// connection monitor.RouterRedialDelay time.DurationRouterRedialAttempts int+ // RouterAuthAttempts is how many times Start, and the handshake after+ // an enrollment, dial the routers before giving up, RouterRedialDelay+ // doubling between attempts. Zero uses the default.+ RouterAuthAttempts int// BiscuitTimeout bounds Datalog evaluation when verifying biscuit tokens.BiscuitTimeout time.Duration// DHT Options@@ -172,6 +181,9 @@ func (o *Options) Default() {if o.RouterRedialAttempts <= 0 {o.RouterRedialAttempts = DefaultRouterRedialAttempts}+ if o.RouterAuthAttempts <= 0 {+ o.RouterAuthAttempts = DefaultRouterAuthAttempts+ }if o.Reachability == "" {o.Reachability = ReachabilityPrivate}diff --git a/internal/node/router_auth_dedupe_test.go b/internal/node/router_auth_dedupe_test.goindex 7942d1e3..66d420bc 100644--- a/internal/node/router_auth_dedupe_test.go+++ b/internal/node/router_auth_dedupe_test.go@@ -23,6 +23,7 @@ import ("net""net/http""net/http/httptest"+ "strings""sync/atomic""testing""time"@@ -51,6 +52,9 @@ type fakeRouter struct {cpPub ed25519.PublicKeycpPriv ed25519.PrivateKeymint func(peerID, role string) []byte+ // admit is the handler installed on AuthProtocolID, for a test that+ // wraps it.+ admit network.StreamHandler}func newFakeRouter(t *testing.T, listenAddrs ...string) *fakeRouter {@@ -89,7 +93,7 @@ func newFakeRouter(t *testing.T, listenAddrs ...string) *fakeRouter {var handshakes atomic.Int32var lastBiscuit atomic.Pointer[[]byte]- h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {+ admit := func(s network.Stream) {defer func() { _ = s.Close() }()handshakes.Add(1)reader := msgio.NewVarintReaderSize(s, 1024*64)@@ -105,7 +109,8 @@ func newFakeRouter(t *testing.T, listenAddrs ...string) *fakeRouter {reader.ReleaseMsg(msg)data, _ := proto.Marshal(&api.AuthResponse{Success: true, Biscuit: routerBiscuit})_ = msgio.NewVarintWriter(s).WriteMsg(data)- })+ }+ h.SetStreamHandler(api.AuthProtocolID, admit)var routerAddrs []multiaddr.Multiaddrfor _, a := range h.Addrs() {@@ -116,7 +121,7 @@ func newFakeRouter(t *testing.T, listenAddrs ...string) *fakeRouter {routerAddrs = append(routerAddrs, ma)}- return &fakeRouter{h: h, routerAddrs: routerAddrs, handshakes: &handshakes, lastBiscuit: &lastBiscuit, cpPub: cpPub, cpPriv: cpPriv, mint: mint}+ return &fakeRouter{h: h, routerAddrs: routerAddrs, handshakes: &handshakes, lastBiscuit: &lastBiscuit, cpPub: cpPub, cpPriv: cpPriv, mint: mint, admit: admit}}// startNode brings up a node enrolled against this router and authenticated.@@ -415,6 +420,90 @@ func TestStartIsReadyOnceOneRouterAdmitsTheNode(t *testing.T) {}}+// A router that is up but too busy to answer a handshake before its+// deadline, as every router is while a fleet joins through it, admits the+// node on the next attempt. Start used to give up at that first deadline,+// and a member that had just enrolled exited.+func TestStartRetriesABusyRouter(t *testing.T) {+ router := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")+ // The first handshake is swallowed: the stream is held open and never+ // answered, so the node's deadline on it fires.+ var busy atomic.Bool+ busy.Store(true)+ router.h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {+ if busy.CompareAndSwap(true, false) {+ <-time.After(2 * time.Second)+ _ = s.Reset()+ return+ }+ router.admit(s)+ })++ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = store.Close() })+ privKey := GetOrGenerateKey(store)+ pid, _ := peer.IDFromPrivateKey(privKey)+ if err := store.SaveIdentity(router.mint(pid.String(), api.RoleNode)); err != nil {+ t.Fatal(err)+ }+ node, err := NewAgentMeshNode(Options{+ PrivKey: privKey,+ Store: store,+ ControlPlanePubKey: router.cpPub,+ RouterAddrs: router.routerAddrs,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ AllowLoopback: true,+ RouterConnectTimeout: time.Second,+ RouterRedialDelay: 100 * time.Millisecond,+ RouterAuthAttempts: 3,+ })+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = node.Teardown() })++ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)+ defer cancel()+ if err := node.Start(ctx); err != nil {+ t.Fatalf("Start against a router that was busy once: %v", err)+ }+ if !node.IsConnected() {+ t.Fatal("Start returned without an authenticated router")+ }+ if busy.Load() {+ t.Fatal("the router never saw the handshake it was to swallow")+ }+ if got := router.handshakes.Load(); got != 1 {+ t.Fatalf("router admitted %d handshakes, want 1 after the swallowed one", got)+ }++ // With no router willing at all, Start still reports it, after the+ // configured attempts.+ fresh, err := NewAgentMeshNode(Options{+ PrivKey: privKey,+ Store: store,+ ControlPlanePubKey: router.cpPub,+ RouterAddrs: router.routerAddrs,+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},+ AllowLoopback: true,+ RouterConnectTimeout: 500 * time.Millisecond,+ RouterRedialDelay: 50 * time.Millisecond,+ RouterAuthAttempts: 2,+ })+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = fresh.Teardown() })+ router.h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) { _ = s.Reset() })+ err = fresh.Start(ctx)+ if err == nil || !strings.Contains(err.Error(), "all connection attempts failed") {+ t.Fatalf("Start with no router admitting: err = %v, want all connection attempts failed", err)+ }+}+// Three entries for one router, as a control plane hands out when a dnsaddr// resolves to every address behind the name: one peer, one dial, one// handshake, and one static relay.diff --git a/site/content/docs/reference/agentmesh-node.md b/site/content/docs/reference/agentmesh-node.mdindex 105fcdcf..619104cb 100644--- a/site/content/docs/reference/agentmesh-node.md+++ b/site/content/docs/reference/agentmesh-node.md@@ -63,7 +63,7 @@ bootstrap token, `join` runs an interactive OIDC login.| `--offline-access` | `false` | Request an OIDC refresh token, so that the node can re-enroll on its own when its credential expires. || `--allow-loopback` | `false` | Advertise and dial loopback and link-local addresses. For several nodes on one host. || `--announce-private` | `true` | Advertise RFC 1918 and ULA addresses. Turn off when peers are only reachable through a relay. |-| `--router-connect-timeout` | `5s` | Dial timeout per router. |+| `--router-connect-timeout` | `5s` | Dial timeout per router. The node dials every router at once and is in the mesh when the first admits it. When none does within the timeout, as happens while a router is busy admitting a fleet, it dials them again after 2 s, then 4 s, then 8 s, and reports the failure after the fourth attempt. |Labels declared in the configuration file are sent with the enrollment.Running `join` for a node that already holds a credential for the same