Skip to content

Agent Mesh

node: retry the routers that did not admit it in time

Antonio Ojeaf6a3e53a7d50
diff --git a/internal/node/enroll.go b/internal/node/enroll.go
index 9240bfcc..78c1700a 100644
--- a/internal/node/enroll.go
+++ b/internal/node/enroll.go
@@ -229,24 +229,33 @@ func (n *AgentMeshNode) connectToRouters(ctx context.Context, addrs []string) er
if len(addrs) == 0 {
return fmt.Errorf("failed to connect and authenticate after HTTP enrollment: control plane returned no router addresses")
}
+ if err := n.joinRouters(ctx, addrs); err != nil {
+ return fmt.Errorf("failed to connect and authenticate with any router after HTTP enrollment: %w", err)
+ }
+ logger.Info("Successfully enrolled via HTTP and stored identity and mesh config.")
+ return nil
+}
- var lastAuthErr error
+// joinRouters is the handshake that follows an enrollment: every router
+// the control plane answered with is dialled at once and the node is in
+// the mesh when one admits it, retried as at Start, since the routers are
+// busiest exactly when a fleet enrolls.
+func (n *AgentMeshNode) joinRouters(ctx context.Context, addrs []string) error {
+ parsed := make([]multiaddr.Multiaddr, 0, len(addrs))
for _, addrStr := range addrs {
addr, err := multiaddr.NewMultiaddr(addrStr)
if err != nil {
logger.Warnf("Failed to parse router address from response: %v", err)
continue
}
- if err := n.ConnectAndAuthWithRouter(ctx, addr); err != nil {
- logger.Warnf("Failed to connect and auth with router after enrollment: %v", err)
- lastAuthErr = err
- } else {
- logger.Info("Successfully enrolled via HTTP and stored identity and mesh config.")
- return nil
- }
+ parsed = append(parsed, addr)
}
-
- return fmt.Errorf("failed to connect and authenticate with any router after HTTP enrollment (last error: %v)", lastAuthErr)
+ routers := routerPeers(ctx, parsed)
+ if len(routers) == 0 {
+ return fmt.Errorf("no router peer behind %v", addrs)
+ }
+ _, err := n.admitByRouters(ctx, routers)
+ return err
}
// EnrollBootstrap enrolls the node with the control plane using a pre-shared bootstrap token.
@@ -417,26 +426,8 @@ func (n *AgentMeshNode) EnrollBootstrap(ctx context.Context, controlPlaneURL str
if len(enrollResp.RouterAddresses) == 0 {
return fmt.Errorf("failed to connect and authenticate after bootstrap enrollment: control plane returned no router addresses")
}
-
- var lastAuthErr error
- var authed bool
- for _, addrStr := range enrollResp.RouterAddresses {
- addr, err := multiaddr.NewMultiaddr(addrStr)
- if err != nil {
- logger.Warnf("Failed to parse router address: %v", err)
- continue
- }
- if err := n.ConnectAndAuthWithRouter(ctx, addr); err != nil {
- logger.Warnf("Failed to connect and auth with router: %v", err)
- lastAuthErr = err
- } else {
- authed = true
- break
- }
- }
-
- if !authed {
- return fmt.Errorf("failed to connect/auth with router after bootstrap enrollment (last error: %v)", lastAuthErr)
+ if err := n.joinRouters(ctx, enrollResp.RouterAddresses); err != nil {
+ return fmt.Errorf("failed to connect/auth with router after bootstrap enrollment: %w", err)
}
logger.Info("Successfully enrolled via Bootstrap token and joined mesh.")
diff --git a/internal/node/node.go b/internal/node/node.go
index 345707bc..97728608 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -687,12 +687,9 @@ func (n *AgentMeshNode) Start(ctx context.Context) error {
n.services.reprovideNow = n.triggerReprovide
n.applyPendingEgress(ctx)
- authenticated, fatalAuthErr := n.authWithRouters(ctx, staticRelays)
- if len(staticRelays) > 0 && !authenticated {
- if fatalAuthErr != nil {
- return fmt.Errorf("fatal auth failure: %w", fatalAuthErr)
- }
- return fmt.Errorf("failed to authenticate with any router: all connection attempts failed")
+ authenticated, err := n.admitByRouters(ctx, staticRelays)
+ if err != nil {
+ return err
}
if authenticated {
@@ -996,6 +993,40 @@ dhtLoop:
return nil
}
+// admitByRouters runs authWithRouters until a router admits this node, with
+// RouterRedialDelay doubling between attempts, up to RouterAuthAttempts of
+// them. A router that is up but slow, as every router is while a fleet
+// joins through it, fails the first handshake on its deadline and admits
+// the node on the next; without the retry a member that had just enrolled
+// exited at that first deadline. A router that refuses the credential ends
+// it at once, and no router at all is reported as before. authenticated is
+// false with a nil error only when there are no routers to admit the node.
+func (n *AgentMeshNode) admitByRouters(ctx context.Context, routers []peer.AddrInfo) (authenticated bool, err error) {
+ if len(routers) == 0 {
+ return false, nil
+ }
+ delay := n.config.RouterRedialDelay
+ for attempt := 1; ; attempt++ {
+ authenticated, fatal := n.authWithRouters(ctx, routers)
+ if authenticated {
+ return true, nil
+ }
+ if fatal != nil {
+ return false, fmt.Errorf("fatal auth failure: %w", fatal)
+ }
+ if attempt >= n.config.RouterAuthAttempts || delay <= 0 {
+ return false, errors.New("failed to authenticate with any router: all connection attempts failed")
+ }
+ logger.Warnf("[AuthN] No router admitted this node on attempt %d of %d; retrying in %s", attempt, n.config.RouterAuthAttempts, delay)
+ select {
+ case <-ctx.Done():
+ return false, ctx.Err()
+ case <-time.After(delay):
+ }
+ delay *= 2
+ }
+}
+
// authWithRouters dials every router at once and returns as soon as one has
// admitted this node, or once all have refused. A router that is down costs
// its dial timeout; paid up front and one router after another, that was
diff --git a/internal/node/options.go b/internal/node/options.go
index e6032a64..91d76e5d 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -35,6 +35,11 @@ const (
// connection monitor's own interval takes over.
DefaultRouterRedialDelay = 2 * time.Second
DefaultRouterRedialAttempts = 6
+ // DefaultRouterAuthAttempts with DefaultRouterRedialDelay doubling
+ // between them waits 14 s in all beyond the dials themselves: enough
+ // for a router that is busy admitting a fleet, short enough that a
+ // member with no router at all still reports it within a minute.
+ DefaultRouterAuthAttempts = 4
// DefaultSocketName is the local API socket the node creates in its data directory.
DefaultSocketName = "agentmesh.sock"
)
@@ -92,6 +97,10 @@ type Options struct {
// connection monitor.
RouterRedialDelay time.Duration
RouterRedialAttempts int
+ // RouterAuthAttempts is how many times Start, and the handshake after
+ // an enrollment, dial the routers before giving up, RouterRedialDelay
+ // doubling between attempts. Zero uses the default.
+ RouterAuthAttempts int
// BiscuitTimeout bounds Datalog evaluation when verifying biscuit tokens.
BiscuitTimeout time.Duration
// DHT Options
@@ -172,6 +181,9 @@ func (o *Options) Default() {
if o.RouterRedialAttempts <= 0 {
o.RouterRedialAttempts = DefaultRouterRedialAttempts
}
+ if o.RouterAuthAttempts <= 0 {
+ o.RouterAuthAttempts = DefaultRouterAuthAttempts
+ }
if o.Reachability == "" {
o.Reachability = ReachabilityPrivate
}
diff --git a/internal/node/router_auth_dedupe_test.go b/internal/node/router_auth_dedupe_test.go
index 7942d1e3..66d420bc 100644
--- a/internal/node/router_auth_dedupe_test.go
+++ b/internal/node/router_auth_dedupe_test.go
@@ -23,6 +23,7 @@ import (
"net"
"net/http"
"net/http/httptest"
+ "strings"
"sync/atomic"
"testing"
"time"
@@ -51,6 +52,9 @@ type fakeRouter struct {
cpPub ed25519.PublicKey
cpPriv ed25519.PrivateKey
mint func(peerID, role string) []byte
+ // admit is the handler installed on AuthProtocolID, for a test that
+ // wraps it.
+ admit network.StreamHandler
}
func newFakeRouter(t *testing.T, listenAddrs ...string) *fakeRouter {
@@ -89,7 +93,7 @@ func newFakeRouter(t *testing.T, listenAddrs ...string) *fakeRouter {
var handshakes atomic.Int32
var lastBiscuit atomic.Pointer[[]byte]
- h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {
+ admit := func(s network.Stream) {
defer func() { _ = s.Close() }()
handshakes.Add(1)
reader := msgio.NewVarintReaderSize(s, 1024*64)
@@ -105,7 +109,8 @@ func newFakeRouter(t *testing.T, listenAddrs ...string) *fakeRouter {
reader.ReleaseMsg(msg)
data, _ := proto.Marshal(&api.AuthResponse{Success: true, Biscuit: routerBiscuit})
_ = msgio.NewVarintWriter(s).WriteMsg(data)
- })
+ }
+ h.SetStreamHandler(api.AuthProtocolID, admit)
var routerAddrs []multiaddr.Multiaddr
for _, a := range h.Addrs() {
@@ -116,7 +121,7 @@ func newFakeRouter(t *testing.T, listenAddrs ...string) *fakeRouter {
routerAddrs = append(routerAddrs, ma)
}
- return &fakeRouter{h: h, routerAddrs: routerAddrs, handshakes: &handshakes, lastBiscuit: &lastBiscuit, cpPub: cpPub, cpPriv: cpPriv, mint: mint}
+ return &fakeRouter{h: h, routerAddrs: routerAddrs, handshakes: &handshakes, lastBiscuit: &lastBiscuit, cpPub: cpPub, cpPriv: cpPriv, mint: mint, admit: admit}
}
// startNode brings up a node enrolled against this router and authenticated.
@@ -415,6 +420,90 @@ func TestStartIsReadyOnceOneRouterAdmitsTheNode(t *testing.T) {
}
}
+// A router that is up but too busy to answer a handshake before its
+// deadline, as every router is while a fleet joins through it, admits the
+// node on the next attempt. Start used to give up at that first deadline,
+// and a member that had just enrolled exited.
+func TestStartRetriesABusyRouter(t *testing.T) {
+ router := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ // The first handshake is swallowed: the stream is held open and never
+ // answered, so the node's deadline on it fires.
+ var busy atomic.Bool
+ busy.Store(true)
+ router.h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {
+ if busy.CompareAndSwap(true, false) {
+ <-time.After(2 * time.Second)
+ _ = s.Reset()
+ return
+ }
+ router.admit(s)
+ })
+
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = store.Close() })
+ privKey := GetOrGenerateKey(store)
+ pid, _ := peer.IDFromPrivateKey(privKey)
+ if err := store.SaveIdentity(router.mint(pid.String(), api.RoleNode)); err != nil {
+ t.Fatal(err)
+ }
+ node, err := NewAgentMeshNode(Options{
+ PrivKey: privKey,
+ Store: store,
+ ControlPlanePubKey: router.cpPub,
+ RouterAddrs: router.routerAddrs,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ AllowLoopback: true,
+ RouterConnectTimeout: time.Second,
+ RouterRedialDelay: 100 * time.Millisecond,
+ RouterAuthAttempts: 3,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = node.Teardown() })
+
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ if err := node.Start(ctx); err != nil {
+ t.Fatalf("Start against a router that was busy once: %v", err)
+ }
+ if !node.IsConnected() {
+ t.Fatal("Start returned without an authenticated router")
+ }
+ if busy.Load() {
+ t.Fatal("the router never saw the handshake it was to swallow")
+ }
+ if got := router.handshakes.Load(); got != 1 {
+ t.Fatalf("router admitted %d handshakes, want 1 after the swallowed one", got)
+ }
+
+ // With no router willing at all, Start still reports it, after the
+ // configured attempts.
+ fresh, err := NewAgentMeshNode(Options{
+ PrivKey: privKey,
+ Store: store,
+ ControlPlanePubKey: router.cpPub,
+ RouterAddrs: router.routerAddrs,
+ ListenAddrs: []string{"/ip4/127.0.0.1/tcp/0"},
+ AllowLoopback: true,
+ RouterConnectTimeout: 500 * time.Millisecond,
+ RouterRedialDelay: 50 * time.Millisecond,
+ RouterAuthAttempts: 2,
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = fresh.Teardown() })
+ router.h.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) { _ = s.Reset() })
+ err = fresh.Start(ctx)
+ if err == nil || !strings.Contains(err.Error(), "all connection attempts failed") {
+ t.Fatalf("Start with no router admitting: err = %v, want all connection attempts failed", err)
+ }
+}
+
// Three entries for one router, as a control plane hands out when a dnsaddr
// resolves to every address behind the name: one peer, one dial, one
// handshake, and one static relay.
diff --git a/site/content/docs/reference/agentmesh-node.md b/site/content/docs/reference/agentmesh-node.md
index 105fcdcf..619104cb 100644
--- a/site/content/docs/reference/agentmesh-node.md
+++ b/site/content/docs/reference/agentmesh-node.md
@@ -63,7 +63,7 @@ bootstrap token, `join` runs an interactive OIDC login.
| `--offline-access` | `false` | Request an OIDC refresh token, so that the node can re-enroll on its own when its credential expires. |
| `--allow-loopback` | `false` | Advertise and dial loopback and link-local addresses. For several nodes on one host. |
| `--announce-private` | `true` | Advertise RFC 1918 and ULA addresses. Turn off when peers are only reachable through a relay. |
-| `--router-connect-timeout` | `5s` | Dial timeout per router. |
+| `--router-connect-timeout` | `5s` | Dial timeout per router. The node dials every router at once and is in the mesh when the first admits it. When none does within the timeout, as happens while a router is busy admitting a fleet, it dials them again after 2 s, then 4 s, then 8 s, and reports the failure after the fourth attempt. |
Labels declared in the configuration file are sent with the enrollment.
Running `join` for a node that already holds a credential for the same