Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
router: close a connection that does not authenticate within a deadline
Antonio Ojeadbc920b0d0a6
diff --git a/api/datalog.go b/api/datalog.goindex 88d33ae1..40eb618b 100644--- a/api/datalog.go+++ b/api/datalog.go@@ -121,6 +121,9 @@ const (// Standard role valuesRoleRouter = "mesh:role:router"RoleNode = "mesh:role:node"+ // RoleControlPlane is the role the control plane's own peer presents to+ // routers when it connects to publish events.+ RoleControlPlane = "mesh:role:control-plane"// FactUser defines the subject (username/userID) claim extracted from the OIDC token.// Contains: biscuit.String(username)diff --git a/cmd/agentmesh-one/main.go b/cmd/agentmesh-one/main.goindex 2e18b6a0..d74ca8e5 100644--- a/cmd/agentmesh-one/main.go+++ b/cmd/agentmesh-one/main.go@@ -256,6 +256,7 @@ func main() {rootCmd.Flags().DurationVar(&routerTunables.RelayLimitDuration, "router-relay-limit-duration", 0, "Relayed connection lifetime (0 keeps the component default of 1h; use e.g. 24h for longer)")rootCmd.Flags().Var(&routerTunables.RelayLimitData, "router-relay-limit-data", "Bytes relayed per direction per connection, e.g. 512MiB (0 keeps the component default: no limit)")rootCmd.Flags().DurationVar(&routerTunables.ShutdownLeaseTTL, "router-shutdown-lease-ttl", router.DefaultShutdownLeaseTTL, "Sent in the embedded router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it (0 sends nothing)")+ rootCmd.Flags().DurationVar(&routerTunables.AuthDeadline, "router-auth-deadline", router.DefaultAuthDeadline, "The embedded router closes an inbound connection whose peer has not passed the mesh handshake within this time (0 keeps every connection)")rootCmd.Flags().BoolVar(&routerAllowLoopback, "router-allow-loopback", true, "Advertise loopback addresses (disable on public deployments)")rootCmd.AddCommand(newAdminSubcommands()...)diff --git a/cmd/agentmesh-router/main.go b/cmd/agentmesh-router/main.goindex f01fe18b..b4d3c4ff 100644--- a/cmd/agentmesh-router/main.go+++ b/cmd/agentmesh-router/main.go@@ -54,6 +54,7 @@ var (relayMaxReservations intrelayMaxCircuits intshutdownLeaseTTL time.Duration+ authDeadline time.DurationlabelFlags []string)@@ -101,6 +102,7 @@ func main() {RelayMaxReservations: relayMaxReservations,RelayMaxCircuits: relayMaxCircuits,ShutdownLeaseTTL: shutdownLeaseTTL,+ AuthDeadline: authDeadline,}labels, err := api.ParseLabelFlags(labelFlags)if err != nil {@@ -151,6 +153,7 @@ func main() {rootCmd.Flags().IntVar(&relayMaxReservations, "relay-max-reservations", 0, "Members that may hold a relay reservation, which is what makes a member reachable through this router; 0 is --high-watermark")rootCmd.Flags().IntVar(&relayMaxCircuits, "relay-max-circuits", 0, fmt.Sprintf("Relayed connections one peer may hold at once through this router, as caller or as destination; 0 is %d", router.DefaultRelayMaxCircuits))rootCmd.Flags().DurationVar(&shutdownLeaseTTL, "shutdown-lease-ttl", router.DefaultShutdownLeaseTTL, "Sent in this router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it. 0 sends nothing and the lease expires on the control plane's schedule")+ rootCmd.Flags().DurationVar(&authDeadline, "auth-deadline", router.DefaultAuthDeadline, "Close an inbound connection whose peer has not passed the mesh handshake within this time; members handshake at once, a DHT query from a member attached elsewhere is done long before. 0 keeps every connection")rootCmd.Flags().StringArrayVar(&labelFlags, "label", nil, "A key=value label declared at enrollment, repeatable; the control plane signs it only if the router's role allows it, and members select or prefer routers by it (e.g. region=eu, zone=us-central1-a)")ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)diff --git a/internal/controlplane/mesh.go b/internal/controlplane/mesh.goindex 08794531..3a5cf176 100644--- a/internal/controlplane/mesh.go+++ b/internal/controlplane/mesh.go@@ -22,16 +22,19 @@ import ("sync""time"+ jwt "github.com/golang-jwt/jwt/v5""github.com/libp2p/go-libp2p"pubsub "github.com/libp2p/go-libp2p-pubsub""github.com/libp2p/go-libp2p/core/host""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-msgio""github.com/multiformats/go-multiaddr""google.golang.org/protobuf/proto""google.golang.org/protobuf/types/known/timestamppb""github.com/google/agentmesh/api"+ "github.com/google/agentmesh/internal/identity""github.com/google/agentmesh/internal/storage")@@ -104,6 +107,13 @@ type P2PMeshAdapter struct {topic *pubsub.Topicstore storage.Storemu sync.Mutex+ // admittedUntil is, per router, when the credential this publisher+ // showed it lapses. A router holds only connections that passed the+ // handshake; the publisher is a member like any other on that point.+ // Its own lock: the disconnect notification that clears it runs while+ // Close holds mu.+ admittedMu sync.Mutex+ admittedUntil map[peer.ID]time.Time// close tears down what NewMeshPublisher built; nil when the host and// topic belong to someone else (agentmesh-one's embedded router).close func() error@@ -124,6 +134,10 @@ func NewP2PMeshAdapter(h host.Host, topic *pubsub.Topic, store storage.Store) (*// RouterDialTimeout bounds each attempt to reach a leased router.const RouterDialTimeout = 10 * time.Second+// publisherCredentialTTL is how long the credential the publisher mints for+// itself lasts; it mints a new one when a quarter of that is left.+const publisherCredentialTTL = time.Hour+// DefaultMeshReconnectInterval is how often the publisher re-reads the lease// table; a router that just enrolled waits at most this long for events. It// is one query and at most a few dials per tick, so it is kept short.@@ -160,7 +174,18 @@ func NewMeshPublisher(ctx context.Context, store storage.Store, reconnect time.D_ = h.Close()return nil, fmt.Errorf("join %s: %w", api.GossipEvents, err)}- p := &P2PMeshAdapter{host: h, topic: topic, store: store}+ p := &P2PMeshAdapter{host: h, topic: topic, store: store, admittedUntil: map[peer.ID]time.Time{}}+ // A router forgets a peer when its last connection closes.+ h.Network().Notify(&network.NotifyBundle{+ DisconnectedF: func(_ network.Network, c network.Conn) {+ if len(h.Network().ConnsToPeer(c.RemotePeer())) > 0 {+ return+ }+ p.admittedMu.Lock()+ delete(p.admittedUntil, c.RemotePeer())+ p.admittedMu.Unlock()+ },+ })var wg sync.WaitGroupwg.Add(1)go func() {@@ -206,18 +231,76 @@ func (p *P2PMeshAdapter) connectRouters(ctx context.Context) {logger.Warnf("[Mesh] Skipping router lease %q: %v", r.PeerID, err)continue}- if p.host.Network().Connectedness(info.ID) == network.Connected {- continue+ if p.host.Network().Connectedness(info.ID) != network.Connected {+ dialCtx, cancel := context.WithTimeout(ctx, RouterDialTimeout)+ err = p.host.Connect(dialCtx, info)+ cancel()+ if err != nil {+ logger.Warnf("[Mesh] Router %s unreachable for event publishing: %v", info.ID, err)+ continue+ }+ logger.Infof("[Mesh] Connected to router %s", info.ID)}- dialCtx, cancel := context.WithTimeout(ctx, RouterDialTimeout)- err = p.host.Connect(dialCtx, info)- cancel()- if err != nil {- logger.Warnf("[Mesh] Router %s unreachable for event publishing: %v", info.ID, err)+ p.admittedMu.Lock()+ until, admitted := p.admittedUntil[info.ID]+ p.admittedMu.Unlock()+ if admitted && time.Until(until) > publisherCredentialTTL/4 {continue}- logger.Infof("[Mesh] Connected to router %s", info.ID)+ if err := p.authenticate(ctx, info.ID); err != nil {+ logger.Warnf("[Mesh] Router %s did not admit the publisher; it will close the connection: %v", info.ID, err)+ }+ }+}++// authenticate runs the mesh handshake with a router, presenting a+// credential the control plane mints for its own peer under its signing+// key, as it mints every member's. Without it the router would hold the+// publisher's connection as a stranger's and close it on its deadline, and+// an event published in the gap would reach nobody.+func (p *P2PMeshAdapter) authenticate(ctx context.Context, router peer.ID) error {+ priv, _, err := p.store.GetCurrentKey(ctx)+ if err != nil {+ return fmt.Errorf("signing key: %w", err)+ }+ expiry := time.Now().Add(publisherCredentialTTL)+ biscuit, _, err := identity.MintBiscuitToken(priv, jwt.MapClaims{"sub": "control-plane"}, nil, p.host.ID(), expiry, []string{api.RoleControlPlane}, nil, nil)+ if err != nil {+ return fmt.Errorf("mint credential: %w", err)}+ ctx, cancel := context.WithTimeout(ctx, RouterDialTimeout)+ defer cancel()+ s, err := p.host.NewStream(ctx, router, api.AuthProtocolID)+ if err != nil {+ return fmt.Errorf("open handshake: %w", err)+ }+ defer func() { _ = s.Close() }()+ _ = s.SetDeadline(time.Now().Add(RouterDialTimeout))+ frame, err := proto.Marshal(&api.AuthFrame{Biscuit: biscuit})+ if err != nil {+ return err+ }+ if err := msgio.NewVarintWriter(s).WriteMsg(frame); err != nil {+ return fmt.Errorf("send credential: %w", err)+ }+ reader := msgio.NewVarintReaderSize(s, 64*1024)+ msg, err := reader.ReadMsg()+ if err != nil {+ return fmt.Errorf("read answer: %w", err)+ }+ defer reader.ReleaseMsg(msg)+ var resp api.AuthResponse+ if err := proto.Unmarshal(msg, &resp); err != nil {+ return fmt.Errorf("decode answer: %w", err)+ }+ if !resp.GetSuccess() {+ return fmt.Errorf("refused: %s", resp.GetError())+ }+ p.admittedMu.Lock()+ p.admittedUntil[router] = expiry+ p.admittedMu.Unlock()+ logger.Infof("[Mesh] Router %s admitted the publisher", router)+ return nil}// routerAddrInfo is the dial target for a lease. Routers announce theirdiff --git a/internal/controlplane/mesh_test.go b/internal/controlplane/mesh_test.goindex bb77c537..e827f2e3 100644--- a/internal/controlplane/mesh_test.go+++ b/internal/controlplane/mesh_test.go@@ -19,17 +19,20 @@ import ("context""crypto/ed25519""crypto/rand"+ "fmt""testing""time"pubsub "github.com/libp2p/go-libp2p-pubsub""github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer"+ "github.com/libp2p/go-msgio""github.com/libp2p/go-libp2p""google.golang.org/protobuf/proto""github.com/google/agentmesh/api"+ "github.com/google/agentmesh/internal/identity""github.com/google/agentmesh/internal/storage")@@ -252,6 +255,33 @@ func TestMeshPublisherReachesLeasedRouter(t *testing.T) {if err != nil {t.Fatalf("failed to subscribe: %v", err)}+ // The router admits peers on the handshake; the publisher is one of them.+ // What it presents must verify under the control plane's key, be bound+ // to the publisher's peer and carry the control-plane role.+ admitted := make(chan error, 4)+ routerHost.SetStreamHandler(api.AuthProtocolID, func(s network.Stream) {+ defer func() { _ = s.Close() }()+ msg, err := msgio.NewVarintReaderSize(s, 64*1024).ReadMsg()+ if err != nil {+ admitted <- err+ return+ }+ var frame api.AuthFrame+ if err := proto.Unmarshal(msg, &frame); err != nil {+ admitted <- err+ return+ }+ if _, err := identity.VerifyBiscuitAndGetExpiry(frame.Biscuit, s.Conn().RemotePeer(), []ed25519.PublicKey{pub}, 5*time.Second); err != nil {+ admitted <- fmt.Errorf("credential: %w", err)+ return+ }+ if err := identity.VerifyBiscuitRole(frame.Biscuit, pub, api.RoleControlPlane, 5*time.Second); err != nil {+ admitted <- fmt.Errorf("role: %w", err)+ return+ }+ out, _ := proto.Marshal(&api.AuthResponse{Success: true})+ admitted <- msgio.NewVarintWriter(s).WriteMsg(out)+ })now := time.Now()lease := storage.RouterLease{PeerID: routerHost.ID().String(), LastRenewal: now, ExpiresAt: now.Add(time.Hour)}for _, a := range routerHost.Addrs() {@@ -298,6 +328,20 @@ func TestMeshPublisherReachesLeasedRouter(t *testing.T) {if peers := routerPS.ListPeers(api.GossipEvents); len(peers) != 0 {t.Errorf("publisher must not subscribe to the topic, router sees %v", peers)}+ select {+ case err := <-admitted:+ if err != nil {+ t.Fatalf("publisher's handshake: %v", err)+ }+ case <-ctx.Done():+ t.Fatal("publisher never ran the handshake with the router")+ }+ publisher.admittedMu.Lock()+ until, ok := publisher.admittedUntil[routerHost.ID()]+ publisher.admittedMu.Unlock()+ if !ok || time.Until(until) < publisherCredentialTTL/2 {+ t.Fatalf("publisher does not record its admission at the router (until %v, %v)", until, ok)+ }_, target := newTestKey(t)if err := publisher.PublishEvent(ctx, api.MeshEvent_BANNED, target.String(), nil); err != nil {diff --git a/internal/router/config.go b/internal/router/config.goindex a9058569..8cb6dfa2 100644--- a/internal/router/config.go+++ b/internal/router/config.go@@ -28,6 +28,12 @@ import (// termination grace period.const DefaultShutdownLeaseTTL = 30 * time.Second+// DefaultAuthDeadline is the CLI default for Options.AuthDeadline: a+// member's first act on a connection is the handshake, which takes well+// under a second; a DHT query from a member attached elsewhere is done in+// less.+const DefaultAuthDeadline = time.Minute+// Options holds configuration details for the agentmesh-router.type Options struct {ControlPlaneURL string@@ -67,6 +73,14 @@ type Options struct {// the control plane's schedule. The CLIs default to// DefaultShutdownLeaseTTL.ShutdownLeaseTTL time.Duration+ // AuthDeadline is how long an inbound connection may stay open without+ // its peer passing the handshake before the router closes it. Members+ // hold a connection for their session; anyone else (a member attached+ // elsewhere whose DHT client queried here, a scanner) is done with the+ // connection long before, and would otherwise count against the+ // watermarks until a trim. Zero keeps every connection. The CLIs+ // default to DefaultAuthDeadline.+ AuthDeadline time.Duration// Labels the router declares at enrollment, the same labels as a// node's: the control plane signs them only if the router's role allows// them (PolicyRole.allowed_labels) and lists them on /info, where adiff --git a/internal/router/goaway.go b/internal/router/goaway.goindex 1ba6aff0..af6f1451 100644--- a/internal/router/goaway.go+++ b/internal/router/goaway.go@@ -162,6 +162,31 @@ func (r *Router) inboundConnections() int {return n}+// enforceAuthDeadline closes an inbound connection whose peer has not+// passed the handshake AuthDeadline after it opened. A peer router is left+// alone whatever its state, as is a peer that authenticated on another+// connection.+func (r *Router) enforceAuthDeadline(c network.Conn) {+ if r.config.AuthDeadline <= 0 || c.Stat().Direction != network.DirInbound {+ return+ }+ time.AfterFunc(r.config.AuthDeadline, func() {+ if c.IsClosed() || (r.ctx != nil && r.ctx.Err() != nil) {+ return+ }+ p := c.RemotePeer()+ if r.isPeerAuthenticated(p) {+ return+ }+ if _, isRouter := r.peerRouters.Load(p); isRouter {+ return+ }+ logger.Debugf("[AuthN] Closing connection from %s: no handshake within %s", p, r.config.AuthDeadline)+ unauthenticatedClosedTotal.Inc()+ _ = c.Close()+ })+}+// shedIfOverloaded sends members away when the router holds at least its// high watermark of them: as many as take it back to the low watermark,// chosen at random so no member is always the one. Members are counted,diff --git a/internal/router/goaway_test.go b/internal/router/goaway_test.goindex cf1bac2c..89d2e44f 100644--- a/internal/router/goaway_test.go+++ b/internal/router/goaway_test.go@@ -264,3 +264,49 @@ func TestTrimTakesUnauthenticatedConnectionsFirst(t *testing.T) {t.Fatal("the trim closed the peer router's connection")}}++// A connection whose peer has not passed the handshake by the deadline is+// closed; one that has, and a peer router's, stay. A member that holds+// several connections is judged as a peer, not per connection.+func TestAuthDeadlineClosesStrangers(t *testing.T) {+ r := newGoAwayRouter(t, 100)+ r.config.AuthDeadline = 300 * time.Millisecond+ r.ctx = context.Background()+ r.Host.Network().Notify(&network.NotifyBundle{+ ConnectedF: func(_ network.Network, c network.Conn) { r.enforceAuthDeadline(c) },+ })++ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ dial := func(p host.Host) {+ if err := p.Connect(ctx, peer.AddrInfo{ID: r.Host.ID(), Addrs: r.Host.Addrs()}); err != nil {+ t.Fatal(err)+ }+ }+ stranger := newGoAwayListener(t)+ dial(stranger.host)+ member := newGoAwayListener(t)+ dial(member.host)+ r.authenticatedPeers.Store(member.host.ID(), time.Now().Add(time.Hour))+ peerRouter := newGoAwayListener(t)+ r.peerRouters.Store(peerRouter.host.ID(), true)+ dial(peerRouter.host)+ // Outbound connections are the router's own business.+ dialed := newGoAwayListener(t)+ if err := r.Host.Connect(ctx, peer.AddrInfo{ID: dialed.host.ID(), Addrs: dialed.host.Addrs()}); err != nil {+ t.Fatal(err)+ }++ deadline := time.Now().Add(5 * time.Second)+ for r.Host.Network().Connectedness(stranger.host.ID()) == network.Connected && time.Now().Before(deadline) {+ time.Sleep(20 * time.Millisecond)+ }+ if r.Host.Network().Connectedness(stranger.host.ID()) == network.Connected {+ t.Fatal("the stranger's connection survived the deadline")+ }+ for name, p := range map[string]host.Host{"member": member.host, "peer router": peerRouter.host, "dialed peer": dialed.host} {+ if r.Host.Network().Connectedness(p.ID()) != network.Connected {+ t.Errorf("the %s's connection was closed", name)+ }+ }+}diff --git a/internal/router/metrics.go b/internal/router/metrics.goindex 6cd56007..0f1e59a9 100644--- a/internal/router/metrics.go+++ b/internal/router/metrics.go@@ -77,6 +77,13 @@ var (},[]string{"reason", "result"},)++ unauthenticatedClosedTotal = promauto.NewCounter(+ prometheus.CounterOpts{+ Name: "agentmesh_router_unauthenticated_connections_closed_total",+ Help: "Inbound connections closed because the peer did not pass the handshake within --auth-deadline",+ },+ ))// Every outcome exists from the first scrape, so a rate() over one that hasdiff --git a/internal/router/router.go b/internal/router/router.goindex 13571b7c..794fa17f 100644--- a/internal/router/router.go+++ b/internal/router/router.go@@ -502,6 +502,9 @@ func (r *Router) Start() (retErr error) {// Clean authenticated status on peer disconnectionhostNode.Network().Notify(&network.NotifyBundle{+ ConnectedF: func(_ network.Network, c network.Conn) {+ r.enforceAuthDeadline(c)+ },DisconnectedF: func(n network.Network, c network.Conn) {p := c.RemotePeer()if len(hostNode.Network().ConnsToPeer(p)) == 0 {diff --git a/internal/standalone/standalone.go b/internal/standalone/standalone.goindex 46077248..230601e9 100644--- a/internal/standalone/standalone.go+++ b/internal/standalone/standalone.go@@ -160,6 +160,9 @@ type RouterTunables struct {// ShutdownLeaseTTL is the lease the embedded router asks for as it stops;// see router.Options.ShutdownLeaseTTL.ShutdownLeaseTTL time.Duration+ // AuthDeadline closes inbound connections that do not pass the handshake+ // in time; see router.Options.AuthDeadline.+ AuthDeadline time.Duration// DisallowLoopback stops advertising loopback addresses (useful on// public deployments; the default keeps local development working).DisallowLoopback bool@@ -376,6 +379,7 @@ func (s *Server) Start(ctx context.Context) (retErr error) {RelayLimitDuration: s.opts.Router.RelayLimitDuration,RelayLimitData: int64(s.opts.Router.RelayLimitData),ShutdownLeaseTTL: s.opts.Router.ShutdownLeaseTTL,+ AuthDeadline: s.opts.Router.AuthDeadline,// Single-port deployments typically sit behind a TLS-terminating// proxy (Cloud Run, L7 LBs) or NAT where every peer shares a few// source IPs; libp2p's default 8-conns-per-IP cap would throttlediff --git a/site/content/docs/guides/scalability.md b/site/content/docs/guides/scalability.mdindex 82f580f7..0101613a 100644--- a/site/content/docs/guides/scalability.md+++ b/site/content/docs/guides/scalability.md@@ -17,14 +17,14 @@ stayed through a rollout of every router and both control plane replicas.A member is one `agentmesh-node` process. It holds a session with tworouters (`--routers`), chosen from those the control plane lists, and arelay reservation on each. Through those sessions it keeps a GossipSub-subscription for control plane events. Its DHT client also keeps a-connection to the other routers, which carries no session: a router-counts members, not connections, against its watermarks, and those are the-connections it closes first when it is full. A member connects to another-member only to call it, through that member's router, where it runs the-credential handshake before asking for the circuit. At rest it uses about-55 MiB of memory, 300 goroutines and 20 file descriptors, and almost no-CPU.+subscription for control plane events. Its DHT client connects to the+other routers when it queries; a router closes such a connection a minute+later (`--auth-deadline`), since no session was opened on it, so a+router's connections are its members' and the watermarks count members.+A member connects to another member only to call it, through that+member's router, where it runs the credential handshake before asking for+the circuit. At rest it uses about 55 MiB of memory, 300 goroutines and+20 file descriptors, and almost no CPU.Joining is the expensive moment. The member enrolls at the control plane(one signed request, one database write, one credential minted), reads the@@ -44,7 +44,7 @@ them.| Bound | Where | Default | At the bound ||---|---|---|---|| Routers per member | `agentmesh-node --routers` | 2 | A member holds this many routers; one is enough to be on the mesh, the second covers a router restart. |-| Connections per router | `agentmesh-router --high-watermark`, `--low-watermark` | 4000, 15% below (3400) | At the high mark of members the router sends members away, at random, down to the low mark; they attach to another router and stay off this one for five minutes. Above the high mark of connections, the connection manager closes those that carry no session first. A mesh of N routers holds about N × 4000 / 2 members. |+| Connections per router | `agentmesh-router --high-watermark`, `--low-watermark` | 4000, 15% below (3400) | At the high mark of members the router sends members away, at random, down to the low mark; they attach to another router and stay off this one for five minutes. A mesh of N routers holds about N × 4000 / 2 members. || Connections per source address | `agentmesh-router --conns-per-source-ip` | a quarter of the high mark (1000) | The router refuses the next connection from that address. Members behind one NAT or one cluster's egress share an address. || Relay reservations per router | `agentmesh-router --relay-max-reservations` | the high mark (4000) | A member that cannot reserve is not reachable through that router. || Relayed connections per peer | `agentmesh-router --relay-max-circuits` | 1024 | A service called through the relay holds one circuit per caller; the next caller is refused. |diff --git a/site/content/docs/reference/agentmesh-one.md b/site/content/docs/reference/agentmesh-one.mdindex dbe1b099..b7f073d9 100644--- a/site/content/docs/reference/agentmesh-one.md+++ b/site/content/docs/reference/agentmesh-one.md@@ -70,6 +70,7 @@ of zero leaves the default of that component unchanged.| `--router-dht-provider-addr-ttl`, `--router-dht-max-record-age` | `--dht-provider-addr-ttl`, `--dht-max-record-age`. `0` keeps `15m` and the library default. || `--router-relay-limit-duration`, `--router-relay-limit-data` | `--relay-limit-duration`, `--relay-limit-data`. `0` keeps `1h` and no data limit, so set a long duration such as `24h` instead of an unlimited one. || `--router-shutdown-lease-ttl` | `--shutdown-lease-ttl`, default `30s`. |+| `--router-auth-deadline` | `--auth-deadline`, default `1m`. || `--router-allow-loopback` | `--allow-loopback`. Defaults to `true` here, for a laptop. Disable it on a public deployment. |## The bannerdiff --git a/site/content/docs/reference/control-plane.md b/site/content/docs/reference/control-plane.mdindex 94210a31..c1c0b75f 100644--- a/site/content/docs/reference/control-plane.md+++ b/site/content/docs/reference/control-plane.md@@ -37,7 +37,7 @@ agentmesh-control-plane admin unban --peer <id> lift a ban| `--key-grace-period` | `1h` | How long a rotated-out key stays accepted. Credentials signed by a retired key cannot be verified or refreshed. Nodes and routers must pull `/keys` well within this window (`agentmesh-node --control-plane-sync-interval`, `agentmesh-router --keys-sync-interval`). || `--lease-duration` | `15m` | How long a router lease lasts without renewal. || `--node-retention` | `720h` (30 days) | How long the record of an enrolled node is kept after its session expires. Banned nodes are kept forever. `0` keeps every record. |-| `--mesh-reconnect-interval` | `30s` | How often the event publisher re-reads the router leases and dials any router it is not connected to. |+| `--mesh-reconnect-interval` | `30s` | How often the event publisher re-reads the router leases and dials any router it is not connected to. On each router it runs the mesh handshake with a credential it mints for its own peer under the signing key, with the role `mesh:role:control-plane`, so the router holds its connection as a member's. || `--log-level` | `info` | `debug`, `info`, `warn`, `error`. `LOG_FORMAT=json` selects JSON output. |## HTTP APIdiff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.mdindex 64e2caa4..cf97c549 100644--- a/site/content/docs/reference/router.md+++ b/site/content/docs/reference/router.md@@ -42,13 +42,14 @@ replica count.| `--lease-renew-interval` | `300s` | How often the lease is renewed. Must be well below the control plane's `--lease-duration`. || `--allow-loopback` | `false` | Announce and accept loopback and link-local addresses. For a router and nodes on one host. || `--conns-per-source-ip` | a quarter of `--high-watermark` (`1000`) | Inbound connections accepted per source address. One address can hold at most this share of the router's connection budget, so filling a router takes at least four addresses. Members behind a NAT or a cluster's SNAT share one address and each holds one connection per router, two while enrolling. |-| `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | How many members the router holds: peers that passed the handshake, other routers aside. At the high mark the router sends members away (see below) until it is back at the low mark, so the gap between the two is what one shedding moves; the default moves 15%. The same numbers bound connections: above the high mark the connection manager closes connections, oldest first among those that carry no session (a member attached elsewhere whose DHT client is connected here, a control plane publisher, a stranger), then members; connections to the other routers are never closed. |+| `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | How many members the router holds: peers that passed the handshake, other routers aside. At the high mark the router sends members away (see below) until it is back at the low mark, so the gap between the two is what one shedding moves; the default moves 15%. The same numbers bound connections: above the high mark the connection manager closes connections, oldest first among those that carry no session, then members; connections to the other routers are never closed. With `--auth-deadline` on, a connection without a session lasts a minute, so the two counts stay close. || `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. || `--dht-max-record-age` | library default | DHT value record lifetime. || `--relay-limit-duration`, `--relay-limit-data` | `1h`, `0` | Caps on each relayed connection: lifetime, and bytes per direction (`512MiB`, `1GB`). The relay cuts the connection when either is reached. `0` means no limit. || `--relay-max-reservations` | `--high-watermark` (`4000`) | Members that may hold a relay reservation on this router at once, which is how many members it can reach on behalf of callers. Per source address the cap is `--conns-per-source-ip`. || `--relay-max-circuits` | `1024` | Relayed connections one peer may hold at once through this router, as caller or as destination. A service that many members call at once needs this many open circuits. || `--shutdown-lease-ttl` | `30s` | Sent in the router's last lease when it stops: how long it expects to be away, which is how long the control plane keeps listing it. The default covers a pod that restarts in place. A router that is being removed for good can send `1s`. `0` sends nothing and the lease expires on the control plane's schedule. |+| `--auth-deadline` | `1m` | How long an inbound connection may stay open without its peer passing the handshake. A member handshakes the moment it connects; a member attached elsewhere whose DHT client came to query is done in less, as is the control plane's event publisher, which presents a credential of its own. What is left at the deadline is closed, so the connections a router holds are its members' and the watermarks mean members. `0` keeps every connection. || `--metrics-addr` | off | Serve `/metrics`, `/healthz` and `/readyz` without authentication on this address. `/readyz` returns `200` once the router is enrolled and the libp2p host is up. Keep this address separate from the libp2p ports and inside the cluster. || `--log-level` | `info` | `debug`, `info`, `warn`, `error`. `LOG_FORMAT=json` selects JSON output. |@@ -116,6 +117,7 @@ The metrics address exposes, next to the Go runtime metrics:| `agentmesh_router_dht_routing_table_size` | DHT routing table size. || `agentmesh_router_auth_handshakes_total{result}`, `agentmesh_router_lease_renewals_total{result}` | Handshakes and lease renewals by outcome. || `agentmesh_router_goaway_sent_total{reason,result}` | Members sent away, by reason (`DRAINING`, `OVERLOADED`) and outcome: `ok` when the member took the message, `unsupported` when it does not speak the protocol and was closed instead, `failed` when the stream broke. A rising `OVERLOADED` count on a router that is not draining is a fleet that has outgrown its routers. |+| `agentmesh_router_unauthenticated_connections_closed_total` | Connections closed at `--auth-deadline` without a handshake. A steady rate is DHT traffic from members attached to other routers; a burst is a scanner. || `agentmesh_router_inbound_connections_refused_total` | Inbound connections refused by the per-address cap. || `libp2p_relaysvc_*` | The relay service's own counters: `reservations_total{type}` and `connections_total{type}` (`opened`, `closed`, `renewed`), requests by response status, rejections and bytes relayed. The difference between `opened` and `closed` is the number in use; read it against `agentmesh_router_relay_limit` to see how close the relay is to its budget. |diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.goindex b3147527..7602fd1f 100644--- a/tests/integration/router_attachment_test.go+++ b/tests/integration/router_attachment_test.go@@ -16,6 +16,7 @@ package integration_testimport ("bytes"+ "context""encoding/json""fmt""io"@@ -31,8 +32,11 @@ import ("time""github.com/google/agentmesh/api"+ "github.com/libp2p/go-libp2p""github.com/libp2p/go-libp2p/core/crypto"+ "github.com/libp2p/go-libp2p/core/network""github.com/libp2p/go-libp2p/core/peer"+ "github.com/multiformats/go-multiaddr")// routerFixture is one agentmesh-router the test started, with the key it@@ -44,6 +48,10 @@ type routerFixture struct {metricsAddr string}+// extraRouterArgs is appended to every router startLabelledRouter starts; a+// test sets it before starting its routers and clears it after.+var extraRouterArgs []string+// gauge reads one metric from the router's /metrics, by its full name with// labels, and fails when it is not there.func (r *routerFixture) gauge(t *testing.T, name string) float64 {@@ -101,6 +109,7 @@ func startLabelledRouter(t *testing.T, routerBin, tmpDir, name string, cpPort infor _, l := range labels {args = append(args, "--label", l)}+ args = append(args, extraRouterArgs...)cmd := exec.Command(routerBin, args...)logPath := filepath.Join(tmpDir, name+".log")logFile, err := os.Create(logPath)@@ -297,6 +306,7 @@ bindings:"--db-dsn", filepath.Join(m.tmpDir, "cp.db")+"?_pragma=journal_mode(DELETE)&_pragma=busy_timeout(5000)","--issuer", oidcURL,"--insecure-skip-tls-verify",+ "--mesh-reconnect-interval", "500ms",)if err := cpCmd.Start(); err != nil {t.Fatalf("start control plane: %v", err)@@ -470,7 +480,8 @@ func mustPeerID(t *testing.T, s string) peer.ID {// else. Three routers, three nodes each pinned to its own router, no// service and no call; after the discovery interval has ticked many times,// no node is connected to another node, and each router has authenticated-// its own node and the two other routers, nobody else. A node that sought+// its own node, the two other routers and the control plane's publisher,+// nobody else. A node that sought// out other members on its own would be connected to them, through their// routers, and would have authenticated at every router on the way, which// is the load of a member on every router rather than on two. Connections@@ -517,9 +528,10 @@ func TestIdleNodesHoldOnlyTheirRouters(t *testing.T) {}}}- // Its own node and the two peer routers.- if got := routers[s].gauge(t, "agentmesh_router_authenticated_peers"); got != 3 {- t.Errorf("router %s authenticated %v peers, want 3 (its node and two routers)", s, got)+ // Its own node, the two peer routers and the control plane's+ // event publisher.+ if got := routers[s].gauge(t, "agentmesh_router_authenticated_peers"); got != 4 {+ t.Errorf("router %s authenticated %v peers, want 4 (its node, two routers, the control plane)", s, got)}if t.Failed() {t.Logf("node %s connected peers: %v", s, connected)@@ -620,15 +632,63 @@ func TestCallerTransitsTheRouterItShares(t *testing.T) {callAcrossMesh(t, caller, "token-c", provider, "shared-tool")- // Router a authenticated the provider and the two routers; the caller- // never came. Router b has both nodes and the two routers.- if got := routerA.gauge(t, "agentmesh_router_authenticated_peers"); got != 3 {- t.Errorf("router a authenticated %v peers, want 3: the caller must transit router b, which it holds", got)+ // Router a authenticated the provider, the two routers and the control+ // plane's publisher; the caller never came. Router b has both nodes.+ if got := routerA.gauge(t, "agentmesh_router_authenticated_peers"); got != 4 {+ t.Errorf("router a authenticated %v peers, want 4: the caller must transit router b, which it holds", got)}- if got := routerB.gauge(t, "agentmesh_router_authenticated_peers"); got != 4 {- t.Errorf("router b authenticated %v peers, want 4", got)+ if got := routerB.gauge(t, "agentmesh_router_authenticated_peers"); got != 5 {+ t.Errorf("router b authenticated %v peers, want 5", got)}if t.Failed() {t.Logf("--- caller ---\n%s", caller.log())}}++// TestRouterKeepsOnlyAdmittedConnections pins what a router holds: a+// connection whose peer does not pass the handshake within --auth-deadline+// is closed, and the control plane's event publisher, which presents a+// credential of its own, is kept. A stranger's connection is gone a+// deadline after it opened; a member's and the publisher's are still there+// several deadlines later.+func TestRouterKeepsOnlyAdmittedConnections(t *testing.T) {+ extraRouterArgs = []string{"--auth-deadline", "1s"}+ t.Cleanup(func() { extraRouterArgs = nil })+ m := startAttachmentMesh(t)+ router := m.startRouter(t, "router")++ node := launchNode(t, m.nodeBin, m.nodeEnv("node"), filepath.Join(m.tmpDir, "node"), append(m.nodeArgs(),+ "--api-token-path", tokenPath(t, "token"),+ "--allow-loopback",+ )...)+ node.waitForAPI(t)++ stranger, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))+ if err != nil {+ t.Fatal(err)+ }+ t.Cleanup(func() { _ = stranger.Close() })+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ routerAddr := multiaddr.StringCast(fmt.Sprintf("/ip4/127.0.0.1/tcp/%d", router.port))+ if err := stranger.Connect(ctx, peer.AddrInfo{ID: router.peerID, Addrs: []multiaddr.Multiaddr{routerAddr}}); err != nil {+ t.Fatalf("stranger could not connect to the router: %v", err)+ }++ // Three deadlines: the stranger is gone, the publisher and the node+ // still there, and the counter says one connection was closed for it.+ time.Sleep(3 * time.Second)+ if stranger.Network().Connectedness(router.peerID) == network.Connected {+ t.Fatal("the stranger's connection survived the deadline")+ }+ if got := router.gauge(t, "agentmesh_router_unauthenticated_connections_closed_total"); got < 1 {+ t.Fatalf("closed %v unauthenticated connections, want at least the stranger's", got)+ }+ // The node and the control plane's publisher: both authenticated.+ if got := router.gauge(t, "agentmesh_router_authenticated_peers"); got != 2 {+ t.Fatalf("router authenticated %v peers, want 2 (the node and the control plane's publisher)", got)+ }+ if got := attachedRouters(t, node); !slices.Equal(got, peerIDStrings(router.peerID)) {+ t.Fatalf("node holds %v after the deadlines, want %s", got, router.peerID)+ }+}