Skip to content

Agent Mesh

router: a trim closes the connections that never authenticated first

Antonio Ojea2529ff841c38
diff --git a/internal/router/goaway.go b/internal/router/goaway.go
index 9364c7ee..0c71da57 100644
--- a/internal/router/goaway.go
+++ b/internal/router/goaway.go
@@ -100,7 +100,7 @@ send:
}
wg.Wait()
for _, p := range peers {
- r.authenticatedPeers.Delete(p)
+ r.forgetPeer(p)
}
select {
case <-time.After(goAwayGrace):
diff --git a/internal/router/goaway_test.go b/internal/router/goaway_test.go
index b98515e3..24924c09 100644
--- a/internal/router/goaway_test.go
+++ b/internal/router/goaway_test.go
@@ -24,6 +24,7 @@ import (
"github.com/libp2p/go-libp2p/core/host"
"github.com/libp2p/go-libp2p/core/network"
"github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/p2p/net/connmgr"
"github.com/libp2p/go-msgio"
"google.golang.org/protobuf/proto"
@@ -182,3 +183,74 @@ func TestDrainMembersSaysDraining(t *testing.T) {
}
}
}
+
+// Past the high watermark the connection manager closes connections, lowest
+// tag first: a member that authenticated carries a tag, a stranger none, so
+// the strangers go and the members stay; a peer router is protected and
+// stays whatever its age. The trim counts protected connections out, so
+// with low 1 and three candidates it closes two.
+func TestTrimTakesUnauthenticatedConnectionsFirst(t *testing.T) {
+ cm, err := connmgr.NewConnManager(1, 3, connmgr.WithGracePeriod(0))
+ if err != nil {
+ t.Fatal(err)
+ }
+ h, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"), libp2p.ConnectionManager(cm))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = h.Close() })
+ r := &Router{Host: h}
+
+ newPeer := func() host.Host {
+ p, err := libp2p.New(libp2p.ListenAddrStrings("/ip4/127.0.0.1/tcp/0"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = p.Close() })
+ return p
+ }
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ connect := func(p host.Host) {
+ if err := p.Connect(ctx, peer.AddrInfo{ID: h.ID(), Addrs: h.Addrs()}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ // Oldest first, the order a trim prefers among equal tags: a stranger,
+ // then a member, then a peer router, then a second stranger that tips
+ // the count over the high watermark.
+ stranger := newPeer()
+ connect(stranger)
+ member := newPeer()
+ connect(member)
+ r.admitPeer(member.ID(), time.Now().Add(time.Hour))
+ peerRouter := newPeer()
+ connect(peerRouter)
+ cm.Protect(peerRouter.ID(), federationTag)
+ late := newPeer()
+ connect(late)
+
+ cm.TrimOpenConns(ctx)
+
+ // The closes propagate through the swarm after TrimOpenConns returns.
+ kept := 2
+ deadline := time.Now().Add(5 * time.Second)
+ for kept > 0 && time.Now().Before(deadline) {
+ kept = 0
+ for _, p := range []host.Host{stranger, late} {
+ if h.Network().Connectedness(p.ID()) == network.Connected {
+ kept++
+ }
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ if kept != 0 {
+ t.Fatalf("the trim kept %d of 2 unauthenticated connections while over the high watermark", kept)
+ }
+ if h.Network().Connectedness(member.ID()) != network.Connected {
+ t.Fatal("the trim closed the member's connection")
+ }
+ if h.Network().Connectedness(peerRouter.ID()) != network.Connected {
+ t.Fatal("the trim closed the peer router's connection")
+ }
+}
diff --git a/internal/router/router.go b/internal/router/router.go
index c81f69f5..13571b7c 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -84,6 +84,10 @@ const (
// federationTag keeps a peer router's connection out of a trim: losing
// it splits the DHT, losing a member's does not.
federationTag = "agentmesh-federation"
+ // memberTag marks the connection of a peer that authenticated; a trim
+ // orders by tag value, lowest first, so the untagged go before members.
+ memberTag = "agentmesh-member"
+ memberTagValue = 100
// DefaultDHTProviderAddrTTL is three of a node's 5-minute reprovides: a
// node that is gone drops out of discovery within it, one late reprovide
// does not. The library default (48h) keeps every past identity listed.
@@ -170,7 +174,7 @@ func (r *Router) isPeerAuthenticated(p peer.ID) bool {
switch exp := v.(type) {
case time.Time:
if !time.Now().Before(exp) {
- r.authenticatedPeers.Delete(p)
+ r.forgetPeer(p)
return false
}
return true
@@ -181,6 +185,24 @@ func (r *Router) isPeerAuthenticated(p peer.ID) bool {
}
}
+// admitPeer records that p proved membership until expiry and tags its
+// connection for the connection manager: a trim takes the lowest tag
+// first, so connections that never authenticated go before members.
+func (r *Router) admitPeer(p peer.ID, expiry time.Time) {
+ r.authenticatedPeers.Store(p, expiry)
+ if r.Host != nil {
+ r.Host.ConnManager().TagPeer(p, memberTag, memberTagValue)
+ }
+}
+
+// forgetPeer is the reverse of admitPeer.
+func (r *Router) forgetPeer(p peer.ID) {
+ r.authenticatedPeers.Delete(p)
+ if r.Host != nil {
+ r.Host.ConnManager().UntagPeer(p, memberTag)
+ }
+}
+
// AdmittedUntil reports when this router's admission of p lapses: the
// expiry of the biscuit p last presented on the auth handshake. A peer that
// never authenticated, or whose admission has lapsed, is not admitted.
@@ -1364,7 +1386,7 @@ func (r *Router) HandleAuthHandshake(s network.Stream) {
return
}
- r.authenticatedPeers.Store(remotePeer, expiry)
+ r.admitPeer(remotePeer, expiry)
authHandshakesTotal.WithLabelValues(handshakeOK).Inc()
logger.Infof("[AuthN] Successfully authenticated peer %s", remotePeer)
@@ -1441,7 +1463,7 @@ func (r *Router) performMutualAuth(s network.Stream) error {
return fmt.Errorf("failed to extract peer router biscuit expiry: %w", err)
}
- r.authenticatedPeers.Store(remotePeer, expiry)
+ r.admitPeer(remotePeer, expiry)
return nil
}
diff --git a/site/content/docs/reference/router.md b/site/content/docs/reference/router.md
index 84470edf..67621ff5 100644
--- a/site/content/docs/reference/router.md
+++ b/site/content/docs/reference/router.md
@@ -42,7 +42,7 @@ replica count.
| `--lease-renew-interval` | `300s` | How often the lease is renewed. Must be well below the control plane's `--lease-duration`. |
| `--allow-loopback` | `false` | Announce and accept loopback and link-local addresses. For a router and nodes on one host. |
| `--conns-per-source-ip` | a quarter of `--high-watermark` (`1000`) | Inbound connections accepted per source address. One address can hold at most this share of the router's connection budget, so filling a router takes at least four addresses. Members behind a NAT or a cluster's SNAT share one address and each holds one connection per router, two while enrolling. |
-| `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | Connection limits. Every member holds one connection, so the high mark is how many members the router holds. At the high mark the router sends members away (see below) until it is back at the low mark, so the gap between the two is what one shedding moves; the default moves 15%. The connection manager closes connections above the high mark as well, for peers that do not take the message. |
+| `--low-watermark`, `--high-watermark` | 15% below the high mark (`3400`), `4000` | Connection limits. Every member holds one connection, so the high mark is how many members the router holds. At the high mark the router sends members away (see below) until it is back at the low mark, so the gap between the two is what one shedding moves; the default moves 15%. The connection manager closes connections above the high mark as well, oldest first among those that never authenticated (a control plane publisher, a member attached elsewhere running a DHT query, a stranger), then members; connections to the other routers are never closed. |
| `--dht-provider-addr-ttl` | `15m` | How long a service announcement lives after a node last made it. Nodes re-announce every 5 minutes, so a node that is gone drops out of discovery within this time. `0` keeps the default. |
| `--dht-max-record-age` | library default | DHT value record lifetime. |
| `--relay-limit-duration`, `--relay-limit-data` | `1h`, `0` | Caps on each relayed connection: lifetime, and bytes per direction (`512MiB`, `1GB`). The relay cuts the connection when either is reached. `0` means no limit. |