Skip to content

Agent Mesh

router: record where a member that reserved with it is reached

Antonio Ojea85bc5d15f18b
diff --git a/internal/router/config.go b/internal/router/config.go
index 8cb6dfa2..51e21f66 100644
--- a/internal/router/config.go
+++ b/internal/router/config.go
@@ -21,6 +21,7 @@ import (
"github.com/google/agentmesh/api"
"github.com/google/agentmesh/internal/identity"
+ "github.com/multiformats/go-multiaddr"
)
// DefaultShutdownLeaseTTL is the CLI default for Options.ShutdownLeaseTTL:
@@ -167,5 +168,10 @@ func (o *Options) Validate() error {
if err := api.ValidateLabels(o.Labels); err != nil {
return fmt.Errorf("labels: %w", err)
}
+ for _, s := range o.ExternalAddrs {
+ if _, err := multiaddr.NewMultiaddr(s); err != nil {
+ return fmt.Errorf("external address %q: %w", s, err)
+ }
+ }
return api.ValidateControlPlaneTransport(o.ControlPlaneURL, o.AllowInsecureControlPlane)
}
diff --git a/internal/router/reservation_addrs_test.go b/internal/router/reservation_addrs_test.go
new file mode 100644
index 00000000..6bdc2a27
--- /dev/null
+++ b/internal/router/reservation_addrs_test.go
@@ -0,0 +1,140 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package router
+
+import (
+ "context"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/google/agentmesh/api"
+ "github.com/libp2p/go-libp2p"
+ dht "github.com/libp2p/go-libp2p-kad-dht"
+ "github.com/libp2p/go-libp2p/core/peer"
+ "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/client"
+ "github.com/libp2p/go-libp2p/p2p/protocol/circuitv2/relay"
+ "github.com/multiformats/go-multiaddr"
+)
+
+// A member that reserved with a router is reached through it, and a DHT
+// lookup for the member that reaches that router must say so: the member
+// told its other routers through identify, but identify ran here before
+// the reservation, and an SDK member listens on nothing else. The router
+// records the circuit address through itself when it grants the
+// reservation, and a peer that queries it finds the member there.
+func TestRouterRecordsWhereAReservedMemberIsReached(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+
+ relayHost := newLoopbackHost(t, libp2p.DisableRelay())
+ r := &Router{Host: relayHost, config: Options{AllowLoopback: true}}
+ if _, err := relay.New(relayHost, relay.WithACL(&relayACL{r: r})); err != nil {
+ t.Fatal(err)
+ }
+ routerDHT, err := dht.New(relayHost, dht.Mode(dht.ModeServer), dht.ProtocolPrefix(api.DHTProtocolPrefix))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = routerDHT.Close() }()
+ relayInfo := peer.AddrInfo{ID: relayHost.ID(), Addrs: relayHost.Addrs()}
+
+ // Like an SDK member: dials out, listens on nothing.
+ member, err := libp2p.New(libp2p.EnableRelay(), libp2p.NoListenAddrs)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = member.Close() }()
+ if err := member.Connect(ctx, relayInfo); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := client.Reserve(ctx, member, relayInfo); err == nil {
+ t.Fatal("a reservation was granted before the member authenticated")
+ }
+ if got := relayHost.Peerstore().Addrs(member.ID()); slices.ContainsFunc(got, hasCircuit) {
+ t.Fatalf("a refused reservation left a circuit address: %v", got)
+ }
+
+ r.authenticatedPeers.Store(member.ID(), true)
+ if _, err := client.Reserve(ctx, member, relayInfo); err != nil {
+ t.Fatal(err)
+ }
+ want := relayHost.Addrs()[0].String() + "/p2p/" + relayHost.ID().String() + "/p2p-circuit"
+ got := relayHost.Peerstore().Addrs(member.ID())
+ if !slices.ContainsFunc(got, func(ma multiaddr.Multiaddr) bool { return ma.String() == want }) {
+ t.Fatalf("router records %v for the member, want %s among them", got, want)
+ }
+
+ // A peer whose lookup reaches the router learns the address. kad-dht
+ // answers FindPeer with "not found" for a peer it could not dial during
+ // the lookup, which this caller cannot (it has no relay transport), but
+ // the lookup leaves what it learned in the caller's peerstore, and the
+ // node reads that; see preparePeerAddrs.
+ caller := newLoopbackHost(t, libp2p.DisableRelay())
+ callerDHT, err := dht.New(caller, dht.Mode(dht.ModeClient), dht.ProtocolPrefix(api.DHTProtocolPrefix))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer func() { _ = callerDHT.Close() }()
+ if err := caller.Connect(ctx, relayInfo); err != nil {
+ t.Fatal(err)
+ }
+ for callerDHT.RoutingTable().Size() == 0 {
+ select {
+ case <-ctx.Done():
+ t.Fatal("the router never entered the caller's routing table")
+ case <-time.After(20 * time.Millisecond):
+ }
+ }
+ _, _ = callerDHT.FindPeer(ctx, member.ID())
+ learned := caller.Peerstore().Addrs(member.ID())
+ if len(learned) != 1 || learned[0].String() != want {
+ t.Errorf("the lookup left %v in the caller's peerstore, want [%s]", learned, want)
+ }
+
+ // The address leaves with the member.
+ r.forgetPeer(member.ID())
+ if got := relayHost.Peerstore().Addrs(member.ID()); slices.ContainsFunc(got, hasCircuit) {
+ t.Errorf("a forgotten member still has a circuit address: %v", got)
+ }
+}
+
+func hasCircuit(ma multiaddr.Multiaddr) bool {
+ return strings.HasSuffix(ma.String(), "/p2p-circuit")
+}
+
+// The circuit addresses follow the addresses the router leases: the
+// configured external ones when set, and a malformed one is a start-up
+// error rather than an address silently dropped from the lease.
+func TestRouterCircuitAddrsFollowTheExternalAddrs(t *testing.T) {
+ h := newLoopbackHost(t, libp2p.DisableRelay())
+ r := &Router{Host: h, config: Options{ExternalAddrs: []string{"/dns4/router.example/tcp/4501", "/ip4/203.0.113.9/udp/4501/quic-v1"}}}
+ var got []string
+ for _, ma := range r.circuitAddrs() {
+ got = append(got, ma.String())
+ }
+ suffix := "/p2p/" + h.ID().String() + "/p2p-circuit"
+ want := []string{"/dns4/router.example/tcp/4501" + suffix, "/ip4/203.0.113.9/udp/4501/quic-v1" + suffix}
+ if !slices.Equal(got, want) {
+ t.Errorf("circuitAddrs = %v, want %v", got, want)
+ }
+
+ o := Options{ControlPlaneURL: "http://127.0.0.1:8080", ExternalAddrs: []string{"203.0.113.9:4501"}}
+ o.Default()
+ if err := o.Validate(); err == nil || !strings.Contains(err.Error(), "external address") {
+ t.Fatalf("malformed external address: err = %v, want rejection", err)
+ }
+}
diff --git a/internal/router/router.go b/internal/router/router.go
index a4706d60..492a0f4c 100644
--- a/internal/router/router.go
+++ b/internal/router/router.go
@@ -140,8 +140,59 @@ func (a *relayACL) AllowReserve(p peer.ID, addr multiaddr.Multiaddr) bool {
ok := a.r.isPeerAuthenticated(p)
if !ok {
logger.Debugf("[Relay] Rejecting reservation for %s: not authenticated", p)
+ return false
}
- return ok
+ a.r.recordReservation(p)
+ return true
+}
+
+// recordReservation puts the addresses p is reached at through this router
+// in the peerstore, so a DHT lookup for p that reaches this router gets
+// them. A member tells its other routers about its reservations through
+// identify, but not the router it reserved with: identify ran before the
+// reservation, and an SDK member listens on nothing else. The entries last
+// as long as the reservation, which the member renews through here;
+// identify, which rewrites a peer's connected-TTL addresses whenever the
+// peer announces new ones, leaves that TTL alone. forgetPeer drops them.
+func (r *Router) recordReservation(p peer.ID) {
+ if r.Host == nil {
+ return
+ }
+ r.Host.Peerstore().AddAddrs(p, r.circuitAddrs(), r.reservationTTL())
+}
+
+// reservationTTL is the relay's reservation lifetime, which the recorded
+// circuit addresses share.
+func (r *Router) reservationTTL() time.Duration {
+ return relay.DefaultResources().ReservationTTL
+}
+
+// advertisedAddrs are the addresses the router leases to the control plane
+// and is reached at: the configured external ones, else what the host
+// listens on.
+func (r *Router) advertisedAddrs() []multiaddr.Multiaddr {
+ if len(r.config.ExternalAddrs) == 0 {
+ return r.Host.Addrs()
+ }
+ addrs := make([]multiaddr.Multiaddr, 0, len(r.config.ExternalAddrs))
+ for _, s := range r.config.ExternalAddrs {
+ if ma, err := multiaddr.NewMultiaddr(s); err == nil {
+ addrs = append(addrs, ma)
+ }
+ }
+ return addrs
+}
+
+// circuitAddrs are advertisedAddrs with /p2p/<router>/p2p-circuit appended:
+// where a peer that reserved with this router is reached.
+func (r *Router) circuitAddrs() []multiaddr.Multiaddr {
+ via := multiaddr.StringCast("/p2p/" + r.Host.ID().String() + "/p2p-circuit")
+ advertised := r.advertisedAddrs()
+ addrs := make([]multiaddr.Multiaddr, 0, len(advertised))
+ for _, a := range advertised {
+ addrs = append(addrs, a.Encapsulate(via))
+ }
+ return addrs
}
func (a *relayACL) AllowConnect(src peer.ID, srcAddr multiaddr.Multiaddr, dest peer.ID) bool {
@@ -197,11 +248,14 @@ func (r *Router) admitPeer(p peer.ID, expiry time.Time) {
}
}
-// forgetPeer is the reverse of admitPeer.
+// forgetPeer is the reverse of admitPeer, and drops the circuit addresses
+// recordReservation put in the peerstore: the relay drops the reservation
+// with the connection.
func (r *Router) forgetPeer(p peer.ID) {
r.authenticatedPeers.Delete(p)
if r.Host != nil {
r.Host.ConnManager().UntagPeer(p, memberTag)
+ r.Host.Peerstore().UpdateAddrs(p, r.reservationTTL(), 0)
}
}
@@ -512,7 +566,7 @@ func (r *Router) Start() (retErr error) {
DisconnectedF: func(n network.Network, c network.Conn) {
p := c.RemotePeer()
if len(hostNode.Network().ConnsToPeer(p)) == 0 {
- r.authenticatedPeers.Delete(p)
+ r.forgetPeer(p)
}
},
})
@@ -948,7 +1002,7 @@ func (r *Router) listenForControlPlaneEvents(ctx context.Context) {
if event.PeerId != "" {
if bannedPeer, err := peer.Decode(event.PeerId); err == nil {
logger.Infof("[Router Event] Received BANNED event for peer %s, evicting from authenticated peers and adding to blocklist", bannedPeer)
- r.authenticatedPeers.Delete(bannedPeer)
+ r.forgetPeer(bannedPeer)
// The timestamp is what stops an /info answer that was
// already in flight from undoing this (see
// reconcileBannedPeers).
@@ -1051,14 +1105,8 @@ func (r *Router) sendLease(ctx context.Context, ttl *time.Duration, attempts int
}
var addrs []string
- if len(r.config.ExternalAddrs) > 0 {
- for _, addr := range r.config.ExternalAddrs {
- addrs = append(addrs, addr+"/p2p/"+r.Host.ID().String())
- }
- } else {
- for _, addr := range r.Host.Addrs() {
- addrs = append(addrs, addr.String()+"/p2p/"+r.Host.ID().String())
- }
+ for _, addr := range r.advertisedAddrs() {
+ addrs = append(addrs, addr.String()+"/p2p/"+r.Host.ID().String())
}
var connectedPeers []string
@@ -1246,7 +1294,7 @@ func (r *Router) reconcileBannedPeers(peerIDs []string, fetchedAt time.Time) {
}
logger.Infof("[Federation] Peer %s is banned, adding to blocklist", p)
r.bannedPeers.Store(p, fetchedAt)
- r.authenticatedPeers.Delete(p)
+ r.forgetPeer(p)
if r.Host != nil {
_ = r.Host.Network().ClosePeer(p)
}
diff --git a/site/content/docs/concepts/networking.md b/site/content/docs/concepts/networking.md
index 8e24f0c7..1851bfe5 100644
--- a/site/content/docs/concepts/networking.md
+++ b/site/content/docs/concepts/networking.md
@@ -76,7 +76,10 @@ one when the NATs allow it.
Routers are the fixed points. A node learns their addresses, labels and
load from the control plane's `/info` at enrollment and again on a
schedule, holds two of them with a relay reservation on each, and replaces
-one that drops or that sends it away. A call to a peer reached through a
+one that drops or that sends it away. A router that grants a reservation
+records the member's address through itself, so a DHT lookup for the
+member that reaches that router learns where it is reached; an SDK member
+listens on nothing else. A call to a peer reached through a
third router goes through that router: the caller runs the credential
handshake there first, since a router relays only between peers it has
verified. The DHT and the relay are closed to anyone who is not enrolled.