Skip to content

Agent Mesh

node: transit one router to reach a peer, the one it shares when it can

Antonio Ojeaa503af8253fe
diff --git a/internal/node/mcp.go b/internal/node/mcp.go
index 75c3b1ec..cc02756a 100644
--- a/internal/node/mcp.go
+++ b/internal/node/mcp.go
@@ -20,6 +20,7 @@ import (
"errors"
"fmt"
"net/http"
+ "sort"
"strings"
"time"
@@ -560,64 +561,102 @@ func (n *AgentMeshNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID
}
}
+ if kept := n.selectTransit(ctx, validAddrs); len(kept) != len(validAddrs) {
+ validAddrs = kept
+ changed = true
+ }
+
if changed && len(validAddrs) > 0 {
// Clear existing addresses to prevent libp2p from hanging on dead private IPs
n.Host.Peerstore().ClearAddrs(targetPeer)
n.Host.Peerstore().AddAddrs(targetPeer, validAddrs, peerstore.TempAddrTTL)
logger.Debugf("[Discovery] Replaced addrs for %s with %d routable/circuit addrs", targetPeer, len(validAddrs))
}
+}
- n.authenticateTransitRouters(ctx, validAddrs)
+// circuitRelay is the relay a circuit address goes through, or "" for any
+// other address.
+func circuitRelay(ma multiaddr.Multiaddr) peer.ID {
+ parts := multiaddr.Split(ma)
+ if len(parts) < 2 || parts[len(parts)-1].Protocol().Code != multiaddr.P_CIRCUIT || parts[len(parts)-2].Protocol().Code != multiaddr.P_P2P {
+ return ""
+ }
+ relayID, err := peer.Decode(parts[len(parts)-2].Value())
+ if err != nil {
+ return ""
+ }
+ return relayID
}
-// authenticateTransitRouters runs the credential handshake with every
-// router the given circuit addresses go through that this node is not yet
-// authenticated to. A router relays only between peers it has admitted,
-// and a node keeps sessions with a few routers, so the destination's router
-// is often not one of them: the handshake is the one the node runs at
-// start, on the connection the circuit will use, before the circuit is
-// asked for. A relay that is not a router (a node with --enable-relay) is
-// left alone; it admits what it has handshaken with on its own terms.
-func (n *AgentMeshNode) authenticateTransitRouters(ctx context.Context, addrs []multiaddr.Multiaddr) {
- var relays []peer.ID
+// selectTransit decides which of a peer's circuit addresses to dial and
+// makes sure the node is admitted at the router behind them. A router
+// relays only between peers it has admitted, and a node keeps sessions
+// with a few routers, so the peer's routers are often not among them. One
+// transit router is enough: a router the node already holds when the peer
+// is on it too, which costs the mesh nothing; otherwise the peer's router
+// with the most room, which the node runs the handshake with, as it does
+// at start, before the circuit is asked for. Circuits through the peer's
+// other routers are dropped: each would be a handshake and a session
+// there, and a service the whole mesh calls would hold every caller at
+// every one of its routers. A relay that is not a router (a node with
+// --enable-relay) is kept and left alone; it admits what it has handshaken
+// with on its own terms. Direct addresses always pass.
+func (n *AgentMeshNode) selectTransit(ctx context.Context, addrs []multiaddr.Multiaddr) []multiaddr.Multiaddr {
+ routers := map[peer.ID]routerCandidate{}
+ for _, c := range n.routerCandidates() {
+ routers[c.ID] = c
+ }
+ var viaRouters []peer.ID
seen := map[peer.ID]bool{}
for _, ma := range addrs {
- parts := multiaddr.Split(ma)
- if len(parts) < 2 || parts[len(parts)-1].Protocol().Code != multiaddr.P_CIRCUIT || parts[len(parts)-2].Protocol().Code != multiaddr.P_P2P {
- continue
- }
- relayID, err := peer.Decode(parts[len(parts)-2].Value())
- if err != nil || seen[relayID] {
+ relay := circuitRelay(ma)
+ if _, isRouter := routers[relay]; relay == "" || !isRouter || seen[relay] {
continue
}
- seen[relayID] = true
- relays = append(relays, relayID)
+ seen[relay] = true
+ viaRouters = append(viaRouters, relay)
}
- if len(relays) == 0 {
- return
+ if len(viaRouters) == 0 {
+ return addrs
}
- routers := map[peer.ID]bool{}
- for _, c := range n.routerCandidates() {
- routers[c.ID] = true
+
+ usable := map[peer.ID]bool{}
+ for _, r := range viaRouters {
+ if n.isAuthenticatedAndConnected(r) {
+ usable[r] = true
+ }
}
- var biscuitBytes []byte
- for _, relayID := range relays {
- if !routers[relayID] || n.isAuthenticatedAndConnected(relayID) {
- continue
+ if len(usable) == 0 {
+ sort.SliceStable(viaRouters, func(i, j int) bool {
+ return routers[viaRouters[i]].loadRatio() < routers[viaRouters[j]].loadRatio()
+ })
+ biscuitBytes, err := n.loadIdentityForAuth()
+ if err != nil {
+ logger.Debugf("[AuthN] Cannot authenticate to a transit router: %v", err)
+ return addrs
}
- if biscuitBytes == nil {
- b, err := n.loadIdentityForAuth()
- if err != nil {
- logger.Debugf("[AuthN] Cannot authenticate to transit router %s: %v", relayID, err)
- return
+ for _, r := range viaRouters {
+ router := peer.AddrInfo{ID: r, Addrs: n.Host.Peerstore().Addrs(r)}
+ if err := n.authRouter(ctx, router, biscuitBytes); err != nil {
+ logger.Warnf("[AuthN] Transit router %s did not admit this node: %v", r, err)
+ continue
}
- biscuitBytes = b
+ logger.Debugf("[AuthN] Authenticated to transit router %s", r)
+ usable[r] = true
+ break
+ }
+ if len(usable) == 0 {
+ return addrs
}
- router := peer.AddrInfo{ID: relayID, Addrs: n.Host.Peerstore().Addrs(relayID)}
- if err := n.authRouter(ctx, router, biscuitBytes); err != nil {
- logger.Warnf("[AuthN] Transit router %s did not admit this node; a circuit through it will be refused: %v", relayID, err)
+ }
+
+ kept := make([]multiaddr.Multiaddr, 0, len(addrs))
+ for _, ma := range addrs {
+ relay := circuitRelay(ma)
+ if _, isRouter := routers[relay]; relay != "" && isRouter && !usable[relay] {
continue
}
- logger.Debugf("[AuthN] Authenticated to transit router %s", relayID)
+ kept = append(kept, ma)
}
+ return kept
}
diff --git a/internal/node/routers.go b/internal/node/routers.go
index 3c7aa9ec..58236ce5 100644
--- a/internal/node/routers.go
+++ b/internal/node/routers.go
@@ -43,6 +43,15 @@ type routerCandidate struct {
ConnectionLimit int
}
+// loadRatio is how full the router is, 0 to 1; 0.5 when its last lease
+// said nothing.
+func (c routerCandidate) loadRatio() float64 {
+ if c.ConnectionLimit <= 0 {
+ return 0.5
+ }
+ return float64(c.Connections) / float64(c.ConnectionLimit)
+}
+
// routerTag protects an attached router's connection from the node's own
// connection manager, which otherwise trims by age and tag value and would
// take the router before a stranger.
@@ -122,11 +131,7 @@ func selectRouters(cands []routerCandidate, k int, selector, prefer map[string]s
preferred++
}
}
- load := 0.5
- if c.ConnectionLimit > 0 {
- load = float64(c.Connections) / float64(c.ConnectionLimit)
- }
- load += rand.Float64() * 0.05
+ load := c.loadRatio() + rand.Float64()*0.05
eligible = append(eligible, scored{c, preferred, load})
}
sort.SliceStable(eligible, func(i, j int) bool {
diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.go
index 4b5baabf..b3147527 100644
--- a/tests/integration/router_attachment_test.go
+++ b/tests/integration/router_attachment_test.go
@@ -193,6 +193,32 @@ func peerIDStrings(ids ...peer.ID) []string {
return out
}
+// announcedCircuitRelays lists the relays the node publishes circuit
+// addresses through, from its own debug endpoint.
+func announcedCircuitRelays(t *testing.T, n *backgroundNode) []string {
+ t.Helper()
+ var info struct {
+ AnnouncedAddresses []string `json:"announced_addresses"`
+ }
+ if err := json.Unmarshal([]byte(debugGetWithToken(t, n.apiAddr, n.token, "/debug/network-info")), &info); err != nil {
+ t.Fatalf("decode network-info: %v", err)
+ }
+ seen := map[string]bool{}
+ var out []string
+ for _, a := range info.AnnouncedAddresses {
+ if i := strings.Index(a, "/p2p-circuit"); i > 0 {
+ parts := strings.Split(a[:i], "/p2p/")
+ relay := parts[len(parts)-1]
+ if !seen[relay] {
+ seen[relay] = true
+ out = append(out, relay)
+ }
+ }
+ }
+ slices.Sort(out)
+ return out
+}
+
// waitForEqual polls get until it returns want.
func waitForEqual(t *testing.T, what string, get func() []string, want []string, timeout time.Duration) {
t.Helper()
@@ -235,7 +261,7 @@ func startAttachmentMesh(t *testing.T) *attachmentMesh {
- name: %s
allowed_services: []
allowed_targets: ["*"]
- allowed_labels: ["site=*"]
+ allowed_labels: ["site=*", "pa=*", "pb=*"]
- name: %s
allowed_services: ["mcp://*"]
allowed_targets: ["*"]
@@ -354,52 +380,10 @@ func TestNodesOnDifferentRoutersReachEachOther(t *testing.T) {
waitForEqual(t, "routers attached by node A", func() []string { return attachedRouters(t, nodeA) }, peerIDStrings(routerA.peerID), 10*time.Second)
waitForEqual(t, "routers attached by node B", func() []string { return attachedRouters(t, nodeB) }, peerIDStrings(routerB.peerID), 10*time.Second)
- // B finds A's service: the DHT spans both routers.
- client := &http.Client{Timeout: 5 * time.Second}
- discover := func() string {
- req, _ := http.NewRequest(http.MethodGet, "http://"+nodeB.apiAddr+"/mesh/service/discover?type=mcp&name=site-tool", nil)
- req.Header.Set(api.HeaderMeshAuthentication, "Bearer token-b")
- resp, err := client.Do(req)
- if err != nil {
- return ""
- }
- defer func() { _ = resp.Body.Close() }()
- body, _ := io.ReadAll(resp.Body)
- return string(body)
- }
- deadline := time.Now().Add(20 * time.Second)
- for !strings.Contains(discover(), "site-tool") {
- if time.Now().After(deadline) {
- t.Fatalf("node B never discovered node A's service.\n--- node A ---\n%s\n--- node B ---\n%s", nodeA.log(), nodeB.log())
- }
- time.Sleep(300 * time.Millisecond)
- }
-
- // B calls A: the circuit goes through router A, where B is a stranger
- // until it runs the handshake there.
- callURL := fmt.Sprintf("http://%s/mesh/%s/mcp/site-tool", nodeB.apiAddr, nodeA.peerID)
- var lastStatus int
- var lastBody string
- deadline = time.Now().Add(20 * time.Second)
- for time.Now().Before(deadline) {
- req, _ := http.NewRequest(http.MethodPost, callURL, bytes.NewBufferString(`{"jsonrpc":"2.0","id":1,"method":"tools/list"}`))
- req.Header.Set("Content-Type", "application/json")
- req.Header.Set("Accept", "application/json, text/event-stream")
- req.Header.Set(api.HeaderMeshAuthentication, "Bearer token-b")
- resp, err := client.Do(req)
- if err == nil {
- body, _ := io.ReadAll(resp.Body)
- _ = resp.Body.Close()
- lastStatus, lastBody = resp.StatusCode, string(body)
- if resp.StatusCode == http.StatusOK {
- break
- }
- }
- time.Sleep(300 * time.Millisecond)
- }
- if lastStatus != http.StatusOK {
- t.Fatalf("call from B to A across routers: %d %s\n--- node A ---\n%s\n--- node B ---\n%s", lastStatus, lastBody, nodeA.log(), nodeB.log())
- }
+ // B finds A's service: the DHT spans both routers. B calls A: the
+ // circuit goes through router A, where B is a stranger until it runs
+ // the handshake there.
+ callAcrossMesh(t, nodeB, "token-b", nodeA, "site-tool")
// The call left B connected to router A as well, without attaching to
// it; A still holds only its own router.
@@ -550,3 +534,101 @@ func TestIdleNodesHoldOnlyTheirRouters(t *testing.T) {
}
}
}
+
+// callAcrossMesh has caller discover provider's MCP service by name and
+// call it, retrying until the call answers 200 or the deadlines pass.
+func callAcrossMesh(t *testing.T, caller *backgroundNode, callerToken string, provider *backgroundNode, service string) {
+ t.Helper()
+ client := &http.Client{Timeout: 5 * time.Second}
+ discover := func() string {
+ req, _ := http.NewRequest(http.MethodGet, "http://"+caller.apiAddr+"/mesh/service/discover?type=mcp&name="+service, nil)
+ req.Header.Set(api.HeaderMeshAuthentication, "Bearer "+callerToken)
+ resp, err := client.Do(req)
+ if err != nil {
+ return ""
+ }
+ defer func() { _ = resp.Body.Close() }()
+ body, _ := io.ReadAll(resp.Body)
+ return string(body)
+ }
+ deadline := time.Now().Add(20 * time.Second)
+ for !strings.Contains(discover(), service) {
+ if time.Now().After(deadline) {
+ t.Fatalf("caller never discovered %s.\n--- provider ---\n%s\n--- caller ---\n%s", service, provider.log(), caller.log())
+ }
+ time.Sleep(300 * time.Millisecond)
+ }
+
+ callURL := fmt.Sprintf("http://%s/mesh/%s/mcp/%s", caller.apiAddr, provider.peerID, service)
+ var lastStatus int
+ var lastBody string
+ deadline = time.Now().Add(20 * time.Second)
+ for time.Now().Before(deadline) {
+ req, _ := http.NewRequest(http.MethodPost, callURL, bytes.NewBufferString(`{"jsonrpc":"2.0","id":1,"method":"tools/list"}`))
+ req.Header.Set("Content-Type", "application/json")
+ req.Header.Set("Accept", "application/json, text/event-stream")
+ req.Header.Set(api.HeaderMeshAuthentication, "Bearer "+callerToken)
+ resp, err := client.Do(req)
+ if err == nil {
+ body, _ := io.ReadAll(resp.Body)
+ _ = resp.Body.Close()
+ lastStatus, lastBody = resp.StatusCode, string(body)
+ if resp.StatusCode == http.StatusOK {
+ return
+ }
+ }
+ time.Sleep(300 * time.Millisecond)
+ }
+ t.Fatalf("call to %s: %d %s\n--- provider ---\n%s\n--- caller ---\n%s", service, lastStatus, lastBody, provider.log(), caller.log())
+}
+
+// TestCallerTransitsTheRouterItShares pins what a call costs the routers.
+// Router b carries both labels, so the provider (pa) holds a and b and the
+// caller (pb) holds b and c. The caller reaches the provider through b, the
+// router both hold, and runs no handshake at a: a service the whole mesh
+// calls must not hold every caller at every one of its routers.
+func TestCallerTransitsTheRouterItShares(t *testing.T) {
+ m := startAttachmentMesh(t)
+ routerA := m.startRouter(t, "router-a", "pa=1")
+ routerB := m.startRouter(t, "router-b", "pa=1", "pb=1")
+ routerC := m.startRouter(t, "router-c", "pb=1")
+ waitForEqual(t, "routers connected to router c", func() []string { return routersConnectedTo(t, m.cpPort, routerC.peerID) },
+ peerIDStrings(routerA.peerID, routerB.peerID), 10*time.Second)
+
+ mcpServer := httptest.NewServer(newBoundaryMCPHandler(t))
+ t.Cleanup(mcpServer.Close)
+ common := append(m.nodeArgs(), "--monitor-bootstrap", "500ms", "--monitor-interval", "500ms")
+ provider := launchNode(t, m.nodeBin, m.nodeEnv("provider"), filepath.Join(m.tmpDir, "provider"), append(slices.Clone(common),
+ "--api-token-path", tokenPath(t, "token-p"),
+ "--router-selector", "pa=1",
+ "--config", writeNodeConfig(t, m.tmpDir, nil, svcDecl{Type: "mcp", Name: "shared-tool", TargetURL: mcpServer.URL}),
+ )...)
+ caller := launchNode(t, m.nodeBin, m.nodeEnv("caller"), filepath.Join(m.tmpDir, "caller"), append(slices.Clone(common),
+ "--api-token-path", tokenPath(t, "token-c"),
+ "--router-selector", "pb=1",
+ )...)
+ provider.waitForAPI(t)
+ caller.waitForAPI(t)
+ waitForEqual(t, "routers attached by the provider", func() []string { return attachedRouters(t, provider) }, peerIDStrings(routerA.peerID, routerB.peerID), 10*time.Second)
+ waitForEqual(t, "routers attached by the caller", func() []string { return attachedRouters(t, caller) }, peerIDStrings(routerB.peerID, routerC.peerID), 10*time.Second)
+ // The provider's record must name both circuits, or the caller has only
+ // router a to go through. A reservation on the second router comes a
+ // moment after the first, and the record follows it after a 2 s debounce.
+ waitForEqual(t, "routers the provider announces circuits through", func() []string { return announcedCircuitRelays(t, provider) },
+ peerIDStrings(routerA.peerID, routerB.peerID), 15*time.Second)
+ time.Sleep(3 * time.Second)
+
+ callAcrossMesh(t, caller, "token-c", provider, "shared-tool")
+
+ // Router a authenticated the provider and the two routers; the caller
+ // never came. Router b has both nodes and the two routers.
+ if got := routerA.gauge(t, "agentmesh_router_authenticated_peers"); got != 3 {
+ t.Errorf("router a authenticated %v peers, want 3: the caller must transit router b, which it holds", got)
+ }
+ if got := routerB.gauge(t, "agentmesh_router_authenticated_peers"); got != 4 {
+ t.Errorf("router b authenticated %v peers, want 4", got)
+ }
+ if t.Failed() {
+ t.Logf("--- caller ---\n%s", caller.log())
+ }
+}