Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: transit one router to reach a peer, the one it shares when it can
Antonio Ojeaa503af8253fe
diff --git a/internal/node/mcp.go b/internal/node/mcp.goindex 75c3b1ec..cc02756a 100644--- a/internal/node/mcp.go+++ b/internal/node/mcp.go@@ -20,6 +20,7 @@ import ("errors""fmt""net/http"+ "sort""strings""time"@@ -560,64 +561,102 @@ func (n *AgentMeshNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID}}+ if kept := n.selectTransit(ctx, validAddrs); len(kept) != len(validAddrs) {+ validAddrs = kept+ changed = true+ }+if changed && len(validAddrs) > 0 {// Clear existing addresses to prevent libp2p from hanging on dead private IPsn.Host.Peerstore().ClearAddrs(targetPeer)n.Host.Peerstore().AddAddrs(targetPeer, validAddrs, peerstore.TempAddrTTL)logger.Debugf("[Discovery] Replaced addrs for %s with %d routable/circuit addrs", targetPeer, len(validAddrs))}+}- n.authenticateTransitRouters(ctx, validAddrs)+// circuitRelay is the relay a circuit address goes through, or "" for any+// other address.+func circuitRelay(ma multiaddr.Multiaddr) peer.ID {+ parts := multiaddr.Split(ma)+ if len(parts) < 2 || parts[len(parts)-1].Protocol().Code != multiaddr.P_CIRCUIT || parts[len(parts)-2].Protocol().Code != multiaddr.P_P2P {+ return ""+ }+ relayID, err := peer.Decode(parts[len(parts)-2].Value())+ if err != nil {+ return ""+ }+ return relayID}-// authenticateTransitRouters runs the credential handshake with every-// router the given circuit addresses go through that this node is not yet-// authenticated to. A router relays only between peers it has admitted,-// and a node keeps sessions with a few routers, so the destination's router-// is often not one of them: the handshake is the one the node runs at-// start, on the connection the circuit will use, before the circuit is-// asked for. A relay that is not a router (a node with --enable-relay) is-// left alone; it admits what it has handshaken with on its own terms.-func (n *AgentMeshNode) authenticateTransitRouters(ctx context.Context, addrs []multiaddr.Multiaddr) {- var relays []peer.ID+// selectTransit decides which of a peer's circuit addresses to dial and+// makes sure the node is admitted at the router behind them. A router+// relays only between peers it has admitted, and a node keeps sessions+// with a few routers, so the peer's routers are often not among them. One+// transit router is enough: a router the node already holds when the peer+// is on it too, which costs the mesh nothing; otherwise the peer's router+// with the most room, which the node runs the handshake with, as it does+// at start, before the circuit is asked for. Circuits through the peer's+// other routers are dropped: each would be a handshake and a session+// there, and a service the whole mesh calls would hold every caller at+// every one of its routers. A relay that is not a router (a node with+// --enable-relay) is kept and left alone; it admits what it has handshaken+// with on its own terms. Direct addresses always pass.+func (n *AgentMeshNode) selectTransit(ctx context.Context, addrs []multiaddr.Multiaddr) []multiaddr.Multiaddr {+ routers := map[peer.ID]routerCandidate{}+ for _, c := range n.routerCandidates() {+ routers[c.ID] = c+ }+ var viaRouters []peer.IDseen := map[peer.ID]bool{}for _, ma := range addrs {- parts := multiaddr.Split(ma)- if len(parts) < 2 || parts[len(parts)-1].Protocol().Code != multiaddr.P_CIRCUIT || parts[len(parts)-2].Protocol().Code != multiaddr.P_P2P {- continue- }- relayID, err := peer.Decode(parts[len(parts)-2].Value())- if err != nil || seen[relayID] {+ relay := circuitRelay(ma)+ if _, isRouter := routers[relay]; relay == "" || !isRouter || seen[relay] {continue}- seen[relayID] = true- relays = append(relays, relayID)+ seen[relay] = true+ viaRouters = append(viaRouters, relay)}- if len(relays) == 0 {- return+ if len(viaRouters) == 0 {+ return addrs}- routers := map[peer.ID]bool{}- for _, c := range n.routerCandidates() {- routers[c.ID] = true++ usable := map[peer.ID]bool{}+ for _, r := range viaRouters {+ if n.isAuthenticatedAndConnected(r) {+ usable[r] = true+ }}- var biscuitBytes []byte- for _, relayID := range relays {- if !routers[relayID] || n.isAuthenticatedAndConnected(relayID) {- continue+ if len(usable) == 0 {+ sort.SliceStable(viaRouters, func(i, j int) bool {+ return routers[viaRouters[i]].loadRatio() < routers[viaRouters[j]].loadRatio()+ })+ biscuitBytes, err := n.loadIdentityForAuth()+ if err != nil {+ logger.Debugf("[AuthN] Cannot authenticate to a transit router: %v", err)+ return addrs}- if biscuitBytes == nil {- b, err := n.loadIdentityForAuth()- if err != nil {- logger.Debugf("[AuthN] Cannot authenticate to transit router %s: %v", relayID, err)- return+ for _, r := range viaRouters {+ router := peer.AddrInfo{ID: r, Addrs: n.Host.Peerstore().Addrs(r)}+ if err := n.authRouter(ctx, router, biscuitBytes); err != nil {+ logger.Warnf("[AuthN] Transit router %s did not admit this node: %v", r, err)+ continue}- biscuitBytes = b+ logger.Debugf("[AuthN] Authenticated to transit router %s", r)+ usable[r] = true+ break+ }+ if len(usable) == 0 {+ return addrs}- router := peer.AddrInfo{ID: relayID, Addrs: n.Host.Peerstore().Addrs(relayID)}- if err := n.authRouter(ctx, router, biscuitBytes); err != nil {- logger.Warnf("[AuthN] Transit router %s did not admit this node; a circuit through it will be refused: %v", relayID, err)+ }++ kept := make([]multiaddr.Multiaddr, 0, len(addrs))+ for _, ma := range addrs {+ relay := circuitRelay(ma)+ if _, isRouter := routers[relay]; relay != "" && isRouter && !usable[relay] {continue}- logger.Debugf("[AuthN] Authenticated to transit router %s", relayID)+ kept = append(kept, ma)}+ return kept}diff --git a/internal/node/routers.go b/internal/node/routers.goindex 3c7aa9ec..58236ce5 100644--- a/internal/node/routers.go+++ b/internal/node/routers.go@@ -43,6 +43,15 @@ type routerCandidate struct {ConnectionLimit int}+// loadRatio is how full the router is, 0 to 1; 0.5 when its last lease+// said nothing.+func (c routerCandidate) loadRatio() float64 {+ if c.ConnectionLimit <= 0 {+ return 0.5+ }+ return float64(c.Connections) / float64(c.ConnectionLimit)+}+// routerTag protects an attached router's connection from the node's own// connection manager, which otherwise trims by age and tag value and would// take the router before a stranger.@@ -122,11 +131,7 @@ func selectRouters(cands []routerCandidate, k int, selector, prefer map[string]spreferred++}}- load := 0.5- if c.ConnectionLimit > 0 {- load = float64(c.Connections) / float64(c.ConnectionLimit)- }- load += rand.Float64() * 0.05+ load := c.loadRatio() + rand.Float64()*0.05eligible = append(eligible, scored{c, preferred, load})}sort.SliceStable(eligible, func(i, j int) bool {diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.goindex 4b5baabf..b3147527 100644--- a/tests/integration/router_attachment_test.go+++ b/tests/integration/router_attachment_test.go@@ -193,6 +193,32 @@ func peerIDStrings(ids ...peer.ID) []string {return out}+// announcedCircuitRelays lists the relays the node publishes circuit+// addresses through, from its own debug endpoint.+func announcedCircuitRelays(t *testing.T, n *backgroundNode) []string {+ t.Helper()+ var info struct {+ AnnouncedAddresses []string `json:"announced_addresses"`+ }+ if err := json.Unmarshal([]byte(debugGetWithToken(t, n.apiAddr, n.token, "/debug/network-info")), &info); err != nil {+ t.Fatalf("decode network-info: %v", err)+ }+ seen := map[string]bool{}+ var out []string+ for _, a := range info.AnnouncedAddresses {+ if i := strings.Index(a, "/p2p-circuit"); i > 0 {+ parts := strings.Split(a[:i], "/p2p/")+ relay := parts[len(parts)-1]+ if !seen[relay] {+ seen[relay] = true+ out = append(out, relay)+ }+ }+ }+ slices.Sort(out)+ return out+}+// waitForEqual polls get until it returns want.func waitForEqual(t *testing.T, what string, get func() []string, want []string, timeout time.Duration) {t.Helper()@@ -235,7 +261,7 @@ func startAttachmentMesh(t *testing.T) *attachmentMesh {- name: %sallowed_services: []allowed_targets: ["*"]- allowed_labels: ["site=*"]+ allowed_labels: ["site=*", "pa=*", "pb=*"]- name: %sallowed_services: ["mcp://*"]allowed_targets: ["*"]@@ -354,52 +380,10 @@ func TestNodesOnDifferentRoutersReachEachOther(t *testing.T) {waitForEqual(t, "routers attached by node A", func() []string { return attachedRouters(t, nodeA) }, peerIDStrings(routerA.peerID), 10*time.Second)waitForEqual(t, "routers attached by node B", func() []string { return attachedRouters(t, nodeB) }, peerIDStrings(routerB.peerID), 10*time.Second)- // B finds A's service: the DHT spans both routers.- client := &http.Client{Timeout: 5 * time.Second}- discover := func() string {- req, _ := http.NewRequest(http.MethodGet, "http://"+nodeB.apiAddr+"/mesh/service/discover?type=mcp&name=site-tool", nil)- req.Header.Set(api.HeaderMeshAuthentication, "Bearer token-b")- resp, err := client.Do(req)- if err != nil {- return ""- }- defer func() { _ = resp.Body.Close() }()- body, _ := io.ReadAll(resp.Body)- return string(body)- }- deadline := time.Now().Add(20 * time.Second)- for !strings.Contains(discover(), "site-tool") {- if time.Now().After(deadline) {- t.Fatalf("node B never discovered node A's service.\n--- node A ---\n%s\n--- node B ---\n%s", nodeA.log(), nodeB.log())- }- time.Sleep(300 * time.Millisecond)- }-- // B calls A: the circuit goes through router A, where B is a stranger- // until it runs the handshake there.- callURL := fmt.Sprintf("http://%s/mesh/%s/mcp/site-tool", nodeB.apiAddr, nodeA.peerID)- var lastStatus int- var lastBody string- deadline = time.Now().Add(20 * time.Second)- for time.Now().Before(deadline) {- req, _ := http.NewRequest(http.MethodPost, callURL, bytes.NewBufferString(`{"jsonrpc":"2.0","id":1,"method":"tools/list"}`))- req.Header.Set("Content-Type", "application/json")- req.Header.Set("Accept", "application/json, text/event-stream")- req.Header.Set(api.HeaderMeshAuthentication, "Bearer token-b")- resp, err := client.Do(req)- if err == nil {- body, _ := io.ReadAll(resp.Body)- _ = resp.Body.Close()- lastStatus, lastBody = resp.StatusCode, string(body)- if resp.StatusCode == http.StatusOK {- break- }- }- time.Sleep(300 * time.Millisecond)- }- if lastStatus != http.StatusOK {- t.Fatalf("call from B to A across routers: %d %s\n--- node A ---\n%s\n--- node B ---\n%s", lastStatus, lastBody, nodeA.log(), nodeB.log())- }+ // B finds A's service: the DHT spans both routers. B calls A: the+ // circuit goes through router A, where B is a stranger until it runs+ // the handshake there.+ callAcrossMesh(t, nodeB, "token-b", nodeA, "site-tool")// The call left B connected to router A as well, without attaching to// it; A still holds only its own router.@@ -550,3 +534,101 @@ func TestIdleNodesHoldOnlyTheirRouters(t *testing.T) {}}}++// callAcrossMesh has caller discover provider's MCP service by name and+// call it, retrying until the call answers 200 or the deadlines pass.+func callAcrossMesh(t *testing.T, caller *backgroundNode, callerToken string, provider *backgroundNode, service string) {+ t.Helper()+ client := &http.Client{Timeout: 5 * time.Second}+ discover := func() string {+ req, _ := http.NewRequest(http.MethodGet, "http://"+caller.apiAddr+"/mesh/service/discover?type=mcp&name="+service, nil)+ req.Header.Set(api.HeaderMeshAuthentication, "Bearer "+callerToken)+ resp, err := client.Do(req)+ if err != nil {+ return ""+ }+ defer func() { _ = resp.Body.Close() }()+ body, _ := io.ReadAll(resp.Body)+ return string(body)+ }+ deadline := time.Now().Add(20 * time.Second)+ for !strings.Contains(discover(), service) {+ if time.Now().After(deadline) {+ t.Fatalf("caller never discovered %s.\n--- provider ---\n%s\n--- caller ---\n%s", service, provider.log(), caller.log())+ }+ time.Sleep(300 * time.Millisecond)+ }++ callURL := fmt.Sprintf("http://%s/mesh/%s/mcp/%s", caller.apiAddr, provider.peerID, service)+ var lastStatus int+ var lastBody string+ deadline = time.Now().Add(20 * time.Second)+ for time.Now().Before(deadline) {+ req, _ := http.NewRequest(http.MethodPost, callURL, bytes.NewBufferString(`{"jsonrpc":"2.0","id":1,"method":"tools/list"}`))+ req.Header.Set("Content-Type", "application/json")+ req.Header.Set("Accept", "application/json, text/event-stream")+ req.Header.Set(api.HeaderMeshAuthentication, "Bearer "+callerToken)+ resp, err := client.Do(req)+ if err == nil {+ body, _ := io.ReadAll(resp.Body)+ _ = resp.Body.Close()+ lastStatus, lastBody = resp.StatusCode, string(body)+ if resp.StatusCode == http.StatusOK {+ return+ }+ }+ time.Sleep(300 * time.Millisecond)+ }+ t.Fatalf("call to %s: %d %s\n--- provider ---\n%s\n--- caller ---\n%s", service, lastStatus, lastBody, provider.log(), caller.log())+}++// TestCallerTransitsTheRouterItShares pins what a call costs the routers.+// Router b carries both labels, so the provider (pa) holds a and b and the+// caller (pb) holds b and c. The caller reaches the provider through b, the+// router both hold, and runs no handshake at a: a service the whole mesh+// calls must not hold every caller at every one of its routers.+func TestCallerTransitsTheRouterItShares(t *testing.T) {+ m := startAttachmentMesh(t)+ routerA := m.startRouter(t, "router-a", "pa=1")+ routerB := m.startRouter(t, "router-b", "pa=1", "pb=1")+ routerC := m.startRouter(t, "router-c", "pb=1")+ waitForEqual(t, "routers connected to router c", func() []string { return routersConnectedTo(t, m.cpPort, routerC.peerID) },+ peerIDStrings(routerA.peerID, routerB.peerID), 10*time.Second)++ mcpServer := httptest.NewServer(newBoundaryMCPHandler(t))+ t.Cleanup(mcpServer.Close)+ common := append(m.nodeArgs(), "--monitor-bootstrap", "500ms", "--monitor-interval", "500ms")+ provider := launchNode(t, m.nodeBin, m.nodeEnv("provider"), filepath.Join(m.tmpDir, "provider"), append(slices.Clone(common),+ "--api-token-path", tokenPath(t, "token-p"),+ "--router-selector", "pa=1",+ "--config", writeNodeConfig(t, m.tmpDir, nil, svcDecl{Type: "mcp", Name: "shared-tool", TargetURL: mcpServer.URL}),+ )...)+ caller := launchNode(t, m.nodeBin, m.nodeEnv("caller"), filepath.Join(m.tmpDir, "caller"), append(slices.Clone(common),+ "--api-token-path", tokenPath(t, "token-c"),+ "--router-selector", "pb=1",+ )...)+ provider.waitForAPI(t)+ caller.waitForAPI(t)+ waitForEqual(t, "routers attached by the provider", func() []string { return attachedRouters(t, provider) }, peerIDStrings(routerA.peerID, routerB.peerID), 10*time.Second)+ waitForEqual(t, "routers attached by the caller", func() []string { return attachedRouters(t, caller) }, peerIDStrings(routerB.peerID, routerC.peerID), 10*time.Second)+ // The provider's record must name both circuits, or the caller has only+ // router a to go through. A reservation on the second router comes a+ // moment after the first, and the record follows it after a 2 s debounce.+ waitForEqual(t, "routers the provider announces circuits through", func() []string { return announcedCircuitRelays(t, provider) },+ peerIDStrings(routerA.peerID, routerB.peerID), 15*time.Second)+ time.Sleep(3 * time.Second)++ callAcrossMesh(t, caller, "token-c", provider, "shared-tool")++ // Router a authenticated the provider and the two routers; the caller+ // never came. Router b has both nodes and the two routers.+ if got := routerA.gauge(t, "agentmesh_router_authenticated_peers"); got != 3 {+ t.Errorf("router a authenticated %v peers, want 3: the caller must transit router b, which it holds", got)+ }+ if got := routerB.gauge(t, "agentmesh_router_authenticated_peers"); got != 4 {+ t.Errorf("router b authenticated %v peers, want 4", got)+ }+ if t.Failed() {+ t.Logf("--- caller ---\n%s", caller.log())+ }+}