Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: announce the node's own record and show what the directory holds
Antonio Ojeaf73afa08e7cb
diff --git a/cmd/agentmesh-node/debug.go b/cmd/agentmesh-node/debug.goindex a87d7bd0..315fd315 100644--- a/cmd/agentmesh-node/debug.go+++ b/cmd/agentmesh-node/debug.go@@ -40,8 +40,8 @@ func newDebugCmd() *cobra.Command {debugCmd.PersistentFlags().StringVar(&socketPathFlag, "socket-path", "", "Unix socket of the running node (defaults to <data-dir>/"+node.DefaultSocketName+")")debugCmd.AddCommand(- newDebugGetCmd("mesh-info", "Show connected peers, the routers the node holds, and DHT size", "/debug/mesh-info"),- newDebugGetCmd("network-info", "Show listener and announced addresses", "/debug/network-info"),+ newDebugGetCmd("mesh-info", "Show connected peers and the routers the node holds", "/debug/mesh-info"),+ newDebugGetCmd("network-info", "Show listener, announced and directory addresses", "/debug/network-info"),newDebugGetCmd("token-info", "Show the local auth token's expiration and status", "/debug/token-info"),newDebugGetCmd("logs", "Show the last few lines of the node's log output", "/debug/logs"),)diff --git a/internal/node/debug_handlers.go b/internal/node/debug_handlers.goindex 765bcd91..07500240 100644--- a/internal/node/debug_handlers.go+++ b/internal/node/debug_handlers.go@@ -156,6 +156,16 @@ type tokenInfoResponse struct {type networkInfoResponse struct {ListenAddresses []string `json:"listen_addresses"`AnnouncedAddresses []string `json:"announced_addresses"`+ // DirectoryAddresses is what the directory holds for this node: the+ // addresses of the record it last announced, which follow+ // AnnouncedAddresses at the next announcement pass.+ DirectoryAddresses []string `json:"directory_addresses"`+}++// recordAnnouncer is a directory that remembers the record it last+// announced.+type recordAnnouncer interface {+ Announced() []multiaddr.Multiaddr}// peerAddrsResponse is what the DHT says about a peer: the addresses it@@ -294,9 +304,17 @@ func (n *AgentMeshNode) networkInfo() networkInfoResponse {announcedAddrs = append(announcedAddrs, a.String())}+ directoryAddrs := []string{}+ if d, ok := n.Directory.(recordAnnouncer); ok {+ for _, a := range d.Announced() {+ directoryAddrs = append(directoryAddrs, a.String())+ }+ }+return networkInfoResponse{ListenAddresses: listenAddrs,AnnouncedAddresses: announcedAddrs,+ DirectoryAddresses: directoryAddrs,}}diff --git a/internal/node/directory_client.go b/internal/node/directory_client.goindex e18a94e1..78903823 100644--- a/internal/node/directory_client.go+++ b/internal/node/directory_client.go@@ -18,6 +18,8 @@ import ("context""errors""fmt"+ "slices"+ "sync""time""github.com/libp2p/go-libp2p/core/host"@@ -101,19 +103,35 @@ type routerDirectory struct {host host.Host // the gated host: a stream to a router follows its handshakerouters func() []peer.IDaddrs func() []multiaddr.Multiaddr++ mu sync.Mutex+ announced []multiaddr.Multiaddr // the addresses of the record last announced}func (d *routerDirectory) Provide(ctx context.Context, key []byte) error {addrs := d.addrs()+ if len(addrs) > api.DirectoryMaxAddrs {+ addrs = addrs[:api.DirectoryMaxAddrs]+ }rec := &api.DirectoryProvide{Key: key, Addrs: make([]string, 0, len(addrs))}for _, a := range addrs {rec.Addrs = append(rec.Addrs, a.String())}- if len(rec.Addrs) > api.DirectoryMaxAddrs {- rec.Addrs = rec.Addrs[:api.DirectoryMaxAddrs]+ if _, err := d.request(ctx, directoryOpProvide, &api.DirectoryRequest{Request: &api.DirectoryRequest_Provide{Provide: rec}}); err != nil {+ return err}- _, err := d.request(ctx, directoryOpProvide, &api.DirectoryRequest{Request: &api.DirectoryRequest_Provide{Provide: rec}})- return err+ d.mu.Lock()+ d.announced = addrs+ d.mu.Unlock()+ return nil+}++// Announced is the addresses of the record this member last announced:+// what the directory holds for it, as against what it would announce now.+func (d *routerDirectory) Announced() []multiaddr.Multiaddr {+ d.mu.Lock()+ defer d.mu.Unlock()+ return slices.Clone(d.announced)}func (d *routerDirectory) FindProviders(ctx context.Context, key []byte, limit int) ([]peer.AddrInfo, error) {diff --git a/internal/node/directory_client_test.go b/internal/node/directory_client_test.goindex 751556ba..dc295d53 100644--- a/internal/node/directory_client_test.go+++ b/internal/node/directory_client_test.go@@ -15,7 +15,9 @@package nodeimport (+ "bytes""context"+ "slices""sync""testing""time"@@ -36,11 +38,12 @@ import (// way a router does: records by key, a peer from the router's own peerstore// or its peer-key record. It fails every request while failing is set.type fakeDirectoryServer struct {- h host.Host- mu sync.Mutex- records map[string]map[peer.ID][]multiaddr.Multiaddr- served int- failing bool+ h host.Host+ mu sync.Mutex+ records map[string]map[peer.ID][]multiaddr.Multiaddr+ provided [][]byte // keys announced here, in order+ served int+ failing bool}func serveDirectory(h host.Host) *fakeDirectoryServer {@@ -71,6 +74,13 @@ func (d *fakeDirectoryServer) providers(key []byte) []peer.ID {return out}+// providedKeys is the keys announced here so far, in order.+func (d *fakeDirectoryServer) providedKeys() [][]byte {+ d.mu.Lock()+ defer d.mu.Unlock()+ return slices.Clone(d.provided)+}+// add puts a record in as if another router had placed it.func (d *fakeDirectoryServer) add(key []byte, p peer.ID, addrs ...multiaddr.Multiaddr) {d.mu.Lock()@@ -111,6 +121,7 @@ func (d *fakeDirectoryServer) handle(s network.Stream) {d.records[string(q.GetKey())] = map[peer.ID][]multiaddr.Multiaddr{}}d.records[string(q.GetKey())][s.Conn().RemotePeer()] = addrs+ d.provided = append(d.provided, q.GetKey())case req.GetFindProviders() != nil:for p, addrs := range d.records[string(req.GetFindProviders().GetKey())] {resp.Providers = append(resp.Providers, recordOf(p, addrs))@@ -229,3 +240,50 @@ func TestRouterDirectoryAsksTheAttachedRouters(t *testing.T) {t.Fatal("FindProviders succeeded with every router failing")}}++// An announcement pass announces the node's services and then the node+// itself under its peer key, so a router that does not hold the node can+// still say where it is reached, and the pass is what the directory holds+// for the node: its last announced addresses are the ones the records+// carry, and a pass no router took leaves them as they were.+func TestAnnounceAllEndsWithThePeerRecord(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)+ defer cancel()+ router := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")+ dir := serveDirectory(router.h)+ member, directory := newDirectoryClient(t, router.h)+ if got := directory.Announced(); len(got) != 0 {+ t.Fatalf("Announced before any announcement = %v, want none", got)+ }+ node := &AgentMeshNode{Host: member, Directory: directory, services: NewServiceRegistry(directory, 0)}+ if err := node.services.Register(ctx, newFakeSvc("calc", api.ServiceType_SERVICE_TYPE_MCP)); err != nil {+ t.Fatalf("Register: %v", err)+ }+ registered := len(dir.providedKeys())++ if withheld := node.announceAll(ctx); withheld != 0 {+ t.Fatalf("announceAll withheld %d services, want 0", withheld)+ }+ nameKey, _ := api.ServiceKey(api.ServiceType_SERVICE_TYPE_MCP, "calc")+ typeKey, _ := api.ServiceKey(api.ServiceType_SERVICE_TYPE_MCP, "")+ pass := dir.providedKeys()[registered:]+ if len(pass) != 3 || !bytes.Equal(pass[0], nameKey) || !bytes.Equal(pass[1], typeKey) || !bytes.Equal(pass[2], api.PeerKey(member.ID())) {+ t.Fatalf("the pass announced %d keys, want the service by name, by type, then the peer key last", len(pass))+ }+ if got := dir.providers(api.PeerKey(member.ID())); len(got) != 1 || got[0] != member.ID() {+ t.Fatalf("the router holds %v under the peer key, want the member", got)+ }+ info, err := directory.FindPeer(ctx, member.ID())+ if err != nil || len(info.Addrs) != len(member.Addrs()) {+ t.Fatalf("FindPeer = %v, %v; want the member at its %d addresses", info, err, len(member.Addrs()))+ }+ if got, want := directory.Announced(), member.Addrs(); !slices.EqualFunc(got, want, multiaddr.Multiaddr.Equal) {+ t.Fatalf("Announced = %v, want the addresses the pass carried, %v", got, want)+ }++ dir.fail(true)+ node.announceAll(ctx)+ if got, want := directory.Announced(), member.Addrs(); !slices.EqualFunc(got, want, multiaddr.Multiaddr.Equal) {+ t.Fatalf("Announced after a pass no router took = %v, want the last announced %v", got, want)+ }+}diff --git a/internal/node/node.go b/internal/node/node.goindex 174c94a1..b3a5d13a 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -798,9 +798,26 @@ func (n *AgentMeshNode) triggerReprovide() {}}-// startReprovideLoop re-announces the node's services every reprovide-// interval of the mesh, read at each cycle so a control plane that changed-// it is followed at the next one.+// announceAll is one announcement pass: the node's services, then the+// node itself under its peer key, so a lookup of the node through a router+// that does not hold it finds it at the addresses its records carry. The+// peer record goes last so that, once it names an address, every service+// record of the pass does too. It returns how many services were withheld+// because their backend did not answer.+func (n *AgentMeshNode) announceAll(ctx context.Context) int {+ withheld := n.services.ReprovideAll(ctx)+ provideCtx, cancel := context.WithTimeout(ctx, 5*time.Second)+ defer cancel()+ if err := n.Directory.Provide(provideCtx, api.PeerKey(n.Host.ID())); err != nil && ctx.Err() == nil && !errors.Is(err, errNoDirectoryRouter) {+ // With no router attached the directory client has said so already.+ logger.Warnf("[Directory] Announcing where this node is reached: %v", err)+ }+ return withheld+}++// startReprovideLoop re-announces the node's services and its own record+// every reprovide interval of the mesh, read at each cycle so a control+// plane that changed it is followed at the next one.func (n *AgentMeshNode) startReprovideLoop(ctx context.Context) {go func() {initialDelay := 5 * time.Second@@ -830,7 +847,7 @@ func (n *AgentMeshNode) startReprovideLoop(ctx context.Context) {interval := n.directorySettings().Reprovidenext := interval- if n.services.ReprovideAll(ctx) > 0 {+ if n.announceAll(ctx) > 0 {// A backend that is not answering yet is usually one that is// still starting. Waiting the full interval would leave it// undiscoverable for minutes after it came up.@@ -946,7 +963,7 @@ func (n *AgentMeshNode) startConnectionMonitor(ctx context.Context, bootstrapDurif reconnected {logger.Infof("[Monitor] Reconnected successfully. Announcing services again...")- n.services.ReprovideAll(ctx)+ n.announceAll(ctx)consecutiveFailures = 0continue}diff --git a/site/content/docs/concepts/networking.md b/site/content/docs/concepts/networking.mdindex b75062ce..b204cb23 100644--- a/site/content/docs/concepts/networking.md+++ b/site/content/docs/concepts/networking.md@@ -44,12 +44,12 @@ The [node API reference](../../reference/node-api/) lists every route.## DiscoveryThe routers keep the mesh's directory: a Kademlia DHT among themselves,-served to members over `/mesh/directory/1.0.0`. A node that publishes a-service announces itself to a router it holds as a provider of that-service name, and where it is reached; the router keeps the record and-places it on the routers closest to the key. A node that looks for a-service, or for a peer, asks a router it holds, and the router runs the-lookup across the routers' DHT. A node keeps no DHT of its own and+served to members over `/mesh/directory/1.0.0`. A node announces where it+is reached, and a node that publishes a service announces itself as a+provider of that service name, to a router it holds; the router keeps the+records and places each on the routers closest to its key. A node that+looks for a service, or for a peer, asks a router it holds, and the router+runs the lookup across the routers' DHT. A node keeps no DHT of its own andconnects to no router it does not hold. It re-announces every reprovideinterval of the mesh (`/info`), and a record a provider stoppedannouncing leaves the directory after three of them.@@ -86,10 +86,12 @@ out what each connection carries and what a NAT or firewall sees.Routers are the fixed points. A node learns their addresses, labels andload from the control plane's `/info` at enrollment and again on aschedule, holds two of them with a relay reservation on each, and replaces-one that drops or that sends it away. A router that grants a reservation-records the member's address through itself, so a directory lookup for-the member learns where it is reached; an SDK member listens on nothing-else. A call to a peer reached through a+one that drops or that sends it away. A router that holds a member answers+a lookup for it from what it sees of the member; any other router answers+from the record the member announced under its own peer ID, so a lookup+for the member learns where it is reached through any router. An SDK+member listens on nothing of its own and is reached through a router it+holds. A call to a peer reached through athird router goes through that router: the caller runs the credentialhandshake there first, since a router relays only between peers it hasverified. The directory, the gossip and the relay are closed to anyone whodiff --git a/site/content/docs/preview/mobile.md b/site/content/docs/preview/mobile.mdindex a2eec96b..640bfbf0 100644--- a/site/content/docs/preview/mobile.md+++ b/site/content/docs/preview/mobile.md@@ -241,13 +241,14 @@ or shows an error, record the status text before retrying.2. Allow up to a minute for the connection to settle.3. Check that the dashboard says **Node is Running**, **Node ID** containsan identifier, and **Connected Peers** is greater than zero.-4. Record the **Node ID**, **Connected Peers**, and **DHT Size** values.- Peer counts can change. A particular DHT size is not a pass criterion.+4. Record the **Node ID**, **Connected Peers**, and **Routers** values.+ Peer counts can change. **Routers** is how many routers the node holds,+ two on a mesh with two or more.**Node is Running** alone does not prove a mesh connection. Report a peercount that remains zero, even if the app shows no error.-{{< figure src="../../../images/agentmesh-connect-running.png" alt="Agent Mesh Connect dashboard showing Node is Running, one connected peer, DHT size one, and a Node ID" caption="A connected node shows Node is Running, a Node ID, and at least one Connected Peer. Your Node ID and peer counts will differ from this example." width="320" >}}+{{< figure src="../../../images/agentmesh-connect-running.png" alt="Agent Mesh Connect dashboard showing Node is Running, one connected peer, one router, and a Node ID" caption="A connected node shows Node is Running, a Node ID, and at least one Connected Peer. Your Node ID and peer counts will differ from this example." width="320" >}}### 5. Check restart and reconnectiondiff --git a/site/content/docs/reference/node-api.md b/site/content/docs/reference/node-api.mdindex b4e35420..6a51f3f3 100644--- a/site/content/docs/reference/node-api.md+++ b/site/content/docs/reference/node-api.md@@ -378,6 +378,10 @@ material is meant for the node's owner and for auditors, not for agents.and `POST /debug/connect-peer` report and probe the node's view of the meshfor troubleshooting, and answer even while the mesh is unreachable.`mesh-info` lists the routers the node holds (`attached_routers`) next to-every peer it is connected to; `peer-addrs` is where the directory says a-peer is reached, looked up and not dialed. `agentmesh-node debug` on the-command line wraps them. Their output is not stable across releases.+every peer it is connected to; `network-info` lists the addresses the node+listens on, the ones it announces, and the ones in the records it last+announced to the directory (`directory_addresses`), which follow the+announced ones at the next announcement pass; `peer-addrs` is where the+directory says a peer is reached, looked up and not dialed. `agentmesh-node+debug` on the command line wraps them. Their output is not stable across+releases.diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.goindex fc3cc180..0268c3d9 100644--- a/tests/integration/router_attachment_test.go+++ b/tests/integration/router_attachment_test.go@@ -233,17 +233,30 @@ func peerIDStrings(ids ...peer.ID) []string {// addresses through, from its own debug endpoint.func announcedCircuitRelays(t *testing.T, n *backgroundNode) []string {t.Helper()- var info struct {- AnnouncedAddresses []string `json:"announced_addresses"`- }+ return circuitRelaysOf(networkInfo(t, n).AnnouncedAddresses)+}++// recordedCircuitRelays lists the relays in the records the node last+// announced to the directory: what a lookup of its services returns, as+// against announcedCircuitRelays, what it will announce at its next pass.+func recordedCircuitRelays(t *testing.T, n *backgroundNode) []string {+ t.Helper()+ return circuitRelaysOf(networkInfo(t, n).DirectoryAddresses)+}++func networkInfo(t *testing.T, n *backgroundNode) (info struct {+ AnnouncedAddresses []string `json:"announced_addresses"`+ DirectoryAddresses []string `json:"directory_addresses"`+}) {+ t.Helper()if err := json.Unmarshal([]byte(debugGetWithToken(t, n.apiAddr, n.token, "/debug/network-info")), &info); err != nil {t.Fatalf("decode network-info: %v", err)}- return circuitRelaysOf(info.AnnouncedAddresses)+ return info}-// circuitRelaysInRecord lists the relays in a peer's DHT record as reader-// looks it up: the addresses a call from reader would choose among.+// circuitRelaysInRecord lists the relays in a peer's record as reader+// looks it up through a router it holds.func circuitRelaysInRecord(t *testing.T, reader *backgroundNode, target peer.ID) []string {t.Helper()req, err := http.NewRequest(http.MethodGet, "http://"+reader.apiAddr+"/debug/peer-addrs?peer_id="+target.String(), nil)@@ -736,21 +749,24 @@ func TestCallerTransitsTheRouterItShares(t *testing.T) {caller.waitForAPI(t)waitForEqual(t, "routers attached by the provider", func() []string { return attachedRouters(t, provider) }, peerIDStrings(routerA.peerID, routerB.peerID), 10*time.Second)waitForEqual(t, "routers attached by the caller", func() []string { return attachedRouters(t, caller) }, peerIDStrings(routerB.peerID, routerC.peerID), 10*time.Second)- // The provider's record must name both circuits, or the caller has only+ // The provider's records must name both circuits, or the caller has only// router a to go through. A reservation on the second router comes a- // moment after the first, and the record follows it after a 2 s debounce;- // the caller's DHT lookup is what the call will read.+ // moment after the first, and the records follow it at the next+ // announcement pass, 2 s later. A router that holds the provider answers+ // a lookup from what it sees of it, which is ahead of the records; the+ // call reads the service record, so the wait is on what the provider+ // announced, then on what the caller reads.waitForEqual(t, "routers the provider announces circuits through", func() []string { return announcedCircuitRelays(t, provider) },peerIDStrings(routerA.peerID, routerB.peerID), 15*time.Second)- waitForEqual(t, "relays in the provider's DHT record as the caller reads it", func() []string { return circuitRelaysInRecord(t, caller, provider.peerID) },+ waitForEqual(t, "relays in the records the provider announced", func() []string { return recordedCircuitRelays(t, provider) },+ peerIDStrings(routerA.peerID, routerB.peerID), 15*time.Second)+ waitForEqual(t, "relays in the provider's record as the caller reads it", func() []string { return circuitRelaysInRecord(t, caller, provider.peerID) },peerIDStrings(routerA.peerID, routerB.peerID), 15*time.Second)callAcrossMesh(t, caller, "token-c", provider, "shared-tool")// The caller holds router b, which the provider's record names, so the- // call went through it and no handshake was run for transit. Router a- // may still hold the caller for its DHT: the provider's key lands on- // the closest routers, which on a mesh of three is all of them.+ // call went through it and no handshake was run for transit.for _, kind := range []string{"listed", "unlisted"} {for _, result := range []string{"ok", "failed"} {series := fmt.Sprintf(`agentmesh_node_transit_handshakes_total{result="%s",router="%s"}`, result, kind)@@ -808,9 +824,10 @@ func TestCallerTransitsARouterItsCatalogMissed(t *testing.T) {waitForEqual(t, "routers attached by the provider", func() []string { return attachedRouters(t, provider) }, peerIDStrings(routerB.peerID), 10*time.Second)waitForEqual(t, "routers the provider announces circuits through", func() []string { return announcedCircuitRelays(t, provider) },peerIDStrings(routerB.peerID), 15*time.Second)- // A peer lookup by ID has to connect the peer, and the only path is the- // circuit through router b, so the record is not readable from here- // before the handshake there; discovery by service name needs no dial.+ // Router a does not hold the provider, so only the record the provider+ // announced under its peer key can answer the caller's lookup of it.+ waitForEqual(t, "relays in the provider's record as the caller reads it", func() []string { return circuitRelaysInRecord(t, caller, provider.peerID) },+ peerIDStrings(routerB.peerID), 15*time.Second)callAcrossMesh(t, caller, "token-c", provider, "late-tool")// Router b admitted the provider, router a, the control plane's publisher