Skip to content

Agent Mesh

node: announce the node's own record and show what the directory holds

Antonio Ojeaf73afa08e7cb
diff --git a/cmd/agentmesh-node/debug.go b/cmd/agentmesh-node/debug.go
index a87d7bd0..315fd315 100644
--- a/cmd/agentmesh-node/debug.go
+++ b/cmd/agentmesh-node/debug.go
@@ -40,8 +40,8 @@ func newDebugCmd() *cobra.Command {
debugCmd.PersistentFlags().StringVar(&socketPathFlag, "socket-path", "", "Unix socket of the running node (defaults to <data-dir>/"+node.DefaultSocketName+")")
debugCmd.AddCommand(
- newDebugGetCmd("mesh-info", "Show connected peers, the routers the node holds, and DHT size", "/debug/mesh-info"),
- newDebugGetCmd("network-info", "Show listener and announced addresses", "/debug/network-info"),
+ newDebugGetCmd("mesh-info", "Show connected peers and the routers the node holds", "/debug/mesh-info"),
+ newDebugGetCmd("network-info", "Show listener, announced and directory addresses", "/debug/network-info"),
newDebugGetCmd("token-info", "Show the local auth token's expiration and status", "/debug/token-info"),
newDebugGetCmd("logs", "Show the last few lines of the node's log output", "/debug/logs"),
)
diff --git a/internal/node/debug_handlers.go b/internal/node/debug_handlers.go
index 765bcd91..07500240 100644
--- a/internal/node/debug_handlers.go
+++ b/internal/node/debug_handlers.go
@@ -156,6 +156,16 @@ type tokenInfoResponse struct {
type networkInfoResponse struct {
ListenAddresses []string `json:"listen_addresses"`
AnnouncedAddresses []string `json:"announced_addresses"`
+ // DirectoryAddresses is what the directory holds for this node: the
+ // addresses of the record it last announced, which follow
+ // AnnouncedAddresses at the next announcement pass.
+ DirectoryAddresses []string `json:"directory_addresses"`
+}
+
+// recordAnnouncer is a directory that remembers the record it last
+// announced.
+type recordAnnouncer interface {
+ Announced() []multiaddr.Multiaddr
}
// peerAddrsResponse is what the DHT says about a peer: the addresses it
@@ -294,9 +304,17 @@ func (n *AgentMeshNode) networkInfo() networkInfoResponse {
announcedAddrs = append(announcedAddrs, a.String())
}
+ directoryAddrs := []string{}
+ if d, ok := n.Directory.(recordAnnouncer); ok {
+ for _, a := range d.Announced() {
+ directoryAddrs = append(directoryAddrs, a.String())
+ }
+ }
+
return networkInfoResponse{
ListenAddresses: listenAddrs,
AnnouncedAddresses: announcedAddrs,
+ DirectoryAddresses: directoryAddrs,
}
}
diff --git a/internal/node/directory_client.go b/internal/node/directory_client.go
index e18a94e1..78903823 100644
--- a/internal/node/directory_client.go
+++ b/internal/node/directory_client.go
@@ -18,6 +18,8 @@ import (
"context"
"errors"
"fmt"
+ "slices"
+ "sync"
"time"
"github.com/libp2p/go-libp2p/core/host"
@@ -101,19 +103,35 @@ type routerDirectory struct {
host host.Host // the gated host: a stream to a router follows its handshake
routers func() []peer.ID
addrs func() []multiaddr.Multiaddr
+
+ mu sync.Mutex
+ announced []multiaddr.Multiaddr // the addresses of the record last announced
}
func (d *routerDirectory) Provide(ctx context.Context, key []byte) error {
addrs := d.addrs()
+ if len(addrs) > api.DirectoryMaxAddrs {
+ addrs = addrs[:api.DirectoryMaxAddrs]
+ }
rec := &api.DirectoryProvide{Key: key, Addrs: make([]string, 0, len(addrs))}
for _, a := range addrs {
rec.Addrs = append(rec.Addrs, a.String())
}
- if len(rec.Addrs) > api.DirectoryMaxAddrs {
- rec.Addrs = rec.Addrs[:api.DirectoryMaxAddrs]
+ if _, err := d.request(ctx, directoryOpProvide, &api.DirectoryRequest{Request: &api.DirectoryRequest_Provide{Provide: rec}}); err != nil {
+ return err
}
- _, err := d.request(ctx, directoryOpProvide, &api.DirectoryRequest{Request: &api.DirectoryRequest_Provide{Provide: rec}})
- return err
+ d.mu.Lock()
+ d.announced = addrs
+ d.mu.Unlock()
+ return nil
+}
+
+// Announced is the addresses of the record this member last announced:
+// what the directory holds for it, as against what it would announce now.
+func (d *routerDirectory) Announced() []multiaddr.Multiaddr {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ return slices.Clone(d.announced)
}
func (d *routerDirectory) FindProviders(ctx context.Context, key []byte, limit int) ([]peer.AddrInfo, error) {
diff --git a/internal/node/directory_client_test.go b/internal/node/directory_client_test.go
index 751556ba..dc295d53 100644
--- a/internal/node/directory_client_test.go
+++ b/internal/node/directory_client_test.go
@@ -15,7 +15,9 @@
package node
import (
+ "bytes"
"context"
+ "slices"
"sync"
"testing"
"time"
@@ -36,11 +38,12 @@ import (
// way a router does: records by key, a peer from the router's own peerstore
// or its peer-key record. It fails every request while failing is set.
type fakeDirectoryServer struct {
- h host.Host
- mu sync.Mutex
- records map[string]map[peer.ID][]multiaddr.Multiaddr
- served int
- failing bool
+ h host.Host
+ mu sync.Mutex
+ records map[string]map[peer.ID][]multiaddr.Multiaddr
+ provided [][]byte // keys announced here, in order
+ served int
+ failing bool
}
func serveDirectory(h host.Host) *fakeDirectoryServer {
@@ -71,6 +74,13 @@ func (d *fakeDirectoryServer) providers(key []byte) []peer.ID {
return out
}
+// providedKeys is the keys announced here so far, in order.
+func (d *fakeDirectoryServer) providedKeys() [][]byte {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ return slices.Clone(d.provided)
+}
+
// add puts a record in as if another router had placed it.
func (d *fakeDirectoryServer) add(key []byte, p peer.ID, addrs ...multiaddr.Multiaddr) {
d.mu.Lock()
@@ -111,6 +121,7 @@ func (d *fakeDirectoryServer) handle(s network.Stream) {
d.records[string(q.GetKey())] = map[peer.ID][]multiaddr.Multiaddr{}
}
d.records[string(q.GetKey())][s.Conn().RemotePeer()] = addrs
+ d.provided = append(d.provided, q.GetKey())
case req.GetFindProviders() != nil:
for p, addrs := range d.records[string(req.GetFindProviders().GetKey())] {
resp.Providers = append(resp.Providers, recordOf(p, addrs))
@@ -229,3 +240,50 @@ func TestRouterDirectoryAsksTheAttachedRouters(t *testing.T) {
t.Fatal("FindProviders succeeded with every router failing")
}
}
+
+// An announcement pass announces the node's services and then the node
+// itself under its peer key, so a router that does not hold the node can
+// still say where it is reached, and the pass is what the directory holds
+// for the node: its last announced addresses are the ones the records
+// carry, and a pass no router took leaves them as they were.
+func TestAnnounceAllEndsWithThePeerRecord(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
+ defer cancel()
+ router := newFakeRouter(t, "/ip4/127.0.0.1/tcp/0")
+ dir := serveDirectory(router.h)
+ member, directory := newDirectoryClient(t, router.h)
+ if got := directory.Announced(); len(got) != 0 {
+ t.Fatalf("Announced before any announcement = %v, want none", got)
+ }
+ node := &AgentMeshNode{Host: member, Directory: directory, services: NewServiceRegistry(directory, 0)}
+ if err := node.services.Register(ctx, newFakeSvc("calc", api.ServiceType_SERVICE_TYPE_MCP)); err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+ registered := len(dir.providedKeys())
+
+ if withheld := node.announceAll(ctx); withheld != 0 {
+ t.Fatalf("announceAll withheld %d services, want 0", withheld)
+ }
+ nameKey, _ := api.ServiceKey(api.ServiceType_SERVICE_TYPE_MCP, "calc")
+ typeKey, _ := api.ServiceKey(api.ServiceType_SERVICE_TYPE_MCP, "")
+ pass := dir.providedKeys()[registered:]
+ if len(pass) != 3 || !bytes.Equal(pass[0], nameKey) || !bytes.Equal(pass[1], typeKey) || !bytes.Equal(pass[2], api.PeerKey(member.ID())) {
+ t.Fatalf("the pass announced %d keys, want the service by name, by type, then the peer key last", len(pass))
+ }
+ if got := dir.providers(api.PeerKey(member.ID())); len(got) != 1 || got[0] != member.ID() {
+ t.Fatalf("the router holds %v under the peer key, want the member", got)
+ }
+ info, err := directory.FindPeer(ctx, member.ID())
+ if err != nil || len(info.Addrs) != len(member.Addrs()) {
+ t.Fatalf("FindPeer = %v, %v; want the member at its %d addresses", info, err, len(member.Addrs()))
+ }
+ if got, want := directory.Announced(), member.Addrs(); !slices.EqualFunc(got, want, multiaddr.Multiaddr.Equal) {
+ t.Fatalf("Announced = %v, want the addresses the pass carried, %v", got, want)
+ }
+
+ dir.fail(true)
+ node.announceAll(ctx)
+ if got, want := directory.Announced(), member.Addrs(); !slices.EqualFunc(got, want, multiaddr.Multiaddr.Equal) {
+ t.Fatalf("Announced after a pass no router took = %v, want the last announced %v", got, want)
+ }
+}
diff --git a/internal/node/node.go b/internal/node/node.go
index 174c94a1..b3a5d13a 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -798,9 +798,26 @@ func (n *AgentMeshNode) triggerReprovide() {
}
}
-// startReprovideLoop re-announces the node's services every reprovide
-// interval of the mesh, read at each cycle so a control plane that changed
-// it is followed at the next one.
+// announceAll is one announcement pass: the node's services, then the
+// node itself under its peer key, so a lookup of the node through a router
+// that does not hold it finds it at the addresses its records carry. The
+// peer record goes last so that, once it names an address, every service
+// record of the pass does too. It returns how many services were withheld
+// because their backend did not answer.
+func (n *AgentMeshNode) announceAll(ctx context.Context) int {
+ withheld := n.services.ReprovideAll(ctx)
+ provideCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ if err := n.Directory.Provide(provideCtx, api.PeerKey(n.Host.ID())); err != nil && ctx.Err() == nil && !errors.Is(err, errNoDirectoryRouter) {
+ // With no router attached the directory client has said so already.
+ logger.Warnf("[Directory] Announcing where this node is reached: %v", err)
+ }
+ return withheld
+}
+
+// startReprovideLoop re-announces the node's services and its own record
+// every reprovide interval of the mesh, read at each cycle so a control
+// plane that changed it is followed at the next one.
func (n *AgentMeshNode) startReprovideLoop(ctx context.Context) {
go func() {
initialDelay := 5 * time.Second
@@ -830,7 +847,7 @@ func (n *AgentMeshNode) startReprovideLoop(ctx context.Context) {
interval := n.directorySettings().Reprovide
next := interval
- if n.services.ReprovideAll(ctx) > 0 {
+ if n.announceAll(ctx) > 0 {
// A backend that is not answering yet is usually one that is
// still starting. Waiting the full interval would leave it
// undiscoverable for minutes after it came up.
@@ -946,7 +963,7 @@ func (n *AgentMeshNode) startConnectionMonitor(ctx context.Context, bootstrapDur
if reconnected {
logger.Infof("[Monitor] Reconnected successfully. Announcing services again...")
- n.services.ReprovideAll(ctx)
+ n.announceAll(ctx)
consecutiveFailures = 0
continue
}
diff --git a/site/content/docs/concepts/networking.md b/site/content/docs/concepts/networking.md
index b75062ce..b204cb23 100644
--- a/site/content/docs/concepts/networking.md
+++ b/site/content/docs/concepts/networking.md
@@ -44,12 +44,12 @@ The [node API reference](../../reference/node-api/) lists every route.
## Discovery
The routers keep the mesh's directory: a Kademlia DHT among themselves,
-served to members over `/mesh/directory/1.0.0`. A node that publishes a
-service announces itself to a router it holds as a provider of that
-service name, and where it is reached; the router keeps the record and
-places it on the routers closest to the key. A node that looks for a
-service, or for a peer, asks a router it holds, and the router runs the
-lookup across the routers' DHT. A node keeps no DHT of its own and
+served to members over `/mesh/directory/1.0.0`. A node announces where it
+is reached, and a node that publishes a service announces itself as a
+provider of that service name, to a router it holds; the router keeps the
+records and places each on the routers closest to its key. A node that
+looks for a service, or for a peer, asks a router it holds, and the router
+runs the lookup across the routers' DHT. A node keeps no DHT of its own and
connects to no router it does not hold. It re-announces every reprovide
interval of the mesh (`/info`), and a record a provider stopped
announcing leaves the directory after three of them.
@@ -86,10 +86,12 @@ out what each connection carries and what a NAT or firewall sees.
Routers are the fixed points. A node learns their addresses, labels and
load from the control plane's `/info` at enrollment and again on a
schedule, holds two of them with a relay reservation on each, and replaces
-one that drops or that sends it away. A router that grants a reservation
-records the member's address through itself, so a directory lookup for
-the member learns where it is reached; an SDK member listens on nothing
-else. A call to a peer reached through a
+one that drops or that sends it away. A router that holds a member answers
+a lookup for it from what it sees of the member; any other router answers
+from the record the member announced under its own peer ID, so a lookup
+for the member learns where it is reached through any router. An SDK
+member listens on nothing of its own and is reached through a router it
+holds. A call to a peer reached through a
third router goes through that router: the caller runs the credential
handshake there first, since a router relays only between peers it has
verified. The directory, the gossip and the relay are closed to anyone who
diff --git a/site/content/docs/preview/mobile.md b/site/content/docs/preview/mobile.md
index a2eec96b..640bfbf0 100644
--- a/site/content/docs/preview/mobile.md
+++ b/site/content/docs/preview/mobile.md
@@ -241,13 +241,14 @@ or shows an error, record the status text before retrying.
2. Allow up to a minute for the connection to settle.
3. Check that the dashboard says **Node is Running**, **Node ID** contains
an identifier, and **Connected Peers** is greater than zero.
-4. Record the **Node ID**, **Connected Peers**, and **DHT Size** values.
- Peer counts can change. A particular DHT size is not a pass criterion.
+4. Record the **Node ID**, **Connected Peers**, and **Routers** values.
+ Peer counts can change. **Routers** is how many routers the node holds,
+ two on a mesh with two or more.
**Node is Running** alone does not prove a mesh connection. Report a peer
count that remains zero, even if the app shows no error.
-{{< figure src="../../../images/agentmesh-connect-running.png" alt="Agent Mesh Connect dashboard showing Node is Running, one connected peer, DHT size one, and a Node ID" caption="A connected node shows Node is Running, a Node ID, and at least one Connected Peer. Your Node ID and peer counts will differ from this example." width="320" >}}
+{{< figure src="../../../images/agentmesh-connect-running.png" alt="Agent Mesh Connect dashboard showing Node is Running, one connected peer, one router, and a Node ID" caption="A connected node shows Node is Running, a Node ID, and at least one Connected Peer. Your Node ID and peer counts will differ from this example." width="320" >}}
### 5. Check restart and reconnection
diff --git a/site/content/docs/reference/node-api.md b/site/content/docs/reference/node-api.md
index b4e35420..6a51f3f3 100644
--- a/site/content/docs/reference/node-api.md
+++ b/site/content/docs/reference/node-api.md
@@ -378,6 +378,10 @@ material is meant for the node's owner and for auditors, not for agents.
and `POST /debug/connect-peer` report and probe the node's view of the mesh
for troubleshooting, and answer even while the mesh is unreachable.
`mesh-info` lists the routers the node holds (`attached_routers`) next to
-every peer it is connected to; `peer-addrs` is where the directory says a
-peer is reached, looked up and not dialed. `agentmesh-node debug` on the
-command line wraps them. Their output is not stable across releases.
+every peer it is connected to; `network-info` lists the addresses the node
+listens on, the ones it announces, and the ones in the records it last
+announced to the directory (`directory_addresses`), which follow the
+announced ones at the next announcement pass; `peer-addrs` is where the
+directory says a peer is reached, looked up and not dialed. `agentmesh-node
+debug` on the command line wraps them. Their output is not stable across
+releases.
diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.go
index fc3cc180..0268c3d9 100644
--- a/tests/integration/router_attachment_test.go
+++ b/tests/integration/router_attachment_test.go
@@ -233,17 +233,30 @@ func peerIDStrings(ids ...peer.ID) []string {
// addresses through, from its own debug endpoint.
func announcedCircuitRelays(t *testing.T, n *backgroundNode) []string {
t.Helper()
- var info struct {
- AnnouncedAddresses []string `json:"announced_addresses"`
- }
+ return circuitRelaysOf(networkInfo(t, n).AnnouncedAddresses)
+}
+
+// recordedCircuitRelays lists the relays in the records the node last
+// announced to the directory: what a lookup of its services returns, as
+// against announcedCircuitRelays, what it will announce at its next pass.
+func recordedCircuitRelays(t *testing.T, n *backgroundNode) []string {
+ t.Helper()
+ return circuitRelaysOf(networkInfo(t, n).DirectoryAddresses)
+}
+
+func networkInfo(t *testing.T, n *backgroundNode) (info struct {
+ AnnouncedAddresses []string `json:"announced_addresses"`
+ DirectoryAddresses []string `json:"directory_addresses"`
+}) {
+ t.Helper()
if err := json.Unmarshal([]byte(debugGetWithToken(t, n.apiAddr, n.token, "/debug/network-info")), &info); err != nil {
t.Fatalf("decode network-info: %v", err)
}
- return circuitRelaysOf(info.AnnouncedAddresses)
+ return info
}
-// circuitRelaysInRecord lists the relays in a peer's DHT record as reader
-// looks it up: the addresses a call from reader would choose among.
+// circuitRelaysInRecord lists the relays in a peer's record as reader
+// looks it up through a router it holds.
func circuitRelaysInRecord(t *testing.T, reader *backgroundNode, target peer.ID) []string {
t.Helper()
req, err := http.NewRequest(http.MethodGet, "http://"+reader.apiAddr+"/debug/peer-addrs?peer_id="+target.String(), nil)
@@ -736,21 +749,24 @@ func TestCallerTransitsTheRouterItShares(t *testing.T) {
caller.waitForAPI(t)
waitForEqual(t, "routers attached by the provider", func() []string { return attachedRouters(t, provider) }, peerIDStrings(routerA.peerID, routerB.peerID), 10*time.Second)
waitForEqual(t, "routers attached by the caller", func() []string { return attachedRouters(t, caller) }, peerIDStrings(routerB.peerID, routerC.peerID), 10*time.Second)
- // The provider's record must name both circuits, or the caller has only
+ // The provider's records must name both circuits, or the caller has only
// router a to go through. A reservation on the second router comes a
- // moment after the first, and the record follows it after a 2 s debounce;
- // the caller's DHT lookup is what the call will read.
+ // moment after the first, and the records follow it at the next
+ // announcement pass, 2 s later. A router that holds the provider answers
+ // a lookup from what it sees of it, which is ahead of the records; the
+ // call reads the service record, so the wait is on what the provider
+ // announced, then on what the caller reads.
waitForEqual(t, "routers the provider announces circuits through", func() []string { return announcedCircuitRelays(t, provider) },
peerIDStrings(routerA.peerID, routerB.peerID), 15*time.Second)
- waitForEqual(t, "relays in the provider's DHT record as the caller reads it", func() []string { return circuitRelaysInRecord(t, caller, provider.peerID) },
+ waitForEqual(t, "relays in the records the provider announced", func() []string { return recordedCircuitRelays(t, provider) },
+ peerIDStrings(routerA.peerID, routerB.peerID), 15*time.Second)
+ waitForEqual(t, "relays in the provider's record as the caller reads it", func() []string { return circuitRelaysInRecord(t, caller, provider.peerID) },
peerIDStrings(routerA.peerID, routerB.peerID), 15*time.Second)
callAcrossMesh(t, caller, "token-c", provider, "shared-tool")
// The caller holds router b, which the provider's record names, so the
- // call went through it and no handshake was run for transit. Router a
- // may still hold the caller for its DHT: the provider's key lands on
- // the closest routers, which on a mesh of three is all of them.
+ // call went through it and no handshake was run for transit.
for _, kind := range []string{"listed", "unlisted"} {
for _, result := range []string{"ok", "failed"} {
series := fmt.Sprintf(`agentmesh_node_transit_handshakes_total{result="%s",router="%s"}`, result, kind)
@@ -808,9 +824,10 @@ func TestCallerTransitsARouterItsCatalogMissed(t *testing.T) {
waitForEqual(t, "routers attached by the provider", func() []string { return attachedRouters(t, provider) }, peerIDStrings(routerB.peerID), 10*time.Second)
waitForEqual(t, "routers the provider announces circuits through", func() []string { return announcedCircuitRelays(t, provider) },
peerIDStrings(routerB.peerID), 15*time.Second)
- // A peer lookup by ID has to connect the peer, and the only path is the
- // circuit through router b, so the record is not readable from here
- // before the handshake there; discovery by service name needs no dial.
+ // Router a does not hold the provider, so only the record the provider
+ // announced under its peer key can answer the caller's lookup of it.
+ waitForEqual(t, "relays in the provider's record as the caller reads it", func() []string { return circuitRelaysInRecord(t, caller, provider.peerID) },
+ peerIDStrings(routerB.peerID), 15*time.Second)
callAcrossMesh(t, caller, "token-c", provider, "late-tool")
// Router b admitted the provider, router a, the control plane's publisher