Skip to content

Agent Mesh

tests: poll instead of sleeping; node: /debug/peer-addrs

Antonio Ojea3bee1c3733dc
diff --git a/cmd/agentmesh-node/debug.go b/cmd/agentmesh-node/debug.go
index 71f65cb8..a87d7bd0 100644
--- a/cmd/agentmesh-node/debug.go
+++ b/cmd/agentmesh-node/debug.go
@@ -40,7 +40,7 @@ func newDebugCmd() *cobra.Command {
debugCmd.PersistentFlags().StringVar(&socketPathFlag, "socket-path", "", "Unix socket of the running node (defaults to <data-dir>/"+node.DefaultSocketName+")")
debugCmd.AddCommand(
- newDebugGetCmd("mesh-info", "Show connected peers, DHT size, and the router peer ID", "/debug/mesh-info"),
+ newDebugGetCmd("mesh-info", "Show connected peers, the routers the node holds, and DHT size", "/debug/mesh-info"),
newDebugGetCmd("network-info", "Show listener and announced addresses", "/debug/network-info"),
newDebugGetCmd("token-info", "Show the local auth token's expiration and status", "/debug/token-info"),
newDebugGetCmd("logs", "Show the last few lines of the node's log output", "/debug/logs"),
@@ -60,6 +60,16 @@ func newDebugCmd() *cobra.Command {
},
})
+ debugCmd.AddCommand(&cobra.Command{
+ Use: "peer-addrs <peer-id>",
+ Short: "Show the addresses the DHT holds for a peer, without dialing it",
+ Args: cobra.ExactArgs(1),
+ SilenceUsage: true,
+ RunE: func(cmd *cobra.Command, args []string) error {
+ return debugRequest(cmd, http.MethodGet, "/debug/peer-addrs?peer_id="+url.QueryEscape(args[0]), nil)
+ },
+ })
+
debugCmd.AddCommand(&cobra.Command{
Use: "connect-peer <multiaddr>",
Short: "Connect the node to a peer by its full multiaddress",
diff --git a/internal/node/debug_handlers.go b/internal/node/debug_handlers.go
index ae546933..411c8728 100644
--- a/internal/node/debug_handlers.go
+++ b/internal/node/debug_handlers.go
@@ -51,6 +51,19 @@ func newDebugHandler(n *AgentMeshNode) http.Handler {
mux.HandleFunc("GET /debug/network-info", func(w http.ResponseWriter, r *http.Request) {
writeDebugJSON(w, n.networkInfo())
})
+ mux.HandleFunc("GET /debug/peer-addrs", func(w http.ResponseWriter, r *http.Request) {
+ pid, err := peer.Decode(r.URL.Query().Get("peer_id"))
+ if err != nil {
+ http.Error(w, "invalid peer id", http.StatusBadRequest)
+ return
+ }
+ resp, err := n.peerAddrs(r.Context(), pid)
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusInternalServerError)
+ return
+ }
+ writeDebugJSON(w, resp)
+ })
mux.HandleFunc("GET /debug/token-info", func(w http.ResponseWriter, r *http.Request) {
writeDebugJSON(w, n.tokenInfo())
})
@@ -145,6 +158,13 @@ type networkInfoResponse struct {
AnnouncedAddresses []string `json:"announced_addresses"`
}
+// peerAddrsResponse is what the DHT says about a peer: the addresses it
+// published, before this node decides which to dial.
+type peerAddrsResponse struct {
+ PeerID string `json:"peer_id"`
+ Addresses []string `json:"addresses"`
+}
+
type logsResponse struct {
Logs []string `json:"logs"`
}
@@ -181,6 +201,27 @@ func (n *AgentMeshNode) meshInfo() (*meshInfoResponse, error) {
}, nil
}
+// peerAddrs backs GET /debug/peer-addrs: the addresses the DHT holds for a
+// peer, looked up and not dialed, so an operator can see what a call would
+// have to work with without making one.
+func (n *AgentMeshNode) peerAddrs(ctx context.Context, pid peer.ID) (*peerAddrsResponse, error) {
+ if err := n.debugReady(); err != nil {
+ return nil, err
+ }
+ ctx, cancel := context.WithTimeout(ctx, dhtLookupTimeout)
+ defer cancel()
+ info, err := n.DHT.FindPeer(ctx, pid)
+ if err != nil {
+ return nil, fmt.Errorf("find %s: %w", pid, err)
+ }
+ resp := &peerAddrsResponse{PeerID: pid.String(), Addresses: make([]string, 0, len(info.Addrs))}
+ for _, a := range info.Addrs {
+ resp.Addresses = append(resp.Addresses, a.String())
+ }
+ slices.Sort(resp.Addresses)
+ return resp, nil
+}
+
// connectivityStats backs GET /debug/connectivity: with a peer ID it pings that
// peer, otherwise it pings the Agent Mesh router.
func (n *AgentMeshNode) connectivityStats(ctx context.Context, peerIDStr string) connectivityResponse {
diff --git a/site/content/docs/reference/node-api.md b/site/content/docs/reference/node-api.md
index 6d4ed5d3..dbffb2e8 100644
--- a/site/content/docs/reference/node-api.md
+++ b/site/content/docs/reference/node-api.md
@@ -366,7 +366,10 @@ material is meant for the node's owner and for auditors, not for agents.
## Debug
`GET /debug/mesh-info`, `GET /debug/connectivity`, `GET /debug/network-info`,
-`GET /debug/token-info`, `GET /debug/logs` and `POST /debug/connect-peer`
-report and probe the node's view of the mesh for troubleshooting, and answer
-even while the mesh is unreachable. `agentmesh-node debug` on the command line
-wraps them. Their output is not stable across releases.
+`GET /debug/peer-addrs?peer_id=`, `GET /debug/token-info`, `GET /debug/logs`
+and `POST /debug/connect-peer` report and probe the node's view of the mesh
+for troubleshooting, and answer even while the mesh is unreachable.
+`mesh-info` lists the routers the node holds (`attached_routers`) next to
+every peer it is connected to; `peer-addrs` is the DHT record of a peer as
+this node reads it, looked up and not dialed. `agentmesh-node debug` on the
+command line wraps them. Their output is not stable across releases.
diff --git a/tests/integration/router_attachment_test.go b/tests/integration/router_attachment_test.go
index 7602fd1f..0f345660 100644
--- a/tests/integration/router_attachment_test.go
+++ b/tests/integration/router_attachment_test.go
@@ -212,9 +212,41 @@ func announcedCircuitRelays(t *testing.T, n *backgroundNode) []string {
if err := json.Unmarshal([]byte(debugGetWithToken(t, n.apiAddr, n.token, "/debug/network-info")), &info); err != nil {
t.Fatalf("decode network-info: %v", err)
}
+ return circuitRelaysOf(info.AnnouncedAddresses)
+}
+
+// circuitRelaysInRecord lists the relays in a peer's DHT record as reader
+// looks it up: the addresses a call from reader would choose among.
+func circuitRelaysInRecord(t *testing.T, reader *backgroundNode, target peer.ID) []string {
+ t.Helper()
+ req, err := http.NewRequest(http.MethodGet, "http://"+reader.apiAddr+"/debug/peer-addrs?peer_id="+target.String(), nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ req.Header.Set(api.HeaderMeshAuthentication, "Bearer "+reader.token)
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatalf("GET /debug/peer-addrs: %v", err)
+ }
+ defer func() { _ = resp.Body.Close() }()
+ body, _ := io.ReadAll(resp.Body)
+ if resp.StatusCode != http.StatusOK {
+ // Not in the DHT yet.
+ return nil
+ }
+ var info struct {
+ Addresses []string `json:"addresses"`
+ }
+ if err := json.Unmarshal(body, &info); err != nil {
+ t.Fatalf("decode peer-addrs: %v", err)
+ }
+ return circuitRelaysOf(info.Addresses)
+}
+
+func circuitRelaysOf(addrs []string) []string {
seen := map[string]bool{}
var out []string
- for _, a := range info.AnnouncedAddresses {
+ for _, a := range addrs {
if i := strings.Index(a, "/p2p-circuit"); i > 0 {
parts := strings.Split(a[:i], "/p2p/")
relay := parts[len(parts)-1]
@@ -513,29 +545,33 @@ func TestIdleNodesHoldOnlyTheirRouters(t *testing.T) {
waitForEqual(t, "routers attached by node "+s, func() []string { return attachedRouters(t, nodes[s]) }, peerIDStrings(routers[s].peerID), 10*time.Second)
}
- // Twenty discovery intervals (200ms) with nothing to do.
- time.Sleep(4 * time.Second)
-
- for _, s := range sites {
- attached, connected := meshInfo(t, nodes[s])
- if !slices.Equal(attached, peerIDStrings(routers[s].peerID)) {
- t.Errorf("node %s holds %v, want its router %s", s, attached, routers[s].peerID)
- }
- for _, p := range connected {
- for _, other := range sites {
- if other != s && p == nodes[other].peerID.String() {
- t.Errorf("idle node %s is connected to node %s", s, other)
+ // Twenty discovery intervals (200ms) with nothing to do; the invariants
+ // are checked throughout, so a node that reaches out fails the test the
+ // moment it does.
+ for end := time.Now().Add(4 * time.Second); time.Now().Before(end); time.Sleep(200 * time.Millisecond) {
+ for _, s := range sites {
+ attached, connected := meshInfo(t, nodes[s])
+ if !slices.Equal(attached, peerIDStrings(routers[s].peerID)) {
+ t.Errorf("node %s holds %v, want its router %s", s, attached, routers[s].peerID)
+ }
+ for _, p := range connected {
+ for _, other := range sites {
+ if other != s && p == nodes[other].peerID.String() {
+ t.Errorf("idle node %s is connected to node %s", s, other)
+ }
}
}
+ if t.Failed() {
+ t.Fatalf("node %s connected peers: %v", s, connected)
+ }
}
+ }
+ for _, s := range sites {
// Its own node, the two peer routers and the control plane's
// event publisher.
if got := routers[s].gauge(t, "agentmesh_router_authenticated_peers"); got != 4 {
t.Errorf("router %s authenticated %v peers, want 4 (its node, two routers, the control plane)", s, got)
}
- if t.Failed() {
- t.Logf("node %s connected peers: %v", s, connected)
- }
}
if t.Failed() {
for _, s := range sites {
@@ -625,10 +661,12 @@ func TestCallerTransitsTheRouterItShares(t *testing.T) {
waitForEqual(t, "routers attached by the caller", func() []string { return attachedRouters(t, caller) }, peerIDStrings(routerB.peerID, routerC.peerID), 10*time.Second)
// The provider's record must name both circuits, or the caller has only
// router a to go through. A reservation on the second router comes a
- // moment after the first, and the record follows it after a 2 s debounce.
+ // moment after the first, and the record follows it after a 2 s debounce;
+ // the caller's DHT lookup is what the call will read.
waitForEqual(t, "routers the provider announces circuits through", func() []string { return announcedCircuitRelays(t, provider) },
peerIDStrings(routerA.peerID, routerB.peerID), 15*time.Second)
- time.Sleep(3 * time.Second)
+ waitForEqual(t, "relays in the provider's DHT record as the caller reads it", func() []string { return circuitRelaysInRecord(t, caller, provider.peerID) },
+ peerIDStrings(routerA.peerID, routerB.peerID), 15*time.Second)
callAcrossMesh(t, caller, "token-c", provider, "shared-tool")
@@ -675,18 +713,27 @@ func TestRouterKeepsOnlyAdmittedConnections(t *testing.T) {
t.Fatalf("stranger could not connect to the router: %v", err)
}
- // Three deadlines: the stranger is gone, the publisher and the node
- // still there, and the counter says one connection was closed for it.
- time.Sleep(3 * time.Second)
+ // The stranger is gone within a deadline or so; the publisher and the
+ // node, both authenticated, are still there after several.
+ deadline := time.Now().Add(5 * time.Second)
+ for stranger.Network().Connectedness(router.peerID) == network.Connected && time.Now().Before(deadline) {
+ time.Sleep(50 * time.Millisecond)
+ }
if stranger.Network().Connectedness(router.peerID) == network.Connected {
t.Fatal("the stranger's connection survived the deadline")
}
if got := router.gauge(t, "agentmesh_router_unauthenticated_connections_closed_total"); got < 1 {
t.Fatalf("closed %v unauthenticated connections, want at least the stranger's", got)
}
- // The node and the control plane's publisher: both authenticated.
- if got := router.gauge(t, "agentmesh_router_authenticated_peers"); got != 2 {
- t.Fatalf("router authenticated %v peers, want 2 (the node and the control plane's publisher)", got)
+ // The node and the control plane's publisher: both authenticated, and
+ // still so two deadlines after the stranger went.
+ waitForEqual(t, "peers the router authenticated", func() []string {
+ return []string{fmt.Sprint(router.gauge(t, "agentmesh_router_authenticated_peers"))}
+ }, []string{"2"}, 5*time.Second)
+ for end := time.Now().Add(2 * time.Second); time.Now().Before(end); time.Sleep(200 * time.Millisecond) {
+ if got := router.gauge(t, "agentmesh_router_authenticated_peers"); got != 2 {
+ t.Fatalf("router authenticated %v peers, want 2 (the node and the control plane's publisher)", got)
+ }
}
if got := attachedRouters(t, node); !slices.Equal(got, peerIDStrings(router.peerID)) {
t.Fatalf("node holds %v after the deadlines, want %s", got, router.peerID)