Skip to content

Agent Mesh

improve node resilience

Antonio Ojeab66d6ba9d442
diff --git a/cmd/sam-node/mcp.go b/cmd/sam-node/mcp.go
index 36edc45a..0d08865d 100644
--- a/cmd/sam-node/mcp.go
+++ b/cmd/sam-node/mcp.go
@@ -145,9 +145,8 @@ func (n *SamNode) CallMCPTool(ctx context.Context, targetPeer peer.ID, toolName
maxRetries := 3
backoff := 1 * time.Second
- // Attempt to resolve direct IPs for any relay nodes in the target's addresses.
- // This helps avoid dial backoff issues when using shared DNS addresses for hubs.
- n.resolveRelayAddresses(ctx, targetPeer)
+ // Attempt to filter private IPs and resolve direct IPs for relay nodes.
+ n.preparePeerAddrs(ctx, targetPeer)
for i := 0; i < maxRetries; i++ {
res, err = n.callMCPToolOnce(ctx, targetPeer, toolName, params)
@@ -171,37 +170,45 @@ func (n *SamNode) CallMCPTool(ctx context.Context, targetPeer peer.ID, toolName
return nil, fmt.Errorf("failed after %d retries: %w", maxRetries, err)
}
-func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolName string, params any) (*mcp.CallToolResult, error) {
+func (n *SamNode) ConnectMCPSession(ctx context.Context, targetPeer peer.ID, targetService string) (*mcp.ClientSession, func(), error) {
// Open stream
logger.Debugf("Dialing %s for MCP...\n", targetPeer)
+ // For discovery, we want a fast failure, but for tool calls, we can wait a bit. We'll use the context's deadline,
+ // but bound it to 15s max for the dial itself to prevent hanging.
dialCtx, cancel := context.WithTimeout(ctx, 15*time.Second)
- // Allow limited connections (like circuit relays) to be used for this stream.
- // Otherwise libp2p will hang waiting for a direct connection!
dialCtx = network.WithAllowLimitedConn(dialCtx, "mcp")
defer cancel()
+
s, err := n.Host.NewStream(dialCtx, targetPeer, api.MCPProtocolID)
if err != nil {
- return nil, fmt.Errorf("failed to open stream to %s: %w", targetPeer, err)
+ // If dial failed completely, mark private IP failed so we skip it next time
+ _ = n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, true)
+ return nil, nil, fmt.Errorf("failed to open stream to %s: %w", targetPeer, err)
+ }
+
+ if remoteAddr := s.Conn().RemoteMultiaddr(); remoteAddr != nil {
+ parts := multiaddr.Split(remoteAddr)
+ isCircuit := len(parts) >= 2 && parts[len(parts)-1].Protocol().Code == multiaddr.P_CIRCUIT
+
+ if isCircuit || !isPrivateIP(remoteAddr) {
+ _ = n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, true)
+ } else {
+ _ = n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, false)
+ }
}
logger.Debugf("Opened stream to %s for MCP\n", targetPeer)
- defer func() {
+
+ cleanup := func() {
if err := s.Close(); err != nil {
logger.Debugf("[MCP] Failed to close stream: %v", err)
}
- }()
+ }
// Load this node's biscuit
biscuitBytes, err := n.Store.LoadIdentity()
if err != nil {
- return nil, fmt.Errorf("failed to load identity biscuit: %w", err)
- }
-
- // Extract target service if it's a federated tool call (service.tool)
- targetService := "/sam/catalog"
- originalToolName := toolName
- if parts := strings.SplitN(toolName, ".", 2); len(parts) == 2 {
- targetService = parts[0]
- originalToolName = parts[1]
+ cleanup()
+ return nil, nil, fmt.Errorf("failed to load identity biscuit: %w", err)
}
// Marshal AuthFrame
@@ -218,7 +225,8 @@ func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolN
logger.Debugf("Writing auth frame to %s...\n", targetPeer)
writer := msgio.NewVarintWriter(s)
if err := writer.WriteMsg(authBytes); err != nil {
- return nil, fmt.Errorf("failed to write auth frame to %s: %w", targetPeer, err)
+ cleanup()
+ return nil, nil, fmt.Errorf("failed to write auth frame to %s: %w", targetPeer, err)
}
// Read ACK
@@ -226,7 +234,8 @@ func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolN
reader := msgio.NewVarintReaderSize(s, 1024*64)
msg, err := reader.ReadMsg()
if err != nil {
- return nil, fmt.Errorf("failed to read auth response from %s: %w", targetPeer, err)
+ cleanup()
+ return nil, nil, fmt.Errorf("failed to read auth response from %s: %w", targetPeer, err)
}
defer reader.ReleaseMsg(msg)
@@ -235,11 +244,13 @@ func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolN
var resp api.AuthResponse
if err := proto.Unmarshal(msg, &resp); err != nil {
- return nil, fmt.Errorf("invalid auth response from %s: %w", targetPeer, err)
+ cleanup()
+ return nil, nil, fmt.Errorf("invalid auth response from %s: %w", targetPeer, err)
}
if !resp.Success {
- return nil, fmt.Errorf("auth rejected by %s: %s", targetPeer, resp.Error)
+ cleanup()
+ return nil, nil, fmt.Errorf("auth rejected by %s: %s", targetPeer, resp.Error)
}
// Handoff to SDK using custom transport
@@ -248,9 +259,31 @@ func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolN
session, err := mcpClient.Connect(ctx, transport, nil)
if err != nil {
- return nil, fmt.Errorf("failed to connect client: %w", err)
+ cleanup()
+ return nil, nil, fmt.Errorf("failed to connect client: %w", err)
+ }
+
+ fullCleanup := func() {
+ _ = session.Close()
+ cleanup()
}
- defer func() { _ = session.Close() }()
+
+ return session, fullCleanup, nil
+}
+
+func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolName string, params any) (*mcp.CallToolResult, error) {
+ targetService := "/sam/catalog"
+ originalToolName := toolName
+ if parts := strings.SplitN(toolName, ".", 2); len(parts) == 2 {
+ targetService = parts[0]
+ originalToolName = parts[1]
+ }
+
+ session, cleanup, err := n.ConnectMCPSession(ctx, targetPeer, targetService)
+ if err != nil {
+ return nil, err
+ }
+ defer cleanup()
// We don't need to marshall the params, the SDK takes care of it
// Passing pre-marshaled []byte triggers encoding/json's base64 which gets rejected
@@ -270,10 +303,24 @@ func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolN
// fetchRemoteServiceCatalog calls the remote peer's list_local_services
// MCP tool with the type filter and returns the parsed catalog.
func (n *SamNode) fetchRemoteServiceCatalog(ctx context.Context, peerID peer.ID, typeStr string) ([]*api.ServiceInfo, error) {
- res, err := n.CallMCPTool(ctx, peerID, "list_local_services", map[string]string{"type": typeStr})
+ n.preparePeerAddrs(ctx, peerID)
+
+ session, cleanup, err := n.ConnectMCPSession(ctx, peerID, "/sam/catalog")
if err != nil {
return nil, err
}
+ defer cleanup()
+
+ callParams := &mcp.CallToolParams{
+ Name: "list_local_services",
+ Arguments: map[string]string{"type": typeStr},
+ }
+
+ res, err := session.CallTool(ctx, callParams)
+ if err != nil {
+ return nil, fmt.Errorf("failed to call tool list_local_services: %w", err)
+ }
+
if res == nil || len(res.Content) == 0 {
return nil, fmt.Errorf("empty response")
}
@@ -289,32 +336,46 @@ func (n *SamNode) fetchRemoteServiceCatalog(ctx context.Context, peerID peer.ID,
return services, nil
}
-// resolveRelayAddresses scans the target peer's addresses for any p2p-circuit relays,
-// looks up the relay's direct IPs via the DHT, and adds them to the target's peerstore.
-// This prevents dial backoff errors when the relay is behind a load-balanced DNS address.
-func (n *SamNode) resolveRelayAddresses(ctx context.Context, targetPeer peer.ID) {
+// preparePeerAddrs scans the target peer's addresses, filters out unroutable private IPs,
+// and ensures relay circuits are available. This prevents dial backoff errors when the
+// relay is behind a load-balanced DNS address or when pods advertise internal IPs.
+func (n *SamNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID) {
if n.Host == nil || n.DHT == nil {
logger.Debugf("[Discovery] Host or DHT is nil")
return
}
addrs := n.Host.Peerstore().Addrs(targetPeer)
- logger.Debugf("[Discovery] resolveRelayAddresses for %s: found %d addrs in peerstore", targetPeer, len(addrs))
+ logger.Debugf("[Discovery] preparePeerAddrs for %s: found %d addrs in peerstore", targetPeer, len(addrs))
- var validCircuitAddrs []multiaddr.Multiaddr
+ var validAddrs []multiaddr.Multiaddr
+ changed := false
for _, ma := range addrs {
parts := multiaddr.Split(ma)
- if len(parts) < 2 {
- continue
- }
- if parts[len(parts)-1].Protocol().Code != multiaddr.P_CIRCUIT {
+ // If it's a circuit relay, we always keep the generic /p2p-circuit form.
+ // We'll reconstruct it below. But we must also check if the address itself is routable.
+ isCircuit := len(parts) >= 2 && parts[len(parts)-1].Protocol().Code == multiaddr.P_CIRCUIT
+
+ if !isCircuit {
+ // Regular address (not a circuit relay). Filter private IPs if loopback isn't allowed,
+ // and we know from a previous dial that private IPs are unreachable for this peer.
+ if isPrivateIP(ma) && !n.AllowLoopback {
+ val, err := n.Host.Peerstore().Get(targetPeer, PeerstoreKeyPrivateIPFailed)
+ if err == nil && val.(bool) {
+ changed = true
+ logger.Debugf("[Discovery] Skipping private IP %s for %s due to previous failure", ma, targetPeer)
+ continue
+ }
+ }
+ validAddrs = append(validAddrs, ma)
continue
}
// Keep the advertised circuit address
- validCircuitAddrs = append(validCircuitAddrs, ma)
+ validAddrs = append(validAddrs, ma)
+ // Also synthesize a generic /p2p-circuit using the relay's peer ID.
p2pPart := parts[len(parts)-2]
if p2pPart.Protocol().Code != multiaddr.P_P2P {
continue
@@ -322,25 +383,20 @@ func (n *SamNode) resolveRelayAddresses(ctx context.Context, targetPeer peer.ID)
relayID, err := peer.Decode(p2pPart.Value())
if err != nil {
- logger.Debugf("[Discovery] Failed to decode relayID: %v", err)
continue
}
- logger.Debugf("[Discovery] Found relay %s for target %s", relayID, targetPeer)
-
circuitAddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/p2p/%s/p2p-circuit", relayID.String()))
if err == nil {
- logger.Debugf("[Discovery] Added generic relay circuit address for %s: %s", targetPeer, circuitAddr.String())
- validCircuitAddrs = append(validCircuitAddrs, circuitAddr)
- } else {
- logger.Errorf("[Discovery] Failed to create circuit addr: %v", err)
+ validAddrs = append(validAddrs, circuitAddr)
+ changed = true
}
}
- if len(validCircuitAddrs) > 0 {
+ if changed && len(validAddrs) > 0 {
// Clear existing addresses to prevent libp2p from hanging on dead private IPs
n.Host.Peerstore().ClearAddrs(targetPeer)
- n.Host.Peerstore().AddAddrs(targetPeer, validCircuitAddrs, peerstore.TempAddrTTL)
- logger.Infof("[Discovery] Replaced addrs for %s with %d circuit addrs", targetPeer, len(validCircuitAddrs))
+ n.Host.Peerstore().AddAddrs(targetPeer, validAddrs, peerstore.TempAddrTTL)
+ logger.Infof("[Discovery] Replaced addrs for %s with %d routable/circuit addrs", targetPeer, len(validAddrs))
}
}
diff --git a/cmd/sam-node/mcp_handlers.go b/cmd/sam-node/mcp_handlers.go
index 632d9a4a..81914f6d 100644
--- a/cmd/sam-node/mcp_handlers.go
+++ b/cmd/sam-node/mcp_handlers.go
@@ -10,10 +10,8 @@ import (
"github.com/google/sam/api"
"github.com/libp2p/go-libp2p/core/peer"
- "github.com/libp2p/go-msgio"
"github.com/modelcontextprotocol/go-sdk/mcp"
"github.com/multiformats/go-multiaddr"
- "google.golang.org/protobuf/proto"
)
// SendMessageParams defines the parameters for the send_message tool.
@@ -331,50 +329,16 @@ func (n *SamNode) fetchRemoteToolCatalogue(ctx context.Context, targetPeer peer.
}
var allTools []*mcp.Tool
- biscuitBytes, err := n.Store.LoadIdentity()
- if err != nil {
- return nil, fmt.Errorf("load identity: %w", err)
- }
for _, svc := range services {
if svc.Type != api.ServiceType_SERVICE_TYPE_MCP {
continue
}
- s, err := n.Host.NewStream(ctx, targetPeer, api.MCPProtocolID)
- if err != nil {
- logger.Debugf("Failed to open stream for service %s: %v", svc.Name, err)
- continue
- }
-
- authFrame := api.AuthFrame{Biscuit: biscuitBytes, TargetService: svc.Name}
- authBytes, _ := proto.Marshal(&authFrame)
-
- writer := msgio.NewVarintWriter(s)
- if err := writer.WriteMsg(authBytes); err != nil {
- _ = s.Close()
- continue
- }
-
- reader := msgio.NewVarintReaderSize(s, 1024*64)
- respMsg, err := reader.ReadMsg()
- if err != nil {
- _ = s.Close()
- continue
- }
- defer reader.ReleaseMsg(respMsg)
-
- var resp api.AuthResponse
- if err := proto.Unmarshal(respMsg, &resp); err != nil || !resp.Success {
- _ = s.Close()
- continue
- }
-
- transport := NewStreamTransport(s)
- client := mcp.NewClient(&mcp.Implementation{Name: "sam-node-find", Version: "0.1.0"}, nil)
- session, err := client.Connect(ctx, transport, nil)
+ n.preparePeerAddrs(ctx, targetPeer)
+ session, cleanup, err := n.ConnectMCPSession(ctx, targetPeer, svc.Name)
if err != nil {
- _ = s.Close()
+ logger.Debugf("Failed to connect MCP session for service %s: %v", svc.Name, err)
continue
}
@@ -386,8 +350,7 @@ func (n *SamNode) fetchRemoteToolCatalogue(ctx context.Context, targetPeer peer.
}
}
- _ = session.Close()
- _ = s.Close()
+ cleanup()
}
return allTools, nil
@@ -495,45 +458,13 @@ func (n *SamNode) handleDescribeRemoteTool(ctx context.Context, req *mcp.CallToo
}
serviceName := parts[0]
actualToolName := parts[1]
+ n.preparePeerAddrs(ctx, pid)
- biscuitBytes, err := n.Store.LoadIdentity()
- if err != nil {
- return nil, nil, err
- }
-
- s, err := n.Host.NewStream(ctx, pid, api.MCPProtocolID)
- if err != nil {
- return nil, nil, fmt.Errorf("open stream: %w", err)
- }
- defer func() { _ = s.Close() }()
-
- authFrame := api.AuthFrame{Biscuit: biscuitBytes, TargetService: serviceName}
- authBytes, _ := proto.Marshal(&authFrame)
-
- writer := msgio.NewVarintWriter(s)
- if err := writer.WriteMsg(authBytes); err != nil {
- return nil, nil, err
- }
-
- reader := msgio.NewVarintReaderSize(s, 1024*64)
- respMsg, err := reader.ReadMsg()
- if err != nil {
- return nil, nil, err
- }
- defer reader.ReleaseMsg(respMsg)
-
- var authResp api.AuthResponse
- if err := proto.Unmarshal(respMsg, &authResp); err != nil || !authResp.Success {
- return nil, nil, fmt.Errorf("auth rejected")
- }
-
- transport := NewStreamTransport(s)
- client := mcp.NewClient(&mcp.Implementation{Name: "sam-node-describe", Version: "0.1.0"}, nil)
- session, err := client.Connect(ctx, transport, nil)
+ session, cleanup, err := n.ConnectMCPSession(ctx, pid, serviceName)
if err != nil {
return nil, nil, err
}
- defer func() { _ = session.Close() }()
+ defer cleanup()
listRes, err := session.ListTools(ctx, nil)
if err != nil {
diff --git a/cmd/sam-node/mcp_test.go b/cmd/sam-node/mcp_test.go
index 311ac3ff..71ed7b79 100644
--- a/cmd/sam-node/mcp_test.go
+++ b/cmd/sam-node/mcp_test.go
@@ -131,7 +131,7 @@ func TestResolveRelayAddresses(t *testing.T) {
localHost.Peerstore().AddAddr(targetID, circuitAddr, peerstore.PermanentAddrTTL)
// Run the function
- node.resolveRelayAddresses(ctx, targetID)
+ node.preparePeerAddrs(ctx, targetID)
// Verify that the direct IP circuit address was added to the target's peerstore
addrs := localHost.Peerstore().Addrs(targetID)
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 2771a986..2ffee741 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -60,6 +60,12 @@ import (
"google.golang.org/protobuf/proto"
)
+// PeerstoreKeyPrivateIPFailed is the key used in the libp2p Peerstore to track
+// if a peer's private IP was previously found to be unreachable or slower than a relay.
+// This allows the node to "try once and remember", avoiding a 15-second timeout on
+// subsequent discovery or tool calls when dialing unroutable private networks.
+const PeerstoreKeyPrivateIPFailed = "private_ip_failed"
+
const (
// Cache sizes
RateLimiterSize = 1000
@@ -71,6 +77,9 @@ const (
// Key pruning
KeyPruningInterval = 1 * time.Hour
+
+ // Reprovide interval
+ ReprovideInterval = 5 * time.Minute
)
var ErrFatalAuth = errors.New("fatal authentication error")
@@ -118,6 +127,7 @@ type SamNode struct {
authSuccess chan struct{}
authOnce sync.Once
currentRelays []peer.AddrInfo
+ reprovideTrigger chan struct{}
}
// UpdateRelays updates the current relays used by AutoRelay.
@@ -173,6 +183,7 @@ func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {
AllowLoopback: cfg.AllowLoopback,
TrustHubRBAC: cfg.TrustHubRBAC,
authSuccess: make(chan struct{}),
+ reprovideTrigger: make(chan struct{}, 1),
}
var err error
@@ -309,10 +320,6 @@ func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {
}
node.DHT = kdht
- if err := kdht.Bootstrap(ctx); err != nil {
- return nil, fmt.Errorf("failed to bootstrap DHT: %w", err)
- }
-
node.services = NewServiceRegistry(node.DHT)
var authenticated bool
@@ -337,6 +344,13 @@ func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {
return nil, fmt.Errorf("failed to authenticate with any hub: all connection attempts failed")
}
+ if authenticated {
+ logger.Infof("[DHT] Bootstrapping DHT with connected hub...")
+ if err := node.DHT.Bootstrap(ctx); err != nil {
+ logger.Warnf("[DHT] Failed to trigger DHT bootstrap: %v", err)
+ }
+ }
+
// Initialize Gossipsub for Hub Events
ps, err := pubsub.NewGossipSub(ctx, h)
if err != nil {
@@ -369,10 +383,13 @@ func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {
}
logger.Infof("[Network] Local addresses updated: %v", addrs)
- // Debounce or reprovide immediately
+ // Debounce and trigger unified reprovide loop
go func() {
time.Sleep(2 * time.Second) // Small debounce
- node.services.ReprovideAll(ctx)
+ select {
+ case node.reprovideTrigger <- struct{}{}:
+ default:
+ }
}()
}
}
@@ -408,9 +425,38 @@ func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {
// Start connection monitor
node.startConnectionMonitor(ctx, cfg.MonitorBootstrap, cfg.MonitorInterval, 3)
+ // Periodically and on-demand reprovide registered services to the DHT
+ node.startReprovideLoop(ctx, ReprovideInterval)
+
return node, nil
}
+func (n *SamNode) startReprovideLoop(ctx context.Context, interval time.Duration) {
+ go func() {
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+
+ // Run an initial reprovide after a short delay to let DHT bootstrap stabilize
+ select {
+ case <-ctx.Done():
+ return
+ case <-time.After(5 * time.Second):
+ n.services.ReprovideAll(ctx)
+ }
+
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-ticker.C:
+ n.services.ReprovideAll(ctx)
+ case <-n.reprovideTrigger:
+ n.services.ReprovideAll(ctx)
+ }
+ }
+ }()
+}
+
func (n *SamNode) IsConnected() bool {
return n.HubPeerID != "" && n.Host.Network().Connectedness(n.HubPeerID) == network.Connected
}
@@ -1336,16 +1382,39 @@ func (n *SamNode) StartIngressServer(ctx context.Context) error {
func isLoopbackOrLinkLocal(addr multiaddr.Multiaddr) bool {
for _, proto := range addr.Protocols() {
- if proto.Code == multiaddr.P_IP4 || proto.Code == multiaddr.P_IP6 {
- value, err := addr.ValueForProtocol(proto.Code)
- if err == nil {
- ip := net.ParseIP(value)
- if ip != nil {
- if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
- return true
- }
- }
- }
+ if proto.Code != multiaddr.P_IP4 && proto.Code != multiaddr.P_IP6 {
+ continue
+ }
+ value, err := addr.ValueForProtocol(proto.Code)
+ if err != nil {
+ continue
+ }
+ ip := net.ParseIP(value)
+ if ip == nil {
+ continue
+ }
+ if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
+ return true
+ }
+ }
+ return false
+}
+
+func isPrivateIP(addr multiaddr.Multiaddr) bool {
+ for _, proto := range addr.Protocols() {
+ if proto.Code != multiaddr.P_IP4 && proto.Code != multiaddr.P_IP6 {
+ continue
+ }
+ value, err := addr.ValueForProtocol(proto.Code)
+ if err != nil {
+ continue
+ }
+ ip := net.ParseIP(value)
+ if ip == nil {
+ continue
+ }
+ if ip.IsPrivate() {
+ return true
}
}
return false