Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
improve node resilience
Antonio Ojeab66d6ba9d442
diff --git a/cmd/sam-node/mcp.go b/cmd/sam-node/mcp.goindex 36edc45a..0d08865d 100644--- a/cmd/sam-node/mcp.go+++ b/cmd/sam-node/mcp.go@@ -145,9 +145,8 @@ func (n *SamNode) CallMCPTool(ctx context.Context, targetPeer peer.ID, toolNamemaxRetries := 3backoff := 1 * time.Second- // Attempt to resolve direct IPs for any relay nodes in the target's addresses.- // This helps avoid dial backoff issues when using shared DNS addresses for hubs.- n.resolveRelayAddresses(ctx, targetPeer)+ // Attempt to filter private IPs and resolve direct IPs for relay nodes.+ n.preparePeerAddrs(ctx, targetPeer)for i := 0; i < maxRetries; i++ {res, err = n.callMCPToolOnce(ctx, targetPeer, toolName, params)@@ -171,37 +170,45 @@ func (n *SamNode) CallMCPTool(ctx context.Context, targetPeer peer.ID, toolNamereturn nil, fmt.Errorf("failed after %d retries: %w", maxRetries, err)}-func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolName string, params any) (*mcp.CallToolResult, error) {+func (n *SamNode) ConnectMCPSession(ctx context.Context, targetPeer peer.ID, targetService string) (*mcp.ClientSession, func(), error) {// Open streamlogger.Debugf("Dialing %s for MCP...\n", targetPeer)+ // For discovery, we want a fast failure, but for tool calls, we can wait a bit. We'll use the context's deadline,+ // but bound it to 15s max for the dial itself to prevent hanging.dialCtx, cancel := context.WithTimeout(ctx, 15*time.Second)- // Allow limited connections (like circuit relays) to be used for this stream.- // Otherwise libp2p will hang waiting for a direct connection!dialCtx = network.WithAllowLimitedConn(dialCtx, "mcp")defer cancel()+s, err := n.Host.NewStream(dialCtx, targetPeer, api.MCPProtocolID)if err != nil {- return nil, fmt.Errorf("failed to open stream to %s: %w", targetPeer, err)+ // If dial failed completely, mark private IP failed so we skip it next time+ _ = n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, true)+ return nil, nil, fmt.Errorf("failed to open stream to %s: %w", targetPeer, err)+ }++ if remoteAddr := s.Conn().RemoteMultiaddr(); remoteAddr != nil {+ parts := multiaddr.Split(remoteAddr)+ isCircuit := len(parts) >= 2 && parts[len(parts)-1].Protocol().Code == multiaddr.P_CIRCUIT++ if isCircuit || !isPrivateIP(remoteAddr) {+ _ = n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, true)+ } else {+ _ = n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, false)+ }}logger.Debugf("Opened stream to %s for MCP\n", targetPeer)- defer func() {++ cleanup := func() {if err := s.Close(); err != nil {logger.Debugf("[MCP] Failed to close stream: %v", err)}- }()+ }// Load this node's biscuitbiscuitBytes, err := n.Store.LoadIdentity()if err != nil {- return nil, fmt.Errorf("failed to load identity biscuit: %w", err)- }-- // Extract target service if it's a federated tool call (service.tool)- targetService := "/sam/catalog"- originalToolName := toolName- if parts := strings.SplitN(toolName, ".", 2); len(parts) == 2 {- targetService = parts[0]- originalToolName = parts[1]+ cleanup()+ return nil, nil, fmt.Errorf("failed to load identity biscuit: %w", err)}// Marshal AuthFrame@@ -218,7 +225,8 @@ func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolNlogger.Debugf("Writing auth frame to %s...\n", targetPeer)writer := msgio.NewVarintWriter(s)if err := writer.WriteMsg(authBytes); err != nil {- return nil, fmt.Errorf("failed to write auth frame to %s: %w", targetPeer, err)+ cleanup()+ return nil, nil, fmt.Errorf("failed to write auth frame to %s: %w", targetPeer, err)}// Read ACK@@ -226,7 +234,8 @@ func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolNreader := msgio.NewVarintReaderSize(s, 1024*64)msg, err := reader.ReadMsg()if err != nil {- return nil, fmt.Errorf("failed to read auth response from %s: %w", targetPeer, err)+ cleanup()+ return nil, nil, fmt.Errorf("failed to read auth response from %s: %w", targetPeer, err)}defer reader.ReleaseMsg(msg)@@ -235,11 +244,13 @@ func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolNvar resp api.AuthResponseif err := proto.Unmarshal(msg, &resp); err != nil {- return nil, fmt.Errorf("invalid auth response from %s: %w", targetPeer, err)+ cleanup()+ return nil, nil, fmt.Errorf("invalid auth response from %s: %w", targetPeer, err)}if !resp.Success {- return nil, fmt.Errorf("auth rejected by %s: %s", targetPeer, resp.Error)+ cleanup()+ return nil, nil, fmt.Errorf("auth rejected by %s: %s", targetPeer, resp.Error)}// Handoff to SDK using custom transport@@ -248,9 +259,31 @@ func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolNsession, err := mcpClient.Connect(ctx, transport, nil)if err != nil {- return nil, fmt.Errorf("failed to connect client: %w", err)+ cleanup()+ return nil, nil, fmt.Errorf("failed to connect client: %w", err)+ }++ fullCleanup := func() {+ _ = session.Close()+ cleanup()}- defer func() { _ = session.Close() }()++ return session, fullCleanup, nil+}++func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolName string, params any) (*mcp.CallToolResult, error) {+ targetService := "/sam/catalog"+ originalToolName := toolName+ if parts := strings.SplitN(toolName, ".", 2); len(parts) == 2 {+ targetService = parts[0]+ originalToolName = parts[1]+ }++ session, cleanup, err := n.ConnectMCPSession(ctx, targetPeer, targetService)+ if err != nil {+ return nil, err+ }+ defer cleanup()// We don't need to marshall the params, the SDK takes care of it// Passing pre-marshaled []byte triggers encoding/json's base64 which gets rejected@@ -270,10 +303,24 @@ func (n *SamNode) callMCPToolOnce(ctx context.Context, targetPeer peer.ID, toolN// fetchRemoteServiceCatalog calls the remote peer's list_local_services// MCP tool with the type filter and returns the parsed catalog.func (n *SamNode) fetchRemoteServiceCatalog(ctx context.Context, peerID peer.ID, typeStr string) ([]*api.ServiceInfo, error) {- res, err := n.CallMCPTool(ctx, peerID, "list_local_services", map[string]string{"type": typeStr})+ n.preparePeerAddrs(ctx, peerID)++ session, cleanup, err := n.ConnectMCPSession(ctx, peerID, "/sam/catalog")if err != nil {return nil, err}+ defer cleanup()++ callParams := &mcp.CallToolParams{+ Name: "list_local_services",+ Arguments: map[string]string{"type": typeStr},+ }++ res, err := session.CallTool(ctx, callParams)+ if err != nil {+ return nil, fmt.Errorf("failed to call tool list_local_services: %w", err)+ }+if res == nil || len(res.Content) == 0 {return nil, fmt.Errorf("empty response")}@@ -289,32 +336,46 @@ func (n *SamNode) fetchRemoteServiceCatalog(ctx context.Context, peerID peer.ID,return services, nil}-// resolveRelayAddresses scans the target peer's addresses for any p2p-circuit relays,-// looks up the relay's direct IPs via the DHT, and adds them to the target's peerstore.-// This prevents dial backoff errors when the relay is behind a load-balanced DNS address.-func (n *SamNode) resolveRelayAddresses(ctx context.Context, targetPeer peer.ID) {+// preparePeerAddrs scans the target peer's addresses, filters out unroutable private IPs,+// and ensures relay circuits are available. This prevents dial backoff errors when the+// relay is behind a load-balanced DNS address or when pods advertise internal IPs.+func (n *SamNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID) {if n.Host == nil || n.DHT == nil {logger.Debugf("[Discovery] Host or DHT is nil")return}addrs := n.Host.Peerstore().Addrs(targetPeer)- logger.Debugf("[Discovery] resolveRelayAddresses for %s: found %d addrs in peerstore", targetPeer, len(addrs))+ logger.Debugf("[Discovery] preparePeerAddrs for %s: found %d addrs in peerstore", targetPeer, len(addrs))- var validCircuitAddrs []multiaddr.Multiaddr+ var validAddrs []multiaddr.Multiaddr+ changed := falsefor _, ma := range addrs {parts := multiaddr.Split(ma)- if len(parts) < 2 {- continue- }- if parts[len(parts)-1].Protocol().Code != multiaddr.P_CIRCUIT {+ // If it's a circuit relay, we always keep the generic /p2p-circuit form.+ // We'll reconstruct it below. But we must also check if the address itself is routable.+ isCircuit := len(parts) >= 2 && parts[len(parts)-1].Protocol().Code == multiaddr.P_CIRCUIT++ if !isCircuit {+ // Regular address (not a circuit relay). Filter private IPs if loopback isn't allowed,+ // and we know from a previous dial that private IPs are unreachable for this peer.+ if isPrivateIP(ma) && !n.AllowLoopback {+ val, err := n.Host.Peerstore().Get(targetPeer, PeerstoreKeyPrivateIPFailed)+ if err == nil && val.(bool) {+ changed = true+ logger.Debugf("[Discovery] Skipping private IP %s for %s due to previous failure", ma, targetPeer)+ continue+ }+ }+ validAddrs = append(validAddrs, ma)continue}// Keep the advertised circuit address- validCircuitAddrs = append(validCircuitAddrs, ma)+ validAddrs = append(validAddrs, ma)+ // Also synthesize a generic /p2p-circuit using the relay's peer ID.p2pPart := parts[len(parts)-2]if p2pPart.Protocol().Code != multiaddr.P_P2P {continue@@ -322,25 +383,20 @@ func (n *SamNode) resolveRelayAddresses(ctx context.Context, targetPeer peer.ID)relayID, err := peer.Decode(p2pPart.Value())if err != nil {- logger.Debugf("[Discovery] Failed to decode relayID: %v", err)continue}- logger.Debugf("[Discovery] Found relay %s for target %s", relayID, targetPeer)-circuitAddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/p2p/%s/p2p-circuit", relayID.String()))if err == nil {- logger.Debugf("[Discovery] Added generic relay circuit address for %s: %s", targetPeer, circuitAddr.String())- validCircuitAddrs = append(validCircuitAddrs, circuitAddr)- } else {- logger.Errorf("[Discovery] Failed to create circuit addr: %v", err)+ validAddrs = append(validAddrs, circuitAddr)+ changed = true}}- if len(validCircuitAddrs) > 0 {+ if changed && len(validAddrs) > 0 {// Clear existing addresses to prevent libp2p from hanging on dead private IPsn.Host.Peerstore().ClearAddrs(targetPeer)- n.Host.Peerstore().AddAddrs(targetPeer, validCircuitAddrs, peerstore.TempAddrTTL)- logger.Infof("[Discovery] Replaced addrs for %s with %d circuit addrs", targetPeer, len(validCircuitAddrs))+ n.Host.Peerstore().AddAddrs(targetPeer, validAddrs, peerstore.TempAddrTTL)+ logger.Infof("[Discovery] Replaced addrs for %s with %d routable/circuit addrs", targetPeer, len(validAddrs))}}diff --git a/cmd/sam-node/mcp_handlers.go b/cmd/sam-node/mcp_handlers.goindex 632d9a4a..81914f6d 100644--- a/cmd/sam-node/mcp_handlers.go+++ b/cmd/sam-node/mcp_handlers.go@@ -10,10 +10,8 @@ import ("github.com/google/sam/api""github.com/libp2p/go-libp2p/core/peer"- "github.com/libp2p/go-msgio""github.com/modelcontextprotocol/go-sdk/mcp""github.com/multiformats/go-multiaddr"- "google.golang.org/protobuf/proto")// SendMessageParams defines the parameters for the send_message tool.@@ -331,50 +329,16 @@ func (n *SamNode) fetchRemoteToolCatalogue(ctx context.Context, targetPeer peer.}var allTools []*mcp.Tool- biscuitBytes, err := n.Store.LoadIdentity()- if err != nil {- return nil, fmt.Errorf("load identity: %w", err)- }for _, svc := range services {if svc.Type != api.ServiceType_SERVICE_TYPE_MCP {continue}- s, err := n.Host.NewStream(ctx, targetPeer, api.MCPProtocolID)- if err != nil {- logger.Debugf("Failed to open stream for service %s: %v", svc.Name, err)- continue- }-- authFrame := api.AuthFrame{Biscuit: biscuitBytes, TargetService: svc.Name}- authBytes, _ := proto.Marshal(&authFrame)-- writer := msgio.NewVarintWriter(s)- if err := writer.WriteMsg(authBytes); err != nil {- _ = s.Close()- continue- }-- reader := msgio.NewVarintReaderSize(s, 1024*64)- respMsg, err := reader.ReadMsg()- if err != nil {- _ = s.Close()- continue- }- defer reader.ReleaseMsg(respMsg)-- var resp api.AuthResponse- if err := proto.Unmarshal(respMsg, &resp); err != nil || !resp.Success {- _ = s.Close()- continue- }-- transport := NewStreamTransport(s)- client := mcp.NewClient(&mcp.Implementation{Name: "sam-node-find", Version: "0.1.0"}, nil)- session, err := client.Connect(ctx, transport, nil)+ n.preparePeerAddrs(ctx, targetPeer)+ session, cleanup, err := n.ConnectMCPSession(ctx, targetPeer, svc.Name)if err != nil {- _ = s.Close()+ logger.Debugf("Failed to connect MCP session for service %s: %v", svc.Name, err)continue}@@ -386,8 +350,7 @@ func (n *SamNode) fetchRemoteToolCatalogue(ctx context.Context, targetPeer peer.}}- _ = session.Close()- _ = s.Close()+ cleanup()}return allTools, nil@@ -495,45 +458,13 @@ func (n *SamNode) handleDescribeRemoteTool(ctx context.Context, req *mcp.CallToo}serviceName := parts[0]actualToolName := parts[1]+ n.preparePeerAddrs(ctx, pid)- biscuitBytes, err := n.Store.LoadIdentity()- if err != nil {- return nil, nil, err- }-- s, err := n.Host.NewStream(ctx, pid, api.MCPProtocolID)- if err != nil {- return nil, nil, fmt.Errorf("open stream: %w", err)- }- defer func() { _ = s.Close() }()-- authFrame := api.AuthFrame{Biscuit: biscuitBytes, TargetService: serviceName}- authBytes, _ := proto.Marshal(&authFrame)-- writer := msgio.NewVarintWriter(s)- if err := writer.WriteMsg(authBytes); err != nil {- return nil, nil, err- }-- reader := msgio.NewVarintReaderSize(s, 1024*64)- respMsg, err := reader.ReadMsg()- if err != nil {- return nil, nil, err- }- defer reader.ReleaseMsg(respMsg)-- var authResp api.AuthResponse- if err := proto.Unmarshal(respMsg, &authResp); err != nil || !authResp.Success {- return nil, nil, fmt.Errorf("auth rejected")- }-- transport := NewStreamTransport(s)- client := mcp.NewClient(&mcp.Implementation{Name: "sam-node-describe", Version: "0.1.0"}, nil)- session, err := client.Connect(ctx, transport, nil)+ session, cleanup, err := n.ConnectMCPSession(ctx, pid, serviceName)if err != nil {return nil, nil, err}- defer func() { _ = session.Close() }()+ defer cleanup()listRes, err := session.ListTools(ctx, nil)if err != nil {diff --git a/cmd/sam-node/mcp_test.go b/cmd/sam-node/mcp_test.goindex 311ac3ff..71ed7b79 100644--- a/cmd/sam-node/mcp_test.go+++ b/cmd/sam-node/mcp_test.go@@ -131,7 +131,7 @@ func TestResolveRelayAddresses(t *testing.T) {localHost.Peerstore().AddAddr(targetID, circuitAddr, peerstore.PermanentAddrTTL)// Run the function- node.resolveRelayAddresses(ctx, targetID)+ node.preparePeerAddrs(ctx, targetID)// Verify that the direct IP circuit address was added to the target's peerstoreaddrs := localHost.Peerstore().Addrs(targetID)diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.goindex 2771a986..2ffee741 100644--- a/cmd/sam-node/node.go+++ b/cmd/sam-node/node.go@@ -60,6 +60,12 @@ import ("google.golang.org/protobuf/proto")+// PeerstoreKeyPrivateIPFailed is the key used in the libp2p Peerstore to track+// if a peer's private IP was previously found to be unreachable or slower than a relay.+// This allows the node to "try once and remember", avoiding a 15-second timeout on+// subsequent discovery or tool calls when dialing unroutable private networks.+const PeerstoreKeyPrivateIPFailed = "private_ip_failed"+const (// Cache sizesRateLimiterSize = 1000@@ -71,6 +77,9 @@ const (// Key pruningKeyPruningInterval = 1 * time.Hour++ // Reprovide interval+ ReprovideInterval = 5 * time.Minute)var ErrFatalAuth = errors.New("fatal authentication error")@@ -118,6 +127,7 @@ type SamNode struct {authSuccess chan struct{}authOnce sync.OncecurrentRelays []peer.AddrInfo+ reprovideTrigger chan struct{}}// UpdateRelays updates the current relays used by AutoRelay.@@ -173,6 +183,7 @@ func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {AllowLoopback: cfg.AllowLoopback,TrustHubRBAC: cfg.TrustHubRBAC,authSuccess: make(chan struct{}),+ reprovideTrigger: make(chan struct{}, 1),}var err error@@ -309,10 +320,6 @@ func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {}node.DHT = kdht- if err := kdht.Bootstrap(ctx); err != nil {- return nil, fmt.Errorf("failed to bootstrap DHT: %w", err)- }-node.services = NewServiceRegistry(node.DHT)var authenticated bool@@ -337,6 +344,13 @@ func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {return nil, fmt.Errorf("failed to authenticate with any hub: all connection attempts failed")}+ if authenticated {+ logger.Infof("[DHT] Bootstrapping DHT with connected hub...")+ if err := node.DHT.Bootstrap(ctx); err != nil {+ logger.Warnf("[DHT] Failed to trigger DHT bootstrap: %v", err)+ }+ }+// Initialize Gossipsub for Hub Eventsps, err := pubsub.NewGossipSub(ctx, h)if err != nil {@@ -369,10 +383,13 @@ func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {}logger.Infof("[Network] Local addresses updated: %v", addrs)- // Debounce or reprovide immediately+ // Debounce and trigger unified reprovide loopgo func() {time.Sleep(2 * time.Second) // Small debounce- node.services.ReprovideAll(ctx)+ select {+ case node.reprovideTrigger <- struct{}{}:+ default:+ }}()}}@@ -408,9 +425,38 @@ func NewSamNode(ctx context.Context, cfg SamNodeConfig) (*SamNode, error) {// Start connection monitornode.startConnectionMonitor(ctx, cfg.MonitorBootstrap, cfg.MonitorInterval, 3)+ // Periodically and on-demand reprovide registered services to the DHT+ node.startReprovideLoop(ctx, ReprovideInterval)+return node, nil}+func (n *SamNode) startReprovideLoop(ctx context.Context, interval time.Duration) {+ go func() {+ ticker := time.NewTicker(interval)+ defer ticker.Stop()++ // Run an initial reprovide after a short delay to let DHT bootstrap stabilize+ select {+ case <-ctx.Done():+ return+ case <-time.After(5 * time.Second):+ n.services.ReprovideAll(ctx)+ }++ for {+ select {+ case <-ctx.Done():+ return+ case <-ticker.C:+ n.services.ReprovideAll(ctx)+ case <-n.reprovideTrigger:+ n.services.ReprovideAll(ctx)+ }+ }+ }()+}+func (n *SamNode) IsConnected() bool {return n.HubPeerID != "" && n.Host.Network().Connectedness(n.HubPeerID) == network.Connected}@@ -1336,16 +1382,39 @@ func (n *SamNode) StartIngressServer(ctx context.Context) error {func isLoopbackOrLinkLocal(addr multiaddr.Multiaddr) bool {for _, proto := range addr.Protocols() {- if proto.Code == multiaddr.P_IP4 || proto.Code == multiaddr.P_IP6 {- value, err := addr.ValueForProtocol(proto.Code)- if err == nil {- ip := net.ParseIP(value)- if ip != nil {- if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {- return true- }- }- }+ if proto.Code != multiaddr.P_IP4 && proto.Code != multiaddr.P_IP6 {+ continue+ }+ value, err := addr.ValueForProtocol(proto.Code)+ if err != nil {+ continue+ }+ ip := net.ParseIP(value)+ if ip == nil {+ continue+ }+ if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {+ return true+ }+ }+ return false+}++func isPrivateIP(addr multiaddr.Multiaddr) bool {+ for _, proto := range addr.Protocols() {+ if proto.Code != multiaddr.P_IP4 && proto.Code != multiaddr.P_IP6 {+ continue+ }+ value, err := addr.ValueForProtocol(proto.Code)+ if err != nil {+ continue+ }+ ip := net.ParseIP(value)+ if ip == nil {+ continue+ }+ if ip.IsPrivate() {+ return true}}return false