Skip to content

Agent Mesh

improve reliability on network connectivity

Antonio Ojea49b5960fed0a
diff --git a/cmd/sam-node/mcp.go b/cmd/sam-node/mcp.go
index cc9c6efc..954367c4 100644
--- a/cmd/sam-node/mcp.go
+++ b/cmd/sam-node/mcp.go
@@ -182,7 +182,9 @@ func (n *SamNode) ConnectMCPSession(ctx context.Context, targetPeer peer.ID, tar
s, err := n.Host.NewStream(dialCtx, targetPeer, api.MCPProtocolID)
if err != nil {
// If dial failed completely, mark private IP failed so we skip it next time
- _ = n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, true)
+ if putErr := n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, true); putErr != nil {
+ logger.Errorf("[Discovery] Failed to put peerstore private IP failed key: %v", putErr)
+ }
return nil, nil, fmt.Errorf("failed to open stream to %s: %w", targetPeer, err)
}
@@ -190,10 +192,14 @@ func (n *SamNode) ConnectMCPSession(ctx context.Context, targetPeer peer.ID, tar
parts := multiaddr.Split(remoteAddr)
isCircuit := len(parts) >= 2 && parts[len(parts)-1].Protocol().Code == multiaddr.P_CIRCUIT
+ var putVal bool
if isCircuit || !isPrivateIP(remoteAddr) {
- _ = n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, true)
+ putVal = true
} else {
- _ = n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, false)
+ putVal = false
+ }
+ if putErr := n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, putVal); putErr != nil {
+ logger.Errorf("[Discovery] Failed to put peerstore private IP failed key: %v", putErr)
}
}
logger.Debugf("Opened stream to %s for MCP\n", targetPeer)
@@ -336,6 +342,15 @@ func (n *SamNode) fetchRemoteServiceCatalog(ctx context.Context, peerID peer.ID,
return services, nil
}
+func hasCircuit(addr multiaddr.Multiaddr) bool {
+ for _, proto := range addr.Protocols() {
+ if proto.Code == multiaddr.P_CIRCUIT {
+ return true
+ }
+ }
+ return false
+}
+
// preparePeerAddrs scans the target peer's addresses, filters out unroutable private IPs,
// and ensures relay circuits are available. This prevents dial backoff errors when the
// relay is behind a load-balanced DNS address or when pods advertise internal IPs.
@@ -344,12 +359,47 @@ func (n *SamNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID) {
logger.Debugf("[Discovery] Host or DHT is nil")
return
}
+ logger.Debugf("[Discovery] connectedness for %s: %s (conns: %d)", targetPeer, n.Host.Network().Connectedness(targetPeer), len(n.Host.Network().ConnsToPeer(targetPeer)))
+
+ // If we are currently connected to the peer, examine active connections to see if any are direct private IPs.
+ cond := n.Host.Network().Connectedness(targetPeer)
+ if cond == network.Connected || cond == network.Limited {
+ conns := n.Host.Network().ConnsToPeer(targetPeer)
+ hasDirectPrivateIP := false
+ for _, c := range conns {
+ remoteAddr := c.RemoteMultiaddr()
+ if isPrivateIP(remoteAddr) && !hasCircuit(remoteAddr) {
+ hasDirectPrivateIP = true
+ break
+ }
+ }
+ if !hasDirectPrivateIP {
+ if err := n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, true); err != nil {
+ logger.Errorf("[Discovery] Failed to put peerstore private IP failed key: %v", err)
+ }
+ logger.Debugf("[Discovery] Peer %s is connected via relay, marking private IP as failed", targetPeer)
+ } else {
+ if err := n.Host.Peerstore().Put(targetPeer, PeerstoreKeyPrivateIPFailed, false); err != nil {
+ logger.Errorf("[Discovery] Failed to put peerstore private IP failed key: %v", err)
+ }
+ }
+ }
+
addrs := n.Host.Peerstore().Addrs(targetPeer)
logger.Debugf("[Discovery] preparePeerAddrs for %s: found %d addrs in peerstore", targetPeer, len(addrs))
var validAddrs []multiaddr.Multiaddr
+ seen := make(map[string]struct{})
changed := false
+ addUnique := func(ma multiaddr.Multiaddr) {
+ str := ma.String()
+ if _, ok := seen[str]; !ok {
+ seen[str] = struct{}{}
+ validAddrs = append(validAddrs, ma)
+ }
+ }
+
for _, ma := range addrs {
parts := multiaddr.Split(ma)
@@ -368,12 +418,12 @@ func (n *SamNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID) {
continue
}
}
- validAddrs = append(validAddrs, ma)
+ addUnique(ma)
continue
}
// Keep the advertised circuit address
- validAddrs = append(validAddrs, ma)
+ addUnique(ma)
// Also synthesize a generic /p2p-circuit using the relay's peer ID.
p2pPart := parts[len(parts)-2]
@@ -388,8 +438,18 @@ func (n *SamNode) preparePeerAddrs(ctx context.Context, targetPeer peer.ID) {
circuitAddr, err := multiaddr.NewMultiaddr(fmt.Sprintf("/p2p/%s/p2p-circuit", relayID.String()))
if err == nil {
- validAddrs = append(validAddrs, circuitAddr)
- changed = true
+ // Check if we already have this address
+ hasIt := false
+ for _, existing := range validAddrs {
+ if existing.Equal(circuitAddr) {
+ hasIt = true
+ break
+ }
+ }
+ if !hasIt {
+ addUnique(circuitAddr)
+ changed = true
+ }
}
}
diff --git a/cmd/sam-node/node.go b/cmd/sam-node/node.go
index 2ffee741..8bbf43c3 100644
--- a/cmd/sam-node/node.go
+++ b/cmd/sam-node/node.go
@@ -916,8 +916,9 @@ func (n *SamNode) startDiscovery(ctx context.Context, meshID string, interval ti
continue
}
- if n.Host.Network().Connectedness(p.ID) != network.Connected {
- logger.Debugf("[Discovery] Found peer not connected via DHT: %s", p.ID)
+ cond := n.Host.Network().Connectedness(p.ID)
+ if cond != network.Connected && cond != network.Limited {
+ logger.Debugf("[Discovery] Found peer not connected via DHT: %s (state: %s)", p.ID, cond)
// Log the addresses returned by DHT to confirm they include p2p-circuit paths
for _, addr := range p.Addrs {
diff --git a/cmd/sam-node/sidecar.go b/cmd/sam-node/sidecar.go
index 184e3cd8..832cf282 100644
--- a/cmd/sam-node/sidecar.go
+++ b/cmd/sam-node/sidecar.go
@@ -30,6 +30,7 @@ import (
"github.com/google/sam/api"
libp2phttp "github.com/libp2p/go-libp2p-http"
+ "github.com/libp2p/go-libp2p/core/peer"
"google.golang.org/protobuf/encoding/protojson"
)
@@ -352,6 +353,10 @@ func createEgressProxy(node *SamNode) http.Handler {
return
}
peerID := parts[2]
+ pid, err := peer.Decode(peerID)
+ if err == nil {
+ node.preparePeerAddrs(req.Context(), pid)
+ }
serviceType := parts[3]
serviceName := parts[4]
upstreamPath := ""
diff --git a/tests/e2e/relay.bats b/tests/e2e/relay.bats
index 228b082f..0934692d 100644
--- a/tests/e2e/relay.bats
+++ b/tests/e2e/relay.bats
@@ -75,5 +75,82 @@ teardown() {
[[ "$status" -eq 0 ]]
MESH_NETWORK=$OLD_NET
+ local node1_peer_id
+ node1_peer_id=$(docker logs "${MESH_PREFIX}-node-1" 2>&1 | grep "PeerID:" | head -n 1 | awk '{print $2}' | tr -d '\r')
+
+ # 1. Setup HTTP Service on Node 1 side (default network)
+ docker run -d \
+ --name "${MESH_PREFIX}-http-service" \
+ --network "${MESH_NETWORK}" \
+ python:3.12 python3 -c '
+from http.server import HTTPServer, BaseHTTPRequestHandler
+class S(BaseHTTPRequestHandler):
+ def do_GET(self):
+ self.send_response(200)
+ self.send_header("Content-type", "application/json")
+ self.end_headers()
+ self.wfile.write(b"{\"status\":\"success\"}")
+HTTPServer(("0.0.0.0", 8000), S).serve_forever()
+'
+ MESH_CONTAINERS+=("${MESH_PREFIX}-http-service")
+
+ # Wait for http-service to be listening
+ local i
+ for ((i=0; i<30; i++)); do
+ if docker run --rm --network "${MESH_NETWORK}" python:3.12 python3 -c "import urllib.request; urllib.request.urlopen('http://${MESH_PREFIX}-http-service:8000')" >/dev/null 2>&1; then
+ break
+ fi
+ sleep 1
+ done
+
+ # Register HTTP service on Node 1
+ run docker run --rm --network "${MESH_NETWORK}" python:3.12 python3 -c "
+import urllib.request
+import json
+
+data = {
+ \"service\": {
+ \"type\": \"SERVICE_TYPE_MCP\",
+ \"name\": \"http-tool\",
+ \"description\": \"test http service\"
+ },
+ \"targetUrl\": \"http://${MESH_PREFIX}-http-service:8000\"
+}
+
+req = urllib.request.Request(
+ \"http://sam-node-1:8080/sam/service/register\",
+ data=json.dumps(data).encode('utf-8'),
+ headers={
+ \"Content-Type\": \"application/json\",
+ \"Authorization\": \"Bearer secret-token\"
+ },
+ method=\"POST\"
+)
+with urllib.request.urlopen(req) as response:
+ print(response.read().decode('utf-8'))
+"
+ [[ "$status" -eq 0 ]]
+ [[ "$output" == *"Service registered"* ]]
+
+ # 2. Test HTTP Datapath: Node 2 calls Node 1's HTTP service via Node 2's egress proxy (isolated network 2)
+ for ((i=0; i<15; i++)); do
+ run docker run --rm --network "${MESH_NETWORK_2}" python:3.12 python3 -c "
+import urllib.request
+req = urllib.request.Request(
+ \"http://sam-node-2:8080/sam/${node1_peer_id}/mcp/http-tool/\",
+ headers={\"Authorization\": \"Bearer secret-token\"}
+)
+with urllib.request.urlopen(req) as response:
+ print(response.read().decode(\"utf-8\"))
+"
+ if [[ "$status" -eq 0 ]] && [[ "$output" == *"{\"status\":\"success\"}"* ]]; then
+ break
+ fi
+ sleep 1
+ done
+
+ [[ "$status" -eq 0 ]]
+ [[ "$output" == *"{\"status\":\"success\"}"* ]]
+
docker network rm "${MESH_NETWORK_2}" >/dev/null 2>&1 || true
}