Skip to content

Agent Mesh

fix: do not advertise an MCP service whose backend is not one

Antonio Ojea50f59450cacc
diff --git a/internal/node/mcp_service.go b/internal/node/mcp_service.go
index 6098e67a..2dc067a7 100644
--- a/internal/node/mcp_service.go
+++ b/internal/node/mcp_service.go
@@ -34,6 +34,41 @@ type MCPService struct {
toolsMu sync.Mutex
cachedTools []string
toolsExpires time.Time
+
+ probeMu sync.Mutex
+ probeErr error
+ probeExpires time.Time
+}
+
+// Probe reports whether the backend actually speaks MCP, by completing an
+// initialize against it.
+//
+// Deliberately weaker than Tools: a backend serving only resources or prompts
+// has no tools and is still a working MCP server, so an empty tool list is no
+// reason to withhold it. Failing to initialize is — that is a backend which is
+// down, or was never an MCP server to begin with.
+func (m *MCPService) Probe(ctx context.Context) error {
+ m.probeMu.Lock()
+ defer m.probeMu.Unlock()
+ if time.Now().Before(m.probeExpires) {
+ return m.probeErr
+ }
+ m.probeErr = m.dialBackend(ctx)
+ m.probeExpires = time.Now().Add(backendProbeTTL)
+ return m.probeErr
+}
+
+func (m *MCPService) dialBackend(ctx context.Context) error {
+ transport, err := m.backendTransport()
+ if err != nil {
+ return err
+ }
+ client := mcp.NewClient(&mcp.Implementation{Name: "sam-node-probe", Version: "0.1.0"}, nil)
+ session, err := client.Connect(ctx, transport, nil)
+ if err != nil {
+ return fmt.Errorf("connect to backend of %q: %w", m.info.GetName(), err)
+ }
+ return session.Close()
}
// Init initializes the base service.
diff --git a/internal/node/service_registry.go b/internal/node/service_registry.go
index 46683e46..bcf4426f 100644
--- a/internal/node/service_registry.go
+++ b/internal/node/service_registry.go
@@ -30,6 +30,34 @@ type dhtProvider interface {
Provide(ctx context.Context, c cid.Cid, broadcast bool) error
}
+// backendProber is implemented by services that can be asked whether their
+// backend is really serving. Services that cannot be asked are advertised
+// unconditionally.
+type backendProber interface {
+ Probe(ctx context.Context) error
+}
+
+// dhtProbeTimeout bounds one backend probe before advertising.
+const dhtProbeTimeout = 2 * time.Second
+
+// advertisable reports whether a service is fit to be published to the DHT.
+//
+// Gossip already withholds announcements for a backend that does not answer,
+// but the DHT is the other half of discovery and had no such check, so a
+// backend that is not an MCP server at all stayed discoverable as one: it was
+// listed by discover_remote_services and only failed later, at initialize, in
+// the caller's face. Advertising is a claim the node makes on the backend's
+// behalf, so it is the node that should verify it.
+func advertisable(ctx context.Context, svc Service) error {
+ prober, ok := svc.(backendProber)
+ if !ok {
+ return nil
+ }
+ probeCtx, cancel := context.WithTimeout(ctx, dhtProbeTimeout)
+ defer cancel()
+ return prober.Probe(probeCtx)
+}
+
// ServiceRegistry is the type-agnostic owner of registered services.
type ServiceRegistry struct {
mu sync.RWMutex
@@ -47,6 +75,10 @@ func NewServiceRegistry(d dhtProvider) *ServiceRegistry {
// Register initialises a service, advertises it on the DHT, and inserts it
// into the map. Init runs before Provide so a failed handler-build never
// briefly advertises an unservable name.
+//
+// A backend that does not answer is registered but not advertised, rather than
+// rejected: backends routinely start after the node does, and the reprovide
+// loop picks them up once they answer.
func (r *ServiceRegistry) Register(ctx context.Context, svc Service) error {
info := svc.Info()
if info.Type == api.ServiceType_SERVICE_TYPE_UNSPECIFIED {
@@ -66,23 +98,30 @@ func (r *ServiceRegistry) Register(ctx context.Context, svc Service) error {
return err
}
- nameCtx, nameCancel := context.WithTimeout(ctx, 5*time.Second)
- defer nameCancel()
- if err := r.dht.Provide(nameCtx, srvNameCID, true); err != nil {
- logger.Warnf("[ServiceRegistry] DHT Provide (name) for %s: %v", info.Name, err)
- }
+ probeErr := advertisable(ctx, svc)
+ if probeErr != nil {
+ logger.Warnf("[ServiceRegistry] Registered %s/%s but not advertising it: backend did not answer: %v", info.Type, info.Name, probeErr)
+ } else {
+ nameCtx, nameCancel := context.WithTimeout(ctx, 5*time.Second)
+ defer nameCancel()
+ if err := r.dht.Provide(nameCtx, srvNameCID, true); err != nil {
+ logger.Warnf("[ServiceRegistry] DHT Provide (name) for %s: %v", info.Name, err)
+ }
- typeCtx, typeCancel := context.WithTimeout(ctx, 5*time.Second)
- defer typeCancel()
- if err := r.dht.Provide(typeCtx, srvTypeCID, true); err != nil {
- logger.Warnf("[ServiceRegistry] DHT Provide (type) for %s: %v", info.Name, err)
+ typeCtx, typeCancel := context.WithTimeout(ctx, 5*time.Second)
+ defer typeCancel()
+ if err := r.dht.Provide(typeCtx, srvTypeCID, true); err != nil {
+ logger.Warnf("[ServiceRegistry] DHT Provide (type) for %s: %v", info.Name, err)
+ }
}
r.mu.Lock()
r.services[info.Name] = svc
r.mu.Unlock()
- logger.Infof("[ServiceRegistry] Registered %s/%s (name CID: %s, type CID: %s)", info.Type, info.Name, srvNameCID, srvTypeCID)
+ if probeErr == nil {
+ logger.Infof("[ServiceRegistry] Registered %s/%s (name CID: %s, type CID: %s)", info.Type, info.Name, srvNameCID, srvTypeCID)
+ }
return nil
}
@@ -150,11 +189,13 @@ func (r *ServiceRegistry) TeardownAll() {
}
// ReprovideAll re-provides all registered services to the DHT concurrently.
+// A service whose backend has stopped answering falls out of the DHT by not
+// being reprovided, and returns on a later tick once it answers again.
func (r *ServiceRegistry) ReprovideAll(ctx context.Context) {
r.mu.Lock()
- var toProvide []*api.ServiceInfo
+ toProvide := make([]Service, 0, len(r.services))
for _, svc := range r.services {
- toProvide = append(toProvide, svc.Info())
+ toProvide = append(toProvide, svc)
}
r.mu.Unlock()
@@ -162,8 +203,8 @@ func (r *ServiceRegistry) ReprovideAll(ctx context.Context) {
sem := make(chan struct{}, 5)
Loop:
- for _, info := range toProvide {
- info := info
+ for _, svc := range toProvide {
+ svc := svc
select {
case <-ctx.Done():
break Loop
@@ -177,6 +218,12 @@ Loop:
wg.Done()
}()
+ info := svc.Info()
+ if err := advertisable(ctx, svc); err != nil {
+ logger.Warnf("[ServiceRegistry] Withholding %s/%s from the DHT: backend did not answer: %v", info.Type, info.Name, err)
+ return
+ }
+
if srvNameCID, err := serviceNameToCID(info.Type, info.Name); err == nil {
nameCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
_ = r.dht.Provide(nameCtx, srvNameCID, true)
diff --git a/internal/node/service_registry_test.go b/internal/node/service_registry_test.go
index 63c986c6..7833fddb 100644
--- a/internal/node/service_registry_test.go
+++ b/internal/node/service_registry_test.go
@@ -17,6 +17,7 @@ import (
"context"
"errors"
"net/http"
+ "sync"
"sync/atomic"
"testing"
@@ -161,3 +162,88 @@ func TestServiceRegistry_TeardownAllContinuesOnError(t *testing.T) {
t.Error("registry not empty after TeardownAll")
}
}
+
+// probingService is a service whose backend can be asked whether it answers.
+type probingService struct {
+ *fakeService
+ mu sync.Mutex
+ probeErr error
+ probeCalls int
+}
+
+func (p *probingService) Probe(context.Context) error {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.probeCalls++
+ return p.probeErr
+}
+
+func (p *probingService) setProbeErr(err error) {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ p.probeErr = err
+}
+
+func newProbingSvc(name string, probeErr error) *probingService {
+ return &probingService{
+ fakeService: newFakeSvc(name, api.ServiceType_SERVICE_TYPE_MCP),
+ probeErr: probeErr,
+ }
+}
+
+// A backend that is not an MCP server at all -- the canary that answers HTTP
+// but fails MCP initialize -- must not be advertised as one. It stays
+// registered so the reprovide loop can pick it up if it starts answering.
+func TestServiceRegistry_UnreachableBackendIsRegisteredButNotAdvertised(t *testing.T) {
+ dht := &fakeDHT{}
+ r := newServiceRegistryForTest(dht)
+
+ svc := newProbingSvc("dummy-http", errors.New(`calling "initialize": EOF`))
+ if err := r.Register(context.Background(), svc); err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+ if len(dht.calls) != 0 {
+ t.Errorf("Provide called %d times for an unreachable backend, want 0", len(dht.calls))
+ }
+ if _, ok := r.Get("dummy-http"); !ok {
+ t.Error("service should stay registered so it can recover")
+ }
+}
+
+func TestServiceRegistry_ReprovideWithholdsUnreachableBackend(t *testing.T) {
+ dht := &fakeDHT{}
+ r := newServiceRegistryForTest(dht)
+
+ svc := newProbingSvc("demo", nil)
+ if err := r.Register(context.Background(), svc); err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+ if len(dht.calls) != 2 {
+ t.Fatalf("Provide called %d times at registration, want 2", len(dht.calls))
+ }
+
+ svc.setProbeErr(errors.New("backend went away"))
+ r.ReprovideAll(context.Background())
+ if len(dht.calls) != 2 {
+ t.Errorf("Provide called %d times total, want 2: a backend that stopped answering must fall out of the DHT", len(dht.calls))
+ }
+}
+
+func TestServiceRegistry_ReprovideResumesWhenBackendRecovers(t *testing.T) {
+ dht := &fakeDHT{}
+ r := newServiceRegistryForTest(dht)
+
+ svc := newProbingSvc("demo", errors.New("not up yet"))
+ if err := r.Register(context.Background(), svc); err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+ if len(dht.calls) != 0 {
+ t.Fatalf("Provide called %d times while the backend was down, want 0", len(dht.calls))
+ }
+
+ svc.setProbeErr(nil)
+ r.ReprovideAll(context.Background())
+ if len(dht.calls) != 2 {
+ t.Errorf("Provide called %d times after recovery, want 2 (name + type CID)", len(dht.calls))
+ }
+}
diff --git a/tests/integration/federation_test.go b/tests/integration/federation_test.go
index 6c767035..0c2ea867 100644
--- a/tests/integration/federation_test.go
+++ b/tests/integration/federation_test.go
@@ -239,17 +239,25 @@ roles:
waitForDHTReady(t, clientBin, apiPortA, "tokenA")
waitForDHTReady(t, clientBin, apiPortB, "tokenB")
- // Start Mock Backend on Node A
- mcpServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ // A real MCP backend: the node probes it before advertising, so a stub
+ // returning a canned JSON-RPC body is deliberately never discoverable.
+ mcpServer := httptest.NewServer(newBoundaryMCPHandler(t))
+ t.Cleanup(func() { mcpServer.Close() })
+
+ // A backend that is not an MCP server. It is never advertised, but stays
+ // reachable when addressed explicitly, which is what the datapath below
+ // exercises: the proxy is a pipe and does not interpret what it carries.
+ rawServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(`{"jsonrpc": "2.0", "id": 1, "result": {"success": true}}`))
}))
- t.Cleanup(func() { mcpServer.Close() })
+ t.Cleanup(func() { rawServer.Close() })
// Publish a service on Node A
t.Log("Publishing tool on Node A...")
registerService(t, fmt.Sprintf("127.0.0.1:%d", apiPortA), "tokenA", "federated-tool", mcpServer.URL)
+ registerService(t, fmt.Sprintf("127.0.0.1:%d", apiPortA), "tokenA", "raw-pipe", rawServer.URL)
time.Sleep(2 * time.Second)
// Search for the service from Node B
@@ -301,8 +309,8 @@ roles:
}
peerIDA_node := matches[1]
- // The proxy path on Node B is /sam/<peerID>/mcp/federated-tool
- proxyURL := fmt.Sprintf("http://127.0.0.1:%d/sam/%s/mcp/federated-tool", apiPortB, peerIDA_node)
+ // The proxy path on Node B is /sam/<peerID>/mcp/raw-pipe
+ proxyURL := fmt.Sprintf("http://127.0.0.1:%d/sam/%s/mcp/raw-pipe", apiPortB, peerIDA_node)
req, _ := http.NewRequest("POST", proxyURL, bytes.NewBuffer([]byte(`{"jsonrpc": "2.0", "id": 1, "method": "test"}`)))
req.Header.Set(api.HeaderSamAuthentication, "Bearer tokenB")
diff --git a/tests/integration/service_discovery_test.go b/tests/integration/service_discovery_test.go
index 90573885..3f49e2f2 100644
--- a/tests/integration/service_discovery_test.go
+++ b/tests/integration/service_discovery_test.go
@@ -95,9 +95,9 @@ func TestServiceDiscovery(t *testing.T) {
t.Fatalf("DHT not ready on Node A (size 0)")
}
- mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- http.Error(w, "Not found", http.StatusNotFound)
- }))
+ // A real MCP backend: the node probes it before advertising, so a stub that
+ // cannot complete an initialize is deliberately never discoverable.
+ mockServer := httptest.NewServer(newBoundaryMCPHandler(t))
defer mockServer.Close()
// Agent A registers a service
@@ -198,9 +198,9 @@ func TestServiceDiscoveryStreaming(t *testing.T) {
t.Fatalf("DHT not ready on Node A")
}
- mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
- http.Error(w, "Not found", http.StatusNotFound)
- }))
+ // A real MCP backend: the node probes it before advertising, so a stub that
+ // cannot complete an initialize is deliberately never discoverable.
+ mockServer := httptest.NewServer(newBoundaryMCPHandler(t))
defer mockServer.Close()
// Agent A registers a service