Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
fix: do not let the probe hide a backend that is merely slow
Antonio Ojeaaf464a462966
diff --git a/internal/node/mcp_service.go b/internal/node/mcp_service.goindex 2dc067a7..ead111a9 100644--- a/internal/node/mcp_service.go+++ b/internal/node/mcp_service.go@@ -48,12 +48,22 @@ type MCPService struct {// reason to withhold it. Failing to initialize is — that is a backend which is// down, or was never an MCP server to begin with.func (m *MCPService) Probe(ctx context.Context) error {+ if err := ctx.Err(); err != nil {+ return err+ }m.probeMu.Lock()defer m.probeMu.Unlock()if time.Now().Before(m.probeExpires) {return m.probeErr}- m.probeErr = m.dialBackend(ctx)+ err := m.dialBackend(ctx)+ // A cancelled or expired context is a fact about the caller, not the+ // backend. Caching it would withhold a merely slow service for the whole+ // TTL -- the failure this gating is meant to avoid.+ if ctxErr := ctx.Err(); ctxErr != nil {+ return ctxErr+ }+ m.probeErr = errm.probeExpires = time.Now().Add(backendProbeTTL)return m.probeErr}@@ -68,7 +78,10 @@ func (m *MCPService) dialBackend(ctx context.Context) error {if err != nil {return fmt.Errorf("connect to backend of %q: %w", m.info.GetName(), err)}- return session.Close()+ // Connect completed the initialize, which is the whole question here; a+ // failure to hang up afterwards does not unmake that.+ _ = session.Close()+ return nil}// Init initializes the base service.diff --git a/internal/node/mcp_service_test.go b/internal/node/mcp_service_test.gonew file mode 100644index 00000000..2c7ff83e--- /dev/null+++ b/internal/node/mcp_service_test.go@@ -0,0 +1,108 @@+// Copyright 2026 Google LLC+//+// Licensed under the Apache License, Version 2.0 (the "License");+// you may not use this file except in compliance with the License.+// You may obtain a copy of the License at+//+// http://www.apache.org/licenses/LICENSE-2.0+//+// Unless required by applicable law or agreed to in writing, software+// distributed under the License is distributed on an "AS IS" BASIS,+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+// See the License for the specific language governing permissions and+// limitations under the License.++package node++import (+ "context"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"++ "github.com/google/sam/api"+ "github.com/modelcontextprotocol/go-sdk/mcp"+)++// newProbeBackend serves a real MCP server that exposes no tools at all.+func newProbeBackend(t *testing.T) *httptest.Server {+ t.Helper()+ srv := mcp.NewServer(&mcp.Implementation{Name: "probe-backend", Version: "0.0.1"}, nil)+ ts := httptest.NewServer(mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return srv }, nil))+ t.Cleanup(ts.Close)+ return ts+}++func newProbeService(url string) *MCPService {+ return &MCPService{baseService: baseService{+ info: &api.ServiceInfo{Name: "probe-me", Type: api.ServiceType_SERVICE_TYPE_MCP},+ backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: url},+ }}+}++// Serving no tools is not a fault: a backend may serve only resources or+// prompts and is still an MCP server, so it must stay advertisable.+func TestMCPService_ProbeAcceptsToollessBackend(t *testing.T) {+ svc := newProbeService(newProbeBackend(t).URL)+ if err := svc.Probe(context.Background()); err != nil {+ t.Errorf("Probe on a tool-less MCP backend: %v", err)+ }+}++// The canary case: answers HTTP, is not an MCP server.+func TestMCPService_ProbeRejectsNonMCPBackend(t *testing.T) {+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ http.Error(w, "not found", http.StatusNotFound)+ }))+ t.Cleanup(ts.Close)++ svc := newProbeService(ts.URL)+ if err := svc.Probe(context.Background()); err == nil {+ t.Fatal("Probe accepted a backend that is not an MCP server")+ }+ if svc.probeExpires.IsZero() {+ t.Error("a real negative result should be cached")+ }+}++// A cancelled context is a fact about the caller, not the backend. Caching it+// would withhold a healthy service for the whole TTL.+func TestMCPService_ProbeDoesNotCacheContextErrors(t *testing.T) {+ svc := newProbeService(newProbeBackend(t).URL)++ cancelled, cancel := context.WithCancel(context.Background())+ cancel()+ if err := svc.Probe(cancelled); err == nil {+ t.Fatal("Probe returned nil for a cancelled context")+ }+ if !svc.probeExpires.IsZero() {+ t.Fatal("a context error was cached")+ }++ if err := svc.Probe(context.Background()); err != nil {+ t.Errorf("Probe after a cancelled one: %v", err)+ }+}++// The same, for a context that is alive on entry and expires mid-dial: this is+// the slow backend the gating is explicitly meant not to punish.+func TestMCPService_ProbeDoesNotCacheTimeouts(t *testing.T) {+ // Answers, but never within the probe's deadline.+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ time.Sleep(150 * time.Millisecond)+ http.Error(w, "too slow", http.StatusServiceUnavailable)+ }))+ t.Cleanup(ts.Close)++ svc := newProbeService(ts.URL)+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)+ defer cancel()++ if err := svc.Probe(ctx); err == nil {+ t.Fatal("Probe returned nil against a backend slower than the deadline")+ }+ if !svc.probeExpires.IsZero() {+ t.Error("a timeout was cached: a slow backend would stay withheld for the whole TTL")+ }+}diff --git a/internal/node/node.go b/internal/node/node.goindex b9b558f1..c4b0bc72 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -90,6 +90,11 @@ const (// Reprovide intervalReprovideInterval = 5 * time.Minute++ // reprovideRetryInterval is the first retry after a backend was withheld,+ // doubling up to ReprovideInterval. Backends commonly start after the node+ // does, and the full interval is far too long to wait for one of those.+ reprovideRetryInterval = 5 * time.Second)var (@@ -543,26 +548,36 @@ func (n *SamNode) Start(ctx context.Context) error {func (n *SamNode) startReprovideLoop(ctx context.Context, interval time.Duration) {go func() {- ticker := time.NewTicker(interval)- defer ticker.Stop()-- // Run an initial reprovide after a short delay to let DHT bootstrap stabilize- select {- case <-ctx.Done():- return- case <-time.After(5 * time.Second):- n.services.ReprovideAll(ctx)- }+ // Initial delay lets DHT bootstrap stabilize.+ timer := time.NewTimer(5 * time.Second)+ defer timer.Stop()+ retry := reprovideRetryIntervalfor {select {case <-ctx.Done():return- case <-ticker.C:- n.services.ReprovideAll(ctx)+ case <-timer.C:case <-n.reprovideTrigger:- n.services.ReprovideAll(ctx)+ if !timer.Stop() {+ select {+ case <-timer.C:+ default:+ }+ }+ }++ next := interval+ if n.services.ReprovideAll(ctx) > 0 {+ // A backend that is not answering yet is usually one that is+ // still starting. Waiting the full interval would leave it+ // undiscoverable for minutes after it came up.+ next = retry+ retry = min(retry*2, interval)+ } else {+ retry = reprovideRetryInterval}+ timer.Reset(next)}}()}diff --git a/internal/node/service_registry.go b/internal/node/service_registry.goindex bcf4426f..79ffca7f 100644--- a/internal/node/service_registry.go+++ b/internal/node/service_registry.go@@ -18,6 +18,7 @@ import ("context""fmt""sync"+ "sync/atomic""time""github.com/google/sam/api"@@ -188,10 +189,11 @@ func (r *ServiceRegistry) TeardownAll() {}}-// ReprovideAll re-provides all registered services to the DHT concurrently.+// ReprovideAll re-provides all registered services to the DHT concurrently and+// reports how many were withheld because their backend did not answer.// A service whose backend has stopped answering falls out of the DHT by not// being reprovided, and returns on a later tick once it answers again.-func (r *ServiceRegistry) ReprovideAll(ctx context.Context) {+func (r *ServiceRegistry) ReprovideAll(ctx context.Context) int {r.mu.Lock()toProvide := make([]Service, 0, len(r.services))for _, svc := range r.services {@@ -199,6 +201,7 @@ func (r *ServiceRegistry) ReprovideAll(ctx context.Context) {}r.mu.Unlock()+ var withheld atomic.Int32var wg sync.WaitGroupsem := make(chan struct{}, 5)@@ -220,7 +223,12 @@ Loop:info := svc.Info()if err := advertisable(ctx, svc); err != nil {- logger.Warnf("[ServiceRegistry] Withholding %s/%s from the DHT: backend did not answer: %v", info.Type, info.Name, err)+ withheld.Add(1)+ // On shutdown every service fails this way, and saying so+ // would blame backends for the node stopping.+ if ctx.Err() == nil {+ logger.Warnf("[ServiceRegistry] Withholding %s/%s from the DHT: backend did not answer: %v", info.Type, info.Name, err)+ }return}@@ -237,4 +245,5 @@ Loop:}()}wg.Wait()+ return int(withheld.Load())}diff --git a/internal/node/service_registry_test.go b/internal/node/service_registry_test.goindex 7833fddb..32e0ee39 100644--- a/internal/node/service_registry_test.go+++ b/internal/node/service_registry_test.go@@ -223,7 +223,9 @@ func TestServiceRegistry_ReprovideWithholdsUnreachableBackend(t *testing.T) {}svc.setProbeErr(errors.New("backend went away"))- r.ReprovideAll(context.Background())+ if withheld := r.ReprovideAll(context.Background()); withheld != 1 {+ t.Errorf("ReprovideAll reported %d withheld, want 1: the retry cadence depends on it", withheld)+ }if len(dht.calls) != 2 {t.Errorf("Provide called %d times total, want 2: a backend that stopped answering must fall out of the DHT", len(dht.calls))}@@ -242,7 +244,9 @@ func TestServiceRegistry_ReprovideResumesWhenBackendRecovers(t *testing.T) {}svc.setProbeErr(nil)- r.ReprovideAll(context.Background())+ if withheld := r.ReprovideAll(context.Background()); withheld != 0 {+ t.Errorf("ReprovideAll reported %d withheld after recovery, want 0", withheld)+ }if len(dht.calls) != 2 {t.Errorf("Provide called %d times after recovery, want 2 (name + type CID)", len(dht.calls))}