Skip to content

Agent Mesh

fix: do not let the probe hide a backend that is merely slow

Antonio Ojeaaf464a462966
diff --git a/internal/node/mcp_service.go b/internal/node/mcp_service.go
index 2dc067a7..ead111a9 100644
--- a/internal/node/mcp_service.go
+++ b/internal/node/mcp_service.go
@@ -48,12 +48,22 @@ type MCPService struct {
// reason to withhold it. Failing to initialize is — that is a backend which is
// down, or was never an MCP server to begin with.
func (m *MCPService) Probe(ctx context.Context) error {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
m.probeMu.Lock()
defer m.probeMu.Unlock()
if time.Now().Before(m.probeExpires) {
return m.probeErr
}
- m.probeErr = m.dialBackend(ctx)
+ err := m.dialBackend(ctx)
+ // A cancelled or expired context is a fact about the caller, not the
+ // backend. Caching it would withhold a merely slow service for the whole
+ // TTL -- the failure this gating is meant to avoid.
+ if ctxErr := ctx.Err(); ctxErr != nil {
+ return ctxErr
+ }
+ m.probeErr = err
m.probeExpires = time.Now().Add(backendProbeTTL)
return m.probeErr
}
@@ -68,7 +78,10 @@ func (m *MCPService) dialBackend(ctx context.Context) error {
if err != nil {
return fmt.Errorf("connect to backend of %q: %w", m.info.GetName(), err)
}
- return session.Close()
+ // Connect completed the initialize, which is the whole question here; a
+ // failure to hang up afterwards does not unmake that.
+ _ = session.Close()
+ return nil
}
// Init initializes the base service.
diff --git a/internal/node/mcp_service_test.go b/internal/node/mcp_service_test.go
new file mode 100644
index 00000000..2c7ff83e
--- /dev/null
+++ b/internal/node/mcp_service_test.go
@@ -0,0 +1,108 @@
+// Copyright 2026 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package node
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+
+ "github.com/google/sam/api"
+ "github.com/modelcontextprotocol/go-sdk/mcp"
+)
+
+// newProbeBackend serves a real MCP server that exposes no tools at all.
+func newProbeBackend(t *testing.T) *httptest.Server {
+ t.Helper()
+ srv := mcp.NewServer(&mcp.Implementation{Name: "probe-backend", Version: "0.0.1"}, nil)
+ ts := httptest.NewServer(mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return srv }, nil))
+ t.Cleanup(ts.Close)
+ return ts
+}
+
+func newProbeService(url string) *MCPService {
+ return &MCPService{baseService: baseService{
+ info: &api.ServiceInfo{Name: "probe-me", Type: api.ServiceType_SERVICE_TYPE_MCP},
+ backend: &api.RegisterServiceRequest_TargetUrl{TargetUrl: url},
+ }}
+}
+
+// Serving no tools is not a fault: a backend may serve only resources or
+// prompts and is still an MCP server, so it must stay advertisable.
+func TestMCPService_ProbeAcceptsToollessBackend(t *testing.T) {
+ svc := newProbeService(newProbeBackend(t).URL)
+ if err := svc.Probe(context.Background()); err != nil {
+ t.Errorf("Probe on a tool-less MCP backend: %v", err)
+ }
+}
+
+// The canary case: answers HTTP, is not an MCP server.
+func TestMCPService_ProbeRejectsNonMCPBackend(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ http.Error(w, "not found", http.StatusNotFound)
+ }))
+ t.Cleanup(ts.Close)
+
+ svc := newProbeService(ts.URL)
+ if err := svc.Probe(context.Background()); err == nil {
+ t.Fatal("Probe accepted a backend that is not an MCP server")
+ }
+ if svc.probeExpires.IsZero() {
+ t.Error("a real negative result should be cached")
+ }
+}
+
+// A cancelled context is a fact about the caller, not the backend. Caching it
+// would withhold a healthy service for the whole TTL.
+func TestMCPService_ProbeDoesNotCacheContextErrors(t *testing.T) {
+ svc := newProbeService(newProbeBackend(t).URL)
+
+ cancelled, cancel := context.WithCancel(context.Background())
+ cancel()
+ if err := svc.Probe(cancelled); err == nil {
+ t.Fatal("Probe returned nil for a cancelled context")
+ }
+ if !svc.probeExpires.IsZero() {
+ t.Fatal("a context error was cached")
+ }
+
+ if err := svc.Probe(context.Background()); err != nil {
+ t.Errorf("Probe after a cancelled one: %v", err)
+ }
+}
+
+// The same, for a context that is alive on entry and expires mid-dial: this is
+// the slow backend the gating is explicitly meant not to punish.
+func TestMCPService_ProbeDoesNotCacheTimeouts(t *testing.T) {
+ // Answers, but never within the probe's deadline.
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ time.Sleep(150 * time.Millisecond)
+ http.Error(w, "too slow", http.StatusServiceUnavailable)
+ }))
+ t.Cleanup(ts.Close)
+
+ svc := newProbeService(ts.URL)
+ ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond)
+ defer cancel()
+
+ if err := svc.Probe(ctx); err == nil {
+ t.Fatal("Probe returned nil against a backend slower than the deadline")
+ }
+ if !svc.probeExpires.IsZero() {
+ t.Error("a timeout was cached: a slow backend would stay withheld for the whole TTL")
+ }
+}
diff --git a/internal/node/node.go b/internal/node/node.go
index b9b558f1..c4b0bc72 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -90,6 +90,11 @@ const (
// Reprovide interval
ReprovideInterval = 5 * time.Minute
+
+ // reprovideRetryInterval is the first retry after a backend was withheld,
+ // doubling up to ReprovideInterval. Backends commonly start after the node
+ // does, and the full interval is far too long to wait for one of those.
+ reprovideRetryInterval = 5 * time.Second
)
var (
@@ -543,26 +548,36 @@ func (n *SamNode) Start(ctx context.Context) error {
func (n *SamNode) startReprovideLoop(ctx context.Context, interval time.Duration) {
go func() {
- ticker := time.NewTicker(interval)
- defer ticker.Stop()
-
- // Run an initial reprovide after a short delay to let DHT bootstrap stabilize
- select {
- case <-ctx.Done():
- return
- case <-time.After(5 * time.Second):
- n.services.ReprovideAll(ctx)
- }
+ // Initial delay lets DHT bootstrap stabilize.
+ timer := time.NewTimer(5 * time.Second)
+ defer timer.Stop()
+ retry := reprovideRetryInterval
for {
select {
case <-ctx.Done():
return
- case <-ticker.C:
- n.services.ReprovideAll(ctx)
+ case <-timer.C:
case <-n.reprovideTrigger:
- n.services.ReprovideAll(ctx)
+ if !timer.Stop() {
+ select {
+ case <-timer.C:
+ default:
+ }
+ }
+ }
+
+ next := interval
+ if n.services.ReprovideAll(ctx) > 0 {
+ // A backend that is not answering yet is usually one that is
+ // still starting. Waiting the full interval would leave it
+ // undiscoverable for minutes after it came up.
+ next = retry
+ retry = min(retry*2, interval)
+ } else {
+ retry = reprovideRetryInterval
}
+ timer.Reset(next)
}
}()
}
diff --git a/internal/node/service_registry.go b/internal/node/service_registry.go
index bcf4426f..79ffca7f 100644
--- a/internal/node/service_registry.go
+++ b/internal/node/service_registry.go
@@ -18,6 +18,7 @@ import (
"context"
"fmt"
"sync"
+ "sync/atomic"
"time"
"github.com/google/sam/api"
@@ -188,10 +189,11 @@ func (r *ServiceRegistry) TeardownAll() {
}
}
-// ReprovideAll re-provides all registered services to the DHT concurrently.
+// ReprovideAll re-provides all registered services to the DHT concurrently and
+// reports how many were withheld because their backend did not answer.
// A service whose backend has stopped answering falls out of the DHT by not
// being reprovided, and returns on a later tick once it answers again.
-func (r *ServiceRegistry) ReprovideAll(ctx context.Context) {
+func (r *ServiceRegistry) ReprovideAll(ctx context.Context) int {
r.mu.Lock()
toProvide := make([]Service, 0, len(r.services))
for _, svc := range r.services {
@@ -199,6 +201,7 @@ func (r *ServiceRegistry) ReprovideAll(ctx context.Context) {
}
r.mu.Unlock()
+ var withheld atomic.Int32
var wg sync.WaitGroup
sem := make(chan struct{}, 5)
@@ -220,7 +223,12 @@ Loop:
info := svc.Info()
if err := advertisable(ctx, svc); err != nil {
- logger.Warnf("[ServiceRegistry] Withholding %s/%s from the DHT: backend did not answer: %v", info.Type, info.Name, err)
+ withheld.Add(1)
+ // On shutdown every service fails this way, and saying so
+ // would blame backends for the node stopping.
+ if ctx.Err() == nil {
+ logger.Warnf("[ServiceRegistry] Withholding %s/%s from the DHT: backend did not answer: %v", info.Type, info.Name, err)
+ }
return
}
@@ -237,4 +245,5 @@ Loop:
}()
}
wg.Wait()
+ return int(withheld.Load())
}
diff --git a/internal/node/service_registry_test.go b/internal/node/service_registry_test.go
index 7833fddb..32e0ee39 100644
--- a/internal/node/service_registry_test.go
+++ b/internal/node/service_registry_test.go
@@ -223,7 +223,9 @@ func TestServiceRegistry_ReprovideWithholdsUnreachableBackend(t *testing.T) {
}
svc.setProbeErr(errors.New("backend went away"))
- r.ReprovideAll(context.Background())
+ if withheld := r.ReprovideAll(context.Background()); withheld != 1 {
+ t.Errorf("ReprovideAll reported %d withheld, want 1: the retry cadence depends on it", withheld)
+ }
if len(dht.calls) != 2 {
t.Errorf("Provide called %d times total, want 2: a backend that stopped answering must fall out of the DHT", len(dht.calls))
}
@@ -242,7 +244,9 @@ func TestServiceRegistry_ReprovideResumesWhenBackendRecovers(t *testing.T) {
}
svc.setProbeErr(nil)
- r.ReprovideAll(context.Background())
+ if withheld := r.ReprovideAll(context.Background()); withheld != 0 {
+ t.Errorf("ReprovideAll reported %d withheld after recovery, want 0", withheld)
+ }
if len(dht.calls) != 2 {
t.Errorf("Provide called %d times after recovery, want 2 (name + type CID)", len(dht.calls))
}