Skip to content

Agent Mesh

node: make the backend probe timeout configurable

Fernando Marinob665247ea502
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.go
index 9c65be62..8c63e131 100644
--- a/cmd/sam-node/main.go
+++ b/cmd/sam-node/main.go
@@ -99,6 +99,7 @@ var (
dhtMaxRecordAgeFlag time.Duration
dhtLookupLimitFlag int
discoveryConcurrencyFlag int
+ backendProbeTimeoutFlag time.Duration
policySyncIntervalFlag time.Duration
)
@@ -466,6 +467,7 @@ func main() {
DHTMaxRecordAge: dhtMaxRecordAgeFlag,
DHTLookupLimit: dhtLookupLimitFlag,
DiscoveryConcurrency: discoveryConcurrencyFlag,
+ BackendProbeTimeout: backendProbeTimeoutFlag,
})
if err != nil {
logger.Fatalf("Failed to initialize mesh node: %v", err)
@@ -533,6 +535,7 @@ func main() {
DHTMaxRecordAge: dhtMaxRecordAgeFlag,
DHTLookupLimit: dhtLookupLimitFlag,
DiscoveryConcurrency: discoveryConcurrencyFlag,
+ BackendProbeTimeout: backendProbeTimeoutFlag,
})
if err != nil {
enrollCancel()
@@ -599,6 +602,7 @@ func main() {
RouterConnectTimeout: routerConnectTimeoutFlag,
RequiredRole: api.RoleNode,
PolicySyncInterval: policySyncIntervalFlag,
+ BackendProbeTimeout: backendProbeTimeoutFlag,
})
if err != nil {
logger.Fatalf("Failed to initialize node after enrollment: %v", err)
@@ -854,6 +858,7 @@ func main() {
runCmd.Flags().IntVar(&dhtLookupLimitFlag, "dht-lookup-limit", 0, "Maximum number of providers to query from the DHT (0 uses default 20)")
runCmd.Flags().IntVar(&discoveryConcurrencyFlag, "discovery-concurrency", 0, "Max concurrent catalog fetches during discovery (0 uses default 10)")
runCmd.Flags().DurationVar(&policySyncIntervalFlag, "policy-sync-interval", 1*time.Hour, "Interval for syncing mesh policy from the control plane")
+ runCmd.Flags().DurationVar(&backendProbeTimeoutFlag, "backend-probe-timeout", 0, "Timeout for probing a command-spawned service backend before advertising it (0 uses default 2s); raise this for backends with slower cold-start times")
rootCmd.PersistentFlags().StringVar(&controlPlaneAddr, "control-plane", "", "Control plane URL")
rootCmd.PersistentFlags().StringVar(&configFile, "config", node.DefaultConfigFile, "Path to sam-node.yaml configuration file")
rootCmd.PersistentFlags().StringVar(&oidcIssuerFlag, "oidc-issuer", "", "OIDC Issuer URL")
diff --git a/internal/node/node.go b/internal/node/node.go
index 685bc613..039084b9 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -520,6 +520,7 @@ func (n *SamNode) Start(ctx context.Context) error {
n.services = NewServiceRegistry(n.DHT)
n.services.reprovideNow = n.triggerReprovide
+ n.services.SetBackendProbeTimeout(n.config.BackendProbeTimeout)
var authenticated bool
var fatalAuthErr error
diff --git a/internal/node/options.go b/internal/node/options.go
index 9f8a0367..2075c7ee 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -78,6 +78,14 @@ type Options struct {
PolicySyncInterval time.Duration
// PolicySyncJitter specifies the maximum jitter delay when scheduling policy syncs on event broadcasts.
PolicySyncJitter time.Duration
+ // BackendProbeTimeout bounds how long a command-spawned service backend
+ // (sam-node.yaml's `command`, spawned as a local subprocess) is given to
+ // answer before the service is registered but withheld from
+ // advertisement. Zero uses the library default (2s). Raise this for
+ // backends with slower cold-start/import costs than that - the default
+ // is tight enough that even simple interpreted-language MCP servers can
+ // miss it on first spawn.
+ BackendProbeTimeout time.Duration
}
// Default applies default values to Options if they are not specified.
diff --git a/internal/node/service_registry.go b/internal/node/service_registry.go
index ffcc2724..280f2718 100644
--- a/internal/node/service_registry.go
+++ b/internal/node/service_registry.go
@@ -38,7 +38,14 @@ type backendProber interface {
Probe(ctx context.Context) error
}
-// dhtProbeTimeout bounds one backend probe before advertising.
+// dhtProbeTimeout is the default bound on one backend probe before
+// advertising, used when a ServiceRegistry isn't given an explicit
+// BackendProbeTimeout (see Options.BackendProbeTimeout / --backend-probe-
+// timeout). Command-spawned backends (sam-node.yaml's `command`, launched as
+// a local subprocess) can need longer than this to answer their first
+// request - a moderately-featured interpreted-language MCP server's own
+// import/startup cost alone can exceed 2s - so this is a floor, not
+// something every backend is expected to meet.
const dhtProbeTimeout = 2 * time.Second
// advertisable reports whether a service is fit to be published to the DHT.
@@ -49,12 +56,12 @@ const dhtProbeTimeout = 2 * time.Second
// listed by discover_remote_services and only failed later, at initialize, in
// the caller's face. Advertising is a claim the node makes on the backend's
// behalf, so it is the node that should verify it.
-func advertisable(ctx context.Context, svc Service) error {
+func advertisable(ctx context.Context, svc Service, probeTimeout time.Duration) error {
prober, ok := svc.(backendProber)
if !ok {
return nil
}
- probeCtx, cancel := context.WithTimeout(ctx, dhtProbeTimeout)
+ probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)
defer cancel()
return prober.Probe(probeCtx)
}
@@ -69,15 +76,40 @@ type ServiceRegistry struct {
// registered after the loop last ran does not wait a whole interval to be
// advertised. Optional; nil outside a running node.
reprovideNow func()
+
+ // backendProbeTimeout bounds each backend probe in advertisable. Defaults
+ // to dhtProbeTimeout; override with SetBackendProbeTimeout.
+ backendProbeTimeout time.Duration
}
func NewServiceRegistry(d dhtProvider) *ServiceRegistry {
return &ServiceRegistry{
- services: map[string]Service{},
- dht: d,
+ services: map[string]Service{},
+ dht: d,
+ backendProbeTimeout: dhtProbeTimeout,
}
}
+// SetBackendProbeTimeout overrides the default backend probe timeout
+// (dhtProbeTimeout). A zero or negative duration is a no-op, so callers can
+// pass an unset Options.BackendProbeTimeout straight through without an
+// explicit zero-check.
+func (r *ServiceRegistry) SetBackendProbeTimeout(d time.Duration) {
+ if d <= 0 {
+ return
+ }
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ r.backendProbeTimeout = d
+}
+
+// probeTimeout returns the current backend probe timeout.
+func (r *ServiceRegistry) probeTimeout() time.Duration {
+ r.mu.RLock()
+ defer r.mu.RUnlock()
+ return r.backendProbeTimeout
+}
+
// Register initialises a service, advertises it on the DHT, and inserts it
// into the map. Init runs before Provide so a failed handler-build never
// briefly advertises an unservable name.
@@ -104,7 +136,7 @@ func (r *ServiceRegistry) Register(ctx context.Context, svc Service) error {
return err
}
- probeErr := advertisable(ctx, svc)
+ probeErr := advertisable(ctx, svc, r.probeTimeout())
if probeErr != nil {
logger.Warnf("[ServiceRegistry] Registered %s/%s but not advertising it: backend did not answer: %v", info.Type, info.Name, probeErr)
} else {
@@ -229,7 +261,7 @@ Loop:
}()
info := svc.Info()
- if err := advertisable(ctx, svc); err != nil {
+ if err := advertisable(ctx, svc, r.probeTimeout()); err != nil {
withheld.Add(1)
// On shutdown every service fails this way, and saying so
// would blame backends for the node stopping.
diff --git a/internal/node/service_registry_test.go b/internal/node/service_registry_test.go
index b0d2f5e5..8d724aae 100644
--- a/internal/node/service_registry_test.go
+++ b/internal/node/service_registry_test.go
@@ -21,6 +21,7 @@ import (
"sync"
"sync/atomic"
"testing"
+ "time"
"github.com/google/sam/api"
"github.com/ipfs/go-cid"
@@ -274,3 +275,82 @@ func TestServiceRegistry_ReprovideResumesWhenBackendRecovers(t *testing.T) {
t.Errorf("Provide called %d times after recovery, want 2 (name + type CID)", len(dht.calls))
}
}
+
+// slowProbingService is a backendProber whose Probe blocks until the given
+// delay elapses or the context is cancelled first, whichever comes first -
+// unlike probingService, it actually respects the probe deadline, which is
+// what a real command-spawned backend with a slow cold-start does.
+type slowProbingService struct {
+ *fakeService
+ delay time.Duration
+}
+
+func newSlowProbingSvc(name string, delay time.Duration) *slowProbingService {
+ return &slowProbingService{
+ fakeService: newFakeSvc(name, api.ServiceType_SERVICE_TYPE_MCP),
+ delay: delay,
+ }
+}
+
+func (p *slowProbingService) Probe(ctx context.Context) error {
+ select {
+ case <-time.After(p.delay):
+ return nil
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+}
+
+// The bug behind #376: dhtProbeTimeout was a hard-coded 2s with no way to
+// raise it, so a backend whose own cold-start cost alone exceeds that -
+// measured in practice for moderately-featured MCP server stacks - could
+// never be advertised on its first registration. SetBackendProbeTimeout
+// (wired from --backend-probe-timeout) is the fix: the same slow backend
+// must fail to advertise under the default and succeed once given more time.
+func TestServiceRegistry_BackendProbeTimeoutIsConfigurable(t *testing.T) {
+ const probeDelay = 60 * time.Millisecond
+
+ t.Run("default timeout is too short for a slow backend", func(t *testing.T) {
+ dht := &fakeDHT{}
+ r := NewServiceRegistry(dht)
+ r.SetBackendProbeTimeout(10 * time.Millisecond) // shorter than probeDelay
+
+ svc := newSlowProbingSvc("slow", probeDelay)
+ if err := r.Register(context.Background(), svc); err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+ if len(dht.calls) != 0 {
+ t.Errorf("Provide called %d times for a backend slower than the probe timeout, want 0", len(dht.calls))
+ }
+ })
+
+ t.Run("raising the timeout lets the same backend advertise", func(t *testing.T) {
+ dht := &fakeDHT{}
+ r := NewServiceRegistry(dht)
+ r.SetBackendProbeTimeout(probeDelay * 5) // comfortably longer than probeDelay
+
+ svc := newSlowProbingSvc("slow", probeDelay)
+ if err := r.Register(context.Background(), svc); err != nil {
+ t.Fatalf("Register: %v", err)
+ }
+ if len(dht.calls) != 2 {
+ t.Errorf("Provide called %d times once given enough time to probe, want 2 (name + type CID)", len(dht.calls))
+ }
+ })
+
+ t.Run("NewServiceRegistry defaults to dhtProbeTimeout unchanged", func(t *testing.T) {
+ r := NewServiceRegistry(&fakeDHT{})
+ if got := r.probeTimeout(); got != dhtProbeTimeout {
+ t.Errorf("default probe timeout = %v, want %v (unchanged default behaviour)", got, dhtProbeTimeout)
+ }
+ })
+
+ t.Run("SetBackendProbeTimeout ignores zero and negative durations", func(t *testing.T) {
+ r := NewServiceRegistry(&fakeDHT{})
+ r.SetBackendProbeTimeout(0)
+ r.SetBackendProbeTimeout(-1 * time.Second)
+ if got := r.probeTimeout(); got != dhtProbeTimeout {
+ t.Errorf("probe timeout after no-op sets = %v, want unchanged %v", got, dhtProbeTimeout)
+ }
+ })
+}