Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
node: make the backend probe timeout configurable
Fernando Marinob665247ea502
diff --git a/cmd/sam-node/main.go b/cmd/sam-node/main.goindex 9c65be62..8c63e131 100644--- a/cmd/sam-node/main.go+++ b/cmd/sam-node/main.go@@ -99,6 +99,7 @@ var (dhtMaxRecordAgeFlag time.DurationdhtLookupLimitFlag intdiscoveryConcurrencyFlag int+ backendProbeTimeoutFlag time.DurationpolicySyncIntervalFlag time.Duration)@@ -466,6 +467,7 @@ func main() {DHTMaxRecordAge: dhtMaxRecordAgeFlag,DHTLookupLimit: dhtLookupLimitFlag,DiscoveryConcurrency: discoveryConcurrencyFlag,+ BackendProbeTimeout: backendProbeTimeoutFlag,})if err != nil {logger.Fatalf("Failed to initialize mesh node: %v", err)@@ -533,6 +535,7 @@ func main() {DHTMaxRecordAge: dhtMaxRecordAgeFlag,DHTLookupLimit: dhtLookupLimitFlag,DiscoveryConcurrency: discoveryConcurrencyFlag,+ BackendProbeTimeout: backendProbeTimeoutFlag,})if err != nil {enrollCancel()@@ -599,6 +602,7 @@ func main() {RouterConnectTimeout: routerConnectTimeoutFlag,RequiredRole: api.RoleNode,PolicySyncInterval: policySyncIntervalFlag,+ BackendProbeTimeout: backendProbeTimeoutFlag,})if err != nil {logger.Fatalf("Failed to initialize node after enrollment: %v", err)@@ -854,6 +858,7 @@ func main() {runCmd.Flags().IntVar(&dhtLookupLimitFlag, "dht-lookup-limit", 0, "Maximum number of providers to query from the DHT (0 uses default 20)")runCmd.Flags().IntVar(&discoveryConcurrencyFlag, "discovery-concurrency", 0, "Max concurrent catalog fetches during discovery (0 uses default 10)")runCmd.Flags().DurationVar(&policySyncIntervalFlag, "policy-sync-interval", 1*time.Hour, "Interval for syncing mesh policy from the control plane")+ runCmd.Flags().DurationVar(&backendProbeTimeoutFlag, "backend-probe-timeout", 0, "Timeout for probing a command-spawned service backend before advertising it (0 uses default 2s); raise this for backends with slower cold-start times")rootCmd.PersistentFlags().StringVar(&controlPlaneAddr, "control-plane", "", "Control plane URL")rootCmd.PersistentFlags().StringVar(&configFile, "config", node.DefaultConfigFile, "Path to sam-node.yaml configuration file")rootCmd.PersistentFlags().StringVar(&oidcIssuerFlag, "oidc-issuer", "", "OIDC Issuer URL")diff --git a/internal/node/node.go b/internal/node/node.goindex 685bc613..039084b9 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -520,6 +520,7 @@ func (n *SamNode) Start(ctx context.Context) error {n.services = NewServiceRegistry(n.DHT)n.services.reprovideNow = n.triggerReprovide+ n.services.SetBackendProbeTimeout(n.config.BackendProbeTimeout)var authenticated boolvar fatalAuthErr errordiff --git a/internal/node/options.go b/internal/node/options.goindex 9f8a0367..2075c7ee 100644--- a/internal/node/options.go+++ b/internal/node/options.go@@ -78,6 +78,14 @@ type Options struct {PolicySyncInterval time.Duration// PolicySyncJitter specifies the maximum jitter delay when scheduling policy syncs on event broadcasts.PolicySyncJitter time.Duration+ // BackendProbeTimeout bounds how long a command-spawned service backend+ // (sam-node.yaml's `command`, spawned as a local subprocess) is given to+ // answer before the service is registered but withheld from+ // advertisement. Zero uses the library default (2s). Raise this for+ // backends with slower cold-start/import costs than that - the default+ // is tight enough that even simple interpreted-language MCP servers can+ // miss it on first spawn.+ BackendProbeTimeout time.Duration}// Default applies default values to Options if they are not specified.diff --git a/internal/node/service_registry.go b/internal/node/service_registry.goindex ffcc2724..280f2718 100644--- a/internal/node/service_registry.go+++ b/internal/node/service_registry.go@@ -38,7 +38,14 @@ type backendProber interface {Probe(ctx context.Context) error}-// dhtProbeTimeout bounds one backend probe before advertising.+// dhtProbeTimeout is the default bound on one backend probe before+// advertising, used when a ServiceRegistry isn't given an explicit+// BackendProbeTimeout (see Options.BackendProbeTimeout / --backend-probe-+// timeout). Command-spawned backends (sam-node.yaml's `command`, launched as+// a local subprocess) can need longer than this to answer their first+// request - a moderately-featured interpreted-language MCP server's own+// import/startup cost alone can exceed 2s - so this is a floor, not+// something every backend is expected to meet.const dhtProbeTimeout = 2 * time.Second// advertisable reports whether a service is fit to be published to the DHT.@@ -49,12 +56,12 @@ const dhtProbeTimeout = 2 * time.Second// listed by discover_remote_services and only failed later, at initialize, in// the caller's face. Advertising is a claim the node makes on the backend's// behalf, so it is the node that should verify it.-func advertisable(ctx context.Context, svc Service) error {+func advertisable(ctx context.Context, svc Service, probeTimeout time.Duration) error {prober, ok := svc.(backendProber)if !ok {return nil}- probeCtx, cancel := context.WithTimeout(ctx, dhtProbeTimeout)+ probeCtx, cancel := context.WithTimeout(ctx, probeTimeout)defer cancel()return prober.Probe(probeCtx)}@@ -69,15 +76,40 @@ type ServiceRegistry struct {// registered after the loop last ran does not wait a whole interval to be// advertised. Optional; nil outside a running node.reprovideNow func()++ // backendProbeTimeout bounds each backend probe in advertisable. Defaults+ // to dhtProbeTimeout; override with SetBackendProbeTimeout.+ backendProbeTimeout time.Duration}func NewServiceRegistry(d dhtProvider) *ServiceRegistry {return &ServiceRegistry{- services: map[string]Service{},- dht: d,+ services: map[string]Service{},+ dht: d,+ backendProbeTimeout: dhtProbeTimeout,}}+// SetBackendProbeTimeout overrides the default backend probe timeout+// (dhtProbeTimeout). A zero or negative duration is a no-op, so callers can+// pass an unset Options.BackendProbeTimeout straight through without an+// explicit zero-check.+func (r *ServiceRegistry) SetBackendProbeTimeout(d time.Duration) {+ if d <= 0 {+ return+ }+ r.mu.Lock()+ defer r.mu.Unlock()+ r.backendProbeTimeout = d+}++// probeTimeout returns the current backend probe timeout.+func (r *ServiceRegistry) probeTimeout() time.Duration {+ r.mu.RLock()+ defer r.mu.RUnlock()+ return r.backendProbeTimeout+}+// Register initialises a service, advertises it on the DHT, and inserts it// into the map. Init runs before Provide so a failed handler-build never// briefly advertises an unservable name.@@ -104,7 +136,7 @@ func (r *ServiceRegistry) Register(ctx context.Context, svc Service) error {return err}- probeErr := advertisable(ctx, svc)+ probeErr := advertisable(ctx, svc, r.probeTimeout())if probeErr != nil {logger.Warnf("[ServiceRegistry] Registered %s/%s but not advertising it: backend did not answer: %v", info.Type, info.Name, probeErr)} else {@@ -229,7 +261,7 @@ Loop:}()info := svc.Info()- if err := advertisable(ctx, svc); err != nil {+ if err := advertisable(ctx, svc, r.probeTimeout()); err != nil {withheld.Add(1)// On shutdown every service fails this way, and saying so// would blame backends for the node stopping.diff --git a/internal/node/service_registry_test.go b/internal/node/service_registry_test.goindex b0d2f5e5..8d724aae 100644--- a/internal/node/service_registry_test.go+++ b/internal/node/service_registry_test.go@@ -21,6 +21,7 @@ import ("sync""sync/atomic""testing"+ "time""github.com/google/sam/api""github.com/ipfs/go-cid"@@ -274,3 +275,82 @@ func TestServiceRegistry_ReprovideResumesWhenBackendRecovers(t *testing.T) {t.Errorf("Provide called %d times after recovery, want 2 (name + type CID)", len(dht.calls))}}++// slowProbingService is a backendProber whose Probe blocks until the given+// delay elapses or the context is cancelled first, whichever comes first -+// unlike probingService, it actually respects the probe deadline, which is+// what a real command-spawned backend with a slow cold-start does.+type slowProbingService struct {+ *fakeService+ delay time.Duration+}++func newSlowProbingSvc(name string, delay time.Duration) *slowProbingService {+ return &slowProbingService{+ fakeService: newFakeSvc(name, api.ServiceType_SERVICE_TYPE_MCP),+ delay: delay,+ }+}++func (p *slowProbingService) Probe(ctx context.Context) error {+ select {+ case <-time.After(p.delay):+ return nil+ case <-ctx.Done():+ return ctx.Err()+ }+}++// The bug behind #376: dhtProbeTimeout was a hard-coded 2s with no way to+// raise it, so a backend whose own cold-start cost alone exceeds that -+// measured in practice for moderately-featured MCP server stacks - could+// never be advertised on its first registration. SetBackendProbeTimeout+// (wired from --backend-probe-timeout) is the fix: the same slow backend+// must fail to advertise under the default and succeed once given more time.+func TestServiceRegistry_BackendProbeTimeoutIsConfigurable(t *testing.T) {+ const probeDelay = 60 * time.Millisecond++ t.Run("default timeout is too short for a slow backend", func(t *testing.T) {+ dht := &fakeDHT{}+ r := NewServiceRegistry(dht)+ r.SetBackendProbeTimeout(10 * time.Millisecond) // shorter than probeDelay++ svc := newSlowProbingSvc("slow", probeDelay)+ if err := r.Register(context.Background(), svc); err != nil {+ t.Fatalf("Register: %v", err)+ }+ if len(dht.calls) != 0 {+ t.Errorf("Provide called %d times for a backend slower than the probe timeout, want 0", len(dht.calls))+ }+ })++ t.Run("raising the timeout lets the same backend advertise", func(t *testing.T) {+ dht := &fakeDHT{}+ r := NewServiceRegistry(dht)+ r.SetBackendProbeTimeout(probeDelay * 5) // comfortably longer than probeDelay++ svc := newSlowProbingSvc("slow", probeDelay)+ if err := r.Register(context.Background(), svc); err != nil {+ t.Fatalf("Register: %v", err)+ }+ if len(dht.calls) != 2 {+ t.Errorf("Provide called %d times once given enough time to probe, want 2 (name + type CID)", len(dht.calls))+ }+ })++ t.Run("NewServiceRegistry defaults to dhtProbeTimeout unchanged", func(t *testing.T) {+ r := NewServiceRegistry(&fakeDHT{})+ if got := r.probeTimeout(); got != dhtProbeTimeout {+ t.Errorf("default probe timeout = %v, want %v (unchanged default behaviour)", got, dhtProbeTimeout)+ }+ })++ t.Run("SetBackendProbeTimeout ignores zero and negative durations", func(t *testing.T) {+ r := NewServiceRegistry(&fakeDHT{})+ r.SetBackendProbeTimeout(0)+ r.SetBackendProbeTimeout(-1 * time.Second)+ if got := r.probeTimeout(); got != dhtProbeTimeout {+ t.Errorf("probe timeout after no-op sets = %v, want unchanged %v", got, dhtProbeTimeout)+ }+ })+}