Agent Mesh
Go a2aa2a-protocolagent-meshagentic-aiinferencemcpmeshmesh-networknetwork-securitynetworking-aip2p-network
fix: jitter the catalog report loop and stop warning once a minute
Antonio Ojea91434d1c1862
diff --git a/internal/node/controlplane_test.go b/internal/node/controlplane_test.goindex d06a46dd..38469539 100644--- a/internal/node/controlplane_test.go+++ b/internal/node/controlplane_test.go@@ -281,3 +281,102 @@ func TestReportNodeCatalog_HTTPError(t *testing.T) {t.Errorf("Expected error to mention status 401, got %v", err)}}++// newCatalogTestNode is a SamNode with just enough state for the catalog+// loop: a store holding the control-plane URL, a cached identity, and a+// registry with one service.+func newCatalogTestNode(t *testing.T, controlPlaneURL string) *SamNode {+ t.Helper()+ store, err := NewStore(t.TempDir())+ if err != nil {+ t.Fatalf("NewStore: %v", err)+ }+ t.Cleanup(func() { _ = store.Close() })+ if controlPlaneURL != "" {+ if err := store.SaveControlPlaneURL(controlPlaneURL); err != nil {+ t.Fatalf("SaveControlPlaneURL: %v", err)+ }+ }+ node := &SamNode{Store: store, services: newServiceRegistryForTest(&fakeDHT{})}+ node.services.insertService(newFakeSvc("calc", api.ServiceType_SERVICE_TYPE_MCP))+ return node+}++func TestReportNodeCatalog_Preconditions(t *testing.T) {+ noURL := newCatalogTestNode(t, "")+ noURL.SetIdentityCache([]byte("biscuit"))+ if err := noURL.reportNodeCatalog(context.Background()); err == nil || !strings.Contains(err.Error(), "control plane URL") {+ t.Errorf("without a control-plane URL: got %v, want a URL error", err)+ }++ noIdentity := newCatalogTestNode(t, "http://127.0.0.1:1")+ if err := noIdentity.reportNodeCatalog(context.Background()); err == nil || !strings.Contains(err.Error(), "identity") {+ t.Errorf("without an identity: got %v, want an identity error", err)+ }+}++// The loop must report once after the initial delay and then keep reporting+// every interval, carrying the live service list each time.+func TestStartCatalogReportLoop(t *testing.T) {+ reports := make(chan *api.NodeCatalogReport, 16)+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/nodes/catalog" || r.Method != http.MethodPost {+ t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)+ }+ body, _ := io.ReadAll(r.Body)+ var report api.NodeCatalogReport+ if err := proto.Unmarshal(body, &report); err != nil {+ t.Errorf("decode report: %v", err)+ }+ reports <- &report+ w.WriteHeader(http.StatusNoContent)+ }))+ defer server.Close()++ node := newCatalogTestNode(t, server.URL)+ node.SetIdentityCache([]byte("biscuit"))++ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ node.startCatalogReportLoop(ctx, 10*time.Millisecond, 20*time.Millisecond)++ for i := 0; i < 3; i++ {+ select {+ case report := <-reports:+ if len(report.Services) != 1 || report.Services[0].Name != "calc" {+ t.Fatalf("report %d: got services %v, want [calc]", i, report.Services)+ }+ case <-time.After(5 * time.Second):+ t.Fatalf("timed out waiting for report %d", i)+ }+ }++ // Cancelling stops the loop: no report after the in-flight one settles.+ cancel()+ time.Sleep(100 * time.Millisecond)+ for len(reports) > 0 {+ <-reports+ }+ select {+ case <-reports:+ t.Fatal("loop kept reporting after context cancellation")+ case <-time.After(150 * time.Millisecond):+ }+}++func TestOptionsDefault_CatalogReport(t *testing.T) {+ var o Options+ o.Default()+ if o.CatalogReportInterval != time.Minute {+ t.Errorf("CatalogReportInterval = %v, want 1m", o.CatalogReportInterval)+ }+ if o.CatalogReportInitialDelay != 5*time.Second {+ t.Errorf("CatalogReportInitialDelay = %v, want 5s", o.CatalogReportInitialDelay)+ }++ custom := Options{CatalogReportInterval: 3 * time.Second, CatalogReportInitialDelay: time.Second}+ custom.Default()+ if custom.CatalogReportInterval != 3*time.Second || custom.CatalogReportInitialDelay != time.Second {+ t.Errorf("Default overwrote explicit values: %+v", custom)+ }+}diff --git a/internal/node/node.go b/internal/node/node.goindex 82343746..b784588d 100644--- a/internal/node/node.go+++ b/internal/node/node.go@@ -633,7 +633,7 @@ func (n *SamNode) Start(ctx context.Context) error {// Periodically self-report local services to the control plane, so an// admin can see mesh-wide service topology.- n.startCatalogReportLoop(ctx, n.config.CatalogReportInterval)+ n.startCatalogReportLoop(ctx, n.config.CatalogReportInitialDelay, n.config.CatalogReportInterval)return nil}@@ -2233,34 +2233,46 @@ func (n *SamNode) reportNodeCatalog(ctx context.Context) error {return nil}-func (n *SamNode) startCatalogReportLoop(ctx context.Context, interval time.Duration) {+// startCatalogReportLoop reports the local catalog once after initialDelay+// and then every interval, each wait stretched by up to a tenth of interval+// so a fleet started together does not hit the control plane in lockstep.+// A failure is logged at Warn once and at Debug while it persists: the+// usual causes (control plane unreachable, path not routed) do not change+// from one tick to the next.+func (n *SamNode) startCatalogReportLoop(ctx context.Context, initialDelay, interval time.Duration) {if interval <= 0 {interval = 1 * time.Minute}+ if initialDelay <= 0 {+ initialDelay = 5 * time.Second+ }go func() {- // Run an initial report after a short delay, once services have had- // a chance to register at startup.- select {- case <-ctx.Done():- return- case <-time.After(5 * time.Second):- if err := n.reportNodeCatalog(ctx); err != nil {- logger.Warnf("Initial node catalog report failed: %v", err)- }- }-- ticker := time.NewTicker(interval)- defer ticker.Stop()-+ wait := initialDelay+ failures := 0for {+ timer := time.NewTimer(wait + time.Duration(rand.Int63n(int64(interval/10)+1)))select {case <-ctx.Done():+ timer.Stop()return- case <-ticker.C:- if err := n.reportNodeCatalog(ctx); err != nil {- logger.Warnf("Periodic node catalog report failed: %v", err)- }+ case <-timer.C:+ }+ wait = interval++ err := n.reportNodeCatalog(ctx)+ switch {+ case err != nil && failures == 0:+ logger.Warnf("Node catalog report failed: %v", err)+ case err != nil:+ logger.Debugf("Node catalog report still failing (%d consecutive): %v", failures+1, err)+ case failures > 0:+ logger.Infof("Node catalog report recovered after %d failures", failures)+ }+ if err != nil {+ failures+++ } else {+ failures = 0}}}()diff --git a/internal/node/options.go b/internal/node/options.goindex 7c4c7cc1..dbc5e30e 100644--- a/internal/node/options.go+++ b/internal/node/options.go@@ -92,6 +92,10 @@ type Options struct {// without the control plane needing DHT/P2P access to every node// itself. Zero uses the default.CatalogReportInterval time.Duration+ // CatalogReportInitialDelay is how long after Start the first catalog+ // report is sent, so services configured at startup have registered by+ // then. Zero uses the default.+ CatalogReportInitialDelay time.Duration}// Default applies default values to Options if they are not specified.@@ -142,9 +146,12 @@ func (o *Options) Default() {if o.PolicySyncInterval == 0 {o.PolicySyncInterval = 1 * time.Hour}- if o.CatalogReportInterval == 0 {+ if o.CatalogReportInterval <= 0 {o.CatalogReportInterval = 1 * time.Minute}+ if o.CatalogReportInitialDelay <= 0 {+ o.CatalogReportInitialDelay = 5 * time.Second+ }if o.PolicySyncJitter <= 0 {o.PolicySyncJitter = 10 * time.Second}