Skip to content

Agent Mesh

fix: jitter the catalog report loop and stop warning once a minute

Antonio Ojea91434d1c1862
diff --git a/internal/node/controlplane_test.go b/internal/node/controlplane_test.go
index d06a46dd..38469539 100644
--- a/internal/node/controlplane_test.go
+++ b/internal/node/controlplane_test.go
@@ -281,3 +281,102 @@ func TestReportNodeCatalog_HTTPError(t *testing.T) {
t.Errorf("Expected error to mention status 401, got %v", err)
}
}
+
+// newCatalogTestNode is a SamNode with just enough state for the catalog
+// loop: a store holding the control-plane URL, a cached identity, and a
+// registry with one service.
+func newCatalogTestNode(t *testing.T, controlPlaneURL string) *SamNode {
+ t.Helper()
+ store, err := NewStore(t.TempDir())
+ if err != nil {
+ t.Fatalf("NewStore: %v", err)
+ }
+ t.Cleanup(func() { _ = store.Close() })
+ if controlPlaneURL != "" {
+ if err := store.SaveControlPlaneURL(controlPlaneURL); err != nil {
+ t.Fatalf("SaveControlPlaneURL: %v", err)
+ }
+ }
+ node := &SamNode{Store: store, services: newServiceRegistryForTest(&fakeDHT{})}
+ node.services.insertService(newFakeSvc("calc", api.ServiceType_SERVICE_TYPE_MCP))
+ return node
+}
+
+func TestReportNodeCatalog_Preconditions(t *testing.T) {
+ noURL := newCatalogTestNode(t, "")
+ noURL.SetIdentityCache([]byte("biscuit"))
+ if err := noURL.reportNodeCatalog(context.Background()); err == nil || !strings.Contains(err.Error(), "control plane URL") {
+ t.Errorf("without a control-plane URL: got %v, want a URL error", err)
+ }
+
+ noIdentity := newCatalogTestNode(t, "http://127.0.0.1:1")
+ if err := noIdentity.reportNodeCatalog(context.Background()); err == nil || !strings.Contains(err.Error(), "identity") {
+ t.Errorf("without an identity: got %v, want an identity error", err)
+ }
+}
+
+// The loop must report once after the initial delay and then keep reporting
+// every interval, carrying the live service list each time.
+func TestStartCatalogReportLoop(t *testing.T) {
+ reports := make(chan *api.NodeCatalogReport, 16)
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/nodes/catalog" || r.Method != http.MethodPost {
+ t.Errorf("unexpected request %s %s", r.Method, r.URL.Path)
+ }
+ body, _ := io.ReadAll(r.Body)
+ var report api.NodeCatalogReport
+ if err := proto.Unmarshal(body, &report); err != nil {
+ t.Errorf("decode report: %v", err)
+ }
+ reports <- &report
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ defer server.Close()
+
+ node := newCatalogTestNode(t, server.URL)
+ node.SetIdentityCache([]byte("biscuit"))
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ node.startCatalogReportLoop(ctx, 10*time.Millisecond, 20*time.Millisecond)
+
+ for i := 0; i < 3; i++ {
+ select {
+ case report := <-reports:
+ if len(report.Services) != 1 || report.Services[0].Name != "calc" {
+ t.Fatalf("report %d: got services %v, want [calc]", i, report.Services)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatalf("timed out waiting for report %d", i)
+ }
+ }
+
+ // Cancelling stops the loop: no report after the in-flight one settles.
+ cancel()
+ time.Sleep(100 * time.Millisecond)
+ for len(reports) > 0 {
+ <-reports
+ }
+ select {
+ case <-reports:
+ t.Fatal("loop kept reporting after context cancellation")
+ case <-time.After(150 * time.Millisecond):
+ }
+}
+
+func TestOptionsDefault_CatalogReport(t *testing.T) {
+ var o Options
+ o.Default()
+ if o.CatalogReportInterval != time.Minute {
+ t.Errorf("CatalogReportInterval = %v, want 1m", o.CatalogReportInterval)
+ }
+ if o.CatalogReportInitialDelay != 5*time.Second {
+ t.Errorf("CatalogReportInitialDelay = %v, want 5s", o.CatalogReportInitialDelay)
+ }
+
+ custom := Options{CatalogReportInterval: 3 * time.Second, CatalogReportInitialDelay: time.Second}
+ custom.Default()
+ if custom.CatalogReportInterval != 3*time.Second || custom.CatalogReportInitialDelay != time.Second {
+ t.Errorf("Default overwrote explicit values: %+v", custom)
+ }
+}
diff --git a/internal/node/node.go b/internal/node/node.go
index 82343746..b784588d 100644
--- a/internal/node/node.go
+++ b/internal/node/node.go
@@ -633,7 +633,7 @@ func (n *SamNode) Start(ctx context.Context) error {
// Periodically self-report local services to the control plane, so an
// admin can see mesh-wide service topology.
- n.startCatalogReportLoop(ctx, n.config.CatalogReportInterval)
+ n.startCatalogReportLoop(ctx, n.config.CatalogReportInitialDelay, n.config.CatalogReportInterval)
return nil
}
@@ -2233,34 +2233,46 @@ func (n *SamNode) reportNodeCatalog(ctx context.Context) error {
return nil
}
-func (n *SamNode) startCatalogReportLoop(ctx context.Context, interval time.Duration) {
+// startCatalogReportLoop reports the local catalog once after initialDelay
+// and then every interval, each wait stretched by up to a tenth of interval
+// so a fleet started together does not hit the control plane in lockstep.
+// A failure is logged at Warn once and at Debug while it persists: the
+// usual causes (control plane unreachable, path not routed) do not change
+// from one tick to the next.
+func (n *SamNode) startCatalogReportLoop(ctx context.Context, initialDelay, interval time.Duration) {
if interval <= 0 {
interval = 1 * time.Minute
}
+ if initialDelay <= 0 {
+ initialDelay = 5 * time.Second
+ }
go func() {
- // Run an initial report after a short delay, once services have had
- // a chance to register at startup.
- select {
- case <-ctx.Done():
- return
- case <-time.After(5 * time.Second):
- if err := n.reportNodeCatalog(ctx); err != nil {
- logger.Warnf("Initial node catalog report failed: %v", err)
- }
- }
-
- ticker := time.NewTicker(interval)
- defer ticker.Stop()
-
+ wait := initialDelay
+ failures := 0
for {
+ timer := time.NewTimer(wait + time.Duration(rand.Int63n(int64(interval/10)+1)))
select {
case <-ctx.Done():
+ timer.Stop()
return
- case <-ticker.C:
- if err := n.reportNodeCatalog(ctx); err != nil {
- logger.Warnf("Periodic node catalog report failed: %v", err)
- }
+ case <-timer.C:
+ }
+ wait = interval
+
+ err := n.reportNodeCatalog(ctx)
+ switch {
+ case err != nil && failures == 0:
+ logger.Warnf("Node catalog report failed: %v", err)
+ case err != nil:
+ logger.Debugf("Node catalog report still failing (%d consecutive): %v", failures+1, err)
+ case failures > 0:
+ logger.Infof("Node catalog report recovered after %d failures", failures)
+ }
+ if err != nil {
+ failures++
+ } else {
+ failures = 0
}
}
}()
diff --git a/internal/node/options.go b/internal/node/options.go
index 7c4c7cc1..dbc5e30e 100644
--- a/internal/node/options.go
+++ b/internal/node/options.go
@@ -92,6 +92,10 @@ type Options struct {
// without the control plane needing DHT/P2P access to every node
// itself. Zero uses the default.
CatalogReportInterval time.Duration
+ // CatalogReportInitialDelay is how long after Start the first catalog
+ // report is sent, so services configured at startup have registered by
+ // then. Zero uses the default.
+ CatalogReportInitialDelay time.Duration
}
// Default applies default values to Options if they are not specified.
@@ -142,9 +146,12 @@ func (o *Options) Default() {
if o.PolicySyncInterval == 0 {
o.PolicySyncInterval = 1 * time.Hour
}
- if o.CatalogReportInterval == 0 {
+ if o.CatalogReportInterval <= 0 {
o.CatalogReportInterval = 1 * time.Minute
}
+ if o.CatalogReportInitialDelay <= 0 {
+ o.CatalogReportInitialDelay = 5 * time.Second
+ }
if o.PolicySyncJitter <= 0 {
o.PolicySyncJitter = 10 * time.Second
}