Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
checker: spread pubsub check starts over a per-monitor window (#3010)
Thibault Le Ouay050a6031a204
diff --git a/apps/checker/cmd/server/main.go b/apps/checker/cmd/server/main.goindex ad76b6e3a..552152fcf 100644--- a/apps/checker/cmd/server/main.go+++ b/apps/checker/cmd/server/main.go@@ -373,6 +373,7 @@ func runConsumer(ctx context.Context, h *handlers.Handler, region string) error},},Drain: consumerDrain,+ Spread: envDuration("PUBSUB_SPREAD", 2*time.Second),OnEvent: logConsumerEvent,}diff --git a/apps/checker/pkg/queue/consumer.go b/apps/checker/pkg/queue/consumer.goindex 34b0db995..2ff11651c 100644--- a/apps/checker/pkg/queue/consumer.go+++ b/apps/checker/pkg/queue/consumer.go@@ -5,6 +5,7 @@ import ("encoding/json""errors""fmt"+ "hash/fnv""time""cloud.google.com/go/pubsub/v2"@@ -58,7 +59,11 @@ type Consumer struct {Loops []Loop// Drain is how long in-flight checks get to finish after shutdown starts// before their message is nacked for redelivery.- Drain time.Duration+ Drain time.Duration+ // Spread offsets each check past its cronTimestamp by a per-monitor delay+ // in [0, Spread), so the checks of one cron tick do not all resolve and+ // handshake in the same instant. Zero disables it.+ Spread time.DurationOnEvent func(ctx context.Context, e Event)}@@ -158,7 +163,8 @@ func (c *Consumer) handle(ctx context.Context, subscription string, slots chan sreturn finish(OutcomePoison, errors.New("missing cronTimestamp"))}- if wait := time.Until(time.UnixMilli(env.CronTimestamp)); wait > 0 {+ startAt := time.UnixMilli(env.CronTimestamp).Add(spread(e.MonitorID, c.Spread))+ if wait := time.Until(startAt); wait > 0 {e.Wait = waittimer := time.NewTimer(wait)select {@@ -212,6 +218,18 @@ func (c *Consumer) handle(ctx context.Context, subscription string, slots chan s}}+// spread maps a monitor to a fixed delay in [0, window): the same monitor keeps+// the same offset every tick, so its checks stay one interval apart.+func spread(monitorID string, window time.Duration) time.Duration {+ if window <= 0 {+ return 0+ }+ h := fnv.New64a()+ h.Write([]byte(monitorID))++ return time.Duration(h.Sum64() % uint64(window))+}+var errUnknownJobType = errors.New("unknown job type")func (c *Consumer) dispatch(ctx context.Context, jobType string, data []byte) error {diff --git a/apps/checker/pkg/queue/consumer_test.go b/apps/checker/pkg/queue/consumer_test.goindex 2ffca2e9d..c56446b83 100644--- a/apps/checker/pkg/queue/consumer_test.go+++ b/apps/checker/pkg/queue/consumer_test.go@@ -4,6 +4,7 @@ import ("context""encoding/json""errors"+ "strconv""sync""sync/atomic""testing"@@ -102,12 +103,19 @@ func (h *harness) publish(t *testing.T, jobType string, monitorID string, cronTifunc (h *harness) start(t *testing.T, runner queue.Runner, loop queue.Loop, drain time.Duration) (context.CancelFunc, <-chan error) {t.Helper()++ return h.startConsumer(t, runner, loop, drain, 0)+}++func (h *harness) startConsumer(t *testing.T, runner queue.Runner, loop queue.Loop, drain time.Duration, spread time.Duration) (context.CancelFunc, <-chan error) {+ t.Helper()loop.Subscription = subscriptionc := &queue.Consumer{Client: h.client,Runner: runner,Loops: []queue.Loop{loop},Drain: drain,+ Spread: spread,OnEvent: func(_ context.Context, e queue.Event) {h.events <- e},@@ -195,6 +203,42 @@ func TestConsumer_WaitsForCronTimestamp(t *testing.T) {assert.GreaterOrEqual(t, ranAt.Load(), at.UnixMilli())}+func TestConsumer_SpreadsStartPastCronTimestamp(t *testing.T) {+ h := newHarness(t)+ const window = time.Second+ // Millisecond precision, as the message carries it: the start is computed+ // from the serialized cronTimestamp, not from the sub-millisecond time.+ at := time.UnixMilli(time.Now().Add(200 * time.Millisecond).UnixMilli())+ h.publish(t, "http", "1", at)+ var ranAt atomic.Int64+ h.startConsumer(t, fakeRunner{run: func(context.Context, string) error {+ ranAt.Store(time.Now().UnixMilli())+ return nil+ }}, queue.Loop{MaxOutstanding: 10}, time.Second, window)++ e := h.next(t)++ assert.Equal(t, queue.OutcomeAck, e.Outcome)+ assert.GreaterOrEqual(t, ranAt.Load(), at.Add(queue.SpreadForTest("1", window)).UnixMilli())+}++func TestSpread(t *testing.T) {+ const window = 2 * time.Second+ assert.Zero(t, queue.SpreadForTest("1", 0))+ assert.Equal(t, queue.SpreadForTest("42", window), queue.SpreadForTest("42", window))++ lo, hi := window, time.Duration(0)+ for i := range 1000 {+ d := queue.SpreadForTest(strconv.Itoa(i), window)+ require.GreaterOrEqual(t, d, time.Duration(0))+ require.Less(t, d, window)+ lo, hi = min(lo, d), max(hi, d)+ }+ // Sequential monitor ids must still cover the whole window.+ assert.Less(t, lo, window/10)+ assert.Greater(t, hi, window*9/10)+}+func TestConsumer_SleepingMessageHoldsNoProbeSlot(t *testing.T) {h := newHarness(t)h.publish(t, "http", "later", time.Now().Add(2*time.Second))diff --git a/apps/checker/pkg/queue/export_test.go b/apps/checker/pkg/queue/export_test.gonew file mode 100644index 000000000..0cf9f09fd--- /dev/null+++ b/apps/checker/pkg/queue/export_test.go@@ -0,0 +1,4 @@+package queue++// SpreadForTest exposes the per-monitor start offset.+var SpreadForTest = spread