Skip to content

Open-source status page, incident response and uptime monitoring

checker: spread pubsub check starts over a per-monitor window (#3010)

Thibault Le Ouay050a6031a204
diff --git a/apps/checker/cmd/server/main.go b/apps/checker/cmd/server/main.go
index ad76b6e3a..552152fcf 100644
--- a/apps/checker/cmd/server/main.go
+++ b/apps/checker/cmd/server/main.go
@@ -373,6 +373,7 @@ func runConsumer(ctx context.Context, h *handlers.Handler, region string) error
},
},
Drain: consumerDrain,
+ Spread: envDuration("PUBSUB_SPREAD", 2*time.Second),
OnEvent: logConsumerEvent,
}
diff --git a/apps/checker/pkg/queue/consumer.go b/apps/checker/pkg/queue/consumer.go
index 34b0db995..2ff11651c 100644
--- a/apps/checker/pkg/queue/consumer.go
+++ b/apps/checker/pkg/queue/consumer.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "hash/fnv"
"time"
"cloud.google.com/go/pubsub/v2"
@@ -58,7 +59,11 @@ type Consumer struct {
Loops []Loop
// Drain is how long in-flight checks get to finish after shutdown starts
// before their message is nacked for redelivery.
- Drain time.Duration
+ Drain time.Duration
+ // Spread offsets each check past its cronTimestamp by a per-monitor delay
+ // in [0, Spread), so the checks of one cron tick do not all resolve and
+ // handshake in the same instant. Zero disables it.
+ Spread time.Duration
OnEvent func(ctx context.Context, e Event)
}
@@ -158,7 +163,8 @@ func (c *Consumer) handle(ctx context.Context, subscription string, slots chan s
return finish(OutcomePoison, errors.New("missing cronTimestamp"))
}
- if wait := time.Until(time.UnixMilli(env.CronTimestamp)); wait > 0 {
+ startAt := time.UnixMilli(env.CronTimestamp).Add(spread(e.MonitorID, c.Spread))
+ if wait := time.Until(startAt); wait > 0 {
e.Wait = wait
timer := time.NewTimer(wait)
select {
@@ -212,6 +218,18 @@ func (c *Consumer) handle(ctx context.Context, subscription string, slots chan s
}
}
+// spread maps a monitor to a fixed delay in [0, window): the same monitor keeps
+// the same offset every tick, so its checks stay one interval apart.
+func spread(monitorID string, window time.Duration) time.Duration {
+ if window <= 0 {
+ return 0
+ }
+ h := fnv.New64a()
+ h.Write([]byte(monitorID))
+
+ return time.Duration(h.Sum64() % uint64(window))
+}
+
var errUnknownJobType = errors.New("unknown job type")
func (c *Consumer) dispatch(ctx context.Context, jobType string, data []byte) error {
diff --git a/apps/checker/pkg/queue/consumer_test.go b/apps/checker/pkg/queue/consumer_test.go
index 2ffca2e9d..c56446b83 100644
--- a/apps/checker/pkg/queue/consumer_test.go
+++ b/apps/checker/pkg/queue/consumer_test.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"errors"
+ "strconv"
"sync"
"sync/atomic"
"testing"
@@ -102,12 +103,19 @@ func (h *harness) publish(t *testing.T, jobType string, monitorID string, cronTi
func (h *harness) start(t *testing.T, runner queue.Runner, loop queue.Loop, drain time.Duration) (context.CancelFunc, <-chan error) {
t.Helper()
+
+ return h.startConsumer(t, runner, loop, drain, 0)
+}
+
+func (h *harness) startConsumer(t *testing.T, runner queue.Runner, loop queue.Loop, drain time.Duration, spread time.Duration) (context.CancelFunc, <-chan error) {
+ t.Helper()
loop.Subscription = subscription
c := &queue.Consumer{
Client: h.client,
Runner: runner,
Loops: []queue.Loop{loop},
Drain: drain,
+ Spread: spread,
OnEvent: func(_ context.Context, e queue.Event) {
h.events <- e
},
@@ -195,6 +203,42 @@ func TestConsumer_WaitsForCronTimestamp(t *testing.T) {
assert.GreaterOrEqual(t, ranAt.Load(), at.UnixMilli())
}
+func TestConsumer_SpreadsStartPastCronTimestamp(t *testing.T) {
+ h := newHarness(t)
+ const window = time.Second
+ // Millisecond precision, as the message carries it: the start is computed
+ // from the serialized cronTimestamp, not from the sub-millisecond time.
+ at := time.UnixMilli(time.Now().Add(200 * time.Millisecond).UnixMilli())
+ h.publish(t, "http", "1", at)
+ var ranAt atomic.Int64
+ h.startConsumer(t, fakeRunner{run: func(context.Context, string) error {
+ ranAt.Store(time.Now().UnixMilli())
+ return nil
+ }}, queue.Loop{MaxOutstanding: 10}, time.Second, window)
+
+ e := h.next(t)
+
+ assert.Equal(t, queue.OutcomeAck, e.Outcome)
+ assert.GreaterOrEqual(t, ranAt.Load(), at.Add(queue.SpreadForTest("1", window)).UnixMilli())
+}
+
+func TestSpread(t *testing.T) {
+ const window = 2 * time.Second
+ assert.Zero(t, queue.SpreadForTest("1", 0))
+ assert.Equal(t, queue.SpreadForTest("42", window), queue.SpreadForTest("42", window))
+
+ lo, hi := window, time.Duration(0)
+ for i := range 1000 {
+ d := queue.SpreadForTest(strconv.Itoa(i), window)
+ require.GreaterOrEqual(t, d, time.Duration(0))
+ require.Less(t, d, window)
+ lo, hi = min(lo, d), max(hi, d)
+ }
+ // Sequential monitor ids must still cover the whole window.
+ assert.Less(t, lo, window/10)
+ assert.Greater(t, hi, window*9/10)
+}
+
func TestConsumer_SleepingMessageHoldsNoProbeSlot(t *testing.T) {
h := newHarness(t)
h.publish(t, "http", "later", time.Now().Add(2*time.Second))
diff --git a/apps/checker/pkg/queue/export_test.go b/apps/checker/pkg/queue/export_test.go
new file mode 100644
index 000000000..0cf9f09fd
--- /dev/null
+++ b/apps/checker/pkg/queue/export_test.go
@@ -0,0 +1,4 @@
+package queue
+
+// SpreadForTest exposes the per-monitor start offset.
+var SpreadForTest = spread