Skip to content

Free, Open Source, Self-Hosted WhatsApp API Gateway

fix(events): take the backlog age from a monotonic clock

Yudhi Armyndharisbc26a73a4f95
diff --git a/src/modules/events/redis-io.adapter.spec.ts b/src/modules/events/redis-io.adapter.spec.ts
index b1a401dde..1529a0569 100644
--- a/src/modules/events/redis-io.adapter.spec.ts
+++ b/src/modules/events/redis-io.adapter.spec.ts
@@ -285,8 +285,8 @@ describe('RedisIoAdapter', () => {
});
it('drops frames once the queue is at the cap and has been above half of it for longer than a command is given', async () => {
- let clock = 1_000_000;
- const now = jest.spyOn(Date, 'now').mockImplementation(() => clock);
+ let clock = 5_000;
+ const now = jest.spyOn(performance, 'now').mockImplementation(() => clock);
const queue = { length: 0 };
try {
await withPublisher(queue, async pub => {
@@ -313,9 +313,34 @@ describe('RedisIoAdapter', () => {
}
});
+ // The age comes from a monotonic clock. The wall clock can step backwards (an NTP correction, a VM resumed), and
+ // an age taken from it would be negative for as long as the step is large, leaving a slow Redis uncapped.
+ it('is not blinded by the wall clock stepping back while the backlog ages', async () => {
+ let monotonic = 5_000;
+ let wall = 1_700_000_000_000;
+ const mono = jest.spyOn(performance, 'now').mockImplementation(() => monotonic);
+ const wallClock = jest.spyOn(Date, 'now').mockImplementation(() => wall);
+ const queue = { length: 0 };
+ try {
+ await withPublisher(queue, async pub => {
+ await pub.publish('socket.io#/events#', 'frame');
+ queue.length = WS_REDIS_MAX_UNANSWERED / 2;
+ monotonic += WS_REDIS_COMMAND_TIMEOUT_MS + 1;
+ wall -= 60_000;
+ queue.length = WS_REDIS_MAX_UNANSWERED;
+
+ await expect(pub.publish('socket.io#/events#', 'frame')).resolves.toBe(0);
+ expect(pub.sent).toHaveBeenCalledTimes(1);
+ });
+ } finally {
+ mono.mockRestore();
+ wallClock.mockRestore();
+ }
+ });
+
it('warns once when an episode starts and once when it ends, each with its own count', async () => {
- let clock = 1_000_000;
- const now = jest.spyOn(Date, 'now').mockImplementation(() => clock);
+ let clock = 5_000;
+ const now = jest.spyOn(performance, 'now').mockImplementation(() => clock);
const warn = jest.spyOn(console, 'warn').mockImplementation(() => undefined);
const lines = (text: string): string[] =>
warn.mock.calls.map(([line]) => String(line)).filter(line => line.includes(text));
diff --git a/src/modules/events/redis-io.adapter.stall.spec.ts b/src/modules/events/redis-io.adapter.stall.spec.ts
index 8bbfda6c8..eefddb7c2 100644
--- a/src/modules/events/redis-io.adapter.stall.spec.ts
+++ b/src/modules/events/redis-io.adapter.stall.spec.ts
@@ -272,8 +272,8 @@ describe('RedisIoAdapter publish client against a Redis that stops answering', (
// deadline never fires and the client would hold the whole backlog. Only the cap on unanswered commands bounds it,
// once the backlog has outlived a command's own deadline. The clock is moved by hand: the rule is about elapsed time.
it('holds a slow peer to the cap, warning once when dropping starts and once when it ends, however often it answers', async () => {
- let clock = 1_000_000;
- jest.spyOn(Date, 'now').mockImplementation(() => clock);
+ let clock = 5_000;
+ jest.spyOn(performance, 'now').mockImplementation(() => clock);
const connections = redis.connections;
redis.trickle = true;
diff --git a/src/modules/events/redis-io.adapter.ts b/src/modules/events/redis-io.adapter.ts
index 925d54f18..39276e081 100644
--- a/src/modules/events/redis-io.adapter.ts
+++ b/src/modules/events/redis-io.adapter.ts
@@ -168,11 +168,13 @@ export class RedisIoAdapter extends IoAdapter {
const publish = pubClient.publish.bind(pubClient);
let dropping = false;
let dropped = 0;
- // When the queue was last seen below half the cap: the backlog is as old as the time since.
+ // When the queue was last seen below half the cap: the backlog is as old as the time since. Read from the
+ // monotonic clock: the wall clock can step back (an NTP correction, a resumed VM), and an age taken from it
+ // would be negative for as long as the step is large, leaving a slow Redis uncapped.
let lastBelowResume = 0;
pubClient.publish = (channel: string, message: string | Buffer) => {
const unanswered = unansweredCommands(pubClient);
- const now = Date.now();
+ const now = performance.now();
if (unanswered < WS_REDIS_RESUME_UNANSWERED) {
lastBelowResume = now;
if (dropping) {