Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

jitter scheduler claim to avoid thundering herd (#3619)

tomsideguide1d7e7a288d60
diff --git a/apps/api/src/services/monitoring/jitter.ts b/apps/api/src/services/monitoring/jitter.ts
new file mode 100644
index 000000000..782937613
--- /dev/null
+++ b/apps/api/src/services/monitoring/jitter.ts
@@ -0,0 +1,13 @@
+import { createHash } from "crypto";
+
+const MAX_JITTER_MS = 5 * 60 * 1000;
+
+export function monitorJitterOffsetMs(
+ monitorId: string,
+ intervalMs: number,
+): number {
+ const jitterMaxMs = Math.min(Math.floor(intervalMs / 2), MAX_JITTER_MS);
+ if (jitterMaxMs <= 0) return 0;
+ const hash = createHash("sha256").update(monitorId).digest();
+ return hash.readUInt32BE(0) % jitterMaxMs;
+}
diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.ts
index 61492e3b6..360f910b9 100644
--- a/apps/api/src/services/monitoring/scheduler.ts
+++ b/apps/api/src/services/monitoring/scheduler.ts
@@ -5,6 +5,7 @@ import {
advanceMonitorAfterSkippedCheck,
claimDueMonitors,
createMonitorCheck,
+ deferMonitorClaim,
dispatchScheduledMonitorCheck,
getMonitorCheck,
updateMonitorCheck,
@@ -12,6 +13,8 @@ import {
} from "./store";
import { autumnService } from "../autumn/autumn.service";
import { isMonitorCheckStale, MONITOR_CHECK_STALE_ERROR } from "./stale";
+import { validateMonitorCron } from "./cron";
+import { monitorJitterOffsetMs } from "./jitter";
import type { MonitorRow } from "./types";
const logger = _logger.child({ module: "monitoring-scheduler" });
@@ -43,6 +46,8 @@ export async function enqueueDueMonitorChecks(
let check: Awaited<ReturnType<typeof createMonitorCheck>> | null = null;
let dispatched = false;
try {
+ if (await deferForJitter(monitor)) continue;
+
if (monitor.current_check_id) {
const cleared = await clearFinishedOrStaleCurrentCheck(monitor);
if (cleared) {
@@ -135,6 +140,28 @@ export async function enqueueDueMonitorChecks(
return enqueued;
}
+async function deferForJitter(monitor: MonitorRow): Promise<boolean> {
+ if (!monitor.next_run_at) return false;
+ const { intervalMs } = validateMonitorCron(
+ monitor.schedule_cron,
+ monitor.schedule_timezone,
+ );
+ const dueAt =
+ new Date(monitor.next_run_at).getTime() +
+ monitorJitterOffsetMs(monitor.id, intervalMs);
+ if (Date.now() >= dueAt) return false;
+ try {
+ await deferMonitorClaim(monitor.id, new Date(dueAt));
+ } catch (error) {
+ logger.warn("Failed to defer monitor claim for jitter", {
+ error,
+ monitorId: monitor.id,
+ });
+ return false;
+ }
+ return true;
+}
+
async function clearFinishedOrStaleCurrentCheck(
monitor: MonitorRow,
): Promise<boolean> {
diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.ts
index 70bb2ee3f..2cec0f79b 100644
--- a/apps/api/src/services/monitoring/store.ts
+++ b/apps/api/src/services/monitoring/store.ts
@@ -698,3 +698,19 @@ export async function claimDueMonitors(params: {
throwIfError(error, "Failed to claim due monitors");
return (data ?? []) as MonitorRow[];
}
+
+export async function deferMonitorClaim(
+ monitorId: string,
+ until: Date,
+): Promise<void> {
+ const { error } = await supabase_service
+ .from("monitors")
+ .update({
+ locked_until: until.toISOString(),
+ locked_at: null,
+ updated_at: new Date().toISOString(),
+ })
+ .eq("id", monitorId);
+
+ throwIfError(error, "Failed to defer monitor claim");
+}