Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
jitter scheduler claim to avoid thundering herd (#3619)
tomsideguide1d7e7a288d60
diff --git a/apps/api/src/services/monitoring/jitter.ts b/apps/api/src/services/monitoring/jitter.tsnew file mode 100644index 000000000..782937613--- /dev/null+++ b/apps/api/src/services/monitoring/jitter.ts@@ -0,0 +1,13 @@+import { createHash } from "crypto";++const MAX_JITTER_MS = 5 * 60 * 1000;++export function monitorJitterOffsetMs(+ monitorId: string,+ intervalMs: number,+): number {+ const jitterMaxMs = Math.min(Math.floor(intervalMs / 2), MAX_JITTER_MS);+ if (jitterMaxMs <= 0) return 0;+ const hash = createHash("sha256").update(monitorId).digest();+ return hash.readUInt32BE(0) % jitterMaxMs;+}diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.tsindex 61492e3b6..360f910b9 100644--- a/apps/api/src/services/monitoring/scheduler.ts+++ b/apps/api/src/services/monitoring/scheduler.ts@@ -5,6 +5,7 @@ import {advanceMonitorAfterSkippedCheck,claimDueMonitors,createMonitorCheck,+ deferMonitorClaim,dispatchScheduledMonitorCheck,getMonitorCheck,updateMonitorCheck,@@ -12,6 +13,8 @@ import {} from "./store";import { autumnService } from "../autumn/autumn.service";import { isMonitorCheckStale, MONITOR_CHECK_STALE_ERROR } from "./stale";+import { validateMonitorCron } from "./cron";+import { monitorJitterOffsetMs } from "./jitter";import type { MonitorRow } from "./types";const logger = _logger.child({ module: "monitoring-scheduler" });@@ -43,6 +46,8 @@ export async function enqueueDueMonitorChecks(let check: Awaited<ReturnType<typeof createMonitorCheck>> | null = null;let dispatched = false;try {+ if (await deferForJitter(monitor)) continue;+if (monitor.current_check_id) {const cleared = await clearFinishedOrStaleCurrentCheck(monitor);if (cleared) {@@ -135,6 +140,28 @@ export async function enqueueDueMonitorChecks(return enqueued;}+async function deferForJitter(monitor: MonitorRow): Promise<boolean> {+ if (!monitor.next_run_at) return false;+ const { intervalMs } = validateMonitorCron(+ monitor.schedule_cron,+ monitor.schedule_timezone,+ );+ const dueAt =+ new Date(monitor.next_run_at).getTime() ++ monitorJitterOffsetMs(monitor.id, intervalMs);+ if (Date.now() >= dueAt) return false;+ try {+ await deferMonitorClaim(monitor.id, new Date(dueAt));+ } catch (error) {+ logger.warn("Failed to defer monitor claim for jitter", {+ error,+ monitorId: monitor.id,+ });+ return false;+ }+ return true;+}+async function clearFinishedOrStaleCurrentCheck(monitor: MonitorRow,): Promise<boolean> {diff --git a/apps/api/src/services/monitoring/store.ts b/apps/api/src/services/monitoring/store.tsindex 70bb2ee3f..2cec0f79b 100644--- a/apps/api/src/services/monitoring/store.ts+++ b/apps/api/src/services/monitoring/store.ts@@ -698,3 +698,19 @@ export async function claimDueMonitors(params: {throwIfError(error, "Failed to claim due monitors");return (data ?? []) as MonitorRow[];}++export async function deferMonitorClaim(+ monitorId: string,+ until: Date,+): Promise<void> {+ const { error } = await supabase_service+ .from("monitors")+ .update({+ locked_until: until.toISOString(),+ locked_at: null,+ updated_at: new Date().toISOString(),+ })+ .eq("id", monitorId);++ throwIfError(error, "Failed to defer monitor claim");+}