Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Spread monitor jitter wider and parallelize the search consumer

Developers Digest3045be830da3
diff --git a/apps/api/src/services/monitoring/jitter.ts b/apps/api/src/services/monitoring/jitter.ts
index 782937613..6bc36722f 100644
--- a/apps/api/src/services/monitoring/jitter.ts
+++ b/apps/api/src/services/monitoring/jitter.ts
@@ -1,6 +1,10 @@
import { createHash } from "crypto";
-const MAX_JITTER_MS = 5 * 60 * 1000;
+// Cap the spread at 30 min: enough that hourly monitors fan out across :00-:30
+// (instead of piling at :00 and starving the consumer) while a daily/weekly cron
+// still fires within tens of minutes of its intended time. Sub-hourly intervals
+// stay bounded by intervalMs/2 below.
+const MAX_JITTER_MS = 30 * 60 * 1000;
export function monitorJitterOffsetMs(
monitorId: string,
diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.ts
index 265e0e760..f14f195f5 100644
--- a/apps/api/src/services/monitoring/queue.ts
+++ b/apps/api/src/services/monitoring/queue.ts
@@ -35,8 +35,16 @@ let consumeChannelPromise: Promise<amqp.Channel> | null = null;
// Registry of consumers so a reconnect can re-attach them; the worker never produces,
// so without this it goes permanently deaf after any RabbitMQ blip until restart.
-const registeredConsumers: Array<{ queue: string; handler: ConsumerHandler }> =
- [];
+const registeredConsumers: Array<{
+ queue: string;
+ handler: ConsumerHandler;
+ prefetch: number;
+}> = [];
+// Search checks run inline (search + judge + scrapes) for 15-50s each, so a single
+// prefetch serializes them and a burst piles up behind one check. Let a worker run a
+// few concurrently; each still fans out SEARCH_SCRAPE_CONCURRENCY scrapes, so keep it
+// modest. Default (page/site/batch) checks just enqueue async jobs, so 1 is fine.
+const SEARCH_CHECK_PREFETCH = 3;
// Queues with a live consumer on the CURRENT consume channel; cleared on drop so a
// reconnect re-subscribes exactly once per queue and can't pile up duplicates.
const subscribedQueues = new Set<string>();
@@ -109,8 +117,8 @@ function scheduleReconnect(delayMs: number): void {
setTimeout(async () => {
try {
const ch = await getConsumeChannel();
- for (const { queue, handler } of registeredConsumers) {
- await subscribe(ch, queue, handler);
+ for (const { queue, handler, prefetch } of registeredConsumers) {
+ await subscribe(ch, queue, handler, prefetch);
}
reconnectInFlight = false;
logger.info("Monitor queue reconnected; consumers re-subscribed", {
@@ -258,23 +266,26 @@ export async function addMonitorCheckJob(
async function consumeQueue(
queue: string,
handler: ConsumerHandler,
+ prefetch: number,
): Promise<void> {
if (!registeredConsumers.some(c => c.queue === queue)) {
- registeredConsumers.push({ queue, handler });
+ registeredConsumers.push({ queue, handler, prefetch });
}
const ch = await getConsumeChannel();
- await subscribe(ch, queue, handler);
+ await subscribe(ch, queue, handler, prefetch);
}
async function subscribe(
ch: amqp.Channel,
queue: string,
handler: ConsumerHandler,
+ prefetch: number,
): Promise<void> {
// Don't add a duplicate consumer if a reconnect re-subscribes an already-subscribed queue.
if (subscribedQueues.has(queue)) return;
- // Per-consumer prefetch (amqplib default) so search and default consumers don't block each other.
- await ch.prefetch(1);
+ // Per-consumer prefetch (global=false) so the search and default consumers each get
+ // their own in-flight budget and don't block each other.
+ await ch.prefetch(prefetch, false);
await ch.consume(
queue,
@@ -314,11 +325,15 @@ async function subscribe(
export async function consumeMonitorCheckJobs(
handler: (data: MonitorCheckJobData) => Promise<void>,
): Promise<void> {
- await consumeQueue(MONITOR_CHECK_QUEUE, handler);
+ await consumeQueue(MONITOR_CHECK_QUEUE, handler, 1);
}
export async function consumeMonitorSearchCheckJobs(
handler: (data: MonitorCheckJobData) => Promise<void>,
): Promise<void> {
- await consumeQueue(MONITOR_SEARCH_CHECK_QUEUE, handler);
+ await consumeQueue(
+ MONITOR_SEARCH_CHECK_QUEUE,
+ handler,
+ SEARCH_CHECK_PREFETCH,
+ );
}