Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

monitor: isolate search checks on a dedicated queue + consumer

Developers Digest89851534e95a
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.ts
index aaec5b0cd..bd141f9fe 100644
--- a/apps/api/src/controllers/v2/monitor.ts
+++ b/apps/api/src/controllers/v2/monitor.ts
@@ -25,7 +25,10 @@ import {
listMonitors,
updateMonitor,
} from "../../services/monitoring/store";
-import { enqueueMonitorCheck } from "../../services/monitoring/scheduler";
+import {
+ enqueueMonitorCheck,
+ monitorIsSearch,
+} from "../../services/monitoring/scheduler";
import {
estimateRunsPerMonth,
validateMonitorCron,
@@ -456,6 +459,7 @@ export async function runMonitorController(
monitorId: monitor.id,
checkId: check.id,
teamId: monitor.team_id,
+ search: monitorIsSearch(monitor),
});
res.status(200).json({
diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.ts
index 77cc8675f..5675bf125 100644
--- a/apps/api/src/services/monitoring/queue.ts
+++ b/apps/api/src/services/monitoring/queue.ts
@@ -6,6 +6,15 @@ const MONITOR_CHECK_QUEUE = "monitor.checks";
const MONITOR_CHECK_DLX = "monitor.checks.dlx";
const MONITOR_CHECK_DLQ = "monitor.checks.dlq";
+// Search-monitor checks are an order of magnitude heavier than scrape/crawl checks
+// (each one scrapes many result pages and runs a multi-stage LLM pipeline). They get
+// a DEDICATED queue + consumer so a burst of search checks drains on its own slot
+// and can't starve the consumer that processes everyone else's page/site/batch
+// checks. prefetch is applied per-consumer, so the two consumers run independently.
+const MONITOR_SEARCH_CHECK_QUEUE = "monitor.checks.search";
+const MONITOR_SEARCH_CHECK_DLX = "monitor.checks.search.dlx";
+const MONITOR_SEARCH_CHECK_DLQ = "monitor.checks.search.dlq";
+
const logger = _logger.child({ module: "monitoring-queue" });
export type MonitorCheckJobData = {
@@ -17,6 +26,33 @@ export type MonitorCheckJobData = {
let connection: amqp.ChannelModel | null = null;
let channel: amqp.Channel | null = null;
+// Declare a check queue with its dead-letter exchange/queue. Identical topology for
+// the default and search queues — only the names differ.
+async function assertCheckQueue(
+ ch: amqp.Channel,
+ queue: string,
+ dlx: string,
+ dlq: string,
+): Promise<void> {
+ await ch.assertExchange(dlx, "direct", { durable: true });
+ await ch.assertQueue(dlq, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ },
+ });
+ await ch.bindQueue(dlq, dlx, queue);
+ await ch.assertQueue(queue, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-dead-letter-exchange": dlx,
+ "x-dead-letter-routing-key": queue,
+ "x-delivery-limit": 1,
+ },
+ });
+}
+
async function getChannel(): Promise<amqp.Channel> {
if (channel) return channel;
@@ -28,28 +64,18 @@ async function getChannel(): Promise<amqp.Channel> {
connection = await amqp.connect(url);
channel = await connection.createChannel();
- await channel.assertExchange(MONITOR_CHECK_DLX, "direct", { durable: true });
- await channel.assertQueue(MONITOR_CHECK_DLQ, {
- durable: true,
- arguments: {
- "x-queue-type": "quorum",
- },
- });
- await channel.bindQueue(
- MONITOR_CHECK_DLQ,
- MONITOR_CHECK_DLX,
+ await assertCheckQueue(
+ channel,
MONITOR_CHECK_QUEUE,
+ MONITOR_CHECK_DLX,
+ MONITOR_CHECK_DLQ,
+ );
+ await assertCheckQueue(
+ channel,
+ MONITOR_SEARCH_CHECK_QUEUE,
+ MONITOR_SEARCH_CHECK_DLX,
+ MONITOR_SEARCH_CHECK_DLQ,
);
-
- await channel.assertQueue(MONITOR_CHECK_QUEUE, {
- durable: true,
- arguments: {
- "x-queue-type": "quorum",
- "x-dead-letter-exchange": MONITOR_CHECK_DLX,
- "x-dead-letter-routing-key": MONITOR_CHECK_QUEUE,
- "x-delivery-limit": 1,
- },
- });
connection.on("close", () => {
logger.warn("Monitor queue connection closed");
@@ -66,22 +92,21 @@ async function getChannel(): Promise<amqp.Channel> {
export async function addMonitorCheckJob(
data: MonitorCheckJobData,
+ opts: { search?: boolean } = {},
): Promise<void> {
const ch = await getChannel();
- const sent = ch.sendToQueue(
- MONITOR_CHECK_QUEUE,
- Buffer.from(JSON.stringify(data)),
- {
- persistent: true,
- contentType: "application/json",
- messageId: data.checkId,
- },
- );
+ const queue = opts.search ? MONITOR_SEARCH_CHECK_QUEUE : MONITOR_CHECK_QUEUE;
+ const sent = ch.sendToQueue(queue, Buffer.from(JSON.stringify(data)), {
+ persistent: true,
+ contentType: "application/json",
+ messageId: data.checkId,
+ });
if (!sent) {
logger.warn("Monitor check message buffer full", {
monitorId: data.monitorId,
checkId: data.checkId,
+ queue,
});
}
@@ -89,17 +114,22 @@ export async function addMonitorCheckJob(
monitorId: data.monitorId,
checkId: data.checkId,
teamId: data.teamId,
+ queue,
});
}
-export async function consumeMonitorCheckJobs(
+async function consumeQueue(
+ queue: string,
handler: (data: MonitorCheckJobData) => Promise<void>,
): Promise<void> {
const ch = await getChannel();
+ // Per-consumer prefetch (amqplib default global=false): each consumer on the
+ // channel gets its own in-flight slot, so the search and default consumers don't
+ // block one another.
await ch.prefetch(1);
await ch.consume(
- MONITOR_CHECK_QUEUE,
+ queue,
async msg => {
if (!msg) return;
@@ -129,5 +159,17 @@ export async function consumeMonitorCheckJobs(
{ noAck: false },
);
- logger.info("Started consuming monitor check jobs");
+ logger.info("Started consuming monitor check jobs", { queue });
+}
+
+export async function consumeMonitorCheckJobs(
+ handler: (data: MonitorCheckJobData) => Promise<void>,
+): Promise<void> {
+ await consumeQueue(MONITOR_CHECK_QUEUE, handler);
+}
+
+export async function consumeMonitorSearchCheckJobs(
+ handler: (data: MonitorCheckJobData) => Promise<void>,
+): Promise<void> {
+ await consumeQueue(MONITOR_SEARCH_CHECK_QUEUE, handler);
}
diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.ts
index 360f910b9..00db2edb4 100644
--- a/apps/api/src/services/monitoring/scheduler.ts
+++ b/apps/api/src/services/monitoring/scheduler.ts
@@ -19,12 +19,26 @@ import type { MonitorRow } from "./types";
const logger = _logger.child({ module: "monitoring-scheduler" });
+// Search monitors ride a dedicated check queue (see queue.ts) so their heavy checks
+// can't starve everyone else's. A monitor is "search" if any of its targets is.
+export function monitorIsSearch(monitor: MonitorRow): boolean {
+ return monitor.targets.some(target => target.type === "search");
+}
+
export async function enqueueMonitorCheck(params: {
monitorId: string;
checkId: string;
teamId: string;
+ search?: boolean;
}): Promise<void> {
- await addMonitorCheckJob(params);
+ await addMonitorCheckJob(
+ {
+ monitorId: params.monitorId,
+ checkId: params.checkId,
+ teamId: params.teamId,
+ },
+ { search: params.search },
+ );
}
export async function enqueueDueMonitorChecks(
@@ -94,6 +108,7 @@ export async function enqueueDueMonitorChecks(
monitorId: monitor.id,
checkId: check.id,
teamId: monitor.team_id,
+ search: monitorIsSearch(monitor),
});
enqueued++;
} catch (error) {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 3f0250278..210193471 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -30,7 +30,10 @@ import {
reconcileRunningMonitorChecks,
} from "./monitoring/runner";
import { enqueueDueMonitorChecks } from "./monitoring/scheduler";
-import { consumeMonitorCheckJobs } from "./monitoring/queue";
+import {
+ consumeMonitorCheckJobs,
+ consumeMonitorSearchCheckJobs,
+} from "./monitoring/queue";
configDotenv();
@@ -483,7 +486,12 @@ app.listen(workerPort, (error?: Error) => {
_logger.error("Failed to reconcile running monitor checks", { error });
});
- await consumeMonitorCheckJobs(processMonitorCheckJob);
+ // Search-monitor checks drain on their own queue/consumer (see queue.ts) so a
+ // burst of heavy search checks can't starve page/site/batch monitor checks.
+ await Promise.all([
+ consumeMonitorCheckJobs(processMonitorCheckJob),
+ consumeMonitorSearchCheckJobs(processMonitorCheckJob),
+ ]);
} else if (!config.USE_DB_AUTHENTICATION) {
_logger.info(
"Skipping monitor worker startup because database authentication is disabled",