Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "Drop dedicated search queue; route search through monitor.checks"

Developers Digest10f936a55d09
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.ts
index aaec5b0cd..bd141f9fe 100644
--- a/apps/api/src/controllers/v2/monitor.ts
+++ b/apps/api/src/controllers/v2/monitor.ts
@@ -25,7 +25,10 @@ import {
listMonitors,
updateMonitor,
} from "../../services/monitoring/store";
-import { enqueueMonitorCheck } from "../../services/monitoring/scheduler";
+import {
+ enqueueMonitorCheck,
+ monitorIsSearch,
+} from "../../services/monitoring/scheduler";
import {
estimateRunsPerMonth,
validateMonitorCron,
@@ -456,6 +459,7 @@ export async function runMonitorController(
monitorId: monitor.id,
checkId: check.id,
teamId: monitor.team_id,
+ search: monitorIsSearch(monitor),
});
res.status(200).json({
diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.ts
index 77cc8675f..6347de42f 100644
--- a/apps/api/src/services/monitoring/queue.ts
+++ b/apps/api/src/services/monitoring/queue.ts
@@ -6,6 +6,12 @@ const MONITOR_CHECK_QUEUE = "monitor.checks";
const MONITOR_CHECK_DLX = "monitor.checks.dlx";
const MONITOR_CHECK_DLQ = "monitor.checks.dlq";
+// Search checks are far heavier than scrape/crawl checks, so they get a dedicated
+// queue + consumer — a burst of them can't starve the page/site/batch checks.
+const MONITOR_SEARCH_CHECK_QUEUE = "monitor.checks.search";
+const MONITOR_SEARCH_CHECK_DLX = "monitor.checks.search.dlx";
+const MONITOR_SEARCH_CHECK_DLQ = "monitor.checks.search.dlq";
+
const logger = _logger.child({ module: "monitoring-queue" });
export type MonitorCheckJobData = {
@@ -17,6 +23,32 @@ export type MonitorCheckJobData = {
let connection: amqp.ChannelModel | null = null;
let channel: amqp.Channel | null = null;
+// Declare a check queue + its dead-letter exchange/queue (same topology for both).
+async function assertCheckQueue(
+ ch: amqp.Channel,
+ queue: string,
+ dlx: string,
+ dlq: string,
+): Promise<void> {
+ await ch.assertExchange(dlx, "direct", { durable: true });
+ await ch.assertQueue(dlq, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ },
+ });
+ await ch.bindQueue(dlq, dlx, queue);
+ await ch.assertQueue(queue, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-dead-letter-exchange": dlx,
+ "x-dead-letter-routing-key": queue,
+ "x-delivery-limit": 1,
+ },
+ });
+}
+
async function getChannel(): Promise<amqp.Channel> {
if (channel) return channel;
@@ -28,28 +60,18 @@ async function getChannel(): Promise<amqp.Channel> {
connection = await amqp.connect(url);
channel = await connection.createChannel();
- await channel.assertExchange(MONITOR_CHECK_DLX, "direct", { durable: true });
- await channel.assertQueue(MONITOR_CHECK_DLQ, {
- durable: true,
- arguments: {
- "x-queue-type": "quorum",
- },
- });
- await channel.bindQueue(
- MONITOR_CHECK_DLQ,
- MONITOR_CHECK_DLX,
+ await assertCheckQueue(
+ channel,
MONITOR_CHECK_QUEUE,
+ MONITOR_CHECK_DLX,
+ MONITOR_CHECK_DLQ,
+ );
+ await assertCheckQueue(
+ channel,
+ MONITOR_SEARCH_CHECK_QUEUE,
+ MONITOR_SEARCH_CHECK_DLX,
+ MONITOR_SEARCH_CHECK_DLQ,
);
-
- await channel.assertQueue(MONITOR_CHECK_QUEUE, {
- durable: true,
- arguments: {
- "x-queue-type": "quorum",
- "x-dead-letter-exchange": MONITOR_CHECK_DLX,
- "x-dead-letter-routing-key": MONITOR_CHECK_QUEUE,
- "x-delivery-limit": 1,
- },
- });
connection.on("close", () => {
logger.warn("Monitor queue connection closed");
@@ -66,22 +88,21 @@ async function getChannel(): Promise<amqp.Channel> {
export async function addMonitorCheckJob(
data: MonitorCheckJobData,
+ opts: { search?: boolean } = {},
): Promise<void> {
const ch = await getChannel();
- const sent = ch.sendToQueue(
- MONITOR_CHECK_QUEUE,
- Buffer.from(JSON.stringify(data)),
- {
- persistent: true,
- contentType: "application/json",
- messageId: data.checkId,
- },
- );
+ const queue = opts.search ? MONITOR_SEARCH_CHECK_QUEUE : MONITOR_CHECK_QUEUE;
+ const sent = ch.sendToQueue(queue, Buffer.from(JSON.stringify(data)), {
+ persistent: true,
+ contentType: "application/json",
+ messageId: data.checkId,
+ });
if (!sent) {
logger.warn("Monitor check message buffer full", {
monitorId: data.monitorId,
checkId: data.checkId,
+ queue,
});
}
@@ -89,17 +110,21 @@ export async function addMonitorCheckJob(
monitorId: data.monitorId,
checkId: data.checkId,
teamId: data.teamId,
+ queue,
});
}
-export async function consumeMonitorCheckJobs(
+async function consumeQueue(
+ queue: string,
handler: (data: MonitorCheckJobData) => Promise<void>,
): Promise<void> {
const ch = await getChannel();
+ // Per-consumer prefetch (amqplib default): each consumer gets its own in-flight
+ // slot, so the search and default consumers don't block each other.
await ch.prefetch(1);
await ch.consume(
- MONITOR_CHECK_QUEUE,
+ queue,
async msg => {
if (!msg) return;
@@ -129,5 +154,17 @@ export async function consumeMonitorCheckJobs(
{ noAck: false },
);
- logger.info("Started consuming monitor check jobs");
+ logger.info("Started consuming monitor check jobs", { queue });
+}
+
+export async function consumeMonitorCheckJobs(
+ handler: (data: MonitorCheckJobData) => Promise<void>,
+): Promise<void> {
+ await consumeQueue(MONITOR_CHECK_QUEUE, handler);
+}
+
+export async function consumeMonitorSearchCheckJobs(
+ handler: (data: MonitorCheckJobData) => Promise<void>,
+): Promise<void> {
+ await consumeQueue(MONITOR_SEARCH_CHECK_QUEUE, handler);
}
diff --git a/apps/api/src/services/monitoring/scheduler.test.ts b/apps/api/src/services/monitoring/scheduler.test.ts
index a64e29789..25e29e7c9 100644
--- a/apps/api/src/services/monitoring/scheduler.test.ts
+++ b/apps/api/src/services/monitoring/scheduler.test.ts
@@ -113,16 +113,38 @@ describe("monitoring scheduler", () => {
monitor,
checkId: check.id,
});
- expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({
- monitorId: monitor.id,
- checkId: check.id,
- teamId: monitor.team_id,
- });
+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(
+ {
+ monitorId: monitor.id,
+ checkId: check.id,
+ teamId: monitor.team_id,
+ },
+ { search: false },
+ );
expect(
mockDispatchScheduledMonitorCheck.mock.invocationCallOrder[0],
).toBeLessThan(mockAddMonitorCheckJob.mock.invocationCallOrder[0]);
});
+ it("routes a search monitor to the dedicated search queue", async () => {
+ mockClaimDueMonitors.mockResolvedValue([
+ { ...monitor, targets: [{ id: "t-1", type: "search" }] } as any,
+ ]);
+
+ await expect(
+ enqueueDueMonitorChecks({ workerId: "worker-1" }),
+ ).resolves.toBe(1);
+
+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(
+ {
+ monitorId: monitor.id,
+ checkId: check.id,
+ teamId: monitor.team_id,
+ },
+ { search: true },
+ );
+ });
+
it("fails and clears a dispatched check when enqueueing fails", async () => {
const error = new Error("queue unavailable");
const failed = { id: check.id, status: "failed" } as any;
@@ -197,10 +219,13 @@ describe("monitoring scheduler", () => {
trigger: "scheduled",
scheduledFor: monitorWithCurrentCheck.next_run_at,
});
- expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({
- monitorId: monitorWithCurrentCheck.id,
- checkId: check.id,
- teamId: monitorWithCurrentCheck.team_id,
- });
+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(
+ {
+ monitorId: monitorWithCurrentCheck.id,
+ checkId: check.id,
+ teamId: monitorWithCurrentCheck.team_id,
+ },
+ { search: false },
+ );
});
});
diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.ts
index 31d1a0278..2d69bcacc 100644
--- a/apps/api/src/services/monitoring/scheduler.ts
+++ b/apps/api/src/services/monitoring/scheduler.ts
@@ -19,16 +19,25 @@ import type { MonitorRow } from "./types";
const logger = _logger.child({ module: "monitoring-scheduler" });
+// Search monitors route to a dedicated check queue (see queue.ts).
+export function monitorIsSearch(monitor: MonitorRow): boolean {
+ return (monitor.targets ?? []).some(target => target.type === "search");
+}
+
export async function enqueueMonitorCheck(params: {
monitorId: string;
checkId: string;
teamId: string;
+ search?: boolean;
}): Promise<void> {
- await addMonitorCheckJob({
- monitorId: params.monitorId,
- checkId: params.checkId,
- teamId: params.teamId,
- });
+ await addMonitorCheckJob(
+ {
+ monitorId: params.monitorId,
+ checkId: params.checkId,
+ teamId: params.teamId,
+ },
+ { search: params.search },
+ );
}
export async function enqueueDueMonitorChecks(
@@ -98,6 +107,7 @@ export async function enqueueDueMonitorChecks(
monitorId: monitor.id,
checkId: check.id,
teamId: monitor.team_id,
+ search: monitorIsSearch(monitor),
});
enqueued++;
} catch (error) {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 3f0250278..f85753f84 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -30,7 +30,10 @@ import {
reconcileRunningMonitorChecks,
} from "./monitoring/runner";
import { enqueueDueMonitorChecks } from "./monitoring/scheduler";
-import { consumeMonitorCheckJobs } from "./monitoring/queue";
+import {
+ consumeMonitorCheckJobs,
+ consumeMonitorSearchCheckJobs,
+} from "./monitoring/queue";
configDotenv();
@@ -483,7 +486,11 @@ app.listen(workerPort, (error?: Error) => {
_logger.error("Failed to reconcile running monitor checks", { error });
});
- await consumeMonitorCheckJobs(processMonitorCheckJob);
+ // Search checks drain on their own consumer so they can't starve the rest.
+ await Promise.all([
+ consumeMonitorCheckJobs(processMonitorCheckJob),
+ consumeMonitorSearchCheckJobs(processMonitorCheckJob),
+ ]);
} else if (!config.USE_DB_AUTHENTICATION) {
_logger.info(
"Skipping monitor worker startup because database authentication is disabled",