Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Drop dedicated search queue; route search through monitor.checks

Developers Digestca38248a1848
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.ts
index bd141f9fe..aaec5b0cd 100644
--- a/apps/api/src/controllers/v2/monitor.ts
+++ b/apps/api/src/controllers/v2/monitor.ts
@@ -25,10 +25,7 @@ import {
listMonitors,
updateMonitor,
} from "../../services/monitoring/store";
-import {
- enqueueMonitorCheck,
- monitorIsSearch,
-} from "../../services/monitoring/scheduler";
+import { enqueueMonitorCheck } from "../../services/monitoring/scheduler";
import {
estimateRunsPerMonth,
validateMonitorCron,
@@ -459,7 +456,6 @@ export async function runMonitorController(
monitorId: monitor.id,
checkId: check.id,
teamId: monitor.team_id,
- search: monitorIsSearch(monitor),
});
res.status(200).json({
diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.ts
index 6347de42f..77cc8675f 100644
--- a/apps/api/src/services/monitoring/queue.ts
+++ b/apps/api/src/services/monitoring/queue.ts
@@ -6,12 +6,6 @@ const MONITOR_CHECK_QUEUE = "monitor.checks";
const MONITOR_CHECK_DLX = "monitor.checks.dlx";
const MONITOR_CHECK_DLQ = "monitor.checks.dlq";
-// Search checks are far heavier than scrape/crawl checks, so they get a dedicated
-// queue + consumer — a burst of them can't starve the page/site/batch checks.
-const MONITOR_SEARCH_CHECK_QUEUE = "monitor.checks.search";
-const MONITOR_SEARCH_CHECK_DLX = "monitor.checks.search.dlx";
-const MONITOR_SEARCH_CHECK_DLQ = "monitor.checks.search.dlq";
-
const logger = _logger.child({ module: "monitoring-queue" });
export type MonitorCheckJobData = {
@@ -23,32 +17,6 @@ export type MonitorCheckJobData = {
let connection: amqp.ChannelModel | null = null;
let channel: amqp.Channel | null = null;
-// Declare a check queue + its dead-letter exchange/queue (same topology for both).
-async function assertCheckQueue(
- ch: amqp.Channel,
- queue: string,
- dlx: string,
- dlq: string,
-): Promise<void> {
- await ch.assertExchange(dlx, "direct", { durable: true });
- await ch.assertQueue(dlq, {
- durable: true,
- arguments: {
- "x-queue-type": "quorum",
- },
- });
- await ch.bindQueue(dlq, dlx, queue);
- await ch.assertQueue(queue, {
- durable: true,
- arguments: {
- "x-queue-type": "quorum",
- "x-dead-letter-exchange": dlx,
- "x-dead-letter-routing-key": queue,
- "x-delivery-limit": 1,
- },
- });
-}
-
async function getChannel(): Promise<amqp.Channel> {
if (channel) return channel;
@@ -60,18 +28,28 @@ async function getChannel(): Promise<amqp.Channel> {
connection = await amqp.connect(url);
channel = await connection.createChannel();
- await assertCheckQueue(
- channel,
- MONITOR_CHECK_QUEUE,
- MONITOR_CHECK_DLX,
+ await channel.assertExchange(MONITOR_CHECK_DLX, "direct", { durable: true });
+ await channel.assertQueue(MONITOR_CHECK_DLQ, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ },
+ });
+ await channel.bindQueue(
MONITOR_CHECK_DLQ,
+ MONITOR_CHECK_DLX,
+ MONITOR_CHECK_QUEUE,
);
- await assertCheckQueue(
- channel,
- MONITOR_SEARCH_CHECK_QUEUE,
- MONITOR_SEARCH_CHECK_DLX,
- MONITOR_SEARCH_CHECK_DLQ,
- );
+
+ await channel.assertQueue(MONITOR_CHECK_QUEUE, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-dead-letter-exchange": MONITOR_CHECK_DLX,
+ "x-dead-letter-routing-key": MONITOR_CHECK_QUEUE,
+ "x-delivery-limit": 1,
+ },
+ });
connection.on("close", () => {
logger.warn("Monitor queue connection closed");
@@ -88,21 +66,22 @@ async function getChannel(): Promise<amqp.Channel> {
export async function addMonitorCheckJob(
data: MonitorCheckJobData,
- opts: { search?: boolean } = {},
): Promise<void> {
const ch = await getChannel();
- const queue = opts.search ? MONITOR_SEARCH_CHECK_QUEUE : MONITOR_CHECK_QUEUE;
- const sent = ch.sendToQueue(queue, Buffer.from(JSON.stringify(data)), {
- persistent: true,
- contentType: "application/json",
- messageId: data.checkId,
- });
+ const sent = ch.sendToQueue(
+ MONITOR_CHECK_QUEUE,
+ Buffer.from(JSON.stringify(data)),
+ {
+ persistent: true,
+ contentType: "application/json",
+ messageId: data.checkId,
+ },
+ );
if (!sent) {
logger.warn("Monitor check message buffer full", {
monitorId: data.monitorId,
checkId: data.checkId,
- queue,
});
}
@@ -110,21 +89,17 @@ export async function addMonitorCheckJob(
monitorId: data.monitorId,
checkId: data.checkId,
teamId: data.teamId,
- queue,
});
}
-async function consumeQueue(
- queue: string,
+export async function consumeMonitorCheckJobs(
handler: (data: MonitorCheckJobData) => Promise<void>,
): Promise<void> {
const ch = await getChannel();
- // Per-consumer prefetch (amqplib default): each consumer gets its own in-flight
- // slot, so the search and default consumers don't block each other.
await ch.prefetch(1);
await ch.consume(
- queue,
+ MONITOR_CHECK_QUEUE,
async msg => {
if (!msg) return;
@@ -154,17 +129,5 @@ async function consumeQueue(
{ noAck: false },
);
- logger.info("Started consuming monitor check jobs", { queue });
-}
-
-export async function consumeMonitorCheckJobs(
- handler: (data: MonitorCheckJobData) => Promise<void>,
-): Promise<void> {
- await consumeQueue(MONITOR_CHECK_QUEUE, handler);
-}
-
-export async function consumeMonitorSearchCheckJobs(
- handler: (data: MonitorCheckJobData) => Promise<void>,
-): Promise<void> {
- await consumeQueue(MONITOR_SEARCH_CHECK_QUEUE, handler);
+ logger.info("Started consuming monitor check jobs");
}
diff --git a/apps/api/src/services/monitoring/scheduler.test.ts b/apps/api/src/services/monitoring/scheduler.test.ts
index 25e29e7c9..a64e29789 100644
--- a/apps/api/src/services/monitoring/scheduler.test.ts
+++ b/apps/api/src/services/monitoring/scheduler.test.ts
@@ -113,38 +113,16 @@ describe("monitoring scheduler", () => {
monitor,
checkId: check.id,
});
- expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(
- {
- monitorId: monitor.id,
- checkId: check.id,
- teamId: monitor.team_id,
- },
- { search: false },
- );
+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({
+ monitorId: monitor.id,
+ checkId: check.id,
+ teamId: monitor.team_id,
+ });
expect(
mockDispatchScheduledMonitorCheck.mock.invocationCallOrder[0],
).toBeLessThan(mockAddMonitorCheckJob.mock.invocationCallOrder[0]);
});
- it("routes a search monitor to the dedicated search queue", async () => {
- mockClaimDueMonitors.mockResolvedValue([
- { ...monitor, targets: [{ id: "t-1", type: "search" }] } as any,
- ]);
-
- await expect(
- enqueueDueMonitorChecks({ workerId: "worker-1" }),
- ).resolves.toBe(1);
-
- expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(
- {
- monitorId: monitor.id,
- checkId: check.id,
- teamId: monitor.team_id,
- },
- { search: true },
- );
- });
-
it("fails and clears a dispatched check when enqueueing fails", async () => {
const error = new Error("queue unavailable");
const failed = { id: check.id, status: "failed" } as any;
@@ -219,13 +197,10 @@ describe("monitoring scheduler", () => {
trigger: "scheduled",
scheduledFor: monitorWithCurrentCheck.next_run_at,
});
- expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(
- {
- monitorId: monitorWithCurrentCheck.id,
- checkId: check.id,
- teamId: monitorWithCurrentCheck.team_id,
- },
- { search: false },
- );
+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({
+ monitorId: monitorWithCurrentCheck.id,
+ checkId: check.id,
+ teamId: monitorWithCurrentCheck.team_id,
+ });
});
});
diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.ts
index 2d69bcacc..31d1a0278 100644
--- a/apps/api/src/services/monitoring/scheduler.ts
+++ b/apps/api/src/services/monitoring/scheduler.ts
@@ -19,25 +19,16 @@ import type { MonitorRow } from "./types";
const logger = _logger.child({ module: "monitoring-scheduler" });
-// Search monitors route to a dedicated check queue (see queue.ts).
-export function monitorIsSearch(monitor: MonitorRow): boolean {
- return (monitor.targets ?? []).some(target => target.type === "search");
-}
-
export async function enqueueMonitorCheck(params: {
monitorId: string;
checkId: string;
teamId: string;
- search?: boolean;
}): Promise<void> {
- await addMonitorCheckJob(
- {
- monitorId: params.monitorId,
- checkId: params.checkId,
- teamId: params.teamId,
- },
- { search: params.search },
- );
+ await addMonitorCheckJob({
+ monitorId: params.monitorId,
+ checkId: params.checkId,
+ teamId: params.teamId,
+ });
}
export async function enqueueDueMonitorChecks(
@@ -107,7 +98,6 @@ export async function enqueueDueMonitorChecks(
monitorId: monitor.id,
checkId: check.id,
teamId: monitor.team_id,
- search: monitorIsSearch(monitor),
});
enqueued++;
} catch (error) {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index f85753f84..3f0250278 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -30,10 +30,7 @@ import {
reconcileRunningMonitorChecks,
} from "./monitoring/runner";
import { enqueueDueMonitorChecks } from "./monitoring/scheduler";
-import {
- consumeMonitorCheckJobs,
- consumeMonitorSearchCheckJobs,
-} from "./monitoring/queue";
+import { consumeMonitorCheckJobs } from "./monitoring/queue";
configDotenv();
@@ -486,11 +483,7 @@ app.listen(workerPort, (error?: Error) => {
_logger.error("Failed to reconcile running monitor checks", { error });
});
- // Search checks drain on their own consumer so they can't starve the rest.
- await Promise.all([
- consumeMonitorCheckJobs(processMonitorCheckJob),
- consumeMonitorSearchCheckJobs(processMonitorCheckJob),
- ]);
+ await consumeMonitorCheckJobs(processMonitorCheckJob);
} else if (!config.USE_DB_AUTHENTICATION) {
_logger.info(
"Skipping monitor worker startup because database authentication is disabled",