Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Drop dedicated search queue; route search through monitor.checks"
Developers Digest10f936a55d09
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.tsindex aaec5b0cd..bd141f9fe 100644--- a/apps/api/src/controllers/v2/monitor.ts+++ b/apps/api/src/controllers/v2/monitor.ts@@ -25,7 +25,10 @@ import {listMonitors,updateMonitor,} from "../../services/monitoring/store";-import { enqueueMonitorCheck } from "../../services/monitoring/scheduler";+import {+ enqueueMonitorCheck,+ monitorIsSearch,+} from "../../services/monitoring/scheduler";import {estimateRunsPerMonth,validateMonitorCron,@@ -456,6 +459,7 @@ export async function runMonitorController(monitorId: monitor.id,checkId: check.id,teamId: monitor.team_id,+ search: monitorIsSearch(monitor),});res.status(200).json({diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.tsindex 77cc8675f..6347de42f 100644--- a/apps/api/src/services/monitoring/queue.ts+++ b/apps/api/src/services/monitoring/queue.ts@@ -6,6 +6,12 @@ const MONITOR_CHECK_QUEUE = "monitor.checks";const MONITOR_CHECK_DLX = "monitor.checks.dlx";const MONITOR_CHECK_DLQ = "monitor.checks.dlq";+// Search checks are far heavier than scrape/crawl checks, so they get a dedicated+// queue + consumer — a burst of them can't starve the page/site/batch checks.+const MONITOR_SEARCH_CHECK_QUEUE = "monitor.checks.search";+const MONITOR_SEARCH_CHECK_DLX = "monitor.checks.search.dlx";+const MONITOR_SEARCH_CHECK_DLQ = "monitor.checks.search.dlq";+const logger = _logger.child({ module: "monitoring-queue" });export type MonitorCheckJobData = {@@ -17,6 +23,32 @@ export type MonitorCheckJobData = {let connection: amqp.ChannelModel | null = null;let channel: amqp.Channel | null = null;+// Declare a check queue + its dead-letter exchange/queue (same topology for both).+async function assertCheckQueue(+ ch: amqp.Channel,+ queue: string,+ dlx: string,+ dlq: string,+): Promise<void> {+ await ch.assertExchange(dlx, "direct", { durable: true });+ await ch.assertQueue(dlq, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ },+ });+ await ch.bindQueue(dlq, dlx, queue);+ await ch.assertQueue(queue, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-dead-letter-exchange": dlx,+ "x-dead-letter-routing-key": queue,+ "x-delivery-limit": 1,+ },+ });+}+async function getChannel(): Promise<amqp.Channel> {if (channel) return channel;@@ -28,28 +60,18 @@ async function getChannel(): Promise<amqp.Channel> {connection = await amqp.connect(url);channel = await connection.createChannel();- await channel.assertExchange(MONITOR_CHECK_DLX, "direct", { durable: true });- await channel.assertQueue(MONITOR_CHECK_DLQ, {- durable: true,- arguments: {- "x-queue-type": "quorum",- },- });- await channel.bindQueue(- MONITOR_CHECK_DLQ,- MONITOR_CHECK_DLX,+ await assertCheckQueue(+ channel,MONITOR_CHECK_QUEUE,+ MONITOR_CHECK_DLX,+ MONITOR_CHECK_DLQ,+ );+ await assertCheckQueue(+ channel,+ MONITOR_SEARCH_CHECK_QUEUE,+ MONITOR_SEARCH_CHECK_DLX,+ MONITOR_SEARCH_CHECK_DLQ,);-- await channel.assertQueue(MONITOR_CHECK_QUEUE, {- durable: true,- arguments: {- "x-queue-type": "quorum",- "x-dead-letter-exchange": MONITOR_CHECK_DLX,- "x-dead-letter-routing-key": MONITOR_CHECK_QUEUE,- "x-delivery-limit": 1,- },- });connection.on("close", () => {logger.warn("Monitor queue connection closed");@@ -66,22 +88,21 @@ async function getChannel(): Promise<amqp.Channel> {export async function addMonitorCheckJob(data: MonitorCheckJobData,+ opts: { search?: boolean } = {},): Promise<void> {const ch = await getChannel();- const sent = ch.sendToQueue(- MONITOR_CHECK_QUEUE,- Buffer.from(JSON.stringify(data)),- {- persistent: true,- contentType: "application/json",- messageId: data.checkId,- },- );+ const queue = opts.search ? MONITOR_SEARCH_CHECK_QUEUE : MONITOR_CHECK_QUEUE;+ const sent = ch.sendToQueue(queue, Buffer.from(JSON.stringify(data)), {+ persistent: true,+ contentType: "application/json",+ messageId: data.checkId,+ });if (!sent) {logger.warn("Monitor check message buffer full", {monitorId: data.monitorId,checkId: data.checkId,+ queue,});}@@ -89,17 +110,21 @@ export async function addMonitorCheckJob(monitorId: data.monitorId,checkId: data.checkId,teamId: data.teamId,+ queue,});}-export async function consumeMonitorCheckJobs(+async function consumeQueue(+ queue: string,handler: (data: MonitorCheckJobData) => Promise<void>,): Promise<void> {const ch = await getChannel();+ // Per-consumer prefetch (amqplib default): each consumer gets its own in-flight+ // slot, so the search and default consumers don't block each other.await ch.prefetch(1);await ch.consume(- MONITOR_CHECK_QUEUE,+ queue,async msg => {if (!msg) return;@@ -129,5 +154,17 @@ export async function consumeMonitorCheckJobs({ noAck: false },);- logger.info("Started consuming monitor check jobs");+ logger.info("Started consuming monitor check jobs", { queue });+}++export async function consumeMonitorCheckJobs(+ handler: (data: MonitorCheckJobData) => Promise<void>,+): Promise<void> {+ await consumeQueue(MONITOR_CHECK_QUEUE, handler);+}++export async function consumeMonitorSearchCheckJobs(+ handler: (data: MonitorCheckJobData) => Promise<void>,+): Promise<void> {+ await consumeQueue(MONITOR_SEARCH_CHECK_QUEUE, handler);}diff --git a/apps/api/src/services/monitoring/scheduler.test.ts b/apps/api/src/services/monitoring/scheduler.test.tsindex a64e29789..25e29e7c9 100644--- a/apps/api/src/services/monitoring/scheduler.test.ts+++ b/apps/api/src/services/monitoring/scheduler.test.ts@@ -113,16 +113,38 @@ describe("monitoring scheduler", () => {monitor,checkId: check.id,});- expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({- monitorId: monitor.id,- checkId: check.id,- teamId: monitor.team_id,- });+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(+ {+ monitorId: monitor.id,+ checkId: check.id,+ teamId: monitor.team_id,+ },+ { search: false },+ );expect(mockDispatchScheduledMonitorCheck.mock.invocationCallOrder[0],).toBeLessThan(mockAddMonitorCheckJob.mock.invocationCallOrder[0]);});+ it("routes a search monitor to the dedicated search queue", async () => {+ mockClaimDueMonitors.mockResolvedValue([+ { ...monitor, targets: [{ id: "t-1", type: "search" }] } as any,+ ]);++ await expect(+ enqueueDueMonitorChecks({ workerId: "worker-1" }),+ ).resolves.toBe(1);++ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(+ {+ monitorId: monitor.id,+ checkId: check.id,+ teamId: monitor.team_id,+ },+ { search: true },+ );+ });+it("fails and clears a dispatched check when enqueueing fails", async () => {const error = new Error("queue unavailable");const failed = { id: check.id, status: "failed" } as any;@@ -197,10 +219,13 @@ describe("monitoring scheduler", () => {trigger: "scheduled",scheduledFor: monitorWithCurrentCheck.next_run_at,});- expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({- monitorId: monitorWithCurrentCheck.id,- checkId: check.id,- teamId: monitorWithCurrentCheck.team_id,- });+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(+ {+ monitorId: monitorWithCurrentCheck.id,+ checkId: check.id,+ teamId: monitorWithCurrentCheck.team_id,+ },+ { search: false },+ );});});diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.tsindex 31d1a0278..2d69bcacc 100644--- a/apps/api/src/services/monitoring/scheduler.ts+++ b/apps/api/src/services/monitoring/scheduler.ts@@ -19,16 +19,25 @@ import type { MonitorRow } from "./types";const logger = _logger.child({ module: "monitoring-scheduler" });+// Search monitors route to a dedicated check queue (see queue.ts).+export function monitorIsSearch(monitor: MonitorRow): boolean {+ return (monitor.targets ?? []).some(target => target.type === "search");+}+export async function enqueueMonitorCheck(params: {monitorId: string;checkId: string;teamId: string;+ search?: boolean;}): Promise<void> {- await addMonitorCheckJob({- monitorId: params.monitorId,- checkId: params.checkId,- teamId: params.teamId,- });+ await addMonitorCheckJob(+ {+ monitorId: params.monitorId,+ checkId: params.checkId,+ teamId: params.teamId,+ },+ { search: params.search },+ );}export async function enqueueDueMonitorChecks(@@ -98,6 +107,7 @@ export async function enqueueDueMonitorChecks(monitorId: monitor.id,checkId: check.id,teamId: monitor.team_id,+ search: monitorIsSearch(monitor),});enqueued++;} catch (error) {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 3f0250278..f85753f84 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -30,7 +30,10 @@ import {reconcileRunningMonitorChecks,} from "./monitoring/runner";import { enqueueDueMonitorChecks } from "./monitoring/scheduler";-import { consumeMonitorCheckJobs } from "./monitoring/queue";+import {+ consumeMonitorCheckJobs,+ consumeMonitorSearchCheckJobs,+} from "./monitoring/queue";configDotenv();@@ -483,7 +486,11 @@ app.listen(workerPort, (error?: Error) => {_logger.error("Failed to reconcile running monitor checks", { error });});- await consumeMonitorCheckJobs(processMonitorCheckJob);+ // Search checks drain on their own consumer so they can't starve the rest.+ await Promise.all([+ consumeMonitorCheckJobs(processMonitorCheckJob),+ consumeMonitorSearchCheckJobs(processMonitorCheckJob),+ ]);} else if (!config.USE_DB_AUTHENTICATION) {_logger.info("Skipping monitor worker startup because database authentication is disabled",