Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Drop dedicated search queue; route search through monitor.checks
Developers Digestca38248a1848
diff --git a/apps/api/src/controllers/v2/monitor.ts b/apps/api/src/controllers/v2/monitor.tsindex bd141f9fe..aaec5b0cd 100644--- a/apps/api/src/controllers/v2/monitor.ts+++ b/apps/api/src/controllers/v2/monitor.ts@@ -25,10 +25,7 @@ import {listMonitors,updateMonitor,} from "../../services/monitoring/store";-import {- enqueueMonitorCheck,- monitorIsSearch,-} from "../../services/monitoring/scheduler";+import { enqueueMonitorCheck } from "../../services/monitoring/scheduler";import {estimateRunsPerMonth,validateMonitorCron,@@ -459,7 +456,6 @@ export async function runMonitorController(monitorId: monitor.id,checkId: check.id,teamId: monitor.team_id,- search: monitorIsSearch(monitor),});res.status(200).json({diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.tsindex 6347de42f..77cc8675f 100644--- a/apps/api/src/services/monitoring/queue.ts+++ b/apps/api/src/services/monitoring/queue.ts@@ -6,12 +6,6 @@ const MONITOR_CHECK_QUEUE = "monitor.checks";const MONITOR_CHECK_DLX = "monitor.checks.dlx";const MONITOR_CHECK_DLQ = "monitor.checks.dlq";-// Search checks are far heavier than scrape/crawl checks, so they get a dedicated-// queue + consumer — a burst of them can't starve the page/site/batch checks.-const MONITOR_SEARCH_CHECK_QUEUE = "monitor.checks.search";-const MONITOR_SEARCH_CHECK_DLX = "monitor.checks.search.dlx";-const MONITOR_SEARCH_CHECK_DLQ = "monitor.checks.search.dlq";-const logger = _logger.child({ module: "monitoring-queue" });export type MonitorCheckJobData = {@@ -23,32 +17,6 @@ export type MonitorCheckJobData = {let connection: amqp.ChannelModel | null = null;let channel: amqp.Channel | null = null;-// Declare a check queue + its dead-letter exchange/queue (same topology for both).-async function assertCheckQueue(- ch: amqp.Channel,- queue: string,- dlx: string,- dlq: string,-): Promise<void> {- await ch.assertExchange(dlx, "direct", { durable: true });- await ch.assertQueue(dlq, {- durable: true,- arguments: {- "x-queue-type": "quorum",- },- });- await ch.bindQueue(dlq, dlx, queue);- await ch.assertQueue(queue, {- durable: true,- arguments: {- "x-queue-type": "quorum",- "x-dead-letter-exchange": dlx,- "x-dead-letter-routing-key": queue,- "x-delivery-limit": 1,- },- });-}-async function getChannel(): Promise<amqp.Channel> {if (channel) return channel;@@ -60,18 +28,28 @@ async function getChannel(): Promise<amqp.Channel> {connection = await amqp.connect(url);channel = await connection.createChannel();- await assertCheckQueue(- channel,- MONITOR_CHECK_QUEUE,- MONITOR_CHECK_DLX,+ await channel.assertExchange(MONITOR_CHECK_DLX, "direct", { durable: true });+ await channel.assertQueue(MONITOR_CHECK_DLQ, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ },+ });+ await channel.bindQueue(MONITOR_CHECK_DLQ,+ MONITOR_CHECK_DLX,+ MONITOR_CHECK_QUEUE,);- await assertCheckQueue(- channel,- MONITOR_SEARCH_CHECK_QUEUE,- MONITOR_SEARCH_CHECK_DLX,- MONITOR_SEARCH_CHECK_DLQ,- );++ await channel.assertQueue(MONITOR_CHECK_QUEUE, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-dead-letter-exchange": MONITOR_CHECK_DLX,+ "x-dead-letter-routing-key": MONITOR_CHECK_QUEUE,+ "x-delivery-limit": 1,+ },+ });connection.on("close", () => {logger.warn("Monitor queue connection closed");@@ -88,21 +66,22 @@ async function getChannel(): Promise<amqp.Channel> {export async function addMonitorCheckJob(data: MonitorCheckJobData,- opts: { search?: boolean } = {},): Promise<void> {const ch = await getChannel();- const queue = opts.search ? MONITOR_SEARCH_CHECK_QUEUE : MONITOR_CHECK_QUEUE;- const sent = ch.sendToQueue(queue, Buffer.from(JSON.stringify(data)), {- persistent: true,- contentType: "application/json",- messageId: data.checkId,- });+ const sent = ch.sendToQueue(+ MONITOR_CHECK_QUEUE,+ Buffer.from(JSON.stringify(data)),+ {+ persistent: true,+ contentType: "application/json",+ messageId: data.checkId,+ },+ );if (!sent) {logger.warn("Monitor check message buffer full", {monitorId: data.monitorId,checkId: data.checkId,- queue,});}@@ -110,21 +89,17 @@ export async function addMonitorCheckJob(monitorId: data.monitorId,checkId: data.checkId,teamId: data.teamId,- queue,});}-async function consumeQueue(- queue: string,+export async function consumeMonitorCheckJobs(handler: (data: MonitorCheckJobData) => Promise<void>,): Promise<void> {const ch = await getChannel();- // Per-consumer prefetch (amqplib default): each consumer gets its own in-flight- // slot, so the search and default consumers don't block each other.await ch.prefetch(1);await ch.consume(- queue,+ MONITOR_CHECK_QUEUE,async msg => {if (!msg) return;@@ -154,17 +129,5 @@ async function consumeQueue({ noAck: false },);- logger.info("Started consuming monitor check jobs", { queue });-}--export async function consumeMonitorCheckJobs(- handler: (data: MonitorCheckJobData) => Promise<void>,-): Promise<void> {- await consumeQueue(MONITOR_CHECK_QUEUE, handler);-}--export async function consumeMonitorSearchCheckJobs(- handler: (data: MonitorCheckJobData) => Promise<void>,-): Promise<void> {- await consumeQueue(MONITOR_SEARCH_CHECK_QUEUE, handler);+ logger.info("Started consuming monitor check jobs");}diff --git a/apps/api/src/services/monitoring/scheduler.test.ts b/apps/api/src/services/monitoring/scheduler.test.tsindex 25e29e7c9..a64e29789 100644--- a/apps/api/src/services/monitoring/scheduler.test.ts+++ b/apps/api/src/services/monitoring/scheduler.test.ts@@ -113,38 +113,16 @@ describe("monitoring scheduler", () => {monitor,checkId: check.id,});- expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(- {- monitorId: monitor.id,- checkId: check.id,- teamId: monitor.team_id,- },- { search: false },- );+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({+ monitorId: monitor.id,+ checkId: check.id,+ teamId: monitor.team_id,+ });expect(mockDispatchScheduledMonitorCheck.mock.invocationCallOrder[0],).toBeLessThan(mockAddMonitorCheckJob.mock.invocationCallOrder[0]);});- it("routes a search monitor to the dedicated search queue", async () => {- mockClaimDueMonitors.mockResolvedValue([- { ...monitor, targets: [{ id: "t-1", type: "search" }] } as any,- ]);-- await expect(- enqueueDueMonitorChecks({ workerId: "worker-1" }),- ).resolves.toBe(1);-- expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(- {- monitorId: monitor.id,- checkId: check.id,- teamId: monitor.team_id,- },- { search: true },- );- });-it("fails and clears a dispatched check when enqueueing fails", async () => {const error = new Error("queue unavailable");const failed = { id: check.id, status: "failed" } as any;@@ -219,13 +197,10 @@ describe("monitoring scheduler", () => {trigger: "scheduled",scheduledFor: monitorWithCurrentCheck.next_run_at,});- expect(mockAddMonitorCheckJob).toHaveBeenCalledWith(- {- monitorId: monitorWithCurrentCheck.id,- checkId: check.id,- teamId: monitorWithCurrentCheck.team_id,- },- { search: false },- );+ expect(mockAddMonitorCheckJob).toHaveBeenCalledWith({+ monitorId: monitorWithCurrentCheck.id,+ checkId: check.id,+ teamId: monitorWithCurrentCheck.team_id,+ });});});diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.tsindex 2d69bcacc..31d1a0278 100644--- a/apps/api/src/services/monitoring/scheduler.ts+++ b/apps/api/src/services/monitoring/scheduler.ts@@ -19,25 +19,16 @@ import type { MonitorRow } from "./types";const logger = _logger.child({ module: "monitoring-scheduler" });-// Search monitors route to a dedicated check queue (see queue.ts).-export function monitorIsSearch(monitor: MonitorRow): boolean {- return (monitor.targets ?? []).some(target => target.type === "search");-}-export async function enqueueMonitorCheck(params: {monitorId: string;checkId: string;teamId: string;- search?: boolean;}): Promise<void> {- await addMonitorCheckJob(- {- monitorId: params.monitorId,- checkId: params.checkId,- teamId: params.teamId,- },- { search: params.search },- );+ await addMonitorCheckJob({+ monitorId: params.monitorId,+ checkId: params.checkId,+ teamId: params.teamId,+ });}export async function enqueueDueMonitorChecks(@@ -107,7 +98,6 @@ export async function enqueueDueMonitorChecks(monitorId: monitor.id,checkId: check.id,teamId: monitor.team_id,- search: monitorIsSearch(monitor),});enqueued++;} catch (error) {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex f85753f84..3f0250278 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -30,10 +30,7 @@ import {reconcileRunningMonitorChecks,} from "./monitoring/runner";import { enqueueDueMonitorChecks } from "./monitoring/scheduler";-import {- consumeMonitorCheckJobs,- consumeMonitorSearchCheckJobs,-} from "./monitoring/queue";+import { consumeMonitorCheckJobs } from "./monitoring/queue";configDotenv();@@ -486,11 +483,7 @@ app.listen(workerPort, (error?: Error) => {_logger.error("Failed to reconcile running monitor checks", { error });});- // Search checks drain on their own consumer so they can't starve the rest.- await Promise.all([- consumeMonitorCheckJobs(processMonitorCheckJob),- consumeMonitorSearchCheckJobs(processMonitorCheckJob),- ]);+ await consumeMonitorCheckJobs(processMonitorCheckJob);} else if (!config.USE_DB_AUTHENTICATION) {_logger.info("Skipping monitor worker startup because database authentication is disabled",