Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add reconnect + resubscribe wrapper to monitoring queue
Developers Digestf4cc7608051e
diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.tsindex 6347de42f..68bdb1e18 100644--- a/apps/api/src/services/monitoring/queue.ts+++ b/apps/api/src/services/monitoring/queue.ts@@ -20,8 +20,24 @@ export type MonitorCheckJobData = {teamId: string;};+type ConsumerHandler = (data: MonitorCheckJobData) => Promise<void>;+let connection: amqp.ChannelModel | null = null;-let channel: amqp.Channel | null = null;+// Publish and consume use separate channels: a channel exception on one side+// (a failed publish, or a PRECONDITION_FAILED on assert) must not tear down the+// other.+let publishChannel: amqp.Channel | null = null;+let consumeChannel: amqp.Channel | null = null;++// Registry of every consumer registered this process, so we can re-attach them+// after a connection/channel drop. Without this the worker β which never+// produces and so never re-triggers channel creation β goes permanently deaf+// after any RabbitMQ blip, and only a restart heals it.+const registeredConsumers: Array<{ queue: string; handler: ConsumerHandler }> =+ [];+let reconnectInFlight = false;+const RECONNECT_BASE_DELAY_MS = 1000;+const RECONNECT_MAX_DELAY_MS = 30000;// Declare a check queue + its dead-letter exchange/queue (same topology for both).async function assertCheckQueue(@@ -49,48 +65,122 @@ async function assertCheckQueue(});}-async function getChannel(): Promise<amqp.Channel> {- if (channel) return channel;-- const url = config.NUQ_RABBITMQ_URL;- if (!url) {- throw new Error("NUQ_RABBITMQ_URL is not configured");- }-- connection = await amqp.connect(url);- channel = await connection.createChannel();-+async function assertTopology(ch: amqp.Channel): Promise<void> {await assertCheckQueue(- channel,+ ch,MONITOR_CHECK_QUEUE,MONITOR_CHECK_DLX,MONITOR_CHECK_DLQ,);await assertCheckQueue(- channel,+ ch,MONITOR_SEARCH_CHECK_QUEUE,MONITOR_SEARCH_CHECK_DLX,MONITOR_SEARCH_CHECK_DLQ,);+}++// Drop all cached state and, if any consumers were registered, kick off a+// backoff reconnect that re-asserts topology and re-subscribes every consumer.+function handleConnectionDrop(reason: string, error?: unknown): void {+ if (connection || publishChannel || consumeChannel) {+ logger.warn("Monitor queue dropped β will reconnect", { reason, error });+ }+ connection = null;+ publishChannel = null;+ consumeChannel = null;+ if (registeredConsumers.length > 0) {+ scheduleReconnect(RECONNECT_BASE_DELAY_MS);+ }+}++function scheduleReconnect(delayMs: number): void {+ if (reconnectInFlight) return;+ reconnectInFlight = true;++ setTimeout(async () => {+ try {+ const ch = await getConsumeChannel();+ for (const { queue, handler } of registeredConsumers) {+ await subscribe(ch, queue, handler);+ }+ reconnectInFlight = false;+ logger.info("Monitor queue reconnected; consumers re-subscribed", {+ consumers: registeredConsumers.length,+ });+ } catch (error) {+ reconnectInFlight = false;+ const nextDelay = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);+ logger.error("Monitor queue reconnect failed; retrying", {+ error,+ nextDelayMs: nextDelay,+ });+ connection = null;+ publishChannel = null;+ consumeChannel = null;+ scheduleReconnect(nextDelay);+ }+ }, delayMs);+}- connection.on("close", () => {- logger.warn("Monitor queue connection closed");- connection = null;- channel = null;+async function getConnection(): Promise<amqp.ChannelModel> {+ if (connection) return connection;++ const url = config.NUQ_RABBITMQ_URL;+ if (!url) {+ throw new Error("NUQ_RABBITMQ_URL is not configured");+ }++ // Name the connection so operators can identify (and isolate) monitoring+ // traffic in `rabbitmqctl list_connections` / the management UI.+ const conn = await amqp.connect(url, {+ clientProperties: { connection_name: "monitoring-queue" },});+ conn.on("close", () => handleConnectionDrop("connection closed"));+ conn.on("error", error =>+ logger.error("Monitor queue connection error", { error }),+ );+ connection = conn;+ return conn;+}- connection.on("error", error => {- logger.error("Monitor queue connection error", { error });+async function createChannel(label: string): Promise<amqp.Channel> {+ const conn = await getConnection();+ const ch = await conn.createChannel();+ await assertTopology(ch);+ // Channels can close independently of the connection (e.g. a+ // PRECONDITION_FAILED from assertQueue). Without these handlers the cached+ // channel would stay non-null and every send/consume would throw forever.+ ch.on("close", () => {+ if (label === "publish" && publishChannel === ch) publishChannel = null;+ if (label === "consume" && consumeChannel === ch) {+ consumeChannel = null;+ handleConnectionDrop("consume channel closed");+ }});+ ch.on("error", error =>+ logger.error("Monitor queue channel error", { label, error }),+ );+ return ch;+}- return channel;+async function getPublishChannel(): Promise<amqp.Channel> {+ if (publishChannel) return publishChannel;+ publishChannel = await createChannel("publish");+ return publishChannel;+}++async function getConsumeChannel(): Promise<amqp.Channel> {+ if (consumeChannel) return consumeChannel;+ consumeChannel = await createChannel("consume");+ return consumeChannel;}export async function addMonitorCheckJob(data: MonitorCheckJobData,opts: { search?: boolean } = {},): Promise<void> {- const ch = await getChannel();+ const ch = await getPublishChannel();const queue = opts.search ? MONITOR_SEARCH_CHECK_QUEUE : MONITOR_CHECK_QUEUE;const sent = ch.sendToQueue(queue, Buffer.from(JSON.stringify(data)), {persistent: true,@@ -116,9 +206,21 @@ export async function addMonitorCheckJob(async function consumeQueue(queue: string,- handler: (data: MonitorCheckJobData) => Promise<void>,+ handler: ConsumerHandler,+): Promise<void> {+ // Record the registration so a reconnect can re-attach this consumer.+ if (!registeredConsumers.some(c => c.queue === queue)) {+ registeredConsumers.push({ queue, handler });+ }+ const ch = await getConsumeChannel();+ await subscribe(ch, queue, handler);+}++async function subscribe(+ ch: amqp.Channel,+ queue: string,+ handler: ConsumerHandler,): Promise<void> {- const ch = await getChannel();// Per-consumer prefetch (amqplib default): each consumer gets its own in-flight// slot, so the search and default consumers don't block each other.await ch.prefetch(1);