Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Memoize MQ connection and use monitor targets
Developers Digest6aea6d03cca5
diff --git a/apps/api/src/services/monitoring/queue.ts b/apps/api/src/services/monitoring/queue.tsindex caed11082..265e0e760 100644--- a/apps/api/src/services/monitoring/queue.ts+++ b/apps/api/src/services/monitoring/queue.ts@@ -22,6 +22,9 @@ export type MonitorCheckJobData = {type ConsumerHandler = (data: MonitorCheckJobData) => Promise<void>;let connection: amqp.ChannelModel | null = null;+// Memoized so concurrent callers share one connect() instead of racing to open+// (and orphan) duplicate connections.+let connectionPromise: Promise<amqp.ChannelModel> | null = null;// Separate publish/consume channels so an exception on one can't tear down the other.let publishChannel: amqp.Channel | null = null;let consumeChannel: amqp.Channel | null = null;@@ -88,6 +91,7 @@ function handleConnectionDrop(reason: string, error?: unknown): void {logger.warn("Monitor queue dropped — will reconnect", { reason, error });}connection = null;+ connectionPromise = null;publishChannel = null;publishChannelPromise = null;consumeChannel = null;@@ -124,6 +128,7 @@ function scheduleReconnect(delayMs: number): void {// make the next reconnect skip (subscribe early-returns) and deafen that queue.await consumeChannel?.close().catch(() => {});connection = null;+ connectionPromise = null;publishChannel = null;publishChannelPromise = null;consumeChannel = null;@@ -136,7 +141,21 @@ function scheduleReconnect(delayMs: number): void {async function getConnection(): Promise<amqp.ChannelModel> {if (connection) return connection;+ if (!connectionPromise) {+ connectionPromise = openConnection()+ .then(conn => {+ connection = conn;+ return conn;+ })+ .catch(err => {+ connectionPromise = null;+ throw err;+ });+ }+ return connectionPromise;+}+async function openConnection(): Promise<amqp.ChannelModel> {const url = config.NUQ_RABBITMQ_URL;if (!url) {throw new Error("NUQ_RABBITMQ_URL is not configured");@@ -150,7 +169,6 @@ async function getConnection(): Promise<amqp.ChannelModel> {conn.on("error", error =>logger.error("Monitor queue connection error", { error }),);- connection = conn;return conn;}diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 070c450c4..0b61e09a9 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -1516,7 +1516,8 @@ async function failStaleMonitorCheck(params: {monitor: MonitorRow;check: MonitorCheckRow;}): Promise<boolean> {- if (!isMonitorCheckStale(params.check)) return false;+ if (!isMonitorCheckStale(params.check, new Date(), params.monitor.targets))+ return false;const error = MONITOR_CHECK_STALE_ERROR;if (params.check.autumn_lock_id) {@@ -1604,7 +1605,7 @@ async function failStaleMonitorCheck(params: {monitorId: params.monitor.id,checkId: params.check.id,startedAt: params.check.started_at,- timeoutMs: monitorCheckStaleTimeoutMs(params.check),+ timeoutMs: monitorCheckStaleTimeoutMs(params.check, params.monitor.targets),});return true;diff --git a/apps/api/src/services/monitoring/scheduler.ts b/apps/api/src/services/monitoring/scheduler.tsindex 2d69bcacc..0994d1fec 100644--- a/apps/api/src/services/monitoring/scheduler.ts+++ b/apps/api/src/services/monitoring/scheduler.ts@@ -189,7 +189,8 @@ async function clearFinishedOrStaleCurrentCheck(if (!current) return false;if (current.status === "running" || current.status === "queued") {- if (!isMonitorCheckStale(current)) return false;+ if (!isMonitorCheckStale(current, new Date(), monitor.targets))+ return false;if (current.autumn_lock_id) {await autumnServicediff --git a/apps/api/src/services/monitoring/stale.ts b/apps/api/src/services/monitoring/stale.tsindex a56065b84..0c0c9c712 100644--- a/apps/api/src/services/monitoring/stale.ts+++ b/apps/api/src/services/monitoring/stale.ts@@ -6,22 +6,33 @@ const MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS = 10 * 60 * 1000;export const MONITOR_CHECK_STALE_ERROR ="Monitor check exceeded the running timeout.";-// Short timeout only when EVERY target is search; a mixed monitor's crawl/scrape fan-out legitimately runs for many minutes.-function isSearchOnlyCheck(check: { target_results?: unknown }): boolean {- const targetResults = check.target_results;- if (!Array.isArray(targetResults) || targetResults.length === 0) return false;- return targetResults.every(- tr =>- tr != null &&- typeof tr === "object" &&- (tr as { type?: unknown }).type === "search",+function allEntriesSearch(entries: unknown): boolean {+ if (!Array.isArray(entries) || entries.length === 0) return false;+ return entries.every(+ e =>+ e != null &&+ typeof e === "object" &&+ (e as { type?: unknown }).type === "search",);}-export function monitorCheckStaleTimeoutMs(check: {- target_results?: unknown;-}): number {- return isSearchOnlyCheck(check)+// Short timeout only when EVERY target is search. Prefer the monitor's configured+// targets so queued checks (empty target_results) still get it; fall back to target_results.+function isSearchOnlyCheck(+ check: { target_results?: unknown },+ monitorTargets?: unknown,+): boolean {+ if (Array.isArray(monitorTargets)) return allEntriesSearch(monitorTargets);+ return allEntriesSearch(check.target_results);+}++export function monitorCheckStaleTimeoutMs(+ check: {+ target_results?: unknown;+ },+ monitorTargets?: unknown,+): number {+ return isSearchOnlyCheck(check, monitorTargets)? MONITOR_SEARCH_CHECK_STALE_TIMEOUT_MS: MONITOR_CHECK_STALE_TIMEOUT_MS;}@@ -31,9 +42,13 @@ export function isMonitorCheckStale(target_results?: unknown;},now: Date = new Date(),+ monitorTargets?: unknown,): boolean {const startedAt = check.started_at ?? check.updated_at ?? check.created_at;const startedAtMs = Date.parse(startedAt);if (!Number.isFinite(startedAtMs)) return false;- return now.getTime() - startedAtMs >= monitorCheckStaleTimeoutMs(check);+ return (+ now.getTime() - startedAtMs >=+ monitorCheckStaleTimeoutMs(check, monitorTargets)+ );}