Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(monitoring): default timeout to 5000 to mitigate false positives on load (#3612)

Developers Digest6bc8369d12c6
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.ts
index 3c2b451f0..6ad84a5e3 100644
--- a/apps/api/src/services/monitoring/runner.ts
+++ b/apps/api/src/services/monitoring/runner.ts
@@ -61,6 +61,10 @@ const logger = _logger.child({ module: "monitoring-runner" });
const poll = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };
+const MONITOR_DEFAULT_WAIT_FOR_MS = 5000;
+const MONITOR_MAX_WAIT_FOR_MS = 60000;
+const MONITOR_MAX_TIMEOUT_MS = MONITOR_MAX_WAIT_FOR_MS * 2;
+
type PageResult = MonitorCheckPageInsert & {
emailStatus?: string;
};
@@ -116,9 +120,18 @@ function withMonitorScrapeDefaults(
const formats = Array.isArray(options.formats)
? normalizeMonitorFormats(options.formats)
: options.formats;
+ const waitFor =
+ typeof options.waitFor === "number"
+ ? options.waitFor
+ : MONITOR_DEFAULT_WAIT_FOR_MS;
+ const timeout =
+ typeof options.timeout === "number"
+ ? Math.min(Math.max(options.timeout, waitFor * 2), MONITOR_MAX_TIMEOUT_MS)
+ : Math.min(waitFor * 2, MONITOR_MAX_TIMEOUT_MS);
+
return {
maxAge: 0,
- ...withMarkdownFormat({ ...options, formats }),
+ ...withMarkdownFormat({ ...options, formats, waitFor, timeout }),
};
}
@@ -351,7 +364,7 @@ async function runCrawlTarget(params: {
const body = crawlRequestSchema.parse({
url: params.target.url,
...(params.target.crawlOptions ?? {}),
- scrapeOptions: withMarkdownFormat(params.target.scrapeOptions ?? {}),
+ scrapeOptions: withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),
origin: "monitor",
}) as CrawlRequest;