Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(monitoring): default timeout to 5000 to mitigate false positives on load (#3612)
Developers Digest6bc8369d12c6
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 3c2b451f0..6ad84a5e3 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -61,6 +61,10 @@ const logger = _logger.child({ module: "monitoring-runner" });const poll = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };+const MONITOR_DEFAULT_WAIT_FOR_MS = 5000;+const MONITOR_MAX_WAIT_FOR_MS = 60000;+const MONITOR_MAX_TIMEOUT_MS = MONITOR_MAX_WAIT_FOR_MS * 2;+type PageResult = MonitorCheckPageInsert & {emailStatus?: string;};@@ -116,9 +120,18 @@ function withMonitorScrapeDefaults(const formats = Array.isArray(options.formats)? normalizeMonitorFormats(options.formats): options.formats;+ const waitFor =+ typeof options.waitFor === "number"+ ? options.waitFor+ : MONITOR_DEFAULT_WAIT_FOR_MS;+ const timeout =+ typeof options.timeout === "number"+ ? Math.min(Math.max(options.timeout, waitFor * 2), MONITOR_MAX_TIMEOUT_MS)+ : Math.min(waitFor * 2, MONITOR_MAX_TIMEOUT_MS);+return {maxAge: 0,- ...withMarkdownFormat({ ...options, formats }),+ ...withMarkdownFormat({ ...options, formats, waitFor, timeout }),};}@@ -351,7 +364,7 @@ async function runCrawlTarget(params: {const body = crawlRequestSchema.parse({url: params.target.url,...(params.target.crawlOptions ?? {}),- scrapeOptions: withMarkdownFormat(params.target.scrapeOptions ?? {}),+ scrapeOptions: withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),origin: "monitor",}) as CrawlRequest;