Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
remove monitor scrape waitfor
Developers Digest03181a0a711c
diff --git a/apps/api/src/services/monitoring/runner.ts b/apps/api/src/services/monitoring/runner.tsindex 6122eb4c0..246e0d6a6 100644--- a/apps/api/src/services/monitoring/runner.ts+++ b/apps/api/src/services/monitoring/runner.ts@@ -62,9 +62,6 @@ const logger = _logger.child({ module: "monitoring-runner" });const poll = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));export { isMonitorCheckStale, MONITOR_CHECK_STALE_TIMEOUT_MS };-const MONITOR_DEFAULT_WAIT_FOR_MS = 5000;-const MONITOR_MAX_WAIT_FOR_MS = 60000;-const MONITOR_MAX_TIMEOUT_MS = MONITOR_MAX_WAIT_FOR_MS * 2;const MONITOR_NOTIFY_CLAIM_TTL_SECONDS = 7 * 24 * 60 * 60;const TERMINAL_CHECK_STATUSES = new Set(["completed",@@ -139,18 +136,9 @@ function withMonitorScrapeDefaults(const formats = Array.isArray(options.formats)? normalizeMonitorFormats(options.formats): options.formats;- const waitFor =- typeof options.waitFor === "number"- ? options.waitFor- : MONITOR_DEFAULT_WAIT_FOR_MS;- const timeout =- typeof options.timeout === "number"- ? Math.min(Math.max(options.timeout, waitFor * 2), MONITOR_MAX_TIMEOUT_MS)- : Math.min(waitFor * 2, MONITOR_MAX_TIMEOUT_MS);-return {maxAge: 0,- ...withMarkdownFormat({ ...options, formats, waitFor, timeout }),+ ...withMarkdownFormat({ ...options, formats }),};}@@ -383,7 +371,7 @@ async function runCrawlTarget(params: {const body = crawlRequestSchema.parse({url: params.target.url,...(params.target.crawlOptions ?? {}),- scrapeOptions: withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),+ scrapeOptions: withMarkdownFormat(params.target.scrapeOptions ?? {}),origin: "monitor",}) as CrawlRequest;@@ -767,7 +755,7 @@ async function enqueueMonitorCrawlTarget(params: {const body = crawlRequestSchema.parse({url: params.target.url,...(params.target.crawlOptions ?? {}),- scrapeOptions: withMonitorScrapeDefaults(params.target.scrapeOptions ?? {}),+ scrapeOptions: withMarkdownFormat(params.target.scrapeOptions ?? {}),origin: "monitor",}) as CrawlRequest;@@ -940,14 +928,19 @@ export async function processMonitorCheckJob(error: error instanceof Error ? error.message : String(error),});- if ((await claimMonitorNotification(check.id).catch(error => {- logger.warn("Failed to claim monitor notification; continuing without dedupe", {- error,- monitorId: monitor.id,- checkId: check.id,- });- return true;- }))) {+ if (+ await claimMonitorNotification(check.id).catch(error => {+ logger.warn(+ "Failed to claim monitor notification; continuing without dedupe",+ {+ error,+ monitorId: monitor.id,+ checkId: check.id,+ },+ );+ return true;+ })+ ) {const notificationStatus = await sendNotifications({monitor,check,