Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/fire-engine): timeout calculation issues
Gergő Móricz47b968fedea4
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex a2deeed2d..2dc134c91 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -123,8 +123,13 @@ export async function scrapeURLWithFireEngineChromeCDP(// Include specified actions...(meta.options.actions ?? []),];++ const totalWait = actions.reduce(+ (a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),+ 0,+ );- const timeout = timeToRun ?? 300000;+ const timeout = (timeToRun ?? 300000) + totalWait;const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestChromeCDP = {@@ -146,18 +151,13 @@ export async function scrapeURLWithFireEngineChromeCDP(// TODO: scrollXPaths};- const totalWait = actions.reduce(- (a, x) => (x.type === "wait" ? (x.milliseconds ?? 1000) + a : a),- 0,- );-let response = await performFireEngineScrape(meta.logger.child({method: "scrapeURLWithFireEngineChromeCDP/callFireEngine",request,}),request,- timeout + totalWait,+ timeout,);specialtyScrapeCheck(@@ -213,7 +213,8 @@ export async function scrapeURLWithFireEnginePlaywright(meta: Meta,timeToRun: number | undefined,): Promise<EngineScrapeResult> {- const timeout = timeToRun ?? 300000;+ const totalWait = meta.options.waitFor;+ const timeout = (timeToRun ?? 300000) + totalWait;const request: FireEngineScrapeRequestCommon &FireEngineScrapeRequestPlaywright = {@@ -237,7 +238,7 @@ export async function scrapeURLWithFireEnginePlaywright(request,}),request,- timeout + meta.options.waitFor,+ timeout,);specialtyScrapeCheck(