Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/fire-engine): wait longer if timeout is not specified
Gergő Móricz63787bc504a8
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 7ad950ad2..17bb40f20 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -96,7 +96,9 @@ export async function scrapeURLWithFireEngineChromeCDP(meta: Meta): Promise<Engilet response = await performFireEngineScrape(meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),request,- defaultTimeout + totalWait,+ meta.options.timeout !== undefined+ ? defaultTimeout + totalWait+ : Infinity, // TODO: better timeout handling);specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/specialtyScrapeCheck" }), response.responseHeaders);@@ -148,7 +150,9 @@ export async function scrapeURLWithFireEnginePlaywright(meta: Meta): Promise<Englet response = await performFireEngineScrape(meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),request,- defaultTimeout + meta.options.waitFor+ meta.options.timeout !== undefined+ ? defaultTimeout + meta.options.waitFor+ : Infinity, // TODO: better timeout handling);specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEnginePlaywright/specialtyScrapeCheck" }), response.responseHeaders);@@ -182,13 +186,16 @@ export async function scrapeURLWithFireEngineTLSClient(meta: Meta): Promise<Engiatsv: meta.internalOptions.atsv,geolocation: meta.options.geolocation,disableJsDom: meta.internalOptions.v0DisableJsDom,-+timeout: meta.options.timeout === undefined ? 300000 : undefined, // TODO: better timeout logic};let response = await performFireEngineScrape(meta.logger.child({ method: "scrapeURLWithFireEngineChromeCDP/callFireEngine", request }),request,+ meta.options.timeout !== undefined+ ? defaultTimeout+ : Infinity, // TODO: better timeout handling);specialtyScrapeCheck(meta.logger.child({ method: "scrapeURLWithFireEngineTLSClient/specialtyScrapeCheck" }), response.responseHeaders);