Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL/fire-engine): timeout-less scrape support (initial)
Gergő Móricz4cddcd5206b1
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 0ea54382c..7ad950ad2 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -87,6 +87,7 @@ export async function scrapeURLWithFireEngineChromeCDP(meta: Meta): Promise<Engipriority: meta.internalOptions.priority,geolocation: meta.options.geolocation,mobile: meta.options.mobile,+ timeout: meta.options.timeout === undefined ? 300000 : undefined, // TODO: better timeout logic// TODO: scrollXPaths};@@ -140,6 +141,8 @@ export async function scrapeURLWithFireEnginePlaywright(meta: Meta): Promise<EngfullPageScreenshot: meta.options.formats.includes("screenshot@fullPage"),wait: meta.options.waitFor,geolocation: meta.options.geolocation,++ timeout: meta.options.timeout === undefined ? 300000 : undefined, // TODO: better timeout logic};let response = await performFireEngineScrape(@@ -179,6 +182,8 @@ export async function scrapeURLWithFireEngineTLSClient(meta: Meta): Promise<Engiatsv: meta.internalOptions.atsv,geolocation: meta.options.geolocation,disableJsDom: meta.internalOptions.v0DisableJsDom,++ timeout: meta.options.timeout === undefined ? 300000 : undefined, // TODO: better timeout logic};let response = await performFireEngineScrape(diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 9332125a6..5f31ba67c 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -25,6 +25,8 @@ export type FireEngineScrapeRequestCommon = {logRequest?: boolean; // default: trueinstantReturn?: boolean; // default: falsegeolocation?: { country?: string; languages?: string[]; };++ timeout?: number;}export type FireEngineScrapeRequestChromeCDP = {