Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update index.ts
Nicolasa5256827c0e9
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 2b67c4d6f..b84b1e900 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -51,7 +51,11 @@ async function performFireEngineScrape<});}- if (Date.now() - startTime > timeout) {+ const userParam = request.timeout ?? 0;+ // Use 70% of the user-provided timeout as the timeout for fire-engine check status+ const fireEngineTimeout = timeout + Math.round(userParam * 0.7);+ const fullTimeout = Math.max(fireEngineTimeout, timeout);+ if (Date.now() - startTime > fullTimeout) {logger.info("Fire-engine was unable to scrape the page before timing out.",{ errors, timeout },