Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: increased timeout for chrome-cdp due to smart wait
Nicolas30058b1da097
diff --git a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.tsindex 295c171c7..dd3577b07 100644--- a/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts+++ b/apps/api/src/scraper/WebScraper/scrapers/fireEngine.ts@@ -131,7 +131,9 @@ export async function scrapWithFireEngine({const waitTotal = (actions ?? []).filter(x => x.type === "wait").reduce((a, x) => (x as { type: "wait"; milliseconds: number; }).milliseconds + a, 0);let checkStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`);- while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitTotal) {++ // added 5 seconds to the timeout to account for 'smart wait'+ while (checkStatusResponse.data.processing && Date.now() - startTime < universalTimeout + waitTotal + 5000) {await new Promise(resolve => setTimeout(resolve, 250)); // wait 0.25 secondscheckStatusResponse = await axiosInstance.get(`${process.env.FIRE_ENGINE_BETA_URL}/scrape/${_response.data.jobId}`);}