Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
post-incident changes (#2095)
Gergő Móricz65780d5cd9fe
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex 54c2f049f..565688ad7 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -190,6 +190,7 @@ export async function fireEngineScrape<body: request,logger: logger.child({ method: "fireEngineScrape/robustFetch" }),tryCount: 3,+ ignoreFailureStatus: true, // sends 500 on processing and various codes on errorsmock,abort,});diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex c1d81fa79..86076e0f6 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -434,7 +434,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {while (remainingEngines.length > 0) {const { engine, unsupportedFeatures } = remainingEngines.shift()!;- const waitUntilWaterfall = getEngineMaxReasonableTime(meta, engine);+ const waitUntilWaterfall =+ getEngineMaxReasonableTime(meta, engine) ++ parseInt(process.env.SCRAPEURL_ENGINE_WATERFALL_DELAY_MS || "0", 10);if (!isFinite(waitUntilWaterfall) ||diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lockindex bb080ac77..8ed791ee7 100644--- a/apps/rust-sdk/Cargo.lock+++ b/apps/rust-sdk/Cargo.lock@@ -685,7 +685,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"[[package]]name = "firecrawl"-version = "1.2.1"+version = "1.2.2"dependencies = ["assert_matches","axum",