Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

post-incident changes (#2095)

Gergő Móricz65780d5cd9fe
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 54c2f049f..565688ad7 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -190,6 +190,7 @@ export async function fireEngineScrape<
body: request,
logger: logger.child({ method: "fireEngineScrape/robustFetch" }),
tryCount: 3,
+ ignoreFailureStatus: true, // sends 500 on processing and various codes on errors
mock,
abort,
});
diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.ts
index c1d81fa79..86076e0f6 100644
--- a/apps/api/src/scraper/scrapeURL/index.ts
+++ b/apps/api/src/scraper/scrapeURL/index.ts
@@ -434,7 +434,9 @@ async function scrapeURLLoop(meta: Meta): Promise<ScrapeUrlResponse> {
while (remainingEngines.length > 0) {
const { engine, unsupportedFeatures } = remainingEngines.shift()!;
- const waitUntilWaterfall = getEngineMaxReasonableTime(meta, engine);
+ const waitUntilWaterfall =
+ getEngineMaxReasonableTime(meta, engine) +
+ parseInt(process.env.SCRAPEURL_ENGINE_WATERFALL_DELAY_MS || "0", 10);
if (
!isFinite(waitUntilWaterfall) ||
diff --git a/apps/rust-sdk/Cargo.lock b/apps/rust-sdk/Cargo.lock
index bb080ac77..8ed791ee7 100644
--- a/apps/rust-sdk/Cargo.lock
+++ b/apps/rust-sdk/Cargo.lock
@@ -685,7 +685,7 @@ checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
[[package]]
name = "firecrawl"
-version = "1.2.1"
+version = "1.2.2"
dependencies = [
"assert_matches",
"axum",