Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrape): retry with stealthProxy on retryWithStealth in sync response (#3432)

firecrawl-spring[bot]e9eb746c4c38
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
index 8cb691601..f1483edf2 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts
@@ -7,6 +7,7 @@ import { MockState } from "../../lib/mock";
import { getDocFromGCS } from "../../../../lib/gcs-jobs";
import {
ActionError,
+ AddFeatureError,
DNSResolutionError,
EngineError,
FEPageLoadFailed,
@@ -152,6 +153,7 @@ const processingSchema = z.object({
const failedSchema = z.object({
error: z.string(),
+ retryWithStealth: z.boolean().optional(),
});
export const fireEngineURL =
@@ -213,6 +215,16 @@ export async function fireEngineScrape<
logger.debug("Scrape job failed", {
status,
});
+ if (
+ failedParse.data.retryWithStealth &&
+ meta.options.proxy === "auto" &&
+ !meta.featureFlags.has("stealthProxy")
+ ) {
+ logger.info(
+ "Scrape signaled retryWithStealth. Adding stealthProxy flag.",
+ );
+ throw new AddFeatureError(["stealthProxy"]);
+ }
if (
typeof status.error === "string" &&
status.error.includes("Chrome error: ")