Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(scrape): retry with stealthProxy on fire-engine retry signal (#3431)

firecrawl-spring[bot]9ba2f1254dce
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
index e062bd734..eb3babd35 100644
--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts
@@ -5,6 +5,7 @@ import { z } from "zod";
import { robustFetch } from "../../lib/fetch";
import {
ActionError,
+ AddFeatureError,
EngineError,
SiteError,
SSLError,
@@ -122,6 +123,7 @@ const failedSchema = z.object({
state: z.literal("failed"),
processing: z.literal(false),
error: z.string(),
+ retryWithStealth: z.boolean().optional(),
});
export class StillProcessingError extends Error {
@@ -175,6 +177,17 @@ export async function fireEngineCheckStatus(
throw new StillProcessingError(jobId);
} else if (failedParse.success) {
logger.debug("Scrape job failed", { status, jobId });
+ if (
+ failedParse.data.retryWithStealth &&
+ meta.options.proxy === "auto" &&
+ !meta.featureFlags.has("stealthProxy")
+ ) {
+ logger.info(
+ "Scrape job signaled retryWithStealth. Adding stealthProxy flag.",
+ { jobId },
+ );
+ throw new AddFeatureError(["stealthProxy"]);
+ }
if (
typeof status.error === "string" &&
status.error.includes("Chrome error: ")