Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrape): retry with stealthProxy on fire-engine retry signal (#3431)
firecrawl-spring[bot]9ba2f1254dce
diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.tsindex e062bd734..eb3babd35 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/checkStatus.ts@@ -5,6 +5,7 @@ import { z } from "zod";import { robustFetch } from "../../lib/fetch";import {ActionError,+ AddFeatureError,EngineError,SiteError,SSLError,@@ -122,6 +123,7 @@ const failedSchema = z.object({state: z.literal("failed"),processing: z.literal(false),error: z.string(),+ retryWithStealth: z.boolean().optional(),});export class StillProcessingError extends Error {@@ -175,6 +177,17 @@ export async function fireEngineCheckStatus(throw new StillProcessingError(jobId);} else if (failedParse.success) {logger.debug("Scrape job failed", { status, jobId });+ if (+ failedParse.data.retryWithStealth &&+ meta.options.proxy === "auto" &&+ !meta.featureFlags.has("stealthProxy")+ ) {+ logger.info(+ "Scrape job signaled retryWithStealth. Adding stealthProxy flag.",+ { jobId },+ );+ throw new AddFeatureError(["stealthProxy"]);+ }if (typeof status.error === "string" &&status.error.includes("Chrome error: ")