Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v1): proxy option / stealthProxy flag (FIR-1050) (#1196)
Gergő Móriczb136e42b53d1
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex 7970d820b..754ef8e0f 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -110,5 +110,27 @@ describe("Scrape tests", () => {expectScrapeToSucceed(response);expect(typeof response.body.data.screenshot).toBe("string");}, 15000);- })+ });++ describe("Proxy API (f-e dependant)", () => {+ it.concurrent("undefined works", async () => {+ await scrape({+ url: "http://firecrawl.dev",+ });+ }, 15000);++ it.concurrent("basic works", async () => {+ await scrape({+ url: "http://firecrawl.dev",+ proxy: "basic",+ });+ }, 15000);++ it.concurrent("stealth works", async () => {+ await scrape({+ url: "http://firecrawl.dev",+ proxy: "stealth",+ });+ }, 15000);+ });});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 858c792ff..b8bf43637 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -217,6 +217,7 @@ const baseScrapeOptions = zfastMode: z.boolean().default(false),useMock: z.string().optional(),blockAds: z.boolean().default(true),+ proxy: z.enum(["basic", "stealth"]).optional(),}).strict(strictMessage);diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 26f7847f1..485b8eeb3 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -207,6 +207,7 @@ export async function scrapeURLWithFireEngineChromeCDP(timeout, // TODO: better timeout logicdisableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,blockAds: meta.options.blockAds,+ mobileProxy: meta.options.proxy === undefined ? undefined : meta.options.proxy === "stealth" ? true : false,// TODO: scrollXPaths};@@ -284,6 +285,7 @@ export async function scrapeURLWithFireEnginePlaywright(wait: meta.options.waitFor,geolocation: meta.options.geolocation ?? meta.options.location,blockAds: meta.options.blockAds,+ mobileProxy: meta.options.proxy === undefined ? undefined : meta.options.proxy === "stealth" ? true : false,timeout,};@@ -338,6 +340,7 @@ export async function scrapeURLWithFireEngineTLSClient(atsv: meta.internalOptions.atsv,geolocation: meta.options.geolocation ?? meta.options.location,disableJsDom: meta.internalOptions.v0DisableJsDom,+ mobileProxy: meta.options.proxy === undefined ? undefined : meta.options.proxy === "stealth" ? true : false,timeout,};diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex f3eedde48..607c0c8d6 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -27,6 +27,8 @@ export type FireEngineScrapeRequestCommon = {instantReturn?: boolean; // default: falsegeolocation?: { country?: string; languages?: string[] };+ mobileProxy?: boolean; // leave it undefined if user doesn't specify+timeout?: number;};diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex 896e177b8..f32708c0d 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -67,6 +67,7 @@ export const featureFlags = ["mobile","skipTlsVerification","useFastMode",+ "stealthProxy",] as const;export type FeatureFlag = (typeof featureFlags)[number];@@ -87,6 +88,7 @@ export const featureFlagOptions: {location: { priority: 10 },mobile: { priority: 10 },skipTlsVerification: { priority: 10 },+ stealthProxy: { priority: 20 },} as const;export type EngineScrapeResult = {@@ -145,6 +147,7 @@ export const engineOptions: {mobile: false,skipTlsVerification: false,useFastMode: false,+ stealthProxy: false,},quality: 1000, // cache should always be tried first},@@ -161,6 +164,7 @@ export const engineOptions: {mobile: true,skipTlsVerification: true,useFastMode: false,+ stealthProxy: true,},quality: 50,},@@ -177,6 +181,7 @@ export const engineOptions: {mobile: false,skipTlsVerification: false,useFastMode: false,+ stealthProxy: true,},quality: 40,},@@ -193,6 +198,7 @@ export const engineOptions: {mobile: false,skipTlsVerification: false,useFastMode: false,+ stealthProxy: false,},quality: 30,},@@ -209,6 +215,7 @@ export const engineOptions: {mobile: false,skipTlsVerification: false,useFastMode: false,+ stealthProxy: false,},quality: 29,},@@ -225,6 +232,7 @@ export const engineOptions: {mobile: false,skipTlsVerification: false,useFastMode: false,+ stealthProxy: false,},quality: 20,},@@ -241,6 +249,7 @@ export const engineOptions: {mobile: false,skipTlsVerification: false,useFastMode: true,+ stealthProxy: true,},quality: 10,},@@ -257,6 +266,7 @@ export const engineOptions: {mobile: false,skipTlsVerification: false,useFastMode: true,+ stealthProxy: false,},quality: 5,},@@ -273,6 +283,7 @@ export const engineOptions: {mobile: false,skipTlsVerification: false,useFastMode: true,+ stealthProxy: true, // kinda...},quality: -10,},@@ -289,6 +300,7 @@ export const engineOptions: {mobile: false,skipTlsVerification: false,useFastMode: true,+ stealthProxy: true, // kinda...},quality: -10,},diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex f95d199f5..416d5731f 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -94,6 +94,10 @@ function buildFeatureFlags(flags.add("useFastMode");}+ if (options.proxy === "stealth") {+ flags.add("stealthProxy");+ }+const urlO = new URL(url);if (urlO.pathname.endsWith(".pdf")) {diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 8e74d2d88..ad22c2a3a 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -94,6 +94,7 @@ export interface CrawlScrapeOptions {skipTlsVerification?: boolean;removeBase64Images?: boolean;blockAds?: boolean;+ proxy?: "basic" | "stealth";}export type Action = {