Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add enhanced proxy option as alias for stealth (#2759)
firecrawl-spring[bot]9e46cbdbd6d8
diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.tsindex 9c905c466..72ed02213 100644--- a/apps/api/src/__tests__/snips/v1/lib.ts+++ b/apps/api/src/__tests__/snips/v1/lib.ts@@ -63,7 +63,7 @@ export async function scrape(): Promise<Document> {const raw = await scrapeRaw(body, identity);expectScrapeToSucceed(raw);- if (body.proxy === "stealth") {+ if (body.proxy === "stealth" || body.proxy === "enhanced") {expect(raw.body.data.metadata.proxyUsed).toBe("stealth");} else if (!body.proxy || body.proxy === "basic") {expect(raw.body.data.metadata.proxyUsed).toBe("basic");diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex f2a6be45c..dc7156407 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -515,7 +515,7 @@ const baseScrapeOptions = z.strictObject({fastMode: z.boolean().prefault(false),useMock: z.string().optional(),blockAds: z.boolean().prefault(true),- proxy: z.enum(["basic", "stealth", "auto"]).prefault("basic"),+ proxy: z.enum(["basic", "stealth", "enhanced", "auto"]).prefault("basic"),maxAge: z.int().gte(0)@@ -575,7 +575,9 @@ const extractTransform = (obj: ScrapeOptions) => {}if (- (obj.proxy === "stealth" || obj.proxy === "auto") &&+ (obj.proxy === "stealth" ||+ obj.proxy === "enhanced" ||+ obj.proxy === "auto") &&obj.timeout === 30000) {obj = { ...obj, timeout: 120000 };diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex f6b813c84..218285bc9 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -541,7 +541,7 @@ const baseScrapeOptions = z.strictObject({fastMode: z.boolean().prefault(false),useMock: z.string().optional(),blockAds: z.boolean().prefault(true),- proxy: z.enum(["basic", "stealth", "auto"]).prefault("auto"),+ proxy: z.enum(["basic", "stealth", "enhanced", "auto"]).prefault("auto"),maxAge: z.int().gte(0).optional(),minAge: z.int().gte(0).optional(),storeInCache: z.boolean().prefault(true),@@ -596,7 +596,9 @@ const extractTransformImpl = <T extends ScrapeOptionsBase | undefined>(}if (- (obj.proxy === "stealth" || obj.proxy === "auto") &&+ (obj.proxy === "stealth" ||+ obj.proxy === "enhanced" ||+ obj.proxy === "auto") &&obj.timeout === 30000) {result = { ...result, timeout: 120000 };diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 42071bfa6..5e2d90e3d 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -175,7 +175,7 @@ function buildFeatureFlags(flags.add("useFastMode");}- if (options.proxy === "stealth") {+ if (options.proxy === "stealth" || options.proxy === "enhanced") {flags.add("stealthProxy");}@@ -254,10 +254,7 @@ async function buildMetaObject(const abortHandle =options.timeout !== undefined? setTimeout(- () =>- abortController.abort(- new ScrapeJobTimeoutError(),- ),+ () => abortController.abort(new ScrapeJobTimeoutError()),options.timeout,): undefined;diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 6b28489f7..941be779b 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.11.5",+ "version": "4.12.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex ed72cb763..77c78e21f 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -131,7 +131,7 @@ export interface CrawlScrapeOptions {skipTlsVerification?: boolean;removeBase64Images?: boolean;blockAds?: boolean;- proxy?: "basic" | "stealth" | "auto";+ proxy?: "basic" | "stealth" | "enhanced" | "auto";storeInCache?: boolean;maxAge?: number;parsePDF?: boolean;diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex b3ec7523e..df9c79de8 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -153,7 +153,7 @@ export interface ScrapeOptions {fastMode?: boolean;useMock?: string;blockAds?: boolean;- proxy?: 'basic' | 'stealth' | 'auto' | string;+ proxy?: 'basic' | 'stealth' | 'enhanced' | 'auto' | string;maxAge?: number;minAge?: number;storeInCache?: boolean;diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex fbea47891..5edca90e0 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.13.4"+__version__ = "4.14.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.pyindex 797cb8b6b..bc21729da 100644--- a/apps/python-sdk/firecrawl/v1/client.py+++ b/apps/python-sdk/firecrawl/v1/client.py@@ -150,7 +150,7 @@ class V1ScrapeOptions(pydantic.BaseModel):skipTlsVerification: Optional[bool] = NoneremoveBase64Images: Optional[bool] = NoneblockAds: Optional[bool] = None- proxy: Optional[Literal["basic", "stealth", "auto"]] = None+ proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = NonechangeTrackingOptions: Optional[V1ChangeTrackingOptions] = NonemaxAge: Optional[int] = NonestoreInCache: Optional[bool] = None@@ -542,7 +542,7 @@ class V1FirecrawlApp:skip_tls_verification: Optional[bool] = None,remove_base64_images: Optional[bool] = None,block_ads: Optional[bool] = None,- proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None,parse_pdf: Optional[bool] = None,extract: Optional[V1JsonConfig] = None,json_options: Optional[V1JsonConfig] = None,@@ -1441,7 +1441,7 @@ class V1FirecrawlApp:skip_tls_verification: Optional[bool] = None,remove_base64_images: Optional[bool] = None,block_ads: Optional[bool] = None,- proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None,extract: Optional[V1JsonConfig] = None,json_options: Optional[V1JsonConfig] = None,actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,@@ -1582,7 +1582,7 @@ class V1FirecrawlApp:skip_tls_verification: Optional[bool] = None,remove_base64_images: Optional[bool] = None,block_ads: Optional[bool] = None,- proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None,extract: Optional[V1JsonConfig] = None,json_options: Optional[V1JsonConfig] = None,actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,@@ -1722,7 +1722,7 @@ class V1FirecrawlApp:skip_tls_verification: Optional[bool] = None,remove_base64_images: Optional[bool] = None,block_ads: Optional[bool] = None,- proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None,extract: Optional[V1JsonConfig] = None,json_options: Optional[V1JsonConfig] = None,actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,@@ -3523,7 +3523,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):skip_tls_verification: Optional[bool] = None,remove_base64_images: Optional[bool] = None,block_ads: Optional[bool] = None,- proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None,parse_pdf: Optional[bool] = None,extract: Optional[V1JsonConfig] = None,json_options: Optional[V1JsonConfig] = None,@@ -3657,7 +3657,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):skip_tls_verification: Optional[bool] = None,remove_base64_images: Optional[bool] = None,block_ads: Optional[bool] = None,- proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None,extract: Optional[V1JsonConfig] = None,json_options: Optional[V1JsonConfig] = None,actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,@@ -3796,7 +3796,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):skip_tls_verification: Optional[bool] = None,remove_base64_images: Optional[bool] = None,block_ads: Optional[bool] = None,- proxy: Optional[Literal["basic", "stealth", "auto"]] = None,+ proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = None,extract: Optional[V1JsonConfig] = None,json_options: Optional[V1JsonConfig] = None,actions: Optional[List[Union[V1WaitAction, V1ScreenshotAction, V1ClickAction, V1WriteAction, V1PressAction, V1ScrollAction, V1ScrapeAction, V1ExecuteJavascriptAction, V1PDFAction]]] = None,diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 6439c3142..33c3e9001 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -523,7 +523,7 @@ class ScrapeOptions(BaseModel):fast_mode: Optional[bool] = Noneuse_mock: Optional[str] = Noneblock_ads: Optional[bool] = None- proxy: Optional[Literal["basic", "stealth", "auto"]] = None+ proxy: Optional[Literal["basic", "stealth", "enhanced", "auto"]] = Nonemax_age: Optional[int] = Nonemin_age: Optional[int] = Nonestore_in_cache: Optional[bool] = None