Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(sdk): add missing deduplicateSimilarURLs param to JS and Python SDKs (#2902)
firecrawl-spring[bot]5c69fc728e03
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex a0bde9fa1..7bcdbac0d 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.13.1",+ "version": "4.13.2","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.tsindex ba110ca37..ce851106a 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts@@ -28,12 +28,14 @@ function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;if (request.sitemap != null) data.sitemap = request.sitemap;if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;+ if (request.deduplicateSimilarURLs != null) data.deduplicateSimilarURLs = request.deduplicateSimilarURLs;if (request.limit != null) data.limit = request.limit;if (request.crawlEntireDomain != null) data.crawlEntireDomain = request.crawlEntireDomain;if (request.allowExternalLinks != null) data.allowExternalLinks = request.allowExternalLinks;if (request.allowSubdomains != null) data.allowSubdomains = request.allowSubdomains;if (request.delay != null) data.delay = request.delay;if (request.maxConcurrency != null) data.maxConcurrency = request.maxConcurrency;+ if (request.regexOnFullURL != null) data.regexOnFullURL = request.regexOnFullURL;if (request.webhook != null) data.webhook = request.webhook;if (request.integration != null && request.integration.trim()) data.integration = request.integration.trim();if (request.scrapeOptions) {diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex d706e3aa7..32666a1a8 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -462,6 +462,7 @@ export interface CrawlOptions {maxDiscoveryDepth?: number | null;sitemap?: 'skip' | 'include' | 'only';ignoreQueryParameters?: boolean;+ deduplicateSimilarURLs?: boolean;limit?: number | null;crawlEntireDomain?: boolean;allowExternalLinks?: boolean;@@ -470,6 +471,7 @@ export interface CrawlOptions {maxConcurrency?: number | null;webhook?: string | WebhookConfig | null;scrapeOptions?: ScrapeOptions | null;+ regexOnFullURL?: boolean;zeroDataRetention?: boolean;integration?: string;}diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 2b95b22de..43c8db18a 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.16.2"+__version__ = "4.16.3"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.pyindex 354735345..2f679bcab 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py@@ -45,11 +45,13 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:"max_discovery_depth": "maxDiscoveryDepth","sitemap": "sitemap","ignore_query_parameters": "ignoreQueryParameters",+ "deduplicate_similar_urls": "deduplicateSimilarURLs","crawl_entire_domain": "crawlEntireDomain","allow_external_links": "allowExternalLinks","allow_subdomains": "allowSubdomains","delay": "delay","max_concurrency": "maxConcurrency",+ "regex_on_full_url": "regexOnFullURL","zero_data_retention": "zeroDataRetention",}for snake, camel in field_mappings.items():@@ -329,6 +331,7 @@ async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequ"maxDiscoveryDepth": "max_discovery_depth","sitemap": "sitemap","ignoreQueryParameters": "ignore_query_parameters",+ "deduplicateSimilarURLs": "deduplicate_similar_urls","crawlEntireDomain": "crawl_entire_domain","allowExternalLinks": "allow_external_links","allowSubdomains": "allow_subdomains",diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex 041d51d46..c4a1488a3 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -84,6 +84,7 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:"max_discovery_depth": "maxDiscoveryDepth","sitemap": "sitemap","ignore_query_parameters": "ignoreQueryParameters",+ "deduplicate_similar_urls": "deduplicateSimilarURLs","crawl_entire_domain": "crawlEntireDomain","allow_external_links": "allowExternalLinks","allow_subdomains": "allowSubdomains",@@ -506,6 +507,7 @@ def crawl_params_preview(client: HttpClient, request: CrawlParamsRequest) -> Cra"maxDiscoveryDepth": "max_discovery_depth","sitemap": "sitemap","ignoreQueryParameters": "ignore_query_parameters",+ "deduplicateSimilarURLs": "deduplicate_similar_urls","crawlEntireDomain": "crawl_entire_domain","allowExternalLinks": "allow_external_links","allowSubdomains": "allow_subdomains",diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex b39d92375..db7eff0d7 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -574,6 +574,7 @@ class CrawlRequest(BaseModel):max_discovery_depth: Optional[int] = Nonesitemap: Literal["skip", "include", "only"] = "include"ignore_query_parameters: bool = False+ deduplicate_similar_urls: bool = Truelimit: Optional[int] = Nonecrawl_entire_domain: bool = Falseallow_external_links: bool = False@@ -667,6 +668,7 @@ class CrawlParamsData(BaseModel):max_discovery_depth: Optional[int] = Nonesitemap: Optional[Literal["skip", "include", "only"]] = Noneignore_query_parameters: bool = False+ deduplicate_similar_urls: bool = Truelimit: Optional[int] = Nonecrawl_entire_domain: bool = Falseallow_external_links: bool = False