Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(sdk): add missing deduplicateSimilarURLs param to JS and Python SDKs (#2902)

firecrawl-spring[bot]5c69fc728e03
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index a0bde9fa1..7bcdbac0d 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.13.1",
+ "version": "4.13.2",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
index ba110ca37..ce851106a 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
@@ -28,12 +28,14 @@ function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {
if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;
if (request.sitemap != null) data.sitemap = request.sitemap;
if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;
+ if (request.deduplicateSimilarURLs != null) data.deduplicateSimilarURLs = request.deduplicateSimilarURLs;
if (request.limit != null) data.limit = request.limit;
if (request.crawlEntireDomain != null) data.crawlEntireDomain = request.crawlEntireDomain;
if (request.allowExternalLinks != null) data.allowExternalLinks = request.allowExternalLinks;
if (request.allowSubdomains != null) data.allowSubdomains = request.allowSubdomains;
if (request.delay != null) data.delay = request.delay;
if (request.maxConcurrency != null) data.maxConcurrency = request.maxConcurrency;
+ if (request.regexOnFullURL != null) data.regexOnFullURL = request.regexOnFullURL;
if (request.webhook != null) data.webhook = request.webhook;
if (request.integration != null && request.integration.trim()) data.integration = request.integration.trim();
if (request.scrapeOptions) {
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index d706e3aa7..32666a1a8 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -462,6 +462,7 @@ export interface CrawlOptions {
maxDiscoveryDepth?: number | null;
sitemap?: 'skip' | 'include' | 'only';
ignoreQueryParameters?: boolean;
+ deduplicateSimilarURLs?: boolean;
limit?: number | null;
crawlEntireDomain?: boolean;
allowExternalLinks?: boolean;
@@ -470,6 +471,7 @@ export interface CrawlOptions {
maxConcurrency?: number | null;
webhook?: string | WebhookConfig | null;
scrapeOptions?: ScrapeOptions | null;
+ regexOnFullURL?: boolean;
zeroDataRetention?: boolean;
integration?: string;
}
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 2b95b22de..43c8db18a 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.16.2"
+__version__ = "4.16.3"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
index 354735345..2f679bcab 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
@@ -45,11 +45,13 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:
"max_discovery_depth": "maxDiscoveryDepth",
"sitemap": "sitemap",
"ignore_query_parameters": "ignoreQueryParameters",
+ "deduplicate_similar_urls": "deduplicateSimilarURLs",
"crawl_entire_domain": "crawlEntireDomain",
"allow_external_links": "allowExternalLinks",
"allow_subdomains": "allowSubdomains",
"delay": "delay",
"max_concurrency": "maxConcurrency",
+ "regex_on_full_url": "regexOnFullURL",
"zero_data_retention": "zeroDataRetention",
}
for snake, camel in field_mappings.items():
@@ -329,6 +331,7 @@ async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequ
"maxDiscoveryDepth": "max_discovery_depth",
"sitemap": "sitemap",
"ignoreQueryParameters": "ignore_query_parameters",
+ "deduplicateSimilarURLs": "deduplicate_similar_urls",
"crawlEntireDomain": "crawl_entire_domain",
"allowExternalLinks": "allow_external_links",
"allowSubdomains": "allow_subdomains",
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
index 041d51d46..c4a1488a3 100644
--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -84,6 +84,7 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:
"max_discovery_depth": "maxDiscoveryDepth",
"sitemap": "sitemap",
"ignore_query_parameters": "ignoreQueryParameters",
+ "deduplicate_similar_urls": "deduplicateSimilarURLs",
"crawl_entire_domain": "crawlEntireDomain",
"allow_external_links": "allowExternalLinks",
"allow_subdomains": "allowSubdomains",
@@ -506,6 +507,7 @@ def crawl_params_preview(client: HttpClient, request: CrawlParamsRequest) -> Cra
"maxDiscoveryDepth": "max_discovery_depth",
"sitemap": "sitemap",
"ignoreQueryParameters": "ignore_query_parameters",
+ "deduplicateSimilarURLs": "deduplicate_similar_urls",
"crawlEntireDomain": "crawl_entire_domain",
"allowExternalLinks": "allow_external_links",
"allowSubdomains": "allow_subdomains",
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index b39d92375..db7eff0d7 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -574,6 +574,7 @@ class CrawlRequest(BaseModel):
max_discovery_depth: Optional[int] = None
sitemap: Literal["skip", "include", "only"] = "include"
ignore_query_parameters: bool = False
+ deduplicate_similar_urls: bool = True
limit: Optional[int] = None
crawl_entire_domain: bool = False
allow_external_links: bool = False
@@ -667,6 +668,7 @@ class CrawlParamsData(BaseModel):
max_discovery_depth: Optional[int] = None
sitemap: Optional[Literal["skip", "include", "only"]] = None
ignore_query_parameters: bool = False
+ deduplicate_similar_urls: bool = True
limit: Optional[int] = None
crawl_entire_domain: bool = False
allow_external_links: bool = False