Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(sdk): add missing deduplicate_similar_urls to v2 Python client crawl/start_crawl signatures (#2914)

devin-ai-integration[bot]95b711da118e
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 1ef61377a..4391f86a7 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.17.0"
+__version__ = "4.17.1"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index dfa529fba..b5a393c78 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -248,6 +248,7 @@ class FirecrawlClient:
webhook: Optional[Union[str, WebhookConfig]] = None,
scrape_options: Optional[ScrapeOptions] = None,
regex_on_full_url: bool = False,
+ deduplicate_similar_urls: bool = True,
zero_data_retention: bool = False,
poll_interval: int = 2,
timeout: Optional[int] = None,
@@ -275,6 +276,7 @@ class FirecrawlClient:
webhook: Webhook configuration for notifications
scrape_options: Page scraping configuration
regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
+ deduplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)
zero_data_retention: Whether to delete data after 24 hours
poll_interval: Seconds between status checks
timeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout)
@@ -308,6 +310,7 @@ class FirecrawlClient:
"webhook": webhook,
"scrape_options": scrape_options,
"regex_on_full_url": regex_on_full_url,
+ "deduplicate_similar_urls": deduplicate_similar_urls,
"zero_data_retention": zero_data_retention,
"integration": integration,
}
@@ -344,6 +347,7 @@ class FirecrawlClient:
webhook: Optional[Union[str, WebhookConfig]] = None,
scrape_options: Optional[ScrapeOptions] = None,
regex_on_full_url: bool = False,
+ deduplicate_similar_urls: bool = True,
zero_data_retention: bool = False,
integration: Optional[str] = None,
) -> CrawlResponse:
@@ -368,6 +372,7 @@ class FirecrawlClient:
webhook: Webhook configuration for notifications
scrape_options: Page scraping configuration
regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
+ deduplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)
zero_data_retention: Whether to delete data after 24 hours
Returns:
@@ -397,6 +402,7 @@ class FirecrawlClient:
"webhook": webhook,
"scrape_options": scrape_options,
"regex_on_full_url": regex_on_full_url,
+ "deduplicate_similar_urls": deduplicate_similar_urls,
"zero_data_retention": zero_data_retention,
"integration": integration,
}