Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(python-sdk): add regex_on_full_url parameter to v2 crawl (#2834)

firecrawl-spring[bot]2ac2afe57b36
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 5edca90e0..b34a05caf 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.14.0"
+__version__ = "4.14.1"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 8a476169e..f461a8122 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -240,6 +240,7 @@ class FirecrawlClient:
max_concurrency: Optional[int] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
scrape_options: Optional[ScrapeOptions] = None,
+ regex_on_full_url: bool = False,
zero_data_retention: bool = False,
poll_interval: int = 2,
timeout: Optional[int] = None,
@@ -248,7 +249,7 @@ class FirecrawlClient:
) -> CrawlJob:
"""
Start a crawl job and wait for it to complete.
-
+
Args:
url: Target URL to start crawling from
prompt: Optional prompt to guide the crawl
@@ -266,6 +267,7 @@ class FirecrawlClient:
max_concurrency: Maximum number of concurrent scrapes
webhook: Webhook configuration for notifications
scrape_options: Page scraping configuration
+ regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
zero_data_retention: Whether to delete data after 24 hours
poll_interval: Seconds between status checks
timeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout)
@@ -298,6 +300,7 @@ class FirecrawlClient:
"max_concurrency": max_concurrency,
"webhook": webhook,
"scrape_options": scrape_options,
+ "regex_on_full_url": regex_on_full_url,
"zero_data_retention": zero_data_retention,
"integration": integration,
}
@@ -305,7 +308,7 @@ class FirecrawlClient:
request_kwargs["sitemap"] = resolved_sitemap
request = CrawlRequest(**request_kwargs)
-
+
return crawl_module.crawl(
self.http_client,
request,
@@ -333,12 +336,13 @@ class FirecrawlClient:
max_concurrency: Optional[int] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
scrape_options: Optional[ScrapeOptions] = None,
+ regex_on_full_url: bool = False,
zero_data_retention: bool = False,
integration: Optional[str] = None,
) -> CrawlResponse:
"""
Start an asynchronous crawl job.
-
+
Args:
url: Target URL to start crawling from
prompt: Optional prompt to guide the crawl
@@ -356,6 +360,7 @@ class FirecrawlClient:
max_concurrency: Maximum number of concurrent scrapes
webhook: Webhook configuration for notifications
scrape_options: Page scraping configuration
+ regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
zero_data_retention: Whether to delete data after 24 hours
Returns:
@@ -384,6 +389,7 @@ class FirecrawlClient:
"max_concurrency": max_concurrency,
"webhook": webhook,
"scrape_options": scrape_options,
+ "regex_on_full_url": regex_on_full_url,
"zero_data_retention": zero_data_retention,
"integration": integration,
}
@@ -391,7 +397,7 @@ class FirecrawlClient:
request_kwargs["sitemap"] = resolved_sitemap
request = CrawlRequest(**request_kwargs)
-
+
return crawl_module.start_crawl(self.http_client, request)
def get_crawl_status(
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
index 1730fc726..041d51d46 100644
--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -89,6 +89,7 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:
"allow_subdomains": "allowSubdomains",
"delay": "delay",
"max_concurrency": "maxConcurrency",
+ "regex_on_full_url": "regexOnFullURL",
"zero_data_retention": "zeroDataRetention"
}
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 33c3e9001..7dd452f7c 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -582,6 +582,7 @@ class CrawlRequest(BaseModel):
max_concurrency: Optional[int] = None
webhook: Optional[Union[str, WebhookConfig]] = None
scrape_options: Optional[ScrapeOptions] = None
+ regex_on_full_url: bool = False
zero_data_retention: bool = False
integration: Optional[str] = None