Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(python-sdk): add regex_on_full_url parameter to v2 crawl (#2834)
firecrawl-spring[bot]2ac2afe57b36
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 5edca90e0..b34a05caf 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.14.0"+__version__ = "4.14.1"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 8a476169e..f461a8122 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -240,6 +240,7 @@ class FirecrawlClient:max_concurrency: Optional[int] = None,webhook: Optional[Union[str, WebhookConfig]] = None,scrape_options: Optional[ScrapeOptions] = None,+ regex_on_full_url: bool = False,zero_data_retention: bool = False,poll_interval: int = 2,timeout: Optional[int] = None,@@ -248,7 +249,7 @@ class FirecrawlClient:) -> CrawlJob:"""Start a crawl job and wait for it to complete.-+Args:url: Target URL to start crawling fromprompt: Optional prompt to guide the crawl@@ -266,6 +267,7 @@ class FirecrawlClient:max_concurrency: Maximum number of concurrent scrapeswebhook: Webhook configuration for notificationsscrape_options: Page scraping configuration+ regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathnamezero_data_retention: Whether to delete data after 24 hourspoll_interval: Seconds between status checkstimeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout)@@ -298,6 +300,7 @@ class FirecrawlClient:"max_concurrency": max_concurrency,"webhook": webhook,"scrape_options": scrape_options,+ "regex_on_full_url": regex_on_full_url,"zero_data_retention": zero_data_retention,"integration": integration,}@@ -305,7 +308,7 @@ class FirecrawlClient:request_kwargs["sitemap"] = resolved_sitemaprequest = CrawlRequest(**request_kwargs)-+return crawl_module.crawl(self.http_client,request,@@ -333,12 +336,13 @@ class FirecrawlClient:max_concurrency: Optional[int] = None,webhook: Optional[Union[str, WebhookConfig]] = None,scrape_options: Optional[ScrapeOptions] = None,+ regex_on_full_url: bool = False,zero_data_retention: bool = False,integration: Optional[str] = None,) -> CrawlResponse:"""Start an asynchronous crawl job.-+Args:url: Target URL to start crawling fromprompt: Optional prompt to guide the crawl@@ -356,6 +360,7 @@ class FirecrawlClient:max_concurrency: Maximum number of concurrent scrapeswebhook: Webhook configuration for notificationsscrape_options: Page scraping configuration+ regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathnamezero_data_retention: Whether to delete data after 24 hoursReturns:@@ -384,6 +389,7 @@ class FirecrawlClient:"max_concurrency": max_concurrency,"webhook": webhook,"scrape_options": scrape_options,+ "regex_on_full_url": regex_on_full_url,"zero_data_retention": zero_data_retention,"integration": integration,}@@ -391,7 +397,7 @@ class FirecrawlClient:request_kwargs["sitemap"] = resolved_sitemaprequest = CrawlRequest(**request_kwargs)-+return crawl_module.start_crawl(self.http_client, request)def get_crawl_status(diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex 1730fc726..041d51d46 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -89,6 +89,7 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:"allow_subdomains": "allowSubdomains","delay": "delay","max_concurrency": "maxConcurrency",+ "regex_on_full_url": "regexOnFullURL","zero_data_retention": "zeroDataRetention"}diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 33c3e9001..7dd452f7c 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -582,6 +582,7 @@ class CrawlRequest(BaseModel):max_concurrency: Optional[int] = Nonewebhook: Optional[Union[str, WebhookConfig]] = Nonescrape_options: Optional[ScrapeOptions] = None+ regex_on_full_url: bool = Falsezero_data_retention: bool = Falseintegration: Optional[str] = None