Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(python-sdk): add missing ignore_robots_txt param to crawl and start_crawl methods (#3305)
firecrawl-spring[bot]fa426d10c389
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex f51951bdc..8da0cc9f9 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.22.0"+__version__ = "4.22.1"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex a145ba9e0..d38a6f423 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -322,6 +322,7 @@ class FirecrawlClient:crawl_entire_domain: bool = False,allow_external_links: bool = False,allow_subdomains: bool = False,+ ignore_robots_txt: bool = False,delay: Optional[int] = None,max_concurrency: Optional[int] = None,webhook: Optional[Union[str, WebhookConfig]] = None,@@ -350,6 +351,7 @@ class FirecrawlClient:crawl_entire_domain: Follow parent directory linksallow_external_links: Follow external domain linksallow_subdomains: Follow subdomains+ ignore_robots_txt: Whether to ignore robots.txt rulesdelay: Delay in seconds between scrapesmax_concurrency: Maximum number of concurrent scrapeswebhook: Webhook configuration for notifications@@ -384,6 +386,7 @@ class FirecrawlClient:"crawl_entire_domain": crawl_entire_domain,"allow_external_links": allow_external_links,"allow_subdomains": allow_subdomains,+ "ignore_robots_txt": ignore_robots_txt,"delay": delay,"max_concurrency": max_concurrency,"webhook": webhook,@@ -421,6 +424,7 @@ class FirecrawlClient:crawl_entire_domain: bool = False,allow_external_links: bool = False,allow_subdomains: bool = False,+ ignore_robots_txt: bool = False,delay: Optional[int] = None,max_concurrency: Optional[int] = None,webhook: Optional[Union[str, WebhookConfig]] = None,@@ -446,6 +450,7 @@ class FirecrawlClient:crawl_entire_domain: Follow parent directory linksallow_external_links: Follow external domain linksallow_subdomains: Follow subdomains+ ignore_robots_txt: Whether to ignore robots.txt rulesdelay: Delay in seconds between scrapesmax_concurrency: Maximum number of concurrent scrapeswebhook: Webhook configuration for notifications@@ -453,7 +458,7 @@ class FirecrawlClient:regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathnamededuplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)zero_data_retention: Whether to delete data after 24 hours-+Returns:CrawlResponse with job information@@ -476,6 +481,7 @@ class FirecrawlClient:"crawl_entire_domain": crawl_entire_domain,"allow_external_links": allow_external_links,"allow_subdomains": allow_subdomains,+ "ignore_robots_txt": ignore_robots_txt,"delay": delay,"max_concurrency": max_concurrency,"webhook": webhook,