Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(python-sdk): add missing ignore_robots_txt param to crawl (#3283)
firecrawl-spring[bot]de32a27a2bd5
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 6c0928dd4..f51951bdc 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.21.1"+__version__ = "4.22.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex c4a1488a3..b05dfd75d 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -88,6 +88,7 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:"crawl_entire_domain": "crawlEntireDomain","allow_external_links": "allowExternalLinks","allow_subdomains": "allowSubdomains",+ "ignore_robots_txt": "ignoreRobotsTxt","delay": "delay","max_concurrency": "maxConcurrency","regex_on_full_url": "regexOnFullURL",diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex bd1a316a2..ac4aec5cd 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -598,6 +598,7 @@ class CrawlRequest(BaseModel):crawl_entire_domain: bool = Falseallow_external_links: bool = Falseallow_subdomains: bool = False+ ignore_robots_txt: bool = Falsedelay: Optional[int] = Nonemax_concurrency: Optional[int] = Nonewebhook: Optional[Union[str, WebhookConfig]] = None@@ -692,6 +693,7 @@ class CrawlParamsData(BaseModel):crawl_entire_domain: bool = Falseallow_external_links: bool = Falseallow_subdomains: bool = False+ ignore_robots_txt: bool = Falsedelay: Optional[int] = Nonemax_concurrency: Optional[int] = Nonewebhook: Optional[Union[str, WebhookConfig]] = None