Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(python-sdk): add missing ignore_robots_txt param to crawl (#3283)

firecrawl-spring[bot]de32a27a2bd5
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 6c0928dd4..f51951bdc 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.21.1"
+__version__ = "4.22.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.py
index c4a1488a3..b05dfd75d 100644
--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py
@@ -88,6 +88,7 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:
"crawl_entire_domain": "crawlEntireDomain",
"allow_external_links": "allowExternalLinks",
"allow_subdomains": "allowSubdomains",
+ "ignore_robots_txt": "ignoreRobotsTxt",
"delay": "delay",
"max_concurrency": "maxConcurrency",
"regex_on_full_url": "regexOnFullURL",
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index bd1a316a2..ac4aec5cd 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -598,6 +598,7 @@ class CrawlRequest(BaseModel):
crawl_entire_domain: bool = False
allow_external_links: bool = False
allow_subdomains: bool = False
+ ignore_robots_txt: bool = False
delay: Optional[int] = None
max_concurrency: Optional[int] = None
webhook: Optional[Union[str, WebhookConfig]] = None
@@ -692,6 +693,7 @@ class CrawlParamsData(BaseModel):
crawl_entire_domain: bool = False
allow_external_links: bool = False
allow_subdomains: bool = False
+ ignore_robots_txt: bool = False
delay: Optional[int] = None
max_concurrency: Optional[int] = None
webhook: Optional[Union[str, WebhookConfig]] = None