Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(python-sdk): clean up _SCRAPE_OPTION_KEYS and add timeout to start_crawl

Himadri Mcea44ea48dfe
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index c3a2015da..f9a526c6e 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -67,13 +67,15 @@ from .methods import browser as browser_module
from .methods import monitor as monitor_module
from .watcher import Watcher
+# Kwargs that map to ScrapeOptions fields. Used by async crawl normalization
+# to extract scrape kwargs from **kwargs before building CrawlRequest.
+# Does not include "integration" (crawl-level param, not a scrape option).
_SCRAPE_OPTION_KEYS = frozenset({
"formats", "headers", "include_tags", "exclude_tags",
"only_main_content", "timeout", "wait_for", "mobile",
"parsers", "actions", "location", "skip_tls_verification",
"remove_base64_images", "fast_mode", "use_mock", "block_ads",
"proxy", "max_age", "store_in_cache", "lockdown", "profile",
- "integration",
})
@@ -548,6 +550,7 @@ class FirecrawlClient:
include_tags: Optional[List[str]] = None,
exclude_tags: Optional[List[str]] = None,
only_main_content: Optional[bool] = None,
+ timeout: Optional[int] = None,
wait_for: Optional[int] = None,
mobile: Optional[bool] = None,
parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,
@@ -594,6 +597,7 @@ class FirecrawlClient:
include_tags: HTML tags to include (convenience kwarg)
exclude_tags: HTML tags to exclude (convenience kwarg)
only_main_content: Restrict to main content (convenience kwarg)
+ timeout: Scrape timeout in milliseconds (convenience kwarg)
wait_for: Wait condition in ms (convenience kwarg)
mobile: Emulate mobile viewport (convenience kwarg)
parsers: Parser list (convenience kwarg)
@@ -624,7 +628,8 @@ class FirecrawlClient:
scrape_kwargs = {k: v for k, v in dict(
formats=formats, headers=headers, include_tags=include_tags,
exclude_tags=exclude_tags, only_main_content=only_main_content,
- wait_for=wait_for, mobile=mobile, parsers=parsers, actions=actions,
+ timeout=timeout, wait_for=wait_for, mobile=mobile,
+ parsers=parsers, actions=actions,
location=location, skip_tls_verification=skip_tls_verification,
remove_base64_images=remove_base64_images, fast_mode=fast_mode,
use_mock=use_mock, block_ads=block_ads, proxy=proxy,
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index 0164a837e..9ac68898a 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -186,14 +186,13 @@ class AsyncFirecrawlClient:
return await async_search.search(self.async_http_client, request)
async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:
- _crawl_scrape_keys = _SCRAPE_OPTION_KEYS - {"integration"}
if kwargs.get("scrape_options") is None:
- scrape_kwargs = {k: kwargs.pop(k) for k in list(kwargs) if k in _crawl_scrape_keys and kwargs[k] is not None}
+ scrape_kwargs = {k: kwargs.pop(k) for k in list(kwargs) if k in _SCRAPE_OPTION_KEYS and kwargs[k] is not None}
if scrape_kwargs:
kwargs["scrape_options"] = ScrapeOptions(**scrape_kwargs)
else:
for k in list(kwargs):
- if k in _crawl_scrape_keys:
+ if k in _SCRAPE_OPTION_KEYS:
kwargs.pop(k)
sitemap = kwargs.pop("sitemap", None)
diff --git a/apps/python-sdk/tests/test_crawl_direct_kwargs.py b/apps/python-sdk/tests/test_crawl_direct_kwargs.py
index 000666bd5..726dfaf52 100644
--- a/apps/python-sdk/tests/test_crawl_direct_kwargs.py
+++ b/apps/python-sdk/tests/test_crawl_direct_kwargs.py
@@ -96,6 +96,38 @@ class TestSyncCrawlDirectKwargs:
assert captured["scrape_options"] is not None
assert captured["scrape_options"].only_main_content is True
+ def test_start_crawl_accepts_scrape_timeout(self, client, monkeypatch):
+ captured = {}
+
+ def fake_start_crawl(http, request):
+ captured["scrape_options"] = request.scrape_options
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
+
+ with pytest.raises(ConnectionError):
+ client.start_crawl("https://example.com", timeout=5000)
+
+ assert captured["scrape_options"] is not None
+ assert captured["scrape_options"].timeout == 5000
+
+ def test_crawl_timeout_is_poll_timeout_not_scrape(self, client, monkeypatch):
+ captured = {}
+
+ def fake_crawl(http, request, poll_interval=2, timeout=None, request_timeout=None):
+ captured["scrape_options"] = request.scrape_options
+ captured["poll_timeout"] = timeout
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(crawl_mod, "crawl", fake_crawl)
+
+ with pytest.raises(ConnectionError):
+ client.crawl("https://example.com", timeout=60, formats=["markdown"])
+
+ assert captured["poll_timeout"] == 60
+ assert captured["scrape_options"] is not None
+ assert captured["scrape_options"].timeout is None
+
class TestAsyncCrawlDirectKwargs:
def test_async_direct_kwargs_build_scrape_options(self, async_client, monkeypatch):