Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(python-sdk): clean up _SCRAPE_OPTION_KEYS and add timeout to start_crawl
Himadri Mcea44ea48dfe
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex c3a2015da..f9a526c6e 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -67,13 +67,15 @@ from .methods import browser as browser_modulefrom .methods import monitor as monitor_modulefrom .watcher import Watcher+# Kwargs that map to ScrapeOptions fields. Used by async crawl normalization+# to extract scrape kwargs from **kwargs before building CrawlRequest.+# Does not include "integration" (crawl-level param, not a scrape option)._SCRAPE_OPTION_KEYS = frozenset({"formats", "headers", "include_tags", "exclude_tags","only_main_content", "timeout", "wait_for", "mobile","parsers", "actions", "location", "skip_tls_verification","remove_base64_images", "fast_mode", "use_mock", "block_ads","proxy", "max_age", "store_in_cache", "lockdown", "profile",- "integration",})@@ -548,6 +550,7 @@ class FirecrawlClient:include_tags: Optional[List[str]] = None,exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,+ timeout: Optional[int] = None,wait_for: Optional[int] = None,mobile: Optional[bool] = None,parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,@@ -594,6 +597,7 @@ class FirecrawlClient:include_tags: HTML tags to include (convenience kwarg)exclude_tags: HTML tags to exclude (convenience kwarg)only_main_content: Restrict to main content (convenience kwarg)+ timeout: Scrape timeout in milliseconds (convenience kwarg)wait_for: Wait condition in ms (convenience kwarg)mobile: Emulate mobile viewport (convenience kwarg)parsers: Parser list (convenience kwarg)@@ -624,7 +628,8 @@ class FirecrawlClient:scrape_kwargs = {k: v for k, v in dict(formats=formats, headers=headers, include_tags=include_tags,exclude_tags=exclude_tags, only_main_content=only_main_content,- wait_for=wait_for, mobile=mobile, parsers=parsers, actions=actions,+ timeout=timeout, wait_for=wait_for, mobile=mobile,+ parsers=parsers, actions=actions,location=location, skip_tls_verification=skip_tls_verification,remove_base64_images=remove_base64_images, fast_mode=fast_mode,use_mock=use_mock, block_ads=block_ads, proxy=proxy,diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 0164a837e..9ac68898a 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -186,14 +186,13 @@ class AsyncFirecrawlClient:return await async_search.search(self.async_http_client, request)async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:- _crawl_scrape_keys = _SCRAPE_OPTION_KEYS - {"integration"}if kwargs.get("scrape_options") is None:- scrape_kwargs = {k: kwargs.pop(k) for k in list(kwargs) if k in _crawl_scrape_keys and kwargs[k] is not None}+ scrape_kwargs = {k: kwargs.pop(k) for k in list(kwargs) if k in _SCRAPE_OPTION_KEYS and kwargs[k] is not None}if scrape_kwargs:kwargs["scrape_options"] = ScrapeOptions(**scrape_kwargs)else:for k in list(kwargs):- if k in _crawl_scrape_keys:+ if k in _SCRAPE_OPTION_KEYS:kwargs.pop(k)sitemap = kwargs.pop("sitemap", None)diff --git a/apps/python-sdk/tests/test_crawl_direct_kwargs.py b/apps/python-sdk/tests/test_crawl_direct_kwargs.pyindex 000666bd5..726dfaf52 100644--- a/apps/python-sdk/tests/test_crawl_direct_kwargs.py+++ b/apps/python-sdk/tests/test_crawl_direct_kwargs.py@@ -96,6 +96,38 @@ class TestSyncCrawlDirectKwargs:assert captured["scrape_options"] is not Noneassert captured["scrape_options"].only_main_content is True+ def test_start_crawl_accepts_scrape_timeout(self, client, monkeypatch):+ captured = {}++ def fake_start_crawl(http, request):+ captured["scrape_options"] = request.scrape_options+ raise ConnectionError("captured")++ monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)++ with pytest.raises(ConnectionError):+ client.start_crawl("https://example.com", timeout=5000)++ assert captured["scrape_options"] is not None+ assert captured["scrape_options"].timeout == 5000++ def test_crawl_timeout_is_poll_timeout_not_scrape(self, client, monkeypatch):+ captured = {}++ def fake_crawl(http, request, poll_interval=2, timeout=None, request_timeout=None):+ captured["scrape_options"] = request.scrape_options+ captured["poll_timeout"] = timeout+ raise ConnectionError("captured")++ monkeypatch.setattr(crawl_mod, "crawl", fake_crawl)++ with pytest.raises(ConnectionError):+ client.crawl("https://example.com", timeout=60, formats=["markdown"])++ assert captured["poll_timeout"] == 60+ assert captured["scrape_options"] is not None+ assert captured["scrape_options"].timeout is None+class TestAsyncCrawlDirectKwargs:def test_async_direct_kwargs_build_scrape_options(self, async_client, monkeypatch):