Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(python-sdk): accept direct scrape kwargs in crawl() and start_crawl()
Himadri Mbb148f83b7a6
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex c10a62020..c3a2015da 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -67,6 +67,16 @@ from .methods import browser as browser_modulefrom .methods import monitor as monitor_modulefrom .watcher import Watcher+_SCRAPE_OPTION_KEYS = frozenset({+ "formats", "headers", "include_tags", "exclude_tags",+ "only_main_content", "timeout", "wait_for", "mobile",+ "parsers", "actions", "location", "skip_tls_verification",+ "remove_base64_images", "fast_mode", "use_mock", "block_ads",+ "proxy", "max_age", "store_in_cache", "lockdown", "profile",+ "integration",+})++class FirecrawlClient:"""Main Firecrawl v2 API client.@@ -377,6 +387,26 @@ class FirecrawlClient:max_concurrency: Optional[int] = None,webhook: Optional[Union[str, WebhookConfig]] = None,scrape_options: Optional[ScrapeOptions] = None,+ formats: Optional[List['FormatOption']] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ mobile: Optional[bool] = None,+ parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,+ actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,+ location: Optional['Location'] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ fast_mode: Optional[bool] = None,+ use_mock: Optional[str] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[str] = None,+ max_age: Optional[int] = None,+ store_in_cache: Optional[bool] = None,+ lockdown: Optional[bool] = None,+ profile: Optional[Dict[str, Any]] = None,regex_on_full_url: bool = False,deduplicate_similar_urls: bool = True,zero_data_retention: bool = False,@@ -405,22 +435,56 @@ class FirecrawlClient:delay: Delay in seconds between scrapesmax_concurrency: Maximum number of concurrent scrapeswebhook: Webhook configuration for notifications- scrape_options: Page scraping configuration+ scrape_options: Page scraping configuration (takes precedence over direct scrape kwargs)+ formats: Output formats (convenience kwarg, ignored if scrape_options is set)+ headers: HTTP headers for scraping (convenience kwarg)+ include_tags: HTML tags to include (convenience kwarg)+ exclude_tags: HTML tags to exclude (convenience kwarg)+ only_main_content: Restrict to main content (convenience kwarg)+ wait_for: Wait condition in ms (convenience kwarg)+ mobile: Emulate mobile viewport (convenience kwarg)+ parsers: Parser list (convenience kwarg)+ actions: Browser actions (convenience kwarg)+ location: Location settings (convenience kwarg)+ skip_tls_verification: Skip TLS verification (convenience kwarg)+ remove_base64_images: Remove base64 images (convenience kwarg)+ fast_mode: Prefer faster modes (convenience kwarg)+ use_mock: Use mock source (convenience kwarg)+ block_ads: Block ads (convenience kwarg)+ proxy: Proxy setting (convenience kwarg)+ max_age: Cache max age (convenience kwarg)+ store_in_cache: Cache results (convenience kwarg)+ lockdown: Serve only cached results (convenience kwarg)+ profile: Browser profile (convenience kwarg)regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathnamededuplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)zero_data_retention: Whether to delete data after 24 hourspoll_interval: Seconds between status checkstimeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout)request_timeout: Timeout (in seconds) for each individual HTTP request, including pagination requests when fetching results. If there are multiple pages, each page request gets this timeout-+Returns:CrawlJob when job completes-+Raises:ValueError: If request is invalidException: If the crawl fails to start or completeTimeoutError: If timeout is reached"""+ if scrape_options is None:+ scrape_kwargs = {k: v for k, v in dict(+ formats=formats, headers=headers, include_tags=include_tags,+ exclude_tags=exclude_tags, only_main_content=only_main_content,+ wait_for=wait_for, mobile=mobile, parsers=parsers, actions=actions,+ location=location, skip_tls_verification=skip_tls_verification,+ remove_base64_images=remove_base64_images, fast_mode=fast_mode,+ use_mock=use_mock, block_ads=block_ads, proxy=proxy,+ max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,+ profile=profile,+ ).items() if v is not None}+ if scrape_kwargs:+ scrape_options = ScrapeOptions(**scrape_kwargs)+resolved_sitemap = sitemapif resolved_sitemap is None and ignore_sitemap is not None:resolved_sitemap = "skip" if ignore_sitemap else "include"@@ -479,6 +543,26 @@ class FirecrawlClient:max_concurrency: Optional[int] = None,webhook: Optional[Union[str, WebhookConfig]] = None,scrape_options: Optional[ScrapeOptions] = None,+ formats: Optional[List['FormatOption']] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ mobile: Optional[bool] = None,+ parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,+ actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,+ location: Optional['Location'] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ fast_mode: Optional[bool] = None,+ use_mock: Optional[str] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[str] = None,+ max_age: Optional[int] = None,+ store_in_cache: Optional[bool] = None,+ lockdown: Optional[bool] = None,+ profile: Optional[Dict[str, Any]] = None,regex_on_full_url: bool = False,deduplicate_similar_urls: bool = True,zero_data_retention: bool = False,@@ -504,18 +588,52 @@ class FirecrawlClient:delay: Delay in seconds between scrapesmax_concurrency: Maximum number of concurrent scrapeswebhook: Webhook configuration for notifications- scrape_options: Page scraping configuration+ scrape_options: Page scraping configuration (takes precedence over direct scrape kwargs)+ formats: Output formats (convenience kwarg, ignored if scrape_options is set)+ headers: HTTP headers for scraping (convenience kwarg)+ include_tags: HTML tags to include (convenience kwarg)+ exclude_tags: HTML tags to exclude (convenience kwarg)+ only_main_content: Restrict to main content (convenience kwarg)+ wait_for: Wait condition in ms (convenience kwarg)+ mobile: Emulate mobile viewport (convenience kwarg)+ parsers: Parser list (convenience kwarg)+ actions: Browser actions (convenience kwarg)+ location: Location settings (convenience kwarg)+ skip_tls_verification: Skip TLS verification (convenience kwarg)+ remove_base64_images: Remove base64 images (convenience kwarg)+ fast_mode: Prefer faster modes (convenience kwarg)+ use_mock: Use mock source (convenience kwarg)+ block_ads: Block ads (convenience kwarg)+ proxy: Proxy setting (convenience kwarg)+ max_age: Cache max age (convenience kwarg)+ store_in_cache: Cache results (convenience kwarg)+ lockdown: Serve only cached results (convenience kwarg)+ profile: Browser profile (convenience kwarg)regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathnamededuplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)zero_data_retention: Whether to delete data after 24 hoursReturns:CrawlResponse with job information-+Raises:ValueError: If request is invalidException: If the crawl operation fails to start"""+ if scrape_options is None:+ scrape_kwargs = {k: v for k, v in dict(+ formats=formats, headers=headers, include_tags=include_tags,+ exclude_tags=exclude_tags, only_main_content=only_main_content,+ wait_for=wait_for, mobile=mobile, parsers=parsers, actions=actions,+ location=location, skip_tls_verification=skip_tls_verification,+ remove_base64_images=remove_base64_images, fast_mode=fast_mode,+ use_mock=use_mock, block_ads=block_ads, proxy=proxy,+ max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,+ profile=profile,+ ).items() if v is not None}+ if scrape_kwargs:+ scrape_options = ScrapeOptions(**scrape_kwargs)+resolved_sitemap = sitemapif resolved_sitemap is None and ignore_sitemap is not None:resolved_sitemap = "skip" if ignore_sitemap else "include"diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 6f6f5fbf8..0164a837e 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -61,6 +61,7 @@ from .methods.aio import agent as async_agent # type: ignore[attr-defined]from .methods.aio import browser as async_browser # type: ignore[attr-defined]from .methods.aio import monitor as async_monitor # type: ignore[attr-defined]+from .client import _SCRAPE_OPTION_KEYSfrom .watcher_async import AsyncWatcherclass AsyncFirecrawlClient:@@ -185,6 +186,16 @@ class AsyncFirecrawlClient:return await async_search.search(self.async_http_client, request)async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:+ _crawl_scrape_keys = _SCRAPE_OPTION_KEYS - {"integration"}+ if kwargs.get("scrape_options") is None:+ scrape_kwargs = {k: kwargs.pop(k) for k in list(kwargs) if k in _crawl_scrape_keys and kwargs[k] is not None}+ if scrape_kwargs:+ kwargs["scrape_options"] = ScrapeOptions(**scrape_kwargs)+ else:+ for k in list(kwargs):+ if k in _crawl_scrape_keys:+ kwargs.pop(k)+sitemap = kwargs.pop("sitemap", None)ignore_sitemap = kwargs.pop("ignore_sitemap", None)if sitemap is None and ignore_sitemap is not None:diff --git a/apps/python-sdk/tests/test_crawl_direct_kwargs.py b/apps/python-sdk/tests/test_crawl_direct_kwargs.pynew file mode 100644index 000000000..000666bd5--- /dev/null+++ b/apps/python-sdk/tests/test_crawl_direct_kwargs.py@@ -0,0 +1,160 @@+"""Tests for direct scrape kwargs on crawl() and start_crawl()."""++import asyncio+import inspect++import pytest++from firecrawl.v2.client import FirecrawlClient, _SCRAPE_OPTION_KEYS+from firecrawl.v2.client_async import AsyncFirecrawlClient+from firecrawl.v2.types import ScrapeOptions+import firecrawl.v2.methods.crawl as crawl_mod+from firecrawl.v2.methods.aio import crawl as async_crawl_mod+++def client():+ return FirecrawlClient(api_key="fc-test", api_url="http://localhost:9")+++def async_client():+ return AsyncFirecrawlClient(api_key="fc-test", api_url="http://localhost:9")+++class TestSyncCrawlDirectKwargs:+ def test_crawl_signature_accepts_formats(self):+ sig = inspect.signature(FirecrawlClient.crawl)+ assert "formats" in sig.parameters++ def test_start_crawl_signature_accepts_formats(self):+ sig = inspect.signature(FirecrawlClient.start_crawl)+ assert "formats" in sig.parameters++ def test_direct_kwargs_build_scrape_options(self, client, monkeypatch):+ captured = {}++ def fake_start_crawl(http, request):+ captured["scrape_options"] = request.scrape_options+ raise ConnectionError("captured")++ monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)++ with pytest.raises(ConnectionError):+ client.start_crawl("https://example.com", formats=["markdown"])++ assert captured["scrape_options"] is not None+ fmt_strs = [str(f) for f in captured["scrape_options"].formats]+ assert "markdown" in fmt_strs++ def test_explicit_scrape_options_wins(self, client, monkeypatch):+ captured = {}++ def fake_start_crawl(http, request):+ captured["scrape_options"] = request.scrape_options+ raise ConnectionError("captured")++ monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)++ with pytest.raises(ConnectionError):+ client.start_crawl(+ "https://example.com",+ scrape_options=ScrapeOptions(formats=["html"]),+ formats=["markdown"],+ )++ fmt_strs = [str(f) for f in captured["scrape_options"].formats]+ assert "html" in fmt_strs+ assert "markdown" not in fmt_strs++ def test_no_scrape_kwargs_means_no_scrape_options(self, client, monkeypatch):+ captured = {}++ def fake_start_crawl(http, request):+ captured["scrape_options"] = request.scrape_options+ raise ConnectionError("captured")++ monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)++ with pytest.raises(ConnectionError):+ client.start_crawl("https://example.com", limit=10)++ assert captured["scrape_options"] is None++ def test_crawl_also_builds_scrape_options(self, client, monkeypatch):+ captured = {}++ def fake_crawl(http, request, poll_interval=2, timeout=None, request_timeout=None):+ captured["scrape_options"] = request.scrape_options+ raise ConnectionError("captured")++ monkeypatch.setattr(crawl_mod, "crawl", fake_crawl)++ with pytest.raises(ConnectionError):+ client.crawl("https://example.com", formats=["markdown"], only_main_content=True)++ assert captured["scrape_options"] is not None+ assert captured["scrape_options"].only_main_content is True+++class TestAsyncCrawlDirectKwargs:+ def test_async_direct_kwargs_build_scrape_options(self, async_client, monkeypatch):+ captured = {}++ async def fake_start_crawl(http, request):+ captured["scrape_options"] = request.scrape_options+ raise ConnectionError("captured")++ monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)++ with pytest.raises(ConnectionError):+ asyncio.get_event_loop().run_until_complete(+ async_client.start_crawl("https://example.com", formats=["markdown"])+ )++ assert captured["scrape_options"] is not None+ fmt_strs = [str(f) for f in captured["scrape_options"].formats]+ assert "markdown" in fmt_strs++ def test_async_explicit_scrape_options_wins(self, async_client, monkeypatch):+ captured = {}++ async def fake_start_crawl(http, request):+ captured["scrape_options"] = request.scrape_options+ raise ConnectionError("captured")++ monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)++ with pytest.raises(ConnectionError):+ asyncio.get_event_loop().run_until_complete(+ async_client.start_crawl(+ "https://example.com",+ scrape_options=ScrapeOptions(formats=["html"]),+ formats=["markdown"],+ )+ )++ fmt_strs = [str(f) for f in captured["scrape_options"].formats]+ assert "html" in fmt_strs++ def test_async_integration_preserved_as_crawl_param(self, async_client, monkeypatch):+ captured = {}++ async def fake_start_crawl(http, request):+ captured["scrape_options"] = request.scrape_options+ captured["integration"] = request.integration+ raise ConnectionError("captured")++ monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)++ with pytest.raises(ConnectionError):+ asyncio.get_event_loop().run_until_complete(+ async_client.start_crawl(+ "https://example.com",+ formats=["markdown"],+ integration="test-int",+ )+ )++ assert captured["integration"] == "test-int"+ assert captured["scrape_options"] is not None