Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(python-sdk): accept direct scrape kwargs in crawl() and start_crawl()

Himadri Mbb148f83b7a6
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index c10a62020..c3a2015da 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -67,6 +67,16 @@ from .methods import browser as browser_module
from .methods import monitor as monitor_module
from .watcher import Watcher
+_SCRAPE_OPTION_KEYS = frozenset({
+ "formats", "headers", "include_tags", "exclude_tags",
+ "only_main_content", "timeout", "wait_for", "mobile",
+ "parsers", "actions", "location", "skip_tls_verification",
+ "remove_base64_images", "fast_mode", "use_mock", "block_ads",
+ "proxy", "max_age", "store_in_cache", "lockdown", "profile",
+ "integration",
+})
+
+
class FirecrawlClient:
"""
Main Firecrawl v2 API client.
@@ -377,6 +387,26 @@ class FirecrawlClient:
max_concurrency: Optional[int] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
scrape_options: Optional[ScrapeOptions] = None,
+ formats: Optional[List['FormatOption']] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ wait_for: Optional[int] = None,
+ mobile: Optional[bool] = None,
+ parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,
+ actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,
+ location: Optional['Location'] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ fast_mode: Optional[bool] = None,
+ use_mock: Optional[str] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[str] = None,
+ max_age: Optional[int] = None,
+ store_in_cache: Optional[bool] = None,
+ lockdown: Optional[bool] = None,
+ profile: Optional[Dict[str, Any]] = None,
regex_on_full_url: bool = False,
deduplicate_similar_urls: bool = True,
zero_data_retention: bool = False,
@@ -405,22 +435,56 @@ class FirecrawlClient:
delay: Delay in seconds between scrapes
max_concurrency: Maximum number of concurrent scrapes
webhook: Webhook configuration for notifications
- scrape_options: Page scraping configuration
+ scrape_options: Page scraping configuration (takes precedence over direct scrape kwargs)
+ formats: Output formats (convenience kwarg, ignored if scrape_options is set)
+ headers: HTTP headers for scraping (convenience kwarg)
+ include_tags: HTML tags to include (convenience kwarg)
+ exclude_tags: HTML tags to exclude (convenience kwarg)
+ only_main_content: Restrict to main content (convenience kwarg)
+ wait_for: Wait condition in ms (convenience kwarg)
+ mobile: Emulate mobile viewport (convenience kwarg)
+ parsers: Parser list (convenience kwarg)
+ actions: Browser actions (convenience kwarg)
+ location: Location settings (convenience kwarg)
+ skip_tls_verification: Skip TLS verification (convenience kwarg)
+ remove_base64_images: Remove base64 images (convenience kwarg)
+ fast_mode: Prefer faster modes (convenience kwarg)
+ use_mock: Use mock source (convenience kwarg)
+ block_ads: Block ads (convenience kwarg)
+ proxy: Proxy setting (convenience kwarg)
+ max_age: Cache max age (convenience kwarg)
+ store_in_cache: Cache results (convenience kwarg)
+ lockdown: Serve only cached results (convenience kwarg)
+ profile: Browser profile (convenience kwarg)
regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
deduplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)
zero_data_retention: Whether to delete data after 24 hours
poll_interval: Seconds between status checks
timeout: Maximum seconds to wait for the entire crawl job to complete (None for no timeout)
request_timeout: Timeout (in seconds) for each individual HTTP request, including pagination requests when fetching results. If there are multiple pages, each page request gets this timeout
-
+
Returns:
CrawlJob when job completes
-
+
Raises:
ValueError: If request is invalid
Exception: If the crawl fails to start or complete
TimeoutError: If timeout is reached
"""
+ if scrape_options is None:
+ scrape_kwargs = {k: v for k, v in dict(
+ formats=formats, headers=headers, include_tags=include_tags,
+ exclude_tags=exclude_tags, only_main_content=only_main_content,
+ wait_for=wait_for, mobile=mobile, parsers=parsers, actions=actions,
+ location=location, skip_tls_verification=skip_tls_verification,
+ remove_base64_images=remove_base64_images, fast_mode=fast_mode,
+ use_mock=use_mock, block_ads=block_ads, proxy=proxy,
+ max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,
+ profile=profile,
+ ).items() if v is not None}
+ if scrape_kwargs:
+ scrape_options = ScrapeOptions(**scrape_kwargs)
+
resolved_sitemap = sitemap
if resolved_sitemap is None and ignore_sitemap is not None:
resolved_sitemap = "skip" if ignore_sitemap else "include"
@@ -479,6 +543,26 @@ class FirecrawlClient:
max_concurrency: Optional[int] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
scrape_options: Optional[ScrapeOptions] = None,
+ formats: Optional[List['FormatOption']] = None,
+ headers: Optional[Dict[str, str]] = None,
+ include_tags: Optional[List[str]] = None,
+ exclude_tags: Optional[List[str]] = None,
+ only_main_content: Optional[bool] = None,
+ wait_for: Optional[int] = None,
+ mobile: Optional[bool] = None,
+ parsers: Optional[Union[List[str], List[Union[str, PDFParser]]]] = None,
+ actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None,
+ location: Optional['Location'] = None,
+ skip_tls_verification: Optional[bool] = None,
+ remove_base64_images: Optional[bool] = None,
+ fast_mode: Optional[bool] = None,
+ use_mock: Optional[str] = None,
+ block_ads: Optional[bool] = None,
+ proxy: Optional[str] = None,
+ max_age: Optional[int] = None,
+ store_in_cache: Optional[bool] = None,
+ lockdown: Optional[bool] = None,
+ profile: Optional[Dict[str, Any]] = None,
regex_on_full_url: bool = False,
deduplicate_similar_urls: bool = True,
zero_data_retention: bool = False,
@@ -504,18 +588,52 @@ class FirecrawlClient:
delay: Delay in seconds between scrapes
max_concurrency: Maximum number of concurrent scrapes
webhook: Webhook configuration for notifications
- scrape_options: Page scraping configuration
+ scrape_options: Page scraping configuration (takes precedence over direct scrape kwargs)
+ formats: Output formats (convenience kwarg, ignored if scrape_options is set)
+ headers: HTTP headers for scraping (convenience kwarg)
+ include_tags: HTML tags to include (convenience kwarg)
+ exclude_tags: HTML tags to exclude (convenience kwarg)
+ only_main_content: Restrict to main content (convenience kwarg)
+ wait_for: Wait condition in ms (convenience kwarg)
+ mobile: Emulate mobile viewport (convenience kwarg)
+ parsers: Parser list (convenience kwarg)
+ actions: Browser actions (convenience kwarg)
+ location: Location settings (convenience kwarg)
+ skip_tls_verification: Skip TLS verification (convenience kwarg)
+ remove_base64_images: Remove base64 images (convenience kwarg)
+ fast_mode: Prefer faster modes (convenience kwarg)
+ use_mock: Use mock source (convenience kwarg)
+ block_ads: Block ads (convenience kwarg)
+ proxy: Proxy setting (convenience kwarg)
+ max_age: Cache max age (convenience kwarg)
+ store_in_cache: Cache results (convenience kwarg)
+ lockdown: Serve only cached results (convenience kwarg)
+ profile: Browser profile (convenience kwarg)
regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
deduplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)
zero_data_retention: Whether to delete data after 24 hours
Returns:
CrawlResponse with job information
-
+
Raises:
ValueError: If request is invalid
Exception: If the crawl operation fails to start
"""
+ if scrape_options is None:
+ scrape_kwargs = {k: v for k, v in dict(
+ formats=formats, headers=headers, include_tags=include_tags,
+ exclude_tags=exclude_tags, only_main_content=only_main_content,
+ wait_for=wait_for, mobile=mobile, parsers=parsers, actions=actions,
+ location=location, skip_tls_verification=skip_tls_verification,
+ remove_base64_images=remove_base64_images, fast_mode=fast_mode,
+ use_mock=use_mock, block_ads=block_ads, proxy=proxy,
+ max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,
+ profile=profile,
+ ).items() if v is not None}
+ if scrape_kwargs:
+ scrape_options = ScrapeOptions(**scrape_kwargs)
+
resolved_sitemap = sitemap
if resolved_sitemap is None and ignore_sitemap is not None:
resolved_sitemap = "skip" if ignore_sitemap else "include"
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index 6f6f5fbf8..0164a837e 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -61,6 +61,7 @@ from .methods.aio import agent as async_agent # type: ignore[attr-defined]
from .methods.aio import browser as async_browser # type: ignore[attr-defined]
from .methods.aio import monitor as async_monitor # type: ignore[attr-defined]
+from .client import _SCRAPE_OPTION_KEYS
from .watcher_async import AsyncWatcher
class AsyncFirecrawlClient:
@@ -185,6 +186,16 @@ class AsyncFirecrawlClient:
return await async_search.search(self.async_http_client, request)
async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:
+ _crawl_scrape_keys = _SCRAPE_OPTION_KEYS - {"integration"}
+ if kwargs.get("scrape_options") is None:
+ scrape_kwargs = {k: kwargs.pop(k) for k in list(kwargs) if k in _crawl_scrape_keys and kwargs[k] is not None}
+ if scrape_kwargs:
+ kwargs["scrape_options"] = ScrapeOptions(**scrape_kwargs)
+ else:
+ for k in list(kwargs):
+ if k in _crawl_scrape_keys:
+ kwargs.pop(k)
+
sitemap = kwargs.pop("sitemap", None)
ignore_sitemap = kwargs.pop("ignore_sitemap", None)
if sitemap is None and ignore_sitemap is not None:
diff --git a/apps/python-sdk/tests/test_crawl_direct_kwargs.py b/apps/python-sdk/tests/test_crawl_direct_kwargs.py
new file mode 100644
index 000000000..000666bd5
--- /dev/null
+++ b/apps/python-sdk/tests/test_crawl_direct_kwargs.py
@@ -0,0 +1,160 @@
+"""Tests for direct scrape kwargs on crawl() and start_crawl()."""
+
+import asyncio
+import inspect
+
+import pytest
+
+from firecrawl.v2.client import FirecrawlClient, _SCRAPE_OPTION_KEYS
+from firecrawl.v2.client_async import AsyncFirecrawlClient
+from firecrawl.v2.types import ScrapeOptions
+import firecrawl.v2.methods.crawl as crawl_mod
+from firecrawl.v2.methods.aio import crawl as async_crawl_mod
+
+
+def client():
+ return FirecrawlClient(api_key="fc-test", api_url="http://localhost:9")
+
+
+def async_client():
+ return AsyncFirecrawlClient(api_key="fc-test", api_url="http://localhost:9")
+
+
+class TestSyncCrawlDirectKwargs:
+ def test_crawl_signature_accepts_formats(self):
+ sig = inspect.signature(FirecrawlClient.crawl)
+ assert "formats" in sig.parameters
+
+ def test_start_crawl_signature_accepts_formats(self):
+ sig = inspect.signature(FirecrawlClient.start_crawl)
+ assert "formats" in sig.parameters
+
+ def test_direct_kwargs_build_scrape_options(self, client, monkeypatch):
+ captured = {}
+
+ def fake_start_crawl(http, request):
+ captured["scrape_options"] = request.scrape_options
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
+
+ with pytest.raises(ConnectionError):
+ client.start_crawl("https://example.com", formats=["markdown"])
+
+ assert captured["scrape_options"] is not None
+ fmt_strs = [str(f) for f in captured["scrape_options"].formats]
+ assert "markdown" in fmt_strs
+
+ def test_explicit_scrape_options_wins(self, client, monkeypatch):
+ captured = {}
+
+ def fake_start_crawl(http, request):
+ captured["scrape_options"] = request.scrape_options
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
+
+ with pytest.raises(ConnectionError):
+ client.start_crawl(
+ "https://example.com",
+ scrape_options=ScrapeOptions(formats=["html"]),
+ formats=["markdown"],
+ )
+
+ fmt_strs = [str(f) for f in captured["scrape_options"].formats]
+ assert "html" in fmt_strs
+ assert "markdown" not in fmt_strs
+
+ def test_no_scrape_kwargs_means_no_scrape_options(self, client, monkeypatch):
+ captured = {}
+
+ def fake_start_crawl(http, request):
+ captured["scrape_options"] = request.scrape_options
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
+
+ with pytest.raises(ConnectionError):
+ client.start_crawl("https://example.com", limit=10)
+
+ assert captured["scrape_options"] is None
+
+ def test_crawl_also_builds_scrape_options(self, client, monkeypatch):
+ captured = {}
+
+ def fake_crawl(http, request, poll_interval=2, timeout=None, request_timeout=None):
+ captured["scrape_options"] = request.scrape_options
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(crawl_mod, "crawl", fake_crawl)
+
+ with pytest.raises(ConnectionError):
+ client.crawl("https://example.com", formats=["markdown"], only_main_content=True)
+
+ assert captured["scrape_options"] is not None
+ assert captured["scrape_options"].only_main_content is True
+
+
+class TestAsyncCrawlDirectKwargs:
+ def test_async_direct_kwargs_build_scrape_options(self, async_client, monkeypatch):
+ captured = {}
+
+ async def fake_start_crawl(http, request):
+ captured["scrape_options"] = request.scrape_options
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
+
+ with pytest.raises(ConnectionError):
+ asyncio.get_event_loop().run_until_complete(
+ async_client.start_crawl("https://example.com", formats=["markdown"])
+ )
+
+ assert captured["scrape_options"] is not None
+ fmt_strs = [str(f) for f in captured["scrape_options"].formats]
+ assert "markdown" in fmt_strs
+
+ def test_async_explicit_scrape_options_wins(self, async_client, monkeypatch):
+ captured = {}
+
+ async def fake_start_crawl(http, request):
+ captured["scrape_options"] = request.scrape_options
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
+
+ with pytest.raises(ConnectionError):
+ asyncio.get_event_loop().run_until_complete(
+ async_client.start_crawl(
+ "https://example.com",
+ scrape_options=ScrapeOptions(formats=["html"]),
+ formats=["markdown"],
+ )
+ )
+
+ fmt_strs = [str(f) for f in captured["scrape_options"].formats]
+ assert "html" in fmt_strs
+
+ def test_async_integration_preserved_as_crawl_param(self, async_client, monkeypatch):
+ captured = {}
+
+ async def fake_start_crawl(http, request):
+ captured["scrape_options"] = request.scrape_options
+ captured["integration"] = request.integration
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
+
+ with pytest.raises(ConnectionError):
+ asyncio.get_event_loop().run_until_complete(
+ async_client.start_crawl(
+ "https://example.com",
+ formats=["markdown"],
+ integration="test-int",
+ )
+ )
+
+ assert captured["integration"] == "test-int"
+ assert captured["scrape_options"] is not None