Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(python-sdk): add get_parse_formats for GET /v2/parse/formats (#4912)
Gergő Móricz37d804c7d230
diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex 708521597..d6ded65d2 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -97,6 +97,13 @@ doc = firecrawl.parse(print(doc.markdown)```+To see which file formats `parse` accepts, call `get_parse_formats` (also available on `AsyncFirecrawl`). Each entry has `format`, `kind` (`"document"` or `"image"`), `extensions`, `mime_types`, and `available`, which is `False` when a format is disabled on the deployment.++```python+for f in firecrawl.get_parse_formats():+ print(f.format, f.kind, f.extensions, f.mime_types, f.available)+```+### Crawling a WebsiteTo crawl a website, use the `crawl` method. It takes the starting URL and optional parameters as arguments. You can control depth, limits, formats, and more.diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 0a3d967a4..1d2aa7d53 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -28,7 +28,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.46.0"+__version__ = "4.46.1"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_parse.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_parse.pyindex 92b4b5eaf..d96f52229 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_parse.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_parse.py@@ -3,6 +3,7 @@ import pytestfrom dotenv import load_dotenvfrom firecrawl import AsyncFirecrawlfrom firecrawl.v2.types import Document, ScrapeOptions+from firecrawl.v2.utils.error_handler import FirecrawlErrorload_dotenv()API_KEY = (os.getenv("API_KEY") or "").strip()@@ -25,3 +26,20 @@ async def test_async_parse_uploaded_html():assert isinstance(doc, Document)assert doc.markdown is not Noneassert "Async Python Parse E2E" in doc.markdown+++ not API_KEY or not API_URL,+ reason="API_KEY and API_URL are required for async parse e2e tests",+)+async def test_async_get_parse_formats():+ client = AsyncFirecrawl(api_key=API_KEY, api_url=API_URL)+ try:+ formats = await client.get_parse_formats()+ except FirecrawlError as e:+ if e.status_code == 404:+ pytest.skip("GET /v2/parse/formats is not deployed on this API")+ raise+ assert len(formats) > 0+ assert any(f.format == "pdf" and f.kind == "document" for f in formats)diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_parse.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_parse.pyindex 6095d313e..8ea0b14ca 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_parse.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_parse.py@@ -3,6 +3,7 @@ import pytestfrom dotenv import load_dotenvfrom firecrawl import Firecrawlfrom firecrawl.v2.types import Document, ScrapeOptions+from firecrawl.v2.utils.error_handler import FirecrawlErrorload_dotenv()API_KEY = (os.getenv("API_KEY") or "").strip()@@ -27,3 +28,13 @@ class TestParseE2E:assert isinstance(doc, Document)assert doc.markdown is not Noneassert "Python Parse E2E" in doc.markdown++ def test_get_parse_formats(self):+ try:+ formats = self.client.get_parse_formats()+ except FirecrawlError as e:+ if e.status_code == 404:+ pytest.skip("GET /v2/parse/formats is not deployed on this API")+ raise+ assert len(formats) > 0+ assert any(f.format == "pdf" and f.kind == "document" for f in formats)diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_formats.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_formats.pynew file mode 100644index 000000000..b2f500ba8--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_parse_formats.py@@ -0,0 +1,163 @@+import asyncio+from unittest.mock import Mock++import httpx+import pytest++from firecrawl import AsyncFirecrawl, Firecrawl+from firecrawl.v2.types import ParseFormat+from firecrawl.v2.utils.error_handler import FirecrawlError, InternalServerError, UnauthorizedError++API_URL = "https://api.example.test"+API_KEY = "fc-test-key"++FORMATS_BODY = {+ "success": True,+ "data": {+ "formats": [+ {+ "format": "pdf",+ "kind": "document",+ "extensions": [".pdf"],+ "mimeTypes": ["application/pdf"],+ "available": True,+ },+ {+ "format": "png",+ "kind": "image",+ "extensions": [".png"],+ "mimeTypes": ["image/png"],+ "available": False,+ },+ {+ "format": "glb",+ "kind": "model",+ "extensions": [".glb"],+ "mimeTypes": ["model/gltf-binary"],+ "available": True,+ "maxBytes": 1024,+ },+ ]+ },+}+++def _assert_formats(formats):+ assert all(isinstance(f, ParseFormat) for f in formats)+ pdf, png, glb = formats+ assert pdf.format == "pdf"+ assert pdf.kind == "document"+ assert pdf.extensions == [".pdf"]+ assert pdf.mime_types == ["application/pdf"]+ assert pdf.available is True+ assert png.kind == "image"+ assert png.mime_types == ["image/png"]+ assert png.available is False+ assert glb.kind == "model"+ assert not hasattr(glb, "maxBytes")+++def _sync_response(status_code, body):+ response = Mock()+ response.status_code = status_code+ response.ok = status_code < 400+ response.json.return_value = body+ return response+++def test_get_parse_formats_sync(monkeypatch):+ get = Mock(return_value=_sync_response(200, FORMATS_BODY))+ monkeypatch.setattr("firecrawl.v2.utils.http_client.requests.get", get)++ formats = Firecrawl(api_key=API_KEY, api_url=API_URL).get_parse_formats()++ get.assert_called_once()+ assert get.call_args.args[0] == f"{API_URL}/v2/parse/formats"+ assert get.call_args.kwargs["headers"]["Authorization"] == f"Bearer {API_KEY}"+ _assert_formats(formats)+++def test_get_parse_formats_sync_v2_surface(monkeypatch):+ monkeypatch.setattr(+ "firecrawl.v2.utils.http_client.requests.get",+ Mock(return_value=_sync_response(200, FORMATS_BODY)),+ )+ _assert_formats(Firecrawl(api_key=API_KEY, api_url=API_URL).v2.get_parse_formats())+++def test_get_parse_formats_sync_unauthorized(monkeypatch):+ monkeypatch.setattr(+ "firecrawl.v2.utils.http_client.requests.get",+ Mock(return_value=_sync_response(401, {"success": False, "error": "Unauthorized"})),+ )+ with pytest.raises(UnauthorizedError):+ Firecrawl(api_key=API_KEY, api_url=API_URL).get_parse_formats()+++def test_get_parse_formats_sync_unsuccessful_body(monkeypatch):+ monkeypatch.setattr(+ "firecrawl.v2.utils.http_client.requests.get",+ Mock(return_value=_sync_response(200, {"success": False, "error": "nope"})),+ )+ with pytest.raises(FirecrawlError):+ Firecrawl(api_key=API_KEY, api_url=API_URL).get_parse_formats()+++def _run_async(handler, call):+ async def run():+ client = AsyncFirecrawl(api_key=API_KEY, api_url=API_URL)+ http = client._v2_client.async_http_client+ headers = http._client.headers+ await http.close()+ http._client = httpx.AsyncClient(+ base_url=API_URL,+ headers=headers,+ transport=httpx.MockTransport(handler),+ )+ try:+ return await call(client)+ finally:+ await http.close()++ return asyncio.run(run())+++def test_get_parse_formats_async():+ requests_seen = []++ def handler(request: httpx.Request) -> httpx.Response:+ requests_seen.append(request)+ return httpx.Response(200, json=FORMATS_BODY)++ formats = _run_async(handler, lambda client: client.get_parse_formats())++ assert len(requests_seen) == 1+ request = requests_seen[0]+ assert request.method == "GET"+ assert str(request.url) == f"{API_URL}/v2/parse/formats"+ assert request.headers["Authorization"] == f"Bearer {API_KEY}"+ _assert_formats(formats)+++def test_get_parse_formats_async_v2_surface():+ formats = _run_async(+ lambda request: httpx.Response(200, json=FORMATS_BODY),+ lambda client: client.v2.get_parse_formats(),+ )+ _assert_formats(formats)+++def test_get_parse_formats_async_server_error():+ with pytest.raises(InternalServerError):+ _run_async(+ lambda request: httpx.Response(500, json={"success": False, "error": "boom"}),+ lambda client: client.get_parse_formats(),+ )+++def test_get_parse_formats_async_unsuccessful_body():+ with pytest.raises(FirecrawlError):+ _run_async(+ lambda request: httpx.Response(200, json={"success": False, "error": "nope"}),+ lambda client: client.get_parse_formats(),+ )diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex b319a4db9..1a3338d47 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -79,6 +79,7 @@ class V2Proxy:self.scrape_execute = self.interactself.delete_scrape_browser = self.stop_interactionself.parse = client_instance.parse+ self.get_parse_formats = client_instance.get_parse_formatsself.search = client_instance.searchself.developer_search = client_instance.developer_searchself.crawl = client_instance.crawl@@ -171,6 +172,7 @@ class AsyncV2Proxy:self.scrape_execute = self.interactself.delete_scrape_browser = self.stop_interactionself.parse = client_instance.parse+ self.get_parse_formats = client_instance.get_parse_formatsself.search = client_instance.searchself.developer_search = client_instance.developer_searchself.crawl = client_instance.crawl@@ -287,6 +289,7 @@ class Firecrawl:self.scrape_execute = self.interactself.delete_scrape_browser = self.stop_interactionself.parse = self._v2_client.parse+ self.get_parse_formats = self._v2_client.get_parse_formatsself.search = self._v2_client.searchself.developer_search = self._v2_client.developer_searchself.map = self._v2_client.map@@ -431,6 +434,7 @@ class AsyncFirecrawl:self.scrape_execute = self.interactself.delete_scrape_browser = self.stop_interactionself.parse = self._v2_client.parse+ self.get_parse_formats = self._v2_client.get_parse_formatsself.search = self._v2_client.searchself.developer_search = self._v2_client.developer_searchself.map = self._v2_client.mapdiff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 2c690d98f..8a6cf6994 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -9,6 +9,7 @@ from pathlib import Pathfrom typing import Optional, List, Dict, Any, Callable, Union, Literal, BinaryIOfrom .types import (ClientConfig,+ ParseFormat,ParseOptions,ScrapeOptions,Document,@@ -468,6 +469,16 @@ class FirecrawlClient:content_type=content_type,)+ def get_parse_formats(self) -> List[ParseFormat]:+ """+ List the file formats the parse endpoint accepts.++ Returns:+ List of ParseFormat entries. ``available`` is False for formats+ that are known but disabled on this deployment.+ """+ return parse_module.get_parse_formats(self.http_client)+def search(self,diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 6a6e44fa1..4d3f552be 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -8,6 +8,7 @@ import timefrom pathlib import Pathfrom typing import Optional, List, Dict, Any, Union, Callable, Literal, BinaryIOfrom .types import (+ ParseFormat,ParseOptions,ScrapeOptions,CrawlRequest,@@ -304,6 +305,10 @@ class AsyncFirecrawlClient:content_type=content_type,)+ async def get_parse_formats(self) -> List[ParseFormat]:+ """List the file formats the parse endpoint accepts."""+ return await async_parse.get_parse_formats(self.async_http_client)+# Searchasync def search(diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/parse.py b/apps/python-sdk/firecrawl/v2/methods/aio/parse.pyindex 0faf86b9c..50c5e5e1d 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/parse.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/parse.py@@ -1,9 +1,9 @@import asynciofrom functools import partialimport json-from typing import Optional, Dict, Any, Tuple+from typing import Optional, Dict, Any, List, Tuple-from ...types import Document, ParseOptions+from ...types import Document, ParseFormat, ParseOptionsfrom ...utils.agent_hints import agent_hint_metadatafrom ...utils.normalize import normalize_document_inputfrom ...utils.error_handler import handle_response_error@@ -64,3 +64,14 @@ async def parse(document_data = body.get("data", {})normalized = {**normalize_document_input(document_data), **agent_hint_metadata(body)}return Document(**normalized)+++async def get_parse_formats(client: AsyncHttpClient) -> List[ParseFormat]:+ resp = await client.get("/v2/parse/formats")+ if resp.status_code >= 400:+ handle_response_error(resp, "get parse formats")+ body = resp.json()+ if not body.get("success"):+ handle_response_error(resp, "get parse formats")+ data = body.get("data") or {}+ return [ParseFormat.model_validate(item) for item in data.get("formats") or []]diff --git a/apps/python-sdk/firecrawl/v2/methods/parse.py b/apps/python-sdk/firecrawl/v2/methods/parse.pyindex eb13fc530..1fec3743f 100644--- a/apps/python-sdk/firecrawl/v2/methods/parse.py+++ b/apps/python-sdk/firecrawl/v2/methods/parse.py@@ -5,9 +5,9 @@ Parse (multipart upload) functionality for Firecrawl v2 API.import jsonimport mimetypesfrom pathlib import Path-from typing import Optional, Dict, Any, BinaryIO, Union, Tuple+from typing import Optional, Dict, Any, BinaryIO, List, Union, Tuple-from ..types import Document, ParseOptions+from ..types import Document, ParseFormat, ParseOptionsfrom ..utils.agent_hints import agent_hint_metadatafrom ..utils.normalize import normalize_document_inputfrom ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options@@ -166,3 +166,14 @@ def parse(document_data = body.get("data", {})normalized = {**normalize_document_input(document_data), **agent_hint_metadata(body)}return Document(**normalized)+++def get_parse_formats(client: HttpClient) -> List[ParseFormat]:+ resp = client.get("/v2/parse/formats")+ if not resp.ok:+ handle_response_error(resp, "get parse formats")+ body = resp.json()+ if not body.get("success"):+ handle_response_error(resp, "get parse formats")+ data = body.get("data") or {}+ return [ParseFormat.model_validate(item) for item in data.get("formats") or []]diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex f01591e24..04a06681a 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -955,6 +955,21 @@ class ScrapeOptions(BaseModel):ParseOptions = ScrapeOptions+ParseFormatKind = Union[Literal["document", "image"], str]+++class ParseFormat(BaseModel):+ """A file format accepted by the parse endpoint."""++ model_config = {"populate_by_name": True}++ format: str+ kind: ParseFormatKind+ extensions: List[str]+ mime_types: List[str] = Field(alias="mimeTypes")+ available: bool++class ScrapeRequest(BaseModel):"""Request for scraping a single URL."""