Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(python-sdk)fix: added normalizer for document.metadata, also added m… (#1986)
Rafael Millerf5d901b9d9be
diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.pyindex 77b038c1d..0396d4601 100644--- a/apps/python-sdk/example.py+++ b/apps/python-sdk/example.py@@ -24,6 +24,8 @@ def main():# Scrapedoc = firecrawl.scrape("https://docs.firecrawl.dev", formats=["markdown"])print("scrape:", doc.markdown)+ # doc.metadata_dict is a dict, doc.metadata_typed is a DocumentMetadata object+ print(doc.metadata_dict.get("source_url"))# Crawl (waits until terminal state)crawl_job = firecrawl.crawl("https://docs.firecrawl.dev", limit=3, poll_interval=1, timeout=120)diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 99e58a1da..5c4e14037 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "3.0.3"+__version__ = "3.1.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.pyindex f174de6bb..da33593fe 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_crawl.py@@ -96,7 +96,6 @@ async def test_async_get_crawl_status_shape():assert status.status in ("scraping", "completed", "failed")assert status.completed >= 0assert status.expires_at is not None- assert status.next is not Noneassert isinstance(status.data, list)diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex fe0476eea..12aa55d30 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -25,6 +25,7 @@ import loggingfrom .v1 import V1FirecrawlApp, AsyncV1FirecrawlAppfrom .v2 import FirecrawlClient as V2FirecrawlClientfrom .v2.client_async import AsyncFirecrawlClient+from .v2.types import Documentlogger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.pyindex fc93ad874..e345b2ce2 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py@@ -14,6 +14,7 @@ from ...types import (from ...utils.error_handler import handle_response_errorfrom ...utils.validation import prepare_scrape_optionsfrom ...utils.http_client_async import AsyncHttpClient+from ...utils.normalize import normalize_document_inputdef _prepare_crawl_request(request: CrawlRequest) -> dict:@@ -76,11 +77,7 @@ async def get_crawl_status(client: AsyncHttpClient, job_id: str) -> CrawlJob:documents = []for doc_data in body.get("data", []):if isinstance(doc_data, dict):- normalized = dict(doc_data)- if 'rawHtml' in normalized and 'raw_html' not in normalized:- normalized['raw_html'] = normalized.pop('rawHtml')- if 'changeTracking' in normalized and 'change_tracking' not in normalized:- normalized['change_tracking'] = normalized.pop('changeTracking')+ normalized = normalize_document_input(doc_data)documents.append(Document(**normalized))return CrawlJob(status=body.get("status"),diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/scrape.py b/apps/python-sdk/firecrawl/v2/methods/aio/scrape.pyindex 956d37b51..848f6e278 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/scrape.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/scrape.py@@ -1,5 +1,6 @@from typing import Optional, Dict, Anyfrom ...types import ScrapeOptions, Document+from ...utils.normalize import normalize_document_inputfrom ...utils.error_handler import handle_response_errorfrom ...utils.validation import prepare_scrape_options, validate_scrape_optionsfrom ...utils.http_client_async import AsyncHttpClient@@ -27,10 +28,6 @@ async def scrape(client: AsyncHttpClient, url: str, options: Optional[ScrapeOptiif not body.get("success"):raise Exception(body.get("error", "Unknown error occurred"))document_data = body.get("data", {})- normalized = dict(document_data)- if 'rawHtml' in normalized and 'raw_html' not in normalized:- normalized['raw_html'] = normalized.pop('rawHtml')- if 'changeTracking' in normalized and 'change_tracking' not in normalized:- normalized['change_tracking'] = normalized.pop('changeTracking')+ normalized = normalize_document_input(document_data)return Document(**normalized)diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/search.py b/apps/python-sdk/firecrawl/v2/methods/aio/search.pyindex d43a522db..bafd7f574 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/search.py@@ -1,5 +1,6 @@from typing import Dict, Anyfrom ...types import SearchRequest, SearchData, SearchResult, Document+from ...utils.normalize import normalize_document_inputfrom ...utils.http_client_async import AsyncHttpClientfrom ...utils.error_handler import handle_response_errorfrom ...utils.validation import prepare_scrape_options, validate_scrape_options@@ -38,11 +39,7 @@ async def search(client: AsyncHttpClient, request: SearchRequest) -> SearchData:if request.scrape_options is not None and any(key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'summary', 'screenshot', 'changeTracking']):- normalized = dict(doc_data)- if 'rawHtml' in normalized and 'raw_html' not in normalized:- normalized['raw_html'] = normalized.pop('rawHtml')- if 'changeTracking' in normalized and 'change_tracking' not in normalized:- normalized['change_tracking'] = normalized.pop('changeTracking')+ normalized = normalize_document_input(doc_data)results.append(Document(**normalized))else:results.append(SearchResult(diff --git a/apps/python-sdk/firecrawl/v2/methods/batch.py b/apps/python-sdk/firecrawl/v2/methods/batch.pyindex 311bea86b..82c8d4460 100644--- a/apps/python-sdk/firecrawl/v2/methods/batch.py+++ b/apps/python-sdk/firecrawl/v2/methods/batch.py@@ -13,6 +13,7 @@ from ..types import (WebhookConfig,)from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options+from ..utils.normalize import normalize_document_inputfrom ..types import CrawlErrorsResponse@@ -107,11 +108,7 @@ def get_batch_scrape_status(documents: List[Document] = []for doc in body.get("data", []) or []:if isinstance(doc, dict):- normalized = dict(doc)- if 'rawHtml' in normalized and 'raw_html' not in normalized:- normalized['raw_html'] = normalized.pop('rawHtml')- if 'changeTracking' in normalized and 'change_tracking' not in normalized:- normalized['change_tracking'] = normalized.pop('changeTracking')+ normalized = normalize_document_input(doc)documents.append(Document(**normalized))return BatchScrapeJob(diff --git a/apps/python-sdk/firecrawl/v2/methods/crawl.py b/apps/python-sdk/firecrawl/v2/methods/crawl.pyindex 6f9fb4d04..99ee81926 100644--- a/apps/python-sdk/firecrawl/v2/methods/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/crawl.py@@ -11,6 +11,7 @@ from ..types import (WebhookConfig, CrawlErrorsResponse, ActiveCrawlsResponse, ActiveCrawl)from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options+from ..utils.normalize import normalize_document_inputdef _validate_crawl_request(request: CrawlRequest) -> None:@@ -173,7 +174,7 @@ def get_crawl_status(client: HttpClient, job_id: str) -> CrawlJob:# but we'll handle it gracefullycontinueelse:- documents.append(Document(**doc_data))+ documents.append(Document(**normalize_document_input(doc_data)))# Create CrawlJob with current status and datareturn CrawlJob(diff --git a/apps/python-sdk/firecrawl/v2/methods/scrape.py b/apps/python-sdk/firecrawl/v2/methods/scrape.pyindex 408d8d378..733f5e318 100644--- a/apps/python-sdk/firecrawl/v2/methods/scrape.py+++ b/apps/python-sdk/firecrawl/v2/methods/scrape.py@@ -4,6 +4,7 @@ Scraping functionality for Firecrawl v2 API.from typing import Optional, Dict, Anyfrom ..types import ScrapeOptions, Document+from ..utils.normalize import normalize_document_inputfrom ..utils import HttpClient, handle_response_error, prepare_scrape_options, validate_scrape_options@@ -59,10 +60,5 @@ def scrape(client: HttpClient, url: str, options: Optional[ScrapeOptions] = Noneraise Exception(body.get("error", "Unknown error occurred"))document_data = body.get("data", {})- # Normalize keys for Document (no Pydantic aliases)- normalized = dict(document_data)- if 'rawHtml' in normalized and 'raw_html' not in normalized:- normalized['raw_html'] = normalized.pop('rawHtml')- if 'changeTracking' in normalized and 'change_tracking' not in normalized:- normalized['change_tracking'] = normalized.pop('changeTracking')+ normalized = normalize_document_input(document_data)return Document(**normalized)\ No newline at end of filediff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.pyindex 13e5fc3f4..d8e05b118 100644--- a/apps/python-sdk/firecrawl/v2/methods/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/search.py@@ -4,6 +4,7 @@ Search functionality for Firecrawl v2 API.from typing import Optional, Dict, Any, Unionfrom ..types import SearchRequest, SearchData, SearchResult, Document+from ..utils.normalize import normalize_document_inputfrom ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options@@ -50,12 +51,7 @@ def search(if request.scrape_options is not None and any(key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'summary', 'screenshot', 'changeTracking']):- # Normalize keys for Document (no Pydantic aliases)- normalized = dict(doc_data)- if 'rawHtml' in normalized and 'raw_html' not in normalized:- normalized['raw_html'] = normalized.pop('rawHtml')- if 'changeTracking' in normalized and 'change_tracking' not in normalized:- normalized['change_tracking'] = normalized.pop('changeTracking')+ normalized = normalize_document_input(doc_data)results.append(Document(**normalized))else:# Minimal search result shapediff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex b13507f80..6c3ba6366 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -7,7 +7,8 @@ This module contains clean, modern type definitions for the v2 API.import warningsfrom datetime import datetimefrom typing import Any, Dict, Generic, List, Literal, Optional, TypeVar, Union-from pydantic import BaseModel, Field, field_validator+import logging+from pydantic import BaseModel, Field, field_validator, ValidationError# Suppress pydantic warnings about schema field shadowing# Tested using schema_field alias="schema" but it doesn't work.@@ -19,6 +20,9 @@ warnings.filterwarnings("ignore", message="Field name \"json\" in \"Document\" sT = TypeVar('T')+# Module logger+logger = logging.getLogger("firecrawl")+# Base response typesclass BaseResponse(BaseModel, Generic[T]):"""Base response structure for all API responses."""@@ -29,18 +33,57 @@ class BaseResponse(BaseModel, Generic[T]):# Document and content typesclass DocumentMetadata(BaseModel):- """Metadata for scraped documents."""+ """Metadata for scraped documents (snake_case only; API camelCase normalized in code)."""+ # Common metadata fieldstitle: Optional[str] = Nonedescription: Optional[str] = None+ url: Optional[str] = Nonelanguage: Optional[str] = Nonekeywords: Optional[Union[str, List[str]]] = Nonerobots: Optional[str] = None++ # OpenGraph and social metadataog_title: Optional[str] = Noneog_description: Optional[str] = Noneog_url: Optional[str] = Noneog_image: Optional[str] = None+ og_audio: Optional[str] = None+ og_determiner: Optional[str] = None+ og_locale: Optional[str] = None+ og_locale_alternate: Optional[List[str]] = None+ og_site_name: Optional[str] = None+ og_video: Optional[str] = None++ # Dublin Core and other site metadata+ favicon: Optional[str] = None+ dc_terms_created: Optional[str] = None+ dc_date_created: Optional[str] = None+ dc_date: Optional[str] = None+ dc_terms_type: Optional[str] = None+ dc_type: Optional[str] = None+ dc_terms_audience: Optional[str] = None+ dc_terms_subject: Optional[str] = None+ dc_subject: Optional[str] = None+ dc_description: Optional[str] = None+ dc_terms_keywords: Optional[str] = None++ modified_time: Optional[str] = None+ published_time: Optional[str] = None+ article_tag: Optional[str] = None+ article_section: Optional[str] = None++ # Response-level metadatasource_url: Optional[str] = Nonestatus_code: Optional[int] = None+ scrape_id: Optional[str] = None+ num_pages: Optional[int] = None+ content_type: Optional[str] = None+ proxy_used: Optional[Literal["basic", "stealth"]] = None+ cache_state: Optional[Literal["hit", "miss"]] = None+ cached_at: Optional[str] = None+ credits_used: Optional[int] = None++ # Error informationerror: Optional[str] = None@staticmethod@@ -85,6 +128,29 @@ class Document(BaseModel):warning: Optional[str] = Nonechange_tracking: Optional[Dict[str, Any]] = None+ @property+ def metadata_typed(self) -> DocumentMetadata:+ """Always returns a DocumentMetadata instance for LSP-friendly access."""+ md = self.metadata+ if isinstance(md, DocumentMetadata):+ return md+ if isinstance(md, dict):+ try:+ return DocumentMetadata(**md)+ except (ValidationError, TypeError) as exc:+ logger.debug("Failed to construct DocumentMetadata from dict: %s", exc)+ return DocumentMetadata()++ @property+ def metadata_dict(self) -> Dict[str, Any]:+ """Returns metadata as a plain dict (exclude None)."""+ md = self.metadata+ if isinstance(md, DocumentMetadata):+ return md.model_dump(exclude_none=True)+ if isinstance(md, dict):+ return {k: v for k, v in md.items() if v is not None}+ return {}+# Webhook typesclass WebhookConfig(BaseModel):"""Configuration for webhooks."""diff --git a/apps/python-sdk/firecrawl/v2/utils/normalize.py b/apps/python-sdk/firecrawl/v2/utils/normalize.pynew file mode 100644index 000000000..54415b555--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/utils/normalize.py@@ -0,0 +1,107 @@+"""+Normalization helpers for v2 API payloads to avoid relying on Pydantic aliases.+"""++from typing import Any, Dict, List+from ..types import DocumentMetadata+++def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:+ """+ Convert API v2 camelCase metadata keys to snake_case expected by DocumentMetadata.+ Leaves unknown keys as-is.+ """+ mapping = {+ # OpenGraph+ "ogTitle": "og_title",+ "ogDescription": "og_description",+ "ogUrl": "og_url",+ "ogImage": "og_image",+ "ogAudio": "og_audio",+ "ogDeterminer": "og_determiner",+ "ogLocale": "og_locale",+ "ogLocaleAlternate": "og_locale_alternate",+ "ogSiteName": "og_site_name",+ "ogVideo": "og_video",+ # Dublin Core and misc+ "dcTermsCreated": "dc_terms_created",+ "dcDateCreated": "dc_date_created",+ "dcDate": "dc_date",+ "dcTermsType": "dc_terms_type",+ "dcType": "dc_type",+ "dcTermsAudience": "dc_terms_audience",+ "dcTermsSubject": "dc_terms_subject",+ "dcSubject": "dc_subject",+ "dcDescription": "dc_description",+ "dcTermsKeywords": "dc_terms_keywords",+ "modifiedTime": "modified_time",+ "publishedTime": "published_time",+ "articleTag": "article_tag",+ "articleSection": "article_section",+ # Response-level+ "sourceURL": "source_url",+ "statusCode": "status_code",+ "scrapeId": "scrape_id",+ "numPages": "num_pages",+ "contentType": "content_type",+ "proxyUsed": "proxy_used",+ "cacheState": "cache_state",+ "cachedAt": "cached_at",+ "creditsUsed": "credits_used",+ }++ out: Dict[str, Any] = {}+ for k, v in md.items():+ snake = mapping.get(k, k)+ out[snake] = v++ # Light coercions where server may send strings/lists+ if isinstance(out.get("status_code"), str):+ try:+ out["status_code"] = int(out["status_code"]) # type: ignore+ except ValueError:+ pass++ # Generic rule: if a value is a list, join with ", " for string-like fields,+ # except for explicit fields we preserve as lists.+ preserve_list_fields: List[str] = [+ "og_locale_alternate",+ ]+ for f, val in list(out.items()):+ if isinstance(val, list) and f not in preserve_list_fields:+ try:+ out[f] = ", ".join(str(x) for x in val)+ except Exception:+ # Fallback: keep original list if join fails+ pass++ return out+++def normalize_document_input(doc: Dict[str, Any]) -> Dict[str, Any]:+ """+ Normalize a raw Document dict from the API into the Python SDK's expected shape:+ - Convert top-level keys rawHtml->raw_html, changeTracking->change_tracking+ - Convert metadata keys from camelCase to snake_case+ """+ normalized = dict(doc)++ if "rawHtml" in normalized and "raw_html" not in normalized:+ normalized["raw_html"] = normalized.pop("rawHtml")++ if "changeTracking" in normalized and "change_tracking" not in normalized:+ normalized["change_tracking"] = normalized.pop("changeTracking")++ md = normalized.get("metadata")+ if isinstance(md, dict):+ mapped = _map_metadata_keys(md)+ # Construct a concrete DocumentMetadata so downstream has a typed object+ try:+ normalized["metadata"] = DocumentMetadata(**mapped)+ except Exception:+ # Fallback to mapped dict if model construction fails for any reason+ normalized["metadata"] = mapped++ return normalized++diff --git a/apps/python-sdk/firecrawl/v2/watcher.py b/apps/python-sdk/firecrawl/v2/watcher.pyindex cb27d747a..086823c41 100644--- a/apps/python-sdk/firecrawl/v2/watcher.py+++ b/apps/python-sdk/firecrawl/v2/watcher.py@@ -15,6 +15,7 @@ from typing import Callable, List, Optional, Literal, Union, Dict, Anyimport websocketsfrom .types import CrawlJob, BatchScrapeJob, Document+from .utils.normalize import normalize_document_inputJobKind = Literal["crawl", "batch"]@@ -172,11 +173,7 @@ class Watcher:docs: List[Document] = []for doc in self.data:if isinstance(doc, dict):- d = dict(doc)- if "rawHtml" in d and "raw_html" not in d:- d["raw_html"] = d.pop("rawHtml")- if "changeTracking" in d and "change_tracking" not in d:- d["change_tracking"] = d.pop("changeTracking")+ d = normalize_document_input(doc)docs.append(Document(**d))if self._kind == "crawl":job = CrawlJob(@@ -212,11 +209,7 @@ class Watcher:docs = []for doc in payload.get("data", []):if isinstance(doc, dict):- d = dict(doc)- if "rawHtml" in d and "raw_html" not in d:- d["raw_html"] = d.pop("rawHtml")- if "changeTracking" in d and "change_tracking" not in d:- d["change_tracking"] = d.pop("changeTracking")+ d = normalize_document_input(doc)docs.append(Document(**d))job = CrawlJob(status=status_str,@@ -241,11 +234,7 @@ class Watcher:docs = []for doc in payload.get("data", []):if isinstance(doc, dict):- d = dict(doc)- if "rawHtml" in d and "raw_html" not in d:- d["raw_html"] = d.pop("rawHtml")- if "changeTracking" in d and "change_tracking" not in d:- d["change_tracking"] = d.pop("changeTracking")+ d = normalize_document_input(doc)docs.append(Document(**d))job = BatchScrapeJob(status=status_str,diff --git a/apps/python-sdk/firecrawl/v2/watcher_async.py b/apps/python-sdk/firecrawl/v2/watcher_async.pyindex c2458d62a..7ed385d2f 100644--- a/apps/python-sdk/firecrawl/v2/watcher_async.py+++ b/apps/python-sdk/firecrawl/v2/watcher_async.py@@ -16,6 +16,7 @@ import websocketsfrom websockets.exceptions import ConnectionClosed, ConnectionClosedOK, ConnectionClosedErrorfrom .types import BatchScrapeJob, CrawlJob, Document+from .utils.normalize import normalize_document_inputJobKind = Literal["crawl", "batch"]@@ -216,11 +217,7 @@ class AsyncWatcher:source_docs = docs_override if docs_override is not None else payload.get("data", []) or []for doc in source_docs:if isinstance(doc, dict):- d = dict(doc)- if "rawHtml" in d and "raw_html" not in d:- d["raw_html"] = d.pop("rawHtml")- if "changeTracking" in d and "change_tracking" not in d:- d["change_tracking"] = d.pop("changeTracking")+ d = normalize_document_input(doc)docs.append(Document(**d))if self._kind == "crawl":