Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(python-sdk) feat: added extra fields to metadata (#2441)
Rafael Miller4c860515dfea
diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.pyindex 0396d4601..25722a0f7 100644--- a/apps/python-sdk/example.py+++ b/apps/python-sdk/example.py@@ -26,6 +26,10 @@ def main():print("scrape:", doc.markdown)# doc.metadata_dict is a dict, doc.metadata_typed is a DocumentMetadata objectprint(doc.metadata_dict.get("source_url"))+ print('metadata_dict.get("title"):', doc.metadata_dict.get("title"))+ print("metadata_typed.title:", doc.metadata_typed.title)+ print("metadata.title", doc.metadata.title if doc.metadata else None)+# Crawl (waits until terminal state)crawl_job = firecrawl.crawl("https://docs.firecrawl.dev", limit=3, poll_interval=1, timeout=120)@@ -47,4 +51,4 @@ def main():print("map links:", len(getattr(map_response, "links", []) or []))if __name__ == "__main__":- main()\ No newline at end of file+ main()diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex d1c736a94..e616fcee9 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.8.0"+__version__ = "4.9.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.pynew file mode 100644index 000000000..8105ff41d--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py@@ -0,0 +1,81 @@+import pytest++from firecrawl.v2.types import Document, DocumentMetadata+from firecrawl.v2.utils.normalize import normalize_document_input+++class TestDocumentMetadataExtras:+ def test_metadata_extras_preserved_in_metadata_dict(self):+ raw = {+ "markdown": "# Hello",+ "metadata": {+ "title": "Example",+ # Known camelCase -> snake_case mapping+ "statusCode": "200",+ # Unknown keys that should be preserved as-is+ "twitter:card": "summary",+ "twitterCard": "summary_large_image",+ "theme-color": "#fff",+ },+ }++ doc = Document(**normalize_document_input(raw))+ md_dict = doc.metadata_dict+ # Known key mapped and coerced+ assert md_dict["status_code"] == 200+ assert md_dict["title"] == "Example"+ # Extras are preserved verbatim+ assert md_dict["twitter:card"] == "summary"+ assert md_dict["twitterCard"] == "summary_large_image"+ assert md_dict["theme-color"] == "#fff"++ def test_metadata_typed_from_plain_dict_preserves_extras(self):+ # Construct Document with raw dict metadata without normalization step+ doc = Document(+ markdown="# Hi",+ metadata={+ "ogTitle": "Hello", # will be treated as extra without normalization+ "x-custom": "ok",+ },+ )++ md = doc.metadata_typed+ assert isinstance(md, DocumentMetadata)+ # Known fields aren't populated without normalization+ assert md.og_title is None+ # Extras are available on the underlying pydantic storage and in metadata_dict+ extras = getattr(md, "__pydantic_extra__", {}) or {}+ assert extras == {"ogTitle": "Hello", "x-custom": "ok"}+ assert doc.metadata_dict["ogTitle"] == "Hello"+ assert doc.metadata_dict["x-custom"] == "ok"++ def test_document_model_dump_includes_metadata_extras(self):+ raw = {+ "markdown": "# Body",+ "metadata": {+ "title": "Page",+ "twitter:site": "@site",+ },+ }+ doc = Document(**normalize_document_input(raw))+ dumped = doc.model_dump(exclude_none=True)+ assert "metadata" in dumped+ assert dumped["metadata"]["title"] == "Page"+ assert dumped["metadata"]["twitter:site"] == "@site"++ def test_unknown_list_metadata_preserved(self):+ raw = {+ "markdown": "# Body",+ "metadata": {+ "title": "Page",+ "x-list": [1, "a", 3],+ },+ }+ doc = Document(**normalize_document_input(raw))+ md = doc.metadata_dict+ assert md["x-list"] == [1, "a", 3]++ def test_metadata_typed_extras_property(self):+ md = DocumentMetadata(title="T", **{"x-foo": "bar"})+ # extras accessor should expose unknown keys+ assert md.extras == {"x-foo": "bar"}diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras_multivalue.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras_multivalue.pynew file mode 100644index 000000000..8bd0b1024--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras_multivalue.py@@ -0,0 +1,22 @@+import pytest++from firecrawl.v2.types import Document+from firecrawl.v2.utils.normalize import normalize_document_input+++class TestMetadataMultiValue:+ def test_article_tag_list_coerced_to_string(self):+ raw = {+ "markdown": "# Body",+ "metadata": {+ "title": "Page",+ "articleTag": ["one", "two"],+ },+ }+ doc = Document(**normalize_document_input(raw))+ # typed access works and is joined as string+ assert doc.metadata is not None+ assert doc.metadata.article_tag == "one, two"+ # dict view shows string+ md = doc.metadata_dict+ assert md["article_tag"] == "one, two"diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 90c085bcd..a6ef1d276 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -8,32 +8,70 @@ import warningsfrom datetime import datetimefrom typing import Any, Dict, Generic, List, Literal, Optional, TypeVar, Unionimport logging-from pydantic import BaseModel, Field, field_validator, ValidationError+from pydantic import (+ BaseModel,+ Field,+ field_validator,+ ValidationError,+ model_serializer,+ model_validator,+)# Suppress pydantic warnings about schema field shadowing# Tested using schema_field alias="schema" but it doesn't work.-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"Format\" shadows an attribute in parent \"BaseModel\"")-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonFormat\" shadows an attribute in parent \"Format\"")-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ChangeTrackingFormat\" shadows an attribute in parent \"Format\"")-warnings.filterwarnings("ignore", message="Field name \"json\" in \"ScrapeFormats\" shadows an attribute in parent \"BaseModel\"")-warnings.filterwarnings("ignore", message="Field name \"json\" in \"Document\" shadows an attribute in parent \"BaseModel\"")--T = TypeVar('T')+warnings.filterwarnings(+ "ignore",+ message='Field name "schema" in "Format" shadows an attribute in parent "BaseModel"',+)+warnings.filterwarnings(+ "ignore",+ message='Field name "schema" in "JsonFormat" shadows an attribute in parent "Format"',+)+warnings.filterwarnings(+ "ignore",+ message='Field name "schema" in "ChangeTrackingFormat" shadows an attribute in parent "Format"',+)+warnings.filterwarnings(+ "ignore",+ message='Field name "json" in "ScrapeFormats" shadows an attribute in parent "BaseModel"',+)+warnings.filterwarnings(+ "ignore",+ message='Field name "json" in "Document" shadows an attribute in parent "BaseModel"',+)++T = TypeVar("T")# Module loggerlogger = logging.getLogger("firecrawl")+# Base response typesclass BaseResponse(BaseModel, Generic[T]):"""Base response structure for all API responses."""+success: booldata: Optional[T] = Noneerror: Optional[str] = Nonewarning: Optional[str] = None+# Document and content typesclass DocumentMetadata(BaseModel):"""Metadata for scraped documents (snake_case only; API camelCase normalized in code)."""++ model_config = {"extra": "allow"}++ @model_serializer(mode="wrap")+ def _serialize(self, handler):+ out = handler(self)+ extra = getattr(self, "__pydantic_extra__", None)+ if isinstance(extra, dict):+ for k, v in extra.items():+ if v is not None:+ out[k] = v+ return out+# Common metadata fieldstitle: Optional[str] = Nonedescription: Optional[str] = None@@ -86,13 +124,19 @@ class DocumentMetadata(BaseModel):# Error informationerror: Optional[str] = None+ @property+ def extras(self) -> Dict[str, Any]:+ """Return unknown metadata keys preserved on the model."""+ extra = getattr(self, "__pydantic_extra__", None)+ return dict(extra) if isinstance(extra, dict) else {}+@staticmethoddef _coerce_list_to_string(value: Any) -> Any:if isinstance(value, list):# Prefer first string if semantically a single-valued field, else joinif len(value) == 1:return str(value[0])- return ', '.join(str(item) for item in value)+ return ", ".join(str(item) for item in value)return value@staticmethod@@ -104,30 +148,101 @@ class DocumentMetadata(BaseModel):return valuereturn value- @field_validator('robots', 'og_title', 'og_description', 'og_url', 'og_image', 'language', mode='before')+ @model_validator(mode="before")+ @classmethod+ def coerce_lists_for_string_fields(cls, data):+ """Before validation: coerce lists to strings for known single-string fields.+ Preserves unknown-key lists.+ """+ if not isinstance(data, dict):+ return data+ single_str_fields = {+ "title",+ "description",+ "url",+ "language",+ "robots",+ "og_title",+ "og_description",+ "og_url",+ "og_image",+ "og_audio",+ "og_determiner",+ "og_locale",+ "og_site_name",+ "og_video",+ "favicon",+ "dc_terms_created",+ "dc_date_created",+ "dc_date",+ "dc_terms_type",+ "dc_type",+ "dc_terms_audience",+ "dc_terms_subject",+ "dc_subject",+ "dc_description",+ "dc_terms_keywords",+ "modified_time",+ "published_time",+ "article_tag",+ "article_section",+ "source_url",+ "scrape_id",+ "content_type",+ "cached_at",+ "error",+ }+ for k, v in list(data.items()):+ if isinstance(v, list) and k in single_str_fields:+ data[k] = cls._coerce_list_to_string(v)+ # For ints that might appear as list, take first+ if isinstance(v, list) and k in {+ "status_code",+ "num_pages",+ "credits_used",+ }:+ first = v[0] if v else None+ data[k] = cls._coerce_string_to_int(first)+ return data++ @field_validator(+ "robots",+ "og_title",+ "og_description",+ "og_url",+ "og_image",+ "language",+ mode="before",+ )@classmethoddef coerce_lists_to_string_fields(cls, v):return cls._coerce_list_to_string(v)- @field_validator('status_code', mode='before')+ @field_validator("status_code", mode="before")@classmethoddef coerce_status_code_to_int(cls, v):return cls._coerce_string_to_int(v)+class AgentOptions(BaseModel):"""Configuration for the agent in extract operations."""+model: Literal["FIRE-1"] = "FIRE-1"+class AttributeResult(BaseModel):"""Result of attribute extraction."""+selector: strattribute: strvalues: List[str]+class BrandingProfile(BaseModel):"""Branding information extracted from a website."""+model_config = {"extra": "allow"}-+color_scheme: Optional[Literal["light", "dark"]] = Nonelogo: Optional[str] = Nonefonts: Optional[List[Dict[str, Any]]] = None@@ -142,8 +257,10 @@ class BrandingProfile(BaseModel):tone: Optional[Dict[str, str]] = Nonepersonality: Optional[Dict[str, Any]] = None+class Document(BaseModel):"""A scraped document."""+markdown: Optional[str] = Nonehtml: Optional[str] = Noneraw_html: Optional[str] = None@@ -166,31 +283,42 @@ class Document(BaseModel):return mdif isinstance(md, dict):try:- return DocumentMetadata(**md)+ return DocumentMetadata.model_validate(md)except (ValidationError, TypeError) as exc:logger.debug("Failed to construct DocumentMetadata from dict: %s", exc)return DocumentMetadata()@propertydef metadata_dict(self) -> Dict[str, Any]:- """Returns metadata as a plain dict (exclude None)."""+ """Returns metadata as a plain dict (exclude None), including extras."""md = self.metadataif isinstance(md, DocumentMetadata):- return md.model_dump(exclude_none=True)+ out = md.model_dump(exclude_none=True)+ # Ensure extras are preserved even if model_dump omits them+ extra = getattr(md, "__pydantic_extra__", None)+ if isinstance(extra, dict):+ for k, v in extra.items():+ if v is not None:+ out[k] = v+ return outif isinstance(md, dict):return {k: v for k, v in md.items() if v is not None}return {}+# Webhook typesclass WebhookConfig(BaseModel):"""Configuration for webhooks."""+url: strheaders: Optional[Dict[str, str]] = Nonemetadata: Optional[Dict[str, str]] = Noneevents: Optional[List[Literal["completed", "failed", "page", "started"]]] = None+class WebhookData(BaseModel):"""Data sent to webhooks."""+job_id: strstatus: strcurrent: Optional[int] = None@@ -198,73 +326,116 @@ class WebhookData(BaseModel):data: Optional[List[Document]] = Noneerror: Optional[str] = None+class Source(BaseModel):"""Configuration for a search source."""+type: str+SourceOption = Union[str, Source]+class Category(BaseModel):"""Configuration for a search category.-+Supported categories:- "github": Filter results to GitHub repositories- "research": Filter results to research papers and academic sites- "pdf": Filter results to PDF files (adds filetype:pdf to search)"""+type: str+CategoryOption = Union[str, Category]FormatString = Literal[# camelCase versions (API format)- "markdown", "html", "rawHtml", "links", "images", "screenshot", "summary", "changeTracking", "json", "attributes", "branding",+ "markdown",+ "html",+ "rawHtml",+ "links",+ "images",+ "screenshot",+ "summary",+ "changeTracking",+ "json",+ "attributes",+ "branding",# snake_case versions (user-friendly)- "raw_html", "change_tracking"+ "raw_html",+ "change_tracking",]+class Viewport(BaseModel):"""Viewport configuration for screenshots."""+width: intheight: int+class Format(BaseModel):"""Configuration for a format."""+type: FormatString+class JsonFormat(Format):"""Configuration for JSON extraction."""+prompt: Optional[str] = Noneschema: Optional[Any] = None+class ChangeTrackingFormat(Format):"""Configuration for change tracking."""+modes: List[Literal["git-diff", "json"]]schema: Optional[Dict[str, Any]] = Noneprompt: Optional[str] = Nonetag: Optional[str] = None+class ScreenshotFormat(BaseModel):"""Configuration for screenshot format."""+type: Literal["screenshot"] = "screenshot"full_page: Optional[bool] = Nonequality: Optional[int] = Noneviewport: Optional[Union[Dict[str, int], Viewport]] = None-++class AttributeSelector(BaseModel):"""Selector and attribute pair for attribute extraction."""+selector: strattribute: str+class AttributesFormat(Format):"""Configuration for attribute extraction."""+type: Literal["attributes"] = "attributes"selectors: List[AttributeSelector]-FormatOption = Union[Dict[str, Any], FormatString, JsonFormat, ChangeTrackingFormat, ScreenshotFormat, AttributesFormat, Format]++FormatOption = Union[+ Dict[str, Any],+ FormatString,+ JsonFormat,+ ChangeTrackingFormat,+ ScreenshotFormat,+ AttributesFormat,+ Format,+]++# Scrape typesclass ScrapeFormats(BaseModel):"""Output formats for scraping."""+formats: Optional[List[FormatOption]] = Nonemarkdown: bool = Truehtml: bool = False@@ -276,13 +447,13 @@ class ScrapeFormats(BaseModel):change_tracking: bool = Falsejson: bool = False- @field_validator('formats')+ @field_validator("formats")@classmethoddef validate_formats(cls, v):"""Validate and normalize formats input."""if v is None:return v-+normalized_formats = []for format_item in v:if isinstance(format_item, str):@@ -294,12 +465,14 @@ class ScrapeFormats(BaseModel):normalized_formats.append(format_item)else:raise ValueError(f"Invalid format format: {format_item}")-+return normalized_formats+class ScrapeOptions(BaseModel):"""Options for scraping operations."""- formats: Optional[Union['ScrapeFormats', List[FormatOption]]] = None++ formats: Optional[Union["ScrapeFormats", List[FormatOption]]] = Noneheaders: Optional[Dict[str, str]] = Noneinclude_tags: Optional[List[str]] = Noneexclude_tags: Optional[List[str]] = None@@ -307,9 +480,23 @@ class ScrapeOptions(BaseModel):timeout: Optional[int] = Nonewait_for: Optional[int] = Nonemobile: Optional[bool] = None- parsers: Optional[Union[List[str], List[Union[str, 'PDFParser']]]] = None- actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None- location: Optional['Location'] = None+ parsers: Optional[Union[List[str], List[Union[str, "PDFParser"]]]] = None+ actions: Optional[+ List[+ Union[+ "WaitAction",+ "ScreenshotAction",+ "ClickAction",+ "WriteAction",+ "PressAction",+ "ScrollAction",+ "ScrapeAction",+ "ExecuteJavascriptAction",+ "PDFAction",+ ]+ ]+ ] = None+ location: Optional["Location"] = Noneskip_tls_verification: Optional[bool] = Noneremove_base64_images: Optional[bool] = Nonefast_mode: Optional[bool] = None@@ -320,7 +507,7 @@ class ScrapeOptions(BaseModel):store_in_cache: Optional[bool] = Noneintegration: Optional[str] = None- @field_validator('formats')+ @field_validator("formats")@classmethoddef validate_formats(cls, v):"""Validate and normalize formats input."""@@ -330,24 +517,34 @@ class ScrapeOptions(BaseModel):return vif isinstance(v, list):return v- raise ValueError(f"Invalid formats type: {type(v)}. Expected ScrapeFormats or List[FormatOption]")+ raise ValueError(+ f"Invalid formats type: {type(v)}. Expected ScrapeFormats or List[FormatOption]"+ )+class ScrapeRequest(BaseModel):"""Request for scraping a single URL."""+url: stroptions: Optional[ScrapeOptions] = None+class ScrapeData(Document):"""Scrape results data."""+pass+class ScrapeResponse(BaseResponse[ScrapeData]):"""Response for scrape operations."""+pass+# Crawl typesclass CrawlRequest(BaseModel):"""Request for crawling a website."""+url: strprompt: Optional[str] = Noneexclude_paths: Optional[List[str]] = None@@ -366,13 +563,17 @@ class CrawlRequest(BaseModel):zero_data_retention: bool = Falseintegration: Optional[str] = None+class CrawlResponse(BaseModel):"""Information about a crawl job."""+id: strurl: str+class CrawlJob(BaseModel):"""Crawl job status and progress data."""+status: Literal["scraping", "completed", "failed", "cancelled"]total: int = 0completed: int = 0@@ -381,55 +582,63 @@ class CrawlJob(BaseModel):next: Optional[str] = Nonedata: List[Document] = []+class CrawlStatusRequest(BaseModel):"""Request to get crawl job status."""+job_id: str+class SearchResultWeb(BaseModel):"""A web search result with URL, title, and description."""+url: strtitle: Optional[str] = Nonedescription: Optional[str] = Nonecategory: Optional[str] = None+class SearchResultNews(BaseModel):- """A news search result with URL, title, snippet, date, image URL, and position."""- title: Optional[str] = None- url: Optional[str] = None- snippet: Optional[str] = None- date: Optional[str] = None- image_url: Optional[str] = None- position: Optional[int] = None- category: Optional[str] = None+ """A news search result with URL, title, snippet, date, image URL, and position."""++ title: Optional[str] = None+ url: Optional[str] = None+ snippet: Optional[str] = None+ date: Optional[str] = None+ image_url: Optional[str] = None+ position: Optional[int] = None+ category: Optional[str] = None+class SearchResultImages(BaseModel):- """An image search result with URL, title, image URL, image width, image height, and position."""- title: Optional[str] = None- image_url: Optional[str] = None- image_width: Optional[int] = None- image_height: Optional[int] = None- url: Optional[str] = None- position: Optional[int] = None+ """An image search result with URL, title, image URL, image width, image height, and position."""++ title: Optional[str] = None+ image_url: Optional[str] = None+ image_width: Optional[int] = None+ image_height: Optional[int] = None+ url: Optional[str] = None+ position: Optional[int] = None-class SearchData(BaseModel):- """Search results grouped by source type."""- web: Optional[List[Union[SearchResultWeb, Document]]] = None- news: Optional[List[Union[SearchResultNews, Document]]] = None- images: Optional[List[Union[SearchResultImages, Document]]] = Noneclass MapDocument(Document):"""A document from a map operation with URL and description."""+url: strdescription: Optional[str] = None+# Crawl params typesclass CrawlParamsRequest(BaseModel):"""Request for getting crawl parameters from LLM."""+url: strprompt: str+class CrawlParamsData(BaseModel):"""Data returned from crawl params endpoint."""+include_paths: Optional[List[str]] = Noneexclude_paths: Optional[List[str]] = Nonemax_discovery_depth: Optional[int] = None@@ -447,13 +656,17 @@ class CrawlParamsData(BaseModel):warning: Optional[str] = Noneintegration: Optional[str] = None+class CrawlParamsResponse(BaseResponse[CrawlParamsData]):"""Response from crawl params endpoint."""+pass+# Batch scrape typesclass BatchScrapeRequest(BaseModel):"""Request for batch scraping multiple URLs (internal helper only)."""+urls: List[str]options: Optional[ScrapeOptions] = Nonewebhook: Optional[Union[str, WebhookConfig]] = None@@ -463,14 +676,18 @@ class BatchScrapeRequest(BaseModel):zero_data_retention: Optional[bool] = Noneintegration: Optional[str] = None+class BatchScrapeResponse(BaseModel):"""Response from starting a batch scrape job (mirrors CrawlResponse naming)."""+id: strurl: strinvalid_urls: Optional[List[str]] = None+class BatchScrapeJob(BaseModel):"""Batch scrape job status and results."""+status: Literal["scraping", "completed", "failed", "cancelled"]completed: inttotal: int@@ -479,43 +696,55 @@ class BatchScrapeJob(BaseModel):next: Optional[str] = Nonedata: List[Document] = []+class BatchScrapeStatusRequest(BaseModel):"""Request to get batch scrape job status."""+job_id: str+class BatchScrapeErrorsRequest(BaseModel):"""Request to get errors for a batch scrape job."""+job_id: str+# Map typesclass MapOptions(BaseModel):"""Options for mapping operations."""+search: Optional[str] = Nonesitemap: Literal["only", "include", "skip"] = "include"include_subdomains: Optional[bool] = Nonelimit: Optional[int] = Nonetimeout: Optional[int] = Noneintegration: Optional[str] = None- location: Optional['Location'] = None+ location: Optional["Location"] = None+class MapRequest(BaseModel):"""Request for mapping a website."""+url: stroptions: Optional[MapOptions] = None-class MapData(BaseModel):"""Map results data."""- links: List['SearchResult']++ links: List["SearchResult"]+class MapResponse(BaseResponse[MapData]):"""Response for map operations."""+pass+# Extract typesclass ExtractRequest(BaseModel):"""Request for extract operations."""+urls: Optional[List[str]] = Noneprompt: Optional[str] = Noneschema_: Optional[Dict[str, Any]] = Field(default=None, alias="schema")@@ -527,9 +756,11 @@ class ExtractRequest(BaseModel):ignore_invalid_urls: Optional[bool] = Noneintegration: Optional[str] = Noneagent: Optional[AgentOptions] = None-++class ExtractResponse(BaseModel):"""Response for extract operations (start/status/final)."""+success: Optional[bool] = Noneid: Optional[str] = Nonestatus: Optional[Literal["processing", "completed", "failed", "cancelled"]] = None@@ -539,122 +770,174 @@ class ExtractResponse(BaseModel):sources: Optional[Dict[str, Any]] = Noneexpires_at: Optional[datetime] = None+# Usage/limits typesclass ConcurrencyCheck(BaseModel):"""Current concurrency and limits for the team/API key."""+concurrency: intmax_concurrency: int+class CreditUsage(BaseModel):"""Remaining credits for the team/API key."""+remaining_credits: intplan_credits: Optional[int] = Nonebilling_period_start: Optional[str] = Nonebilling_period_end: Optional[str] = None+class TokenUsage(BaseModel):"""Recent token usage metrics (if available)."""+remaining_tokens: intplan_tokens: Optional[int] = Nonebilling_period_start: Optional[str] = Nonebilling_period_end: Optional[str] = None+class QueueStatusRequest(BaseModel):"""Request to retrieve queue status."""+pass+class QueueStatusResponse(BaseModel):"""Metrics about the team's scrape queue."""+jobs_in_queue: intactive_jobs_in_queue: intwaiting_jobs_in_queue: intmax_concurrency: intmost_recent_success: Optional[datetime] = None+class CreditUsageHistoricalPeriod(BaseModel):startDate: Optional[str] = NoneendDate: Optional[str] = NoneapiKey: Optional[str] = NonecreditsUsed: int+class CreditUsageHistoricalResponse(BaseModel):success: boolperiods: List[CreditUsageHistoricalPeriod]+class TokenUsageHistoricalPeriod(BaseModel):startDate: Optional[str] = NoneendDate: Optional[str] = NoneapiKey: Optional[str] = NonetokensUsed: int+class TokenUsageHistoricalResponse(BaseModel):success: boolperiods: List[TokenUsageHistoricalPeriod]+# Action typesclass WaitAction(BaseModel):"""Wait action to perform during scraping."""+type: Literal["wait"] = "wait"milliseconds: Optional[int] = Noneselector: Optional[str] = None+class ScreenshotAction(BaseModel):"""Screenshot action to perform during scraping."""+type: Literal["screenshot"] = "screenshot"full_page: Optional[bool] = Nonequality: Optional[int] = Noneviewport: Optional[Union[Dict[str, int], Viewport]] = None+class ClickAction(BaseModel):"""Click action to perform during scraping."""+type: Literal["click"] = "click"selector: str+class WriteAction(BaseModel):"""Write action to perform during scraping."""+type: Literal["write"] = "write"text: str+class PressAction(BaseModel):"""Press action to perform during scraping."""+type: Literal["press"] = "press"key: str+class ScrollAction(BaseModel):"""Scroll action to perform during scraping."""+type: Literal["scroll"] = "scroll"direction: Literal["up", "down"]selector: Optional[str] = None+class ScrapeAction(BaseModel):"""Scrape action to perform during scraping."""+type: Literal["scrape"] = "scrape"+class ExecuteJavascriptAction(BaseModel):"""Execute javascript action to perform during scraping."""+type: Literal["executeJavascript"] = "executeJavascript"script: str+class PDFAction(BaseModel):"""PDF action to perform during scraping."""+type: Literal["pdf"] = "pdf"- format: Optional[Literal["A0", "A1", "A2", "A3", "A4", "A5", "A6", "Letter", "Legal", "Tabloid", "Ledger"]] = None+ format: Optional[+ Literal[+ "A0",+ "A1",+ "A2",+ "A3",+ "A4",+ "A5",+ "A6",+ "Letter",+ "Legal",+ "Tabloid",+ "Ledger",+ ]+ ] = Nonelandscape: Optional[bool] = Nonescale: Optional[float] = None+class PDFParser(BaseModel):"""PDF parser configuration with optional page limit."""+type: Literal["pdf"] = "pdf"max_pages: Optional[int] = None+# Location typesclass Location(BaseModel):"""Location configuration for scraping."""+country: Optional[str] = Nonelanguages: Optional[List[str]] = None+class SearchRequest(BaseModel):"""Request for search operations."""+query: strsources: Optional[List[SourceOption]] = Nonecategories: Optional[List[CategoryOption]] = None@@ -666,13 +949,13 @@ class SearchRequest(BaseModel):scrape_options: Optional[ScrapeOptions] = Noneintegration: Optional[str] = None- @field_validator('sources')+ @field_validator("sources")@classmethoddef validate_sources(cls, v):"""Validate and normalize sources input."""if v is None:return v-+normalized_sources = []for source in v:if isinstance(source, str):@@ -683,16 +966,16 @@ class SearchRequest(BaseModel):normalized_sources.append(source)else:raise ValueError(f"Invalid source format: {source}")-+return normalized_sources-- @field_validator('categories')++ @field_validator("categories")@classmethoddef validate_categories(cls, v):"""Validate and normalize categories input."""if v is None:return v-+normalized_categories = []for category in v:if isinstance(category, str):@@ -703,46 +986,59 @@ class SearchRequest(BaseModel):normalized_categories.append(category)else:raise ValueError(f"Invalid category format: {category}")-+return normalized_categories# NOTE: parsers validation does not belong on SearchRequest; it is part of ScrapeOptions.+class LinkResult(BaseModel):"""A generic link result with optional metadata (used by search and map)."""+url: strtitle: Optional[str] = Nonedescription: Optional[str] = None-++# Backward-compatible alias for existing tests/usagesSearchResult = LinkResult+class SearchData(BaseModel):"""Search results grouped by source type."""+web: Optional[List[Union[SearchResultWeb, Document]]] = Nonenews: Optional[List[Union[SearchResultNews, Document]]] = Noneimages: Optional[List[Union[SearchResultImages, Document]]] = None+class SearchResponse(BaseResponse[SearchData]):"""Response from search operation."""+pass+# Error typesclass ErrorDetails(BaseModel):"""Detailed error information."""+code: Optional[str] = Nonemessage: strdetails: Optional[Dict[str, Any]] = None+class ErrorResponse(BaseModel):"""Error response structure."""+success: bool = Falseerror: strdetails: Optional[ErrorDetails] = None+# Job management typesclass JobStatus(BaseModel):"""Generic job status information."""+id: strstatus: Literal["pending", "scraping", "completed", "failed"]current: Optional[int] = None@@ -751,54 +1047,71 @@ class JobStatus(BaseModel):completed_at: Optional[datetime] = Noneexpires_at: Optional[datetime] = None+class CrawlError(BaseModel):"""A crawl error."""+id: strtimestamp: Optional[datetime] = Noneurl: strcode: Optional[str] = Noneerror: str+class CrawlErrorsResponse(BaseModel):"""Response from crawl error monitoring."""+errors: List[CrawlError]robots_blocked: List[str]+class CrawlErrorsRequest(BaseModel):"""Request for crawl error monitoring."""+crawl_id: str+class ActiveCrawl(BaseModel):"""Information about an active crawl job."""+id: strteam_id: strurl: stroptions: Optional[Dict[str, Any]] = None+class ActiveCrawlsResponse(BaseModel):"""Response from active crawls endpoint."""+success: bool = Truecrawls: List[ActiveCrawl]+class ActiveCrawlsRequest(BaseModel):"""Request for listing active crawl jobs."""+pass+# Configuration typesclass ClientConfig(BaseModel):"""Configuration for the Firecrawl client."""+api_key: Optional[str] = Noneapi_url: str = "https://api.firecrawl.dev"timeout: Optional[float] = Nonemax_retries: int = 3backoff_factor: float = 0.5+class PaginationConfig(BaseModel):"""Configuration for pagination behavior."""+auto_paginate: bool = Truemax_pages: Optional[int] = Field(default=None, ge=0)max_results: Optional[int] = Field(default=None, ge=0)- max_wait_time: Optional[int] = Field(default=None, ge=0) # seconds+ max_wait_time: Optional[int] = Field(default=None, ge=0) # seconds+# Response union typesAnyResponse = Union[diff --git a/apps/python-sdk/firecrawl/v2/utils/normalize.py b/apps/python-sdk/firecrawl/v2/utils/normalize.pyindex 2b523c7ec..296933a7c 100644--- a/apps/python-sdk/firecrawl/v2/utils/normalize.py+++ b/apps/python-sdk/firecrawl/v2/utils/normalize.py@@ -62,19 +62,7 @@ def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:except ValueError:pass- # Generic rule: if a value is a list, join with ", " for string-like fields,- # except for explicit fields we preserve as lists.- preserve_list_fields: List[str] = [- "og_locale_alternate",- ]- for f, val in list(out.items()):- if isinstance(val, list) and f not in preserve_list_fields:- try:- out[f] = ", ".join(str(x) for x in val)- except Exception:- # Fallback: keep original list if join fails- pass-+ # Preserve list values for unknown keys; only lightweight coercions abovereturn out@@ -96,11 +84,10 @@ def normalize_document_input(doc: Dict[str, Any]) -> Dict[str, Any]:md = normalized.get("metadata")if isinstance(md, dict):mapped = _map_metadata_keys(md)- # Construct a concrete DocumentMetadata so downstream has a typed object+ # Construct a typed DocumentMetadata; extras allowed/preservedtry:- normalized["metadata"] = DocumentMetadata(**mapped)+ normalized["metadata"] = DocumentMetadata.model_validate(mapped)except Exception:- # Fallback to mapped dict if model construction fails for any reasonnormalized["metadata"] = mapped# Normalize branding top-level camelCase keys@@ -134,5 +121,3 @@ def _map_search_result_keys(result: Dict[str, Any], result_type: str) -> Dict[stout[snake] = vreturn out--