Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(python-sdk) feat: added extra fields to metadata (#2441)

Rafael Miller4c860515dfea
diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.py
index 0396d4601..25722a0f7 100644
--- a/apps/python-sdk/example.py
+++ b/apps/python-sdk/example.py
@@ -26,6 +26,10 @@ def main():
print("scrape:", doc.markdown)
# doc.metadata_dict is a dict, doc.metadata_typed is a DocumentMetadata object
print(doc.metadata_dict.get("source_url"))
+ print('metadata_dict.get("title"):', doc.metadata_dict.get("title"))
+ print("metadata_typed.title:", doc.metadata_typed.title)
+ print("metadata.title", doc.metadata.title if doc.metadata else None)
+
# Crawl (waits until terminal state)
crawl_job = firecrawl.crawl("https://docs.firecrawl.dev", limit=3, poll_interval=1, timeout=120)
@@ -47,4 +51,4 @@ def main():
print("map links:", len(getattr(map_response, "links", []) or []))
if __name__ == "__main__":
- main()
\ No newline at end of file
+ main()
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index d1c736a94..e616fcee9 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.8.0"
+__version__ = "4.9.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py
new file mode 100644
index 000000000..8105ff41d
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras.py
@@ -0,0 +1,81 @@
+import pytest
+
+from firecrawl.v2.types import Document, DocumentMetadata
+from firecrawl.v2.utils.normalize import normalize_document_input
+
+
+class TestDocumentMetadataExtras:
+ def test_metadata_extras_preserved_in_metadata_dict(self):
+ raw = {
+ "markdown": "# Hello",
+ "metadata": {
+ "title": "Example",
+ # Known camelCase -> snake_case mapping
+ "statusCode": "200",
+ # Unknown keys that should be preserved as-is
+ "twitter:card": "summary",
+ "twitterCard": "summary_large_image",
+ "theme-color": "#fff",
+ },
+ }
+
+ doc = Document(**normalize_document_input(raw))
+ md_dict = doc.metadata_dict
+ # Known key mapped and coerced
+ assert md_dict["status_code"] == 200
+ assert md_dict["title"] == "Example"
+ # Extras are preserved verbatim
+ assert md_dict["twitter:card"] == "summary"
+ assert md_dict["twitterCard"] == "summary_large_image"
+ assert md_dict["theme-color"] == "#fff"
+
+ def test_metadata_typed_from_plain_dict_preserves_extras(self):
+ # Construct Document with raw dict metadata without normalization step
+ doc = Document(
+ markdown="# Hi",
+ metadata={
+ "ogTitle": "Hello", # will be treated as extra without normalization
+ "x-custom": "ok",
+ },
+ )
+
+ md = doc.metadata_typed
+ assert isinstance(md, DocumentMetadata)
+ # Known fields aren't populated without normalization
+ assert md.og_title is None
+ # Extras are available on the underlying pydantic storage and in metadata_dict
+ extras = getattr(md, "__pydantic_extra__", {}) or {}
+ assert extras == {"ogTitle": "Hello", "x-custom": "ok"}
+ assert doc.metadata_dict["ogTitle"] == "Hello"
+ assert doc.metadata_dict["x-custom"] == "ok"
+
+ def test_document_model_dump_includes_metadata_extras(self):
+ raw = {
+ "markdown": "# Body",
+ "metadata": {
+ "title": "Page",
+ "twitter:site": "@site",
+ },
+ }
+ doc = Document(**normalize_document_input(raw))
+ dumped = doc.model_dump(exclude_none=True)
+ assert "metadata" in dumped
+ assert dumped["metadata"]["title"] == "Page"
+ assert dumped["metadata"]["twitter:site"] == "@site"
+
+ def test_unknown_list_metadata_preserved(self):
+ raw = {
+ "markdown": "# Body",
+ "metadata": {
+ "title": "Page",
+ "x-list": [1, "a", 3],
+ },
+ }
+ doc = Document(**normalize_document_input(raw))
+ md = doc.metadata_dict
+ assert md["x-list"] == [1, "a", 3]
+
+ def test_metadata_typed_extras_property(self):
+ md = DocumentMetadata(title="T", **{"x-foo": "bar"})
+ # extras accessor should expose unknown keys
+ assert md.extras == {"x-foo": "bar"}
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras_multivalue.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras_multivalue.py
new file mode 100644
index 000000000..8bd0b1024
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_metadata_extras_multivalue.py
@@ -0,0 +1,22 @@
+import pytest
+
+from firecrawl.v2.types import Document
+from firecrawl.v2.utils.normalize import normalize_document_input
+
+
+class TestMetadataMultiValue:
+ def test_article_tag_list_coerced_to_string(self):
+ raw = {
+ "markdown": "# Body",
+ "metadata": {
+ "title": "Page",
+ "articleTag": ["one", "two"],
+ },
+ }
+ doc = Document(**normalize_document_input(raw))
+ # typed access works and is joined as string
+ assert doc.metadata is not None
+ assert doc.metadata.article_tag == "one, two"
+ # dict view shows string
+ md = doc.metadata_dict
+ assert md["article_tag"] == "one, two"
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 90c085bcd..a6ef1d276 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -8,32 +8,70 @@ import warnings
from datetime import datetime
from typing import Any, Dict, Generic, List, Literal, Optional, TypeVar, Union
import logging
-from pydantic import BaseModel, Field, field_validator, ValidationError
+from pydantic import (
+ BaseModel,
+ Field,
+ field_validator,
+ ValidationError,
+ model_serializer,
+ model_validator,
+)
# Suppress pydantic warnings about schema field shadowing
# Tested using schema_field alias="schema" but it doesn't work.
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"Format\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"JsonFormat\" shadows an attribute in parent \"Format\"")
-warnings.filterwarnings("ignore", message="Field name \"schema\" in \"ChangeTrackingFormat\" shadows an attribute in parent \"Format\"")
-warnings.filterwarnings("ignore", message="Field name \"json\" in \"ScrapeFormats\" shadows an attribute in parent \"BaseModel\"")
-warnings.filterwarnings("ignore", message="Field name \"json\" in \"Document\" shadows an attribute in parent \"BaseModel\"")
-
-T = TypeVar('T')
+warnings.filterwarnings(
+ "ignore",
+ message='Field name "schema" in "Format" shadows an attribute in parent "BaseModel"',
+)
+warnings.filterwarnings(
+ "ignore",
+ message='Field name "schema" in "JsonFormat" shadows an attribute in parent "Format"',
+)
+warnings.filterwarnings(
+ "ignore",
+ message='Field name "schema" in "ChangeTrackingFormat" shadows an attribute in parent "Format"',
+)
+warnings.filterwarnings(
+ "ignore",
+ message='Field name "json" in "ScrapeFormats" shadows an attribute in parent "BaseModel"',
+)
+warnings.filterwarnings(
+ "ignore",
+ message='Field name "json" in "Document" shadows an attribute in parent "BaseModel"',
+)
+
+T = TypeVar("T")
# Module logger
logger = logging.getLogger("firecrawl")
+
# Base response types
class BaseResponse(BaseModel, Generic[T]):
"""Base response structure for all API responses."""
+
success: bool
data: Optional[T] = None
error: Optional[str] = None
warning: Optional[str] = None
+
# Document and content types
class DocumentMetadata(BaseModel):
"""Metadata for scraped documents (snake_case only; API camelCase normalized in code)."""
+
+ model_config = {"extra": "allow"}
+
+ @model_serializer(mode="wrap")
+ def _serialize(self, handler):
+ out = handler(self)
+ extra = getattr(self, "__pydantic_extra__", None)
+ if isinstance(extra, dict):
+ for k, v in extra.items():
+ if v is not None:
+ out[k] = v
+ return out
+
# Common metadata fields
title: Optional[str] = None
description: Optional[str] = None
@@ -86,13 +124,19 @@ class DocumentMetadata(BaseModel):
# Error information
error: Optional[str] = None
+ @property
+ def extras(self) -> Dict[str, Any]:
+ """Return unknown metadata keys preserved on the model."""
+ extra = getattr(self, "__pydantic_extra__", None)
+ return dict(extra) if isinstance(extra, dict) else {}
+
@staticmethod
def _coerce_list_to_string(value: Any) -> Any:
if isinstance(value, list):
# Prefer first string if semantically a single-valued field, else join
if len(value) == 1:
return str(value[0])
- return ', '.join(str(item) for item in value)
+ return ", ".join(str(item) for item in value)
return value
@staticmethod
@@ -104,30 +148,101 @@ class DocumentMetadata(BaseModel):
return value
return value
- @field_validator('robots', 'og_title', 'og_description', 'og_url', 'og_image', 'language', mode='before')
+ @model_validator(mode="before")
+ @classmethod
+ def coerce_lists_for_string_fields(cls, data):
+ """Before validation: coerce lists to strings for known single-string fields.
+ Preserves unknown-key lists.
+ """
+ if not isinstance(data, dict):
+ return data
+ single_str_fields = {
+ "title",
+ "description",
+ "url",
+ "language",
+ "robots",
+ "og_title",
+ "og_description",
+ "og_url",
+ "og_image",
+ "og_audio",
+ "og_determiner",
+ "og_locale",
+ "og_site_name",
+ "og_video",
+ "favicon",
+ "dc_terms_created",
+ "dc_date_created",
+ "dc_date",
+ "dc_terms_type",
+ "dc_type",
+ "dc_terms_audience",
+ "dc_terms_subject",
+ "dc_subject",
+ "dc_description",
+ "dc_terms_keywords",
+ "modified_time",
+ "published_time",
+ "article_tag",
+ "article_section",
+ "source_url",
+ "scrape_id",
+ "content_type",
+ "cached_at",
+ "error",
+ }
+ for k, v in list(data.items()):
+ if isinstance(v, list) and k in single_str_fields:
+ data[k] = cls._coerce_list_to_string(v)
+ # For ints that might appear as list, take first
+ if isinstance(v, list) and k in {
+ "status_code",
+ "num_pages",
+ "credits_used",
+ }:
+ first = v[0] if v else None
+ data[k] = cls._coerce_string_to_int(first)
+ return data
+
+ @field_validator(
+ "robots",
+ "og_title",
+ "og_description",
+ "og_url",
+ "og_image",
+ "language",
+ mode="before",
+ )
@classmethod
def coerce_lists_to_string_fields(cls, v):
return cls._coerce_list_to_string(v)
- @field_validator('status_code', mode='before')
+ @field_validator("status_code", mode="before")
@classmethod
def coerce_status_code_to_int(cls, v):
return cls._coerce_string_to_int(v)
+
class AgentOptions(BaseModel):
"""Configuration for the agent in extract operations."""
+
model: Literal["FIRE-1"] = "FIRE-1"
+
class AttributeResult(BaseModel):
"""Result of attribute extraction."""
+
selector: str
attribute: str
values: List[str]
+
class BrandingProfile(BaseModel):
"""Branding information extracted from a website."""
+
model_config = {"extra": "allow"}
-
+
color_scheme: Optional[Literal["light", "dark"]] = None
logo: Optional[str] = None
fonts: Optional[List[Dict[str, Any]]] = None
@@ -142,8 +257,10 @@ class BrandingProfile(BaseModel):
tone: Optional[Dict[str, str]] = None
personality: Optional[Dict[str, Any]] = None
+
class Document(BaseModel):
"""A scraped document."""
+
markdown: Optional[str] = None
html: Optional[str] = None
raw_html: Optional[str] = None
@@ -166,31 +283,42 @@ class Document(BaseModel):
return md
if isinstance(md, dict):
try:
- return DocumentMetadata(**md)
+ return DocumentMetadata.model_validate(md)
except (ValidationError, TypeError) as exc:
logger.debug("Failed to construct DocumentMetadata from dict: %s", exc)
return DocumentMetadata()
@property
def metadata_dict(self) -> Dict[str, Any]:
- """Returns metadata as a plain dict (exclude None)."""
+ """Returns metadata as a plain dict (exclude None), including extras."""
md = self.metadata
if isinstance(md, DocumentMetadata):
- return md.model_dump(exclude_none=True)
+ out = md.model_dump(exclude_none=True)
+ # Ensure extras are preserved even if model_dump omits them
+ extra = getattr(md, "__pydantic_extra__", None)
+ if isinstance(extra, dict):
+ for k, v in extra.items():
+ if v is not None:
+ out[k] = v
+ return out
if isinstance(md, dict):
return {k: v for k, v in md.items() if v is not None}
return {}
+
# Webhook types
class WebhookConfig(BaseModel):
"""Configuration for webhooks."""
+
url: str
headers: Optional[Dict[str, str]] = None
metadata: Optional[Dict[str, str]] = None
events: Optional[List[Literal["completed", "failed", "page", "started"]]] = None
+
class WebhookData(BaseModel):
"""Data sent to webhooks."""
+
job_id: str
status: str
current: Optional[int] = None
@@ -198,73 +326,116 @@ class WebhookData(BaseModel):
data: Optional[List[Document]] = None
error: Optional[str] = None
+
class Source(BaseModel):
"""Configuration for a search source."""
+
type: str
+
SourceOption = Union[str, Source]
+
class Category(BaseModel):
"""Configuration for a search category.
-
+
Supported categories:
- "github": Filter results to GitHub repositories
- "research": Filter results to research papers and academic sites
- "pdf": Filter results to PDF files (adds filetype:pdf to search)
"""
+
type: str
+
CategoryOption = Union[str, Category]
FormatString = Literal[
# camelCase versions (API format)
- "markdown", "html", "rawHtml", "links", "images", "screenshot", "summary", "changeTracking", "json", "attributes", "branding",
+ "markdown",
+ "html",
+ "rawHtml",
+ "links",
+ "images",
+ "screenshot",
+ "summary",
+ "changeTracking",
+ "json",
+ "attributes",
+ "branding",
# snake_case versions (user-friendly)
- "raw_html", "change_tracking"
+ "raw_html",
+ "change_tracking",
]
+
class Viewport(BaseModel):
"""Viewport configuration for screenshots."""
+
width: int
height: int
+
class Format(BaseModel):
"""Configuration for a format."""
+
type: FormatString
+
class JsonFormat(Format):
"""Configuration for JSON extraction."""
+
prompt: Optional[str] = None
schema: Optional[Any] = None
+
class ChangeTrackingFormat(Format):
"""Configuration for change tracking."""
+
modes: List[Literal["git-diff", "json"]]
schema: Optional[Dict[str, Any]] = None
prompt: Optional[str] = None
tag: Optional[str] = None
+
class ScreenshotFormat(BaseModel):
"""Configuration for screenshot format."""
+
type: Literal["screenshot"] = "screenshot"
full_page: Optional[bool] = None
quality: Optional[int] = None
viewport: Optional[Union[Dict[str, int], Viewport]] = None
-
+
+
class AttributeSelector(BaseModel):
"""Selector and attribute pair for attribute extraction."""
+
selector: str
attribute: str
+
class AttributesFormat(Format):
"""Configuration for attribute extraction."""
+
type: Literal["attributes"] = "attributes"
selectors: List[AttributeSelector]
-FormatOption = Union[Dict[str, Any], FormatString, JsonFormat, ChangeTrackingFormat, ScreenshotFormat, AttributesFormat, Format]
+
+FormatOption = Union[
+ Dict[str, Any],
+ FormatString,
+ JsonFormat,
+ ChangeTrackingFormat,
+ ScreenshotFormat,
+ AttributesFormat,
+ Format,
+]
+
+
# Scrape types
class ScrapeFormats(BaseModel):
"""Output formats for scraping."""
+
formats: Optional[List[FormatOption]] = None
markdown: bool = True
html: bool = False
@@ -276,13 +447,13 @@ class ScrapeFormats(BaseModel):
change_tracking: bool = False
json: bool = False
- @field_validator('formats')
+ @field_validator("formats")
@classmethod
def validate_formats(cls, v):
"""Validate and normalize formats input."""
if v is None:
return v
-
+
normalized_formats = []
for format_item in v:
if isinstance(format_item, str):
@@ -294,12 +465,14 @@ class ScrapeFormats(BaseModel):
normalized_formats.append(format_item)
else:
raise ValueError(f"Invalid format format: {format_item}")
-
+
return normalized_formats
+
class ScrapeOptions(BaseModel):
"""Options for scraping operations."""
- formats: Optional[Union['ScrapeFormats', List[FormatOption]]] = None
+
+ formats: Optional[Union["ScrapeFormats", List[FormatOption]]] = None
headers: Optional[Dict[str, str]] = None
include_tags: Optional[List[str]] = None
exclude_tags: Optional[List[str]] = None
@@ -307,9 +480,23 @@ class ScrapeOptions(BaseModel):
timeout: Optional[int] = None
wait_for: Optional[int] = None
mobile: Optional[bool] = None
- parsers: Optional[Union[List[str], List[Union[str, 'PDFParser']]]] = None
- actions: Optional[List[Union['WaitAction', 'ScreenshotAction', 'ClickAction', 'WriteAction', 'PressAction', 'ScrollAction', 'ScrapeAction', 'ExecuteJavascriptAction', 'PDFAction']]] = None
- location: Optional['Location'] = None
+ parsers: Optional[Union[List[str], List[Union[str, "PDFParser"]]]] = None
+ actions: Optional[
+ List[
+ Union[
+ "WaitAction",
+ "ScreenshotAction",
+ "ClickAction",
+ "WriteAction",
+ "PressAction",
+ "ScrollAction",
+ "ScrapeAction",
+ "ExecuteJavascriptAction",
+ "PDFAction",
+ ]
+ ]
+ ] = None
+ location: Optional["Location"] = None
skip_tls_verification: Optional[bool] = None
remove_base64_images: Optional[bool] = None
fast_mode: Optional[bool] = None
@@ -320,7 +507,7 @@ class ScrapeOptions(BaseModel):
store_in_cache: Optional[bool] = None
integration: Optional[str] = None
- @field_validator('formats')
+ @field_validator("formats")
@classmethod
def validate_formats(cls, v):
"""Validate and normalize formats input."""
@@ -330,24 +517,34 @@ class ScrapeOptions(BaseModel):
return v
if isinstance(v, list):
return v
- raise ValueError(f"Invalid formats type: {type(v)}. Expected ScrapeFormats or List[FormatOption]")
+ raise ValueError(
+ f"Invalid formats type: {type(v)}. Expected ScrapeFormats or List[FormatOption]"
+ )
+
class ScrapeRequest(BaseModel):
"""Request for scraping a single URL."""
+
url: str
options: Optional[ScrapeOptions] = None
+
class ScrapeData(Document):
"""Scrape results data."""
+
pass
+
class ScrapeResponse(BaseResponse[ScrapeData]):
"""Response for scrape operations."""
+
pass
+
# Crawl types
class CrawlRequest(BaseModel):
"""Request for crawling a website."""
+
url: str
prompt: Optional[str] = None
exclude_paths: Optional[List[str]] = None
@@ -366,13 +563,17 @@ class CrawlRequest(BaseModel):
zero_data_retention: bool = False
integration: Optional[str] = None
+
class CrawlResponse(BaseModel):
"""Information about a crawl job."""
+
id: str
url: str
+
class CrawlJob(BaseModel):
"""Crawl job status and progress data."""
+
status: Literal["scraping", "completed", "failed", "cancelled"]
total: int = 0
completed: int = 0
@@ -381,55 +582,63 @@ class CrawlJob(BaseModel):
next: Optional[str] = None
data: List[Document] = []
+
class CrawlStatusRequest(BaseModel):
"""Request to get crawl job status."""
+
job_id: str
+
class SearchResultWeb(BaseModel):
"""A web search result with URL, title, and description."""
+
url: str
title: Optional[str] = None
description: Optional[str] = None
category: Optional[str] = None
+
class SearchResultNews(BaseModel):
- """A news search result with URL, title, snippet, date, image URL, and position."""
- title: Optional[str] = None
- url: Optional[str] = None
- snippet: Optional[str] = None
- date: Optional[str] = None
- image_url: Optional[str] = None
- position: Optional[int] = None
- category: Optional[str] = None
+ """A news search result with URL, title, snippet, date, image URL, and position."""
+
+ title: Optional[str] = None
+ url: Optional[str] = None
+ snippet: Optional[str] = None
+ date: Optional[str] = None
+ image_url: Optional[str] = None
+ position: Optional[int] = None
+ category: Optional[str] = None
+
class SearchResultImages(BaseModel):
- """An image search result with URL, title, image URL, image width, image height, and position."""
- title: Optional[str] = None
- image_url: Optional[str] = None
- image_width: Optional[int] = None
- image_height: Optional[int] = None
- url: Optional[str] = None
- position: Optional[int] = None
+ """An image search result with URL, title, image URL, image width, image height, and position."""
+
+ title: Optional[str] = None
+ image_url: Optional[str] = None
+ image_width: Optional[int] = None
+ image_height: Optional[int] = None
+ url: Optional[str] = None
+ position: Optional[int] = None
-class SearchData(BaseModel):
- """Search results grouped by source type."""
- web: Optional[List[Union[SearchResultWeb, Document]]] = None
- news: Optional[List[Union[SearchResultNews, Document]]] = None
- images: Optional[List[Union[SearchResultImages, Document]]] = None
class MapDocument(Document):
"""A document from a map operation with URL and description."""
+
url: str
description: Optional[str] = None
+
# Crawl params types
class CrawlParamsRequest(BaseModel):
"""Request for getting crawl parameters from LLM."""
+
url: str
prompt: str
+
class CrawlParamsData(BaseModel):
"""Data returned from crawl params endpoint."""
+
include_paths: Optional[List[str]] = None
exclude_paths: Optional[List[str]] = None
max_discovery_depth: Optional[int] = None
@@ -447,13 +656,17 @@ class CrawlParamsData(BaseModel):
warning: Optional[str] = None
integration: Optional[str] = None
+
class CrawlParamsResponse(BaseResponse[CrawlParamsData]):
"""Response from crawl params endpoint."""
+
pass
+
# Batch scrape types
class BatchScrapeRequest(BaseModel):
"""Request for batch scraping multiple URLs (internal helper only)."""
+
urls: List[str]
options: Optional[ScrapeOptions] = None
webhook: Optional[Union[str, WebhookConfig]] = None
@@ -463,14 +676,18 @@ class BatchScrapeRequest(BaseModel):
zero_data_retention: Optional[bool] = None
integration: Optional[str] = None
+
class BatchScrapeResponse(BaseModel):
"""Response from starting a batch scrape job (mirrors CrawlResponse naming)."""
+
id: str
url: str
invalid_urls: Optional[List[str]] = None
+
class BatchScrapeJob(BaseModel):
"""Batch scrape job status and results."""
+
status: Literal["scraping", "completed", "failed", "cancelled"]
completed: int
total: int
@@ -479,43 +696,55 @@ class BatchScrapeJob(BaseModel):
next: Optional[str] = None
data: List[Document] = []
+
class BatchScrapeStatusRequest(BaseModel):
"""Request to get batch scrape job status."""
+
job_id: str
+
class BatchScrapeErrorsRequest(BaseModel):
"""Request to get errors for a batch scrape job."""
+
job_id: str
+
# Map types
class MapOptions(BaseModel):
"""Options for mapping operations."""
+
search: Optional[str] = None
sitemap: Literal["only", "include", "skip"] = "include"
include_subdomains: Optional[bool] = None
limit: Optional[int] = None
timeout: Optional[int] = None
integration: Optional[str] = None
- location: Optional['Location'] = None
+ location: Optional["Location"] = None
+
class MapRequest(BaseModel):
"""Request for mapping a website."""
+
url: str
options: Optional[MapOptions] = None
-
class MapData(BaseModel):
"""Map results data."""
- links: List['SearchResult']
+
+ links: List["SearchResult"]
+
class MapResponse(BaseResponse[MapData]):
"""Response for map operations."""
+
pass
+
# Extract types
class ExtractRequest(BaseModel):
"""Request for extract operations."""
+
urls: Optional[List[str]] = None
prompt: Optional[str] = None
schema_: Optional[Dict[str, Any]] = Field(default=None, alias="schema")
@@ -527,9 +756,11 @@ class ExtractRequest(BaseModel):
ignore_invalid_urls: Optional[bool] = None
integration: Optional[str] = None
agent: Optional[AgentOptions] = None
-
+
+
class ExtractResponse(BaseModel):
"""Response for extract operations (start/status/final)."""
+
success: Optional[bool] = None
id: Optional[str] = None
status: Optional[Literal["processing", "completed", "failed", "cancelled"]] = None
@@ -539,122 +770,174 @@ class ExtractResponse(BaseModel):
sources: Optional[Dict[str, Any]] = None
expires_at: Optional[datetime] = None
+
# Usage/limits types
class ConcurrencyCheck(BaseModel):
"""Current concurrency and limits for the team/API key."""
+
concurrency: int
max_concurrency: int
+
class CreditUsage(BaseModel):
"""Remaining credits for the team/API key."""
+
remaining_credits: int
plan_credits: Optional[int] = None
billing_period_start: Optional[str] = None
billing_period_end: Optional[str] = None
+
class TokenUsage(BaseModel):
"""Recent token usage metrics (if available)."""
+
remaining_tokens: int
plan_tokens: Optional[int] = None
billing_period_start: Optional[str] = None
billing_period_end: Optional[str] = None
+
class QueueStatusRequest(BaseModel):
"""Request to retrieve queue status."""
+
pass
+
class QueueStatusResponse(BaseModel):
"""Metrics about the team's scrape queue."""
+
jobs_in_queue: int
active_jobs_in_queue: int
waiting_jobs_in_queue: int
max_concurrency: int
most_recent_success: Optional[datetime] = None
+
class CreditUsageHistoricalPeriod(BaseModel):
startDate: Optional[str] = None
endDate: Optional[str] = None
apiKey: Optional[str] = None
creditsUsed: int
+
class CreditUsageHistoricalResponse(BaseModel):
success: bool
periods: List[CreditUsageHistoricalPeriod]
+
class TokenUsageHistoricalPeriod(BaseModel):
startDate: Optional[str] = None
endDate: Optional[str] = None
apiKey: Optional[str] = None
tokensUsed: int
+
class TokenUsageHistoricalResponse(BaseModel):
success: bool
periods: List[TokenUsageHistoricalPeriod]
+
# Action types
class WaitAction(BaseModel):
"""Wait action to perform during scraping."""
+
type: Literal["wait"] = "wait"
milliseconds: Optional[int] = None
selector: Optional[str] = None
+
class ScreenshotAction(BaseModel):
"""Screenshot action to perform during scraping."""
+
type: Literal["screenshot"] = "screenshot"
full_page: Optional[bool] = None
quality: Optional[int] = None
viewport: Optional[Union[Dict[str, int], Viewport]] = None
+
class ClickAction(BaseModel):
"""Click action to perform during scraping."""
+
type: Literal["click"] = "click"
selector: str
+
class WriteAction(BaseModel):
"""Write action to perform during scraping."""
+
type: Literal["write"] = "write"
text: str
+
class PressAction(BaseModel):
"""Press action to perform during scraping."""
+
type: Literal["press"] = "press"
key: str
+
class ScrollAction(BaseModel):
"""Scroll action to perform during scraping."""
+
type: Literal["scroll"] = "scroll"
direction: Literal["up", "down"]
selector: Optional[str] = None
+
class ScrapeAction(BaseModel):
"""Scrape action to perform during scraping."""
+
type: Literal["scrape"] = "scrape"
+
class ExecuteJavascriptAction(BaseModel):
"""Execute javascript action to perform during scraping."""
+
type: Literal["executeJavascript"] = "executeJavascript"
script: str
+
class PDFAction(BaseModel):
"""PDF action to perform during scraping."""
+
type: Literal["pdf"] = "pdf"
- format: Optional[Literal["A0", "A1", "A2", "A3", "A4", "A5", "A6", "Letter", "Legal", "Tabloid", "Ledger"]] = None
+ format: Optional[
+ Literal[
+ "A0",
+ "A1",
+ "A2",
+ "A3",
+ "A4",
+ "A5",
+ "A6",
+ "Letter",
+ "Legal",
+ "Tabloid",
+ "Ledger",
+ ]
+ ] = None
landscape: Optional[bool] = None
scale: Optional[float] = None
+
class PDFParser(BaseModel):
"""PDF parser configuration with optional page limit."""
+
type: Literal["pdf"] = "pdf"
max_pages: Optional[int] = None
+
# Location types
class Location(BaseModel):
"""Location configuration for scraping."""
+
country: Optional[str] = None
languages: Optional[List[str]] = None
+
class SearchRequest(BaseModel):
"""Request for search operations."""
+
query: str
sources: Optional[List[SourceOption]] = None
categories: Optional[List[CategoryOption]] = None
@@ -666,13 +949,13 @@ class SearchRequest(BaseModel):
scrape_options: Optional[ScrapeOptions] = None
integration: Optional[str] = None
- @field_validator('sources')
+ @field_validator("sources")
@classmethod
def validate_sources(cls, v):
"""Validate and normalize sources input."""
if v is None:
return v
-
+
normalized_sources = []
for source in v:
if isinstance(source, str):
@@ -683,16 +966,16 @@ class SearchRequest(BaseModel):
normalized_sources.append(source)
else:
raise ValueError(f"Invalid source format: {source}")
-
+
return normalized_sources
-
- @field_validator('categories')
+
+ @field_validator("categories")
@classmethod
def validate_categories(cls, v):
"""Validate and normalize categories input."""
if v is None:
return v
-
+
normalized_categories = []
for category in v:
if isinstance(category, str):
@@ -703,46 +986,59 @@ class SearchRequest(BaseModel):
normalized_categories.append(category)
else:
raise ValueError(f"Invalid category format: {category}")
-
+
return normalized_categories
# NOTE: parsers validation does not belong on SearchRequest; it is part of ScrapeOptions.
+
class LinkResult(BaseModel):
"""A generic link result with optional metadata (used by search and map)."""
+
url: str
title: Optional[str] = None
description: Optional[str] = None
-
+
+
# Backward-compatible alias for existing tests/usages
SearchResult = LinkResult
+
class SearchData(BaseModel):
"""Search results grouped by source type."""
+
web: Optional[List[Union[SearchResultWeb, Document]]] = None
news: Optional[List[Union[SearchResultNews, Document]]] = None
images: Optional[List[Union[SearchResultImages, Document]]] = None
+
class SearchResponse(BaseResponse[SearchData]):
"""Response from search operation."""
+
pass
+
# Error types
class ErrorDetails(BaseModel):
"""Detailed error information."""
+
code: Optional[str] = None
message: str
details: Optional[Dict[str, Any]] = None
+
class ErrorResponse(BaseModel):
"""Error response structure."""
+
success: bool = False
error: str
details: Optional[ErrorDetails] = None
+
# Job management types
class JobStatus(BaseModel):
"""Generic job status information."""
+
id: str
status: Literal["pending", "scraping", "completed", "failed"]
current: Optional[int] = None
@@ -751,54 +1047,71 @@ class JobStatus(BaseModel):
completed_at: Optional[datetime] = None
expires_at: Optional[datetime] = None
+
class CrawlError(BaseModel):
"""A crawl error."""
+
id: str
timestamp: Optional[datetime] = None
url: str
code: Optional[str] = None
error: str
+
class CrawlErrorsResponse(BaseModel):
"""Response from crawl error monitoring."""
+
errors: List[CrawlError]
robots_blocked: List[str]
+
class CrawlErrorsRequest(BaseModel):
"""Request for crawl error monitoring."""
+
crawl_id: str
+
class ActiveCrawl(BaseModel):
"""Information about an active crawl job."""
+
id: str
team_id: str
url: str
options: Optional[Dict[str, Any]] = None
+
class ActiveCrawlsResponse(BaseModel):
"""Response from active crawls endpoint."""
+
success: bool = True
crawls: List[ActiveCrawl]
+
class ActiveCrawlsRequest(BaseModel):
"""Request for listing active crawl jobs."""
+
pass
+
# Configuration types
class ClientConfig(BaseModel):
"""Configuration for the Firecrawl client."""
+
api_key: Optional[str] = None
api_url: str = "https://api.firecrawl.dev"
timeout: Optional[float] = None
max_retries: int = 3
backoff_factor: float = 0.5
+
class PaginationConfig(BaseModel):
"""Configuration for pagination behavior."""
+
auto_paginate: bool = True
max_pages: Optional[int] = Field(default=None, ge=0)
max_results: Optional[int] = Field(default=None, ge=0)
- max_wait_time: Optional[int] = Field(default=None, ge=0) # seconds
+ max_wait_time: Optional[int] = Field(default=None, ge=0) # seconds
+
# Response union types
AnyResponse = Union[
diff --git a/apps/python-sdk/firecrawl/v2/utils/normalize.py b/apps/python-sdk/firecrawl/v2/utils/normalize.py
index 2b523c7ec..296933a7c 100644
--- a/apps/python-sdk/firecrawl/v2/utils/normalize.py
+++ b/apps/python-sdk/firecrawl/v2/utils/normalize.py
@@ -62,19 +62,7 @@ def _map_metadata_keys(md: Dict[str, Any]) -> Dict[str, Any]:
except ValueError:
pass
- # Generic rule: if a value is a list, join with ", " for string-like fields,
- # except for explicit fields we preserve as lists.
- preserve_list_fields: List[str] = [
- "og_locale_alternate",
- ]
- for f, val in list(out.items()):
- if isinstance(val, list) and f not in preserve_list_fields:
- try:
- out[f] = ", ".join(str(x) for x in val)
- except Exception:
- # Fallback: keep original list if join fails
- pass
-
+ # Preserve list values for unknown keys; only lightweight coercions above
return out
@@ -96,11 +84,10 @@ def normalize_document_input(doc: Dict[str, Any]) -> Dict[str, Any]:
md = normalized.get("metadata")
if isinstance(md, dict):
mapped = _map_metadata_keys(md)
- # Construct a concrete DocumentMetadata so downstream has a typed object
+ # Construct a typed DocumentMetadata; extras allowed/preserved
try:
- normalized["metadata"] = DocumentMetadata(**mapped)
+ normalized["metadata"] = DocumentMetadata.model_validate(mapped)
except Exception:
- # Fallback to mapped dict if model construction fails for any reason
normalized["metadata"] = mapped
# Normalize branding top-level camelCase keys
@@ -134,5 +121,3 @@ def _map_search_result_keys(result: Dict[str, Any], result_type: str) -> Dict[st
out[snake] = v
return out
-
-