Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
add query format support to js and python sdks (#3118)
Developers Digest011fbf7bb609
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex d652d6d74..9a8ed5a17 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.15.4",+ "version": "4.16.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex a801c0931..21e294a7c 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -51,13 +51,19 @@ export interface AttributesFormat extends Format {}>;}+export interface QueryFormat {+ type: 'query';+ prompt: string;+}+export type FormatOption =| FormatString| Format| JsonFormat| ChangeTrackingFormat| ScreenshotFormat- | AttributesFormat;+ | AttributesFormat+ | QueryFormat;export interface LocationConfig {country?: string;@@ -388,6 +394,7 @@ export interface Document {values: string[];}>;actions?: Record<string, unknown>;+ answer?: string;warning?: string;changeTracking?: Record<string, unknown>;branding?: BrandingProfile;diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 5219818cb..c2f626428 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.18.1"+__version__ = "4.19.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 9876d94f0..6f5bf37f6 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -275,6 +275,7 @@ class Document(BaseModel):images: Optional[List[str]] = Nonescreenshot: Optional[str] = Noneactions: Optional[Dict[str, Any]] = None+ answer: Optional[str] = Nonewarning: Optional[str] = Nonechange_tracking: Optional[Dict[str, Any]] = Nonebranding: Optional[BrandingProfile] = None@@ -384,6 +385,7 @@ FormatString = Literal["json","attributes","branding",+ "query",# snake_case versions (user-friendly)"raw_html","change_tracking",@@ -442,6 +444,13 @@ class AttributesFormat(Format):selectors: List[AttributeSelector]+class QueryFormat(Format):+ """Configuration for query format - ask a question about the page content."""++ type: Literal["query"] = "query"+ prompt: str++FormatOption = Union[Dict[str, Any],FormatString,@@ -449,6 +458,7 @@ FormatOption = Union[ChangeTrackingFormat,ScreenshotFormat,AttributesFormat,+ QueryFormat,Format,]@@ -478,8 +488,14 @@ class ScrapeFormats(BaseModel):normalized_formats = []for format_item in v:if isinstance(format_item, str):+ if format_item == "query":+ raise ValueError("query format must be an object with 'type' and 'prompt' fields")normalized_formats.append(Format(type=format_item))elif isinstance(format_item, dict):+ # Reject query dicts missing prompt early+ prompt = format_item.get('prompt')+ if format_item.get('type') == 'query' and (not isinstance(prompt, str) or not prompt.strip()):+ raise ValueError("query format requires a non-empty 'prompt' string")# Preserve dicts as-is to avoid dropping custom fields like 'schema'normalized_formats.append(format_item)elif isinstance(format_item, Format):diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.pyindex 4e5fd2c1b..2de93f69c 100644--- a/apps/python-sdk/firecrawl/v2/utils/validation.py+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py@@ -435,6 +435,28 @@ def _validate_json_format(format_obj: Any) -> Dict[str, Any]:return normalized+def _validate_query_format(format_obj: Any) -> Dict[str, Any]:+ """+ Validate and prepare query format object.++ Args:+ format_obj: Format object that should be query type++ Returns:+ Validated query format dict++ Raises:+ ValueError: If query format is missing required 'prompt' field+ """+ if not isinstance(format_obj, dict):+ raise ValueError("query format must be an object with 'type' and 'prompt' fields")++ if not isinstance(format_obj.get('prompt'), str) or not format_obj['prompt'].strip():+ raise ValueError("query format requires a non-empty 'prompt' string")++ return format_obj++def validate_scrape_options(options: Optional[ScrapeOptions]) -> Optional[ScrapeOptions]:"""Validate and normalize scrape options.@@ -542,12 +564,16 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[stif isinstance(fmt, str):if fmt == "json":raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")+ if fmt == "query":+ raise ValueError("query format must be an object with 'type' and 'prompt' fields")converted_formats.append(_convert_format_string(fmt))elif isinstance(fmt, dict):fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else Noneif fmt_type == 'json':validated_json = _validate_json_format({**fmt, 'type': 'json'})converted_formats.append(validated_json)+ elif fmt_type == 'query':+ converted_formats.append(_validate_query_format(fmt))elif fmt_type == 'screenshot':# Normalize screenshot optionsnormalized = {**fmt, 'type': 'screenshot'}@@ -565,6 +591,8 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[stelif hasattr(fmt, 'type'):if fmt.type == 'json':converted_formats.append(_validate_json_format(fmt.model_dump()))+ elif fmt.type == 'query':+ converted_formats.append(_validate_query_format(fmt.model_dump(exclude_none=True)))else:converted_formats.append(_convert_format_string(fmt.type))else:@@ -593,12 +621,16 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[stif isinstance(fmt, str):if fmt == "json":raise ValueError("json format must be an object with 'type', 'prompt', and 'schema' fields")+ if fmt == "query":+ raise ValueError("query format must be an object with 'type' and 'prompt' fields")converted_formats.append(_convert_format_string(fmt))elif isinstance(fmt, dict):fmt_type = _convert_format_string(fmt.get('type')) if fmt.get('type') else Noneif fmt_type == 'json':validated_json = _validate_json_format({**fmt, 'type': 'json'})converted_formats.append(validated_json)+ elif fmt_type == 'query':+ converted_formats.append(_validate_query_format(fmt))elif fmt_type == 'screenshot':normalized = {**fmt, 'type': 'screenshot'}if 'full_page' in normalized:@@ -624,6 +656,8 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[stif vp is not None:normalized['viewport'] = vp.model_dump(exclude_none=True) if hasattr(vp, 'model_dump') else vpconverted_formats.append(normalized)+ elif fmt.type == 'query':+ converted_formats.append(_validate_query_format(fmt.model_dump(exclude_none=True)))else:converted_formats.append(_convert_format_string(fmt.type))else: