Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

(python-sdk): fixed search types + added max_keepalive_connections to fix event loop is closed bug (#2005)

Rafael Miller61c3abcb5b37
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 7f0ecc4cf..424b24bcd 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "3.1.1"
+__version__ = "3.2.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_search.py
index 6d0b6daa9..12a21bda5 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_search.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_search.py
@@ -2,8 +2,19 @@ import os
import pytest
from dotenv import load_dotenv
from firecrawl import AsyncFirecrawl
-from firecrawl.v2.types import ScrapeOptions, ScrapeFormats, SearchData, SearchResult, Document
+from firecrawl.types import (
+ SearchData,
+ Document,
+ ScrapeOptions,
+ ScrapeFormats,
+ SearchResultWeb,
+ SearchResultNews,
+ SearchResultImages,
+)
+load_dotenv()
+
+firecrawl = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
def _collect_texts(entries):
texts = []
@@ -36,148 +47,202 @@ def _is_document(entry) -> bool:
hasattr(entry, 'change_tracking') or \
hasattr(entry, 'summary')
+async def test_async_search_minimal_request():
+ results = await firecrawl.search(
+ query="What is the capital of France?"
+ )
+ assert isinstance(results, SearchData)
+ assert hasattr(results, 'web')
+ assert results.web is not None
+ assert len(results.web) > 0
+ assert hasattr(results, 'news')
+ assert results.news is None
+ assert hasattr(results, 'images')
+ assert results.images is None
+
+ for result in results.web:
+ assert isinstance(result, SearchResultWeb)
+ assert hasattr(result, 'url')
+ assert hasattr(result, 'title')
+ assert hasattr(result, 'description')
+ assert result.url.startswith('http')
+ assert result.title is not None
+ assert result.description is not None
+
+ all_text = ' '.join(_collect_texts(results.web))
+ assert 'paris' in all_text
+
+ assert results.news is None
+ assert results.images is None
-load_dotenv()
-
-if not os.getenv("API_KEY"):
- raise ValueError("API_KEY is not set")
-
-if not os.getenv("API_URL"):
- raise ValueError("API_URL is not set")
+async def test_async_search_with_sources():
+ results = await firecrawl.search(
+ query="firecrawl",
+ sources=["web", "news", "images"],
+ limit=3
+ )
+ assert isinstance(results, SearchData)
+ assert results.web is not None
+ assert len(results.web) <= 3
+ assert isinstance(results.web[0], SearchResultWeb)
+ if results.news is not None:
+ assert len(results.news) <= 3
+ assert isinstance(results.news[0], SearchResultNews)
-async def test_async_search_minimal():
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
- data = await client.search("What is the capital of France?")
- # Assert sections like sync tests
- assert hasattr(data, "web")
- assert hasattr(data, "news")
- assert hasattr(data, "images")
- assert data.web is not None
- assert len(data.web) > 0
- titles = [getattr(r, "title", None) for r in data.web]
- descs = [getattr(r, "description", None) for r in data.web]
- all_text = " ".join([t.lower() for t in titles if t] + [d.lower() for d in descs if d])
- assert "paris" in all_text
- assert data.news is None
- assert data.images is None
+ if results.images is not None:
+ assert len(results.images) <= 3
+ assert isinstance(results.images[0], SearchResultImages)
+ web_titles = [result.title.lower() for result in results.web]
+ web_descriptions = [result.description.lower() for result in results.web]
+ all_web_text = ' '.join(web_titles + web_descriptions)
+ assert 'firecrawl' in all_web_text
@pytest.mark.asyncio
-async def test_async_search_with_sources_and_limit():
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
- data = await client.search("firecrawl", sources=["web", "news"], limit=3)
- # Sections present
- assert hasattr(data, "web") and hasattr(data, "news") and hasattr(data, "images")
- # Web present, images absent, news optional but if present respects limit
- if data.web is not None:
- assert len(data.web) <= 3
- if data.news is not None:
- assert len(data.news) <= 3
- assert data.images is None
-
+async def test_async_search_result_structure():
+ results = await firecrawl.search(
+ query="test query",
+ limit=1
+ )
+ if results.web and len(results.web) > 0:
+ result = results.web[0]
+ assert hasattr(result, 'url')
+ assert hasattr(result, 'title')
+ assert hasattr(result, 'description')
+ assert isinstance(result.url, str)
+ assert isinstance(result.title, str) or result.title is None
+ assert isinstance(result.description, str) or result.description is None
+ assert result.url.startswith('http')
@pytest.mark.asyncio
-async def test_async_search_with_all_params():
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
- data = await client.search(
- "artificial intelligence",
- sources=["web", "news"],
+async def test_async_search_all_parameters():
+ from firecrawl.types import ScrapeOptions, Location, WaitAction
+ schema = {
+ "type": "object",
+ "properties": {
+ "title": {"type": "string"},
+ "description": {"type": "string"},
+ "url": {"type": "string"}
+ },
+ "required": ["title", "description"]
+ }
+ results = await firecrawl.search(
+ query="artificial intelligence",
+ sources=[
+ {"type": "web"},
+ {"type": "news"}
+ ],
limit=3,
- tbs="qdr:w",
+ tbs="qdr:m",
location="US",
- ignore_invalid_urls=False,
- timeout=30000,
- scrape_options={
- "formats": ["markdown"],
- "headers": {"User-Agent": "E2E-AIO"},
- "include_tags": ["h1"],
- "exclude_tags": ["nav"],
- "only_main_content": False,
- "timeout": 15000,
- "wait_for": 2000,
- "mobile": True,
- "skip_tls_verification": True,
- "remove_base64_images": False,
- },
+ ignore_invalid_urls=True,
+ timeout=60000,
+ scrape_options=ScrapeOptions(
+ formats=[
+ "markdown",
+ "html",
+ {
+ "type": "json",
+ "prompt": "Extract the title and description from the page",
+ "schema": schema
+ },
+ {"type": "summary"}
+ ],
+ headers={"User-Agent": "Firecrawl-Test/1.0"},
+ include_tags=["h1", "h2", "p"],
+ exclude_tags=["nav", "footer"],
+ only_main_content=True,
+ wait_for=2000,
+ mobile=False,
+ skip_tls_verification=False,
+ remove_base64_images=True,
+ block_ads=True,
+ proxy="basic",
+ max_age=3600000,
+ store_in_cache=True,
+ location=Location(
+ country="US",
+ languages=["en"]
+ ),
+ actions=[
+ WaitAction(milliseconds=1000)
+ ]
+ )
)
- # Structure and type assertions mirroring sync
- assert isinstance(data, SearchData)
- assert hasattr(data, "web") and hasattr(data, "news") and hasattr(data, "images")
- assert data.web is not None
- assert len(data.web) <= 3
- non_doc = [r for r in (data.web or []) if not _is_document(r)]
- if non_doc:
- combined = " ".join(_collect_texts(non_doc))
- ai_terms = ["artificial", "intelligence", "ai", "machine", "learning"]
- assert any(term in combined for term in ai_terms)
- for r in data.web:
- assert isinstance(r, (SearchResult, Document))
- if isinstance(r, Document):
- assert (r.markdown is not None) or (r.html is not None)
+ assert isinstance(results, SearchData)
+ assert hasattr(results, 'web')
+ assert hasattr(results, 'news')
+ assert hasattr(results, 'images')
+ assert results.web is not None
+ assert len(results.web) <= 3
+
+ non_doc_entries = [r for r in (results.web or []) if not _is_document(r)]
+ if non_doc_entries:
+ all_web_text = ' '.join(_collect_texts(non_doc_entries))
+ ai_terms = ['artificial', 'intelligence', 'ai', 'machine', 'learning']
+ assert any(term in all_web_text for term in ai_terms)
+
+ for result in results.web:
+ assert isinstance(result, (SearchResultWeb, Document))
+ if isinstance(result, Document):
+ assert (result.markdown is not None) or (result.html is not None)
else:
- assert hasattr(r, "url")
- assert isinstance(r.url, str) and r.url.startswith("http")
- if data.news is not None:
- assert len(data.news) <= 10
- for r in data.news:
- assert isinstance(r, (SearchResult, Document))
- if isinstance(r, Document):
- assert (r.markdown is not None) or (r.html is not None)
+ assert hasattr(result, 'url')
+ assert isinstance(result.url, str) and result.url.startswith('http')
+
+ if results.news is not None:
+ assert len(results.news) <= 3
+ for result in results.news:
+ assert isinstance(result, (SearchResultNews, Document))
+ if isinstance(result, Document):
+ assert (result.markdown is not None) or (result.html is not None)
else:
- assert isinstance(r.url, str) and r.url.startswith("http")
- assert data.images is None
-
-
-async def test_async_search_minimal_content_check():
- """Stronger assertion similar to sync: content check on a known query."""
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
- data = await client.search("What is the capital of France?")
- assert hasattr(data, "web") and data.web is not None
- non_doc = [r for r in (data.web or []) if not _is_document(r)]
- if non_doc:
- combined = " ".join(_collect_texts(non_doc))
- assert "paris" in combined
-
-
-async def test_async_search_result_structure():
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
- data = await client.search("test query", limit=1)
- if data.web and len(data.web) > 0:
- result = data.web[0]
- assert hasattr(result, "url")
- assert hasattr(result, "title")
- assert hasattr(result, "description")
- assert isinstance(result.url, str) and result.url.startswith("http")
- assert isinstance(getattr(result, "title", None), (str, type(None)))
- assert isinstance(getattr(result, "description", None), (str, type(None)))
+ assert hasattr(result, 'url')
+ assert isinstance(result.url, str) and result.url.startswith('http')
+ assert results.images is None
@pytest.mark.asyncio
async def test_async_search_formats_flexibility():
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
- # list string
- res1 = await client.search("python programming", limit=1, scrape_options=ScrapeOptions(formats=["markdown"]))
- # list objects
- res2 = await client.search("python programming", limit=1, scrape_options=ScrapeOptions(formats=[{"type": "markdown"}]))
- # ScrapeFormats object
- res3 = await client.search("python programming", limit=1, scrape_options=ScrapeOptions(formats=ScrapeFormats(markdown=True)))
- assert isinstance(res1, SearchData) and hasattr(res1, "web")
- assert isinstance(res2, SearchData) and hasattr(res2, "web")
- assert isinstance(res3, SearchData) and hasattr(res3, "web")
-
+ # Test with list format
+ results1 = await firecrawl.search(
+ query="python programming",
+ limit=1,
+ scrape_options=ScrapeOptions(
+ formats=["markdown"]
+ )
+ )
+ # Test with ScrapeFormats object
+ results2 = await firecrawl.search(
+ query="python programming",
+ limit=1,
+ scrape_options=ScrapeOptions(
+ formats=ScrapeFormats(markdown=True)
+ )
+ )
+ assert isinstance(results1, SearchData)
+ assert isinstance(results2, SearchData)
+ assert results1.web is not None
+ assert results2.web is not None
@pytest.mark.asyncio
-async def test_async_search_json_format_object():
- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
- json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}
- data = await client.search(
- "site:docs.firecrawl.dev",
+async def test_async_search_with_json_format_object():
+ json_schema = {
+ "type": "object",
+ "properties": {
+ "title": {"type": "string"}
+ },
+ "required": ["title"],
+ }
+ results = await firecrawl.search(
+ query="site:docs.firecrawl.dev",
limit=1,
- scrape_options={"formats": [{"type": "json", "prompt": "Extract page title", "schema": json_schema}]},
+ scrape_options=ScrapeOptions(
+ formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]
+ ),
)
- assert hasattr(data, "web")
-
+ assert isinstance(results, SearchData)
+ assert results.web is not None and len(results.web) >= 0
diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
index 85837f725..54b494cb9 100644
--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py
@@ -1,7 +1,7 @@
from firecrawl import Firecrawl
import os
from dotenv import load_dotenv
-from firecrawl.types import SearchData, SearchResult, Document, ScrapeFormats, ScrapeOptions
+from firecrawl.types import SearchData, Document, ScrapeOptions, SearchResultWeb, SearchResultNews, SearchResultImages
load_dotenv()
@@ -53,7 +53,7 @@ def test_search_minimal_request():
assert results.images is None
for result in results.web:
- assert isinstance(result, SearchResult)
+ assert isinstance(result, SearchResultWeb)
assert hasattr(result, 'url')
assert hasattr(result, 'title')
assert hasattr(result, 'description')
@@ -73,7 +73,7 @@ def test_search_with_sources():
"""Test search with specific sources."""
results = firecrawl.search(
query="firecrawl",
- sources=["web", "news"],
+ sources=["web", "news", "images"],
limit=3
)
@@ -81,11 +81,15 @@ def test_search_with_sources():
assert results.web is not None
assert len(results.web) <= 3
+ assert isinstance(results.web[0], SearchResultWeb)
if results.news is not None:
assert len(results.news) <= 3
+ assert isinstance(results.news[0], SearchResultNews)
- assert results.images is None
+ if results.images is not None:
+ assert len(results.images) <= 3
+ assert isinstance(results.images[0], SearchResultImages)
web_titles = [result.title.lower() for result in results.web]
web_descriptions = [result.description.lower() for result in results.web]
@@ -193,7 +197,7 @@ def test_search_all_parameters():
# Test that each result has proper structure
for result in results.web:
- assert isinstance(result, (SearchResult, Document))
+ assert isinstance(result, (SearchResultWeb, Document))
if isinstance(result, Document):
# Document path: ensure content present
assert (result.markdown is not None) or (result.html is not None)
@@ -206,7 +210,7 @@ def test_search_all_parameters():
if results.news is not None:
assert len(results.news) <= 3
for result in results.news:
- assert isinstance(result, (SearchResult, Document))
+ assert isinstance(result, (SearchResultNews, Document))
if isinstance(result, Document):
assert (result.markdown is not None) or (result.html is not None)
else:
diff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.py
index d46b43722..0fcaa8a74 100644
--- a/apps/python-sdk/firecrawl/types.py
+++ b/apps/python-sdk/firecrawl/types.py
@@ -48,7 +48,9 @@ from .v2.types import (
JsonFormat,
FormatOption,
SearchRequest,
- SearchResult,
+ SearchResultWeb,
+ SearchResultNews,
+ SearchResultImages,
SearchData,
SearchResponse,
@@ -124,7 +126,9 @@ __all__ = [
'JsonFormat',
'FormatOption',
'SearchRequest',
- 'SearchResult',
+ 'SearchResultWeb',
+ 'SearchResultNews',
+ 'SearchResultImages',
'SearchData',
'SearchResponse',
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/search.py b/apps/python-sdk/firecrawl/v2/methods/aio/search.py
index bafd7f574..a86454db4 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/search.py
@@ -1,55 +1,172 @@
-from typing import Dict, Any
-from ...types import SearchRequest, SearchData, SearchResult, Document
-from ...utils.normalize import normalize_document_input
+import re
+from typing import Dict, Any, Union, List, TypeVar, Type
+from ...types import (
+ SearchRequest,
+ SearchData,
+ Document,
+ SearchResultWeb,
+ SearchResultNews,
+ SearchResultImages,
+)
from ...utils.http_client_async import AsyncHttpClient
from ...utils.error_handler import handle_response_error
-from ...utils.validation import prepare_scrape_options, validate_scrape_options
+from ...utils.validation import validate_scrape_options, prepare_scrape_options
+T = TypeVar("T")
+
+async def search(
+ client: AsyncHttpClient,
+ request: SearchRequest
+) -> SearchData:
+ """
+ Async search for documents.
+
+ Args:
+ client: Async HTTP client instance
+ request: Search request
+
+ Returns:
+ SearchData with search results grouped by source type
+
+ Raises:
+ FirecrawlError: If the search operation fails
+ """
+ request_data = _prepare_search_request(request)
+ try:
+ response = await client.post("/v2/search", request_data)
+ if response.status_code != 200:
+ handle_response_error(response, "search")
+ response_data = response.json()
+ if not response_data.get("success"):
+ handle_response_error(response, "search")
+ data = response_data.get("data", {}) or {}
+ out = SearchData()
+ if "web" in data:
+ out.web = _transform_array(data["web"], SearchResultWeb)
+ if "news" in data:
+ out.news = _transform_array(data["news"], SearchResultNews)
+ if "images" in data:
+ out.images = _transform_array(data["images"], SearchResultImages)
+ return out
+ except Exception as err:
+ if hasattr(err, "response"):
+ handle_response_error(getattr(err, "response"), "search")
+ raise err
+
+def _transform_array(arr: List[Any], result_type: Type[T]) -> List[Union[T, Document]]:
+ """
+ Transforms an array of items into a list of result_type or Document.
+ If the item dict contains any of the special keys, it is treated as a Document.
+ Otherwise, it is treated as result_type.
+ If the item is not a dict, it is wrapped as result_type with url=item.
+ """
+ results: List[Union[T, Document]] = []
+ for item in arr:
+ if item and isinstance(item, dict):
+ if (
+ "markdown" in item or
+ "html" in item or
+ "rawHtml" in item or
+ "links" in item or
+ "screenshot" in item or
+ "changeTracking" in item or
+ "summary" in item or
+ "json" in item
+ ):
+ results.append(Document(**item))
+ else:
+ results.append(result_type(**item))
+ else:
+ results.append(result_type(url=item))
+ return results
+
+def _validate_search_request(request: SearchRequest) -> SearchRequest:
+ """
+ Validate and normalize search request.
+
+ Args:
+ request: Search request to validate
+
+ Returns:
+ Validated request
+
+ Raises:
+ ValueError: If request is invalid
+ """
+ if not request.query or not request.query.strip():
+ raise ValueError("Query cannot be empty")
+
+ if request.limit is not None:
+ if request.limit <= 0:
+ raise ValueError("Limit must be positive")
+ if request.limit > 100:
+ raise ValueError("Limit cannot exceed 100")
+
+ if request.timeout is not None:
+ if request.timeout <= 0:
+ raise ValueError("Timeout must be positive")
+ if request.timeout > 300000:
+ raise ValueError("Timeout cannot exceed 300000ms (5 minutes)")
+
+ if request.sources is not None:
+ valid_sources = {"web", "news", "images"}
+ for source in request.sources:
+ if isinstance(source, str):
+ if source not in valid_sources:
+ raise ValueError(f"Invalid source type: {source}. Valid types: {valid_sources}")
+ elif hasattr(source, 'type'):
+ if source.type not in valid_sources:
+ raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")
+
+ if request.location is not None:
+ if not isinstance(request.location, str) or len(request.location.strip()) == 0:
+ raise ValueError("Location must be a non-empty string")
+
+ if request.tbs is not None:
+ valid_tbs_values = {
+ "qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y",
+ "d", "w", "m", "y"
+ }
+ if request.tbs in valid_tbs_values:
+ pass
+ elif request.tbs.startswith("cdr:"):
+ custom_date_pattern = r"^cdr:1,cd_min:\d{1,2}/\d{1,2}/\d{4},cd_max:\d{1,2}/\d{1,2}/\d{4}$"
+ if not re.match(custom_date_pattern, request.tbs):
+ raise ValueError(f"Invalid custom date range format: {request.tbs}. Expected format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")
+ else:
+ raise ValueError(f"Invalid tbs value: {request.tbs}. Valid values: {valid_tbs_values} or custom date range format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")
-def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
- data = request.model_dump(exclude_none=True)
- if request.ignore_invalid_urls is not None:
- data["ignoreInvalidURLs"] = request.ignore_invalid_urls
- data.pop("ignore_invalid_urls", None)
if request.scrape_options is not None:
validate_scrape_options(request.scrape_options)
- scrape_data = prepare_scrape_options(request.scrape_options)
+
+ return request
+
+def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
+ """
+ Prepare a search request payload.
+
+ Args:
+ request: Search request
+
+ Returns:
+ Request payload dictionary
+ """
+ validated_request = _validate_search_request(request)
+ data = validated_request.model_dump(exclude_none=True, by_alias=True)
+
+ if "limit" not in data and validated_request.limit is not None:
+ data["limit"] = validated_request.limit
+ if "timeout" not in data and validated_request.timeout is not None:
+ data["timeout"] = validated_request.timeout
+
+ if validated_request.ignore_invalid_urls is not None:
+ data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls
+ data.pop("ignore_invalid_urls", None)
+
+ if validated_request.scrape_options is not None:
+ scrape_data = prepare_scrape_options(validated_request.scrape_options)
if scrape_data:
data["scrapeOptions"] = scrape_data
data.pop("scrape_options", None)
- return data
-
-
-async def search(client: AsyncHttpClient, request: SearchRequest) -> SearchData:
- payload = _prepare_search_request(request)
- response = await client.post("/v2/search", payload)
- if response.status_code >= 400:
- handle_response_error(response, "search")
- body = response.json()
- if not body.get("success"):
- raise Exception(body.get("error", "Unknown error occurred"))
-
- data = body.get("data", {})
- search_data = SearchData()
- for source_type, source_documents in data.items():
- if isinstance(source_documents, list):
- results = []
- for doc_data in source_documents:
- if isinstance(doc_data, dict):
- if request.scrape_options is not None and any(
- key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'summary', 'screenshot', 'changeTracking']
- ):
- normalized = normalize_document_input(doc_data)
- results.append(Document(**normalized))
- else:
- results.append(SearchResult(
- url=doc_data.get('url', ''),
- title=doc_data.get('title'),
- description=doc_data.get('description')
- ))
- elif isinstance(doc_data, str):
- results.append(SearchResult(url=doc_data))
- if hasattr(search_data, source_type):
- setattr(search_data, source_type, results)
- return search_data
+ return data
diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.py
index b91683a5a..68d120092 100644
--- a/apps/python-sdk/firecrawl/v2/methods/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/search.py
@@ -3,11 +3,12 @@ Search functionality for Firecrawl v2 API.
"""
import re
-from typing import Optional, Dict, Any, Union
-from ..types import SearchRequest, SearchData, SearchResult, Document
+from typing import Dict, Any, Union, List, TypeVar, Type
+from ..types import SearchRequest, SearchData, Document, SearchResultWeb, SearchResultNews, SearchResultImages
from ..utils.normalize import normalize_document_input
from ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options
+T = TypeVar("T")
def search(
client: HttpClient,
@@ -27,48 +28,56 @@ def search(
FirecrawlError: If the search operation fails
"""
request_data = _prepare_search_request(request)
-
- response = client.post("/v2/search", request_data)
-
- if not response.ok:
- handle_response_error(response, "search")
-
- response_data = response.json()
-
- if not response_data.get("success"):
- # Handle error case
- error_msg = response_data.get("error", "Unknown error occurred")
- raise Exception(f"Search failed: {error_msg}")
-
- data = response_data.get("data", {})
- search_data = SearchData()
-
- for source_type, source_documents in data.items():
- if isinstance(source_documents, list):
- results = []
- for doc_data in source_documents:
- if isinstance(doc_data, dict):
- # If page scraping options were provided, API returns full Document objects
- if request.scrape_options is not None and any(
- key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'summary', 'screenshot', 'changeTracking']
- ):
- normalized = normalize_document_input(doc_data)
- results.append(Document(**normalized))
- else:
- # Minimal search result shape
- results.append(SearchResult(
- url=doc_data.get('url', ''),
- title=doc_data.get('title'),
- description=doc_data.get('description')
- ))
- elif isinstance(doc_data, str):
- results.append(SearchResult(url=doc_data))
-
- if hasattr(search_data, source_type):
- setattr(search_data, source_type, results)
-
- return search_data
+ try:
+ response = client.post("/v2/search", request_data)
+ if response.status_code != 200:
+ handle_response_error(response, "search")
+ response_data = response.json()
+ if not response_data.get("success"):
+ handle_response_error(response, "search")
+ data = response_data.get("data", {}) or {}
+ out = SearchData()
+ if "web" in data:
+ out.web = _transform_array(data["web"], SearchResultWeb)
+ if "news" in data:
+ out.news = _transform_array(data["news"], SearchResultNews)
+ if "images" in data:
+ out.images = _transform_array(data["images"], SearchResultImages)
+ return out
+ except Exception as err:
+ # If the error is an HTTP error from requests, handle it
+ # (simulate isAxiosError by checking for requests' HTTPError or Response)
+ if hasattr(err, "response"):
+ handle_response_error(getattr(err, "response"), "search")
+ raise err
+def _transform_array(arr: List[Any], result_type: Type[T]) -> List[Union[T, 'Document']]:
+ """
+ Transforms an array of items into a list of result_type or Document.
+ If the item dict contains any of the special keys, it is treated as a Document.
+ Otherwise, it is treated as result_type.
+ If the item is not a dict, it is wrapped as result_type with url=item.
+ """
+ results: List[Union[T, 'Document']] = []
+ for item in arr:
+ if item and isinstance(item, dict):
+ if (
+ "markdown" in item or
+ "html" in item or
+ "rawHtml" in item or
+ "links" in item or
+ "screenshot" in item or
+ "changeTracking" in item or
+ "summary" in item or
+ "json" in item
+ ):
+ results.append(Document(**item))
+ else:
+ results.append(result_type(**item))
+ else:
+ # For non-dict items, assume it's a URL and wrap in result_type
+ results.append(result_type(url=item))
+ return results
def _validate_search_request(request: SearchRequest) -> SearchRequest:
"""
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 6c3ba6366..159fa9cc6 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -327,11 +327,35 @@ class CrawlJob(BaseModel):
next: Optional[str] = None
data: List[Document] = []
-class SearchDocument(Document):
- """A document from a search operation with URL and description."""
+class SearchResultWeb(BaseModel):
+ """A web search result with URL, title, and description."""
url: str
title: Optional[str] = None
- description: Optional[str] = None
+ description: Optional[str] = None
+
+class SearchResultNews(BaseModel):
+ """A news search result with URL, title, snippet, date, image URL, and position."""
+ title: Optional[str] = None
+ url: Optional[str] = None
+ snippet: Optional[str] = None
+ date: Optional[str] = None
+ image_url: Optional[str] = None
+ position: Optional[int] = None
+
+class SearchResultImages(BaseModel):
+ """An image search result with URL, title, image URL, image width, image height, and position."""
+ title: Optional[str] = None
+ image_url: Optional[str] = None
+ image_width: Optional[int] = None
+ image_height: Optional[int] = None
+ url: Optional[str] = None
+ position: Optional[int] = None
+
+class SearchData(BaseModel):
+ """Search results grouped by source type."""
+ web: Optional[List[Union[SearchResultWeb, Document]]] = None
+ news: Optional[List[Union[SearchResultNews, Document]]] = None
+ images: Optional[List[Union[SearchResultImages, Document]]] = None
class MapDocument(Document):
"""A document from a map operation with URL and description."""
@@ -535,9 +559,9 @@ SearchResult = LinkResult
class SearchData(BaseModel):
"""Search results grouped by source type."""
- web: Optional[List[Union[LinkResult, SearchDocument]]] = None
- news: Optional[List[Union[LinkResult, SearchDocument]]] = None
- images: Optional[List[Union[LinkResult, SearchDocument]]] = None
+ web: Optional[List[Union[SearchResultWeb, Document]]] = None
+ news: Optional[List[Union[SearchResultNews, Document]]] = None
+ images: Optional[List[Union[SearchResultImages, Document]]] = None
class SearchResponse(BaseResponse[SearchData]):
"""Response from search operation."""
diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
index dc51ddcef..73e7e729f 100644
--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
@@ -15,6 +15,7 @@ class AsyncHttpClient:
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
},
+ limits=httpx.Limits(max_keepalive_connections=0),
)
async def close(self) -> None: