Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
(python-sdk): fixed search types + added max_keepalive_connections to fix event loop is closed bug (#2005)
Rafael Miller61c3abcb5b37
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 7f0ecc4cf..424b24bcd 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "3.1.1"+__version__ = "3.2.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_search.pyindex 6d0b6daa9..12a21bda5 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_search.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_search.py@@ -2,8 +2,19 @@ import osimport pytestfrom dotenv import load_dotenvfrom firecrawl import AsyncFirecrawl-from firecrawl.v2.types import ScrapeOptions, ScrapeFormats, SearchData, SearchResult, Document+from firecrawl.types import (+ SearchData,+ Document,+ ScrapeOptions,+ ScrapeFormats,+ SearchResultWeb,+ SearchResultNews,+ SearchResultImages,+)+load_dotenv()++firecrawl = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))def _collect_texts(entries):texts = []@@ -36,148 +47,202 @@ def _is_document(entry) -> bool:hasattr(entry, 'change_tracking') or \hasattr(entry, 'summary')+async def test_async_search_minimal_request():+ results = await firecrawl.search(+ query="What is the capital of France?"+ )+ assert isinstance(results, SearchData)+ assert hasattr(results, 'web')+ assert results.web is not None+ assert len(results.web) > 0+ assert hasattr(results, 'news')+ assert results.news is None+ assert hasattr(results, 'images')+ assert results.images is None++ for result in results.web:+ assert isinstance(result, SearchResultWeb)+ assert hasattr(result, 'url')+ assert hasattr(result, 'title')+ assert hasattr(result, 'description')+ assert result.url.startswith('http')+ assert result.title is not None+ assert result.description is not None++ all_text = ' '.join(_collect_texts(results.web))+ assert 'paris' in all_text++ assert results.news is None+ assert results.images is None-load_dotenv()--if not os.getenv("API_KEY"):- raise ValueError("API_KEY is not set")--if not os.getenv("API_URL"):- raise ValueError("API_URL is not set")+async def test_async_search_with_sources():+ results = await firecrawl.search(+ query="firecrawl",+ sources=["web", "news", "images"],+ limit=3+ )+ assert isinstance(results, SearchData)+ assert results.web is not None+ assert len(results.web) <= 3+ assert isinstance(results.web[0], SearchResultWeb)+ if results.news is not None:+ assert len(results.news) <= 3+ assert isinstance(results.news[0], SearchResultNews)-async def test_async_search_minimal():- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))- data = await client.search("What is the capital of France?")- # Assert sections like sync tests- assert hasattr(data, "web")- assert hasattr(data, "news")- assert hasattr(data, "images")- assert data.web is not None- assert len(data.web) > 0- titles = [getattr(r, "title", None) for r in data.web]- descs = [getattr(r, "description", None) for r in data.web]- all_text = " ".join([t.lower() for t in titles if t] + [d.lower() for d in descs if d])- assert "paris" in all_text- assert data.news is None- assert data.images is None+ if results.images is not None:+ assert len(results.images) <= 3+ assert isinstance(results.images[0], SearchResultImages)+ web_titles = [result.title.lower() for result in results.web]+ web_descriptions = [result.description.lower() for result in results.web]+ all_web_text = ' '.join(web_titles + web_descriptions)+ assert 'firecrawl' in all_web_text@pytest.mark.asyncio-async def test_async_search_with_sources_and_limit():- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))- data = await client.search("firecrawl", sources=["web", "news"], limit=3)- # Sections present- assert hasattr(data, "web") and hasattr(data, "news") and hasattr(data, "images")- # Web present, images absent, news optional but if present respects limit- if data.web is not None:- assert len(data.web) <= 3- if data.news is not None:- assert len(data.news) <= 3- assert data.images is None-+async def test_async_search_result_structure():+ results = await firecrawl.search(+ query="test query",+ limit=1+ )+ if results.web and len(results.web) > 0:+ result = results.web[0]+ assert hasattr(result, 'url')+ assert hasattr(result, 'title')+ assert hasattr(result, 'description')+ assert isinstance(result.url, str)+ assert isinstance(result.title, str) or result.title is None+ assert isinstance(result.description, str) or result.description is None+ assert result.url.startswith('http')@pytest.mark.asyncio-async def test_async_search_with_all_params():- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))- data = await client.search(- "artificial intelligence",- sources=["web", "news"],+async def test_async_search_all_parameters():+ from firecrawl.types import ScrapeOptions, Location, WaitAction+ schema = {+ "type": "object",+ "properties": {+ "title": {"type": "string"},+ "description": {"type": "string"},+ "url": {"type": "string"}+ },+ "required": ["title", "description"]+ }+ results = await firecrawl.search(+ query="artificial intelligence",+ sources=[+ {"type": "web"},+ {"type": "news"}+ ],limit=3,- tbs="qdr:w",+ tbs="qdr:m",location="US",- ignore_invalid_urls=False,- timeout=30000,- scrape_options={- "formats": ["markdown"],- "headers": {"User-Agent": "E2E-AIO"},- "include_tags": ["h1"],- "exclude_tags": ["nav"],- "only_main_content": False,- "timeout": 15000,- "wait_for": 2000,- "mobile": True,- "skip_tls_verification": True,- "remove_base64_images": False,- },+ ignore_invalid_urls=True,+ timeout=60000,+ scrape_options=ScrapeOptions(+ formats=[+ "markdown",+ "html",+ {+ "type": "json",+ "prompt": "Extract the title and description from the page",+ "schema": schema+ },+ {"type": "summary"}+ ],+ headers={"User-Agent": "Firecrawl-Test/1.0"},+ include_tags=["h1", "h2", "p"],+ exclude_tags=["nav", "footer"],+ only_main_content=True,+ wait_for=2000,+ mobile=False,+ skip_tls_verification=False,+ remove_base64_images=True,+ block_ads=True,+ proxy="basic",+ max_age=3600000,+ store_in_cache=True,+ location=Location(+ country="US",+ languages=["en"]+ ),+ actions=[+ WaitAction(milliseconds=1000)+ ]+ ))- # Structure and type assertions mirroring sync- assert isinstance(data, SearchData)- assert hasattr(data, "web") and hasattr(data, "news") and hasattr(data, "images")- assert data.web is not None- assert len(data.web) <= 3- non_doc = [r for r in (data.web or []) if not _is_document(r)]- if non_doc:- combined = " ".join(_collect_texts(non_doc))- ai_terms = ["artificial", "intelligence", "ai", "machine", "learning"]- assert any(term in combined for term in ai_terms)- for r in data.web:- assert isinstance(r, (SearchResult, Document))- if isinstance(r, Document):- assert (r.markdown is not None) or (r.html is not None)+ assert isinstance(results, SearchData)+ assert hasattr(results, 'web')+ assert hasattr(results, 'news')+ assert hasattr(results, 'images')+ assert results.web is not None+ assert len(results.web) <= 3++ non_doc_entries = [r for r in (results.web or []) if not _is_document(r)]+ if non_doc_entries:+ all_web_text = ' '.join(_collect_texts(non_doc_entries))+ ai_terms = ['artificial', 'intelligence', 'ai', 'machine', 'learning']+ assert any(term in all_web_text for term in ai_terms)++ for result in results.web:+ assert isinstance(result, (SearchResultWeb, Document))+ if isinstance(result, Document):+ assert (result.markdown is not None) or (result.html is not None)else:- assert hasattr(r, "url")- assert isinstance(r.url, str) and r.url.startswith("http")- if data.news is not None:- assert len(data.news) <= 10- for r in data.news:- assert isinstance(r, (SearchResult, Document))- if isinstance(r, Document):- assert (r.markdown is not None) or (r.html is not None)+ assert hasattr(result, 'url')+ assert isinstance(result.url, str) and result.url.startswith('http')++ if results.news is not None:+ assert len(results.news) <= 3+ for result in results.news:+ assert isinstance(result, (SearchResultNews, Document))+ if isinstance(result, Document):+ assert (result.markdown is not None) or (result.html is not None)else:- assert isinstance(r.url, str) and r.url.startswith("http")- assert data.images is None---async def test_async_search_minimal_content_check():- """Stronger assertion similar to sync: content check on a known query."""- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))- data = await client.search("What is the capital of France?")- assert hasattr(data, "web") and data.web is not None- non_doc = [r for r in (data.web or []) if not _is_document(r)]- if non_doc:- combined = " ".join(_collect_texts(non_doc))- assert "paris" in combined---async def test_async_search_result_structure():- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))- data = await client.search("test query", limit=1)- if data.web and len(data.web) > 0:- result = data.web[0]- assert hasattr(result, "url")- assert hasattr(result, "title")- assert hasattr(result, "description")- assert isinstance(result.url, str) and result.url.startswith("http")- assert isinstance(getattr(result, "title", None), (str, type(None)))- assert isinstance(getattr(result, "description", None), (str, type(None)))+ assert hasattr(result, 'url')+ assert isinstance(result.url, str) and result.url.startswith('http')+ assert results.images is None@pytest.mark.asyncioasync def test_async_search_formats_flexibility():- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))- # list string- res1 = await client.search("python programming", limit=1, scrape_options=ScrapeOptions(formats=["markdown"]))- # list objects- res2 = await client.search("python programming", limit=1, scrape_options=ScrapeOptions(formats=[{"type": "markdown"}]))- # ScrapeFormats object- res3 = await client.search("python programming", limit=1, scrape_options=ScrapeOptions(formats=ScrapeFormats(markdown=True)))- assert isinstance(res1, SearchData) and hasattr(res1, "web")- assert isinstance(res2, SearchData) and hasattr(res2, "web")- assert isinstance(res3, SearchData) and hasattr(res3, "web")-+ # Test with list format+ results1 = await firecrawl.search(+ query="python programming",+ limit=1,+ scrape_options=ScrapeOptions(+ formats=["markdown"]+ )+ )+ # Test with ScrapeFormats object+ results2 = await firecrawl.search(+ query="python programming",+ limit=1,+ scrape_options=ScrapeOptions(+ formats=ScrapeFormats(markdown=True)+ )+ )+ assert isinstance(results1, SearchData)+ assert isinstance(results2, SearchData)+ assert results1.web is not None+ assert results2.web is not None@pytest.mark.asyncio-async def test_async_search_json_format_object():- client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))- json_schema = {"type": "object", "properties": {"title": {"type": "string"}}, "required": ["title"]}- data = await client.search(- "site:docs.firecrawl.dev",+async def test_async_search_with_json_format_object():+ json_schema = {+ "type": "object",+ "properties": {+ "title": {"type": "string"}+ },+ "required": ["title"],+ }+ results = await firecrawl.search(+ query="site:docs.firecrawl.dev",limit=1,- scrape_options={"formats": [{"type": "json", "prompt": "Extract page title", "schema": json_schema}]},+ scrape_options=ScrapeOptions(+ formats=[{"type": "json", "prompt": "Extract page title", "schema": json_schema}]+ ),)- assert hasattr(data, "web")-+ assert isinstance(results, SearchData)+ assert results.web is not None and len(results.web) >= 0diff --git a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.pyindex 85837f725..54b494cb9 100644--- a/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py+++ b/apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py@@ -1,7 +1,7 @@from firecrawl import Firecrawlimport osfrom dotenv import load_dotenv-from firecrawl.types import SearchData, SearchResult, Document, ScrapeFormats, ScrapeOptions+from firecrawl.types import SearchData, Document, ScrapeOptions, SearchResultWeb, SearchResultNews, SearchResultImagesload_dotenv()@@ -53,7 +53,7 @@ def test_search_minimal_request():assert results.images is Nonefor result in results.web:- assert isinstance(result, SearchResult)+ assert isinstance(result, SearchResultWeb)assert hasattr(result, 'url')assert hasattr(result, 'title')assert hasattr(result, 'description')@@ -73,7 +73,7 @@ def test_search_with_sources():"""Test search with specific sources."""results = firecrawl.search(query="firecrawl",- sources=["web", "news"],+ sources=["web", "news", "images"],limit=3)@@ -81,11 +81,15 @@ def test_search_with_sources():assert results.web is not Noneassert len(results.web) <= 3+ assert isinstance(results.web[0], SearchResultWeb)if results.news is not None:assert len(results.news) <= 3+ assert isinstance(results.news[0], SearchResultNews)- assert results.images is None+ if results.images is not None:+ assert len(results.images) <= 3+ assert isinstance(results.images[0], SearchResultImages)web_titles = [result.title.lower() for result in results.web]web_descriptions = [result.description.lower() for result in results.web]@@ -193,7 +197,7 @@ def test_search_all_parameters():# Test that each result has proper structurefor result in results.web:- assert isinstance(result, (SearchResult, Document))+ assert isinstance(result, (SearchResultWeb, Document))if isinstance(result, Document):# Document path: ensure content presentassert (result.markdown is not None) or (result.html is not None)@@ -206,7 +210,7 @@ def test_search_all_parameters():if results.news is not None:assert len(results.news) <= 3for result in results.news:- assert isinstance(result, (SearchResult, Document))+ assert isinstance(result, (SearchResultNews, Document))if isinstance(result, Document):assert (result.markdown is not None) or (result.html is not None)else:diff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.pyindex d46b43722..0fcaa8a74 100644--- a/apps/python-sdk/firecrawl/types.py+++ b/apps/python-sdk/firecrawl/types.py@@ -48,7 +48,9 @@ from .v2.types import (JsonFormat,FormatOption,SearchRequest,- SearchResult,+ SearchResultWeb,+ SearchResultNews,+ SearchResultImages,SearchData,SearchResponse,@@ -124,7 +126,9 @@ __all__ = ['JsonFormat','FormatOption','SearchRequest',- 'SearchResult',+ 'SearchResultWeb',+ 'SearchResultNews',+ 'SearchResultImages','SearchData','SearchResponse',diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/search.py b/apps/python-sdk/firecrawl/v2/methods/aio/search.pyindex bafd7f574..a86454db4 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/search.py@@ -1,55 +1,172 @@-from typing import Dict, Any-from ...types import SearchRequest, SearchData, SearchResult, Document-from ...utils.normalize import normalize_document_input+import re+from typing import Dict, Any, Union, List, TypeVar, Type+from ...types import (+ SearchRequest,+ SearchData,+ Document,+ SearchResultWeb,+ SearchResultNews,+ SearchResultImages,+)from ...utils.http_client_async import AsyncHttpClientfrom ...utils.error_handler import handle_response_error-from ...utils.validation import prepare_scrape_options, validate_scrape_options+from ...utils.validation import validate_scrape_options, prepare_scrape_options+T = TypeVar("T")++async def search(+ client: AsyncHttpClient,+ request: SearchRequest+) -> SearchData:+ """+ Async search for documents.++ Args:+ client: Async HTTP client instance+ request: Search request++ Returns:+ SearchData with search results grouped by source type++ Raises:+ FirecrawlError: If the search operation fails+ """+ request_data = _prepare_search_request(request)+ try:+ response = await client.post("/v2/search", request_data)+ if response.status_code != 200:+ handle_response_error(response, "search")+ response_data = response.json()+ if not response_data.get("success"):+ handle_response_error(response, "search")+ data = response_data.get("data", {}) or {}+ out = SearchData()+ if "web" in data:+ out.web = _transform_array(data["web"], SearchResultWeb)+ if "news" in data:+ out.news = _transform_array(data["news"], SearchResultNews)+ if "images" in data:+ out.images = _transform_array(data["images"], SearchResultImages)+ return out+ except Exception as err:+ if hasattr(err, "response"):+ handle_response_error(getattr(err, "response"), "search")+ raise err++def _transform_array(arr: List[Any], result_type: Type[T]) -> List[Union[T, Document]]:+ """+ Transforms an array of items into a list of result_type or Document.+ If the item dict contains any of the special keys, it is treated as a Document.+ Otherwise, it is treated as result_type.+ If the item is not a dict, it is wrapped as result_type with url=item.+ """+ results: List[Union[T, Document]] = []+ for item in arr:+ if item and isinstance(item, dict):+ if (+ "markdown" in item or+ "html" in item or+ "rawHtml" in item or+ "links" in item or+ "screenshot" in item or+ "changeTracking" in item or+ "summary" in item or+ "json" in item+ ):+ results.append(Document(**item))+ else:+ results.append(result_type(**item))+ else:+ results.append(result_type(url=item))+ return results++def _validate_search_request(request: SearchRequest) -> SearchRequest:+ """+ Validate and normalize search request.++ Args:+ request: Search request to validate++ Returns:+ Validated request++ Raises:+ ValueError: If request is invalid+ """+ if not request.query or not request.query.strip():+ raise ValueError("Query cannot be empty")++ if request.limit is not None:+ if request.limit <= 0:+ raise ValueError("Limit must be positive")+ if request.limit > 100:+ raise ValueError("Limit cannot exceed 100")++ if request.timeout is not None:+ if request.timeout <= 0:+ raise ValueError("Timeout must be positive")+ if request.timeout > 300000:+ raise ValueError("Timeout cannot exceed 300000ms (5 minutes)")++ if request.sources is not None:+ valid_sources = {"web", "news", "images"}+ for source in request.sources:+ if isinstance(source, str):+ if source not in valid_sources:+ raise ValueError(f"Invalid source type: {source}. Valid types: {valid_sources}")+ elif hasattr(source, 'type'):+ if source.type not in valid_sources:+ raise ValueError(f"Invalid source type: {source.type}. Valid types: {valid_sources}")++ if request.location is not None:+ if not isinstance(request.location, str) or len(request.location.strip()) == 0:+ raise ValueError("Location must be a non-empty string")++ if request.tbs is not None:+ valid_tbs_values = {+ "qdr:h", "qdr:d", "qdr:w", "qdr:m", "qdr:y",+ "d", "w", "m", "y"+ }+ if request.tbs in valid_tbs_values:+ pass+ elif request.tbs.startswith("cdr:"):+ custom_date_pattern = r"^cdr:1,cd_min:\d{1,2}/\d{1,2}/\d{4},cd_max:\d{1,2}/\d{1,2}/\d{4}$"+ if not re.match(custom_date_pattern, request.tbs):+ raise ValueError(f"Invalid custom date range format: {request.tbs}. Expected format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")+ else:+ raise ValueError(f"Invalid tbs value: {request.tbs}. Valid values: {valid_tbs_values} or custom date range format: cdr:1,cd_min:MM/DD/YYYY,cd_max:MM/DD/YYYY")-def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:- data = request.model_dump(exclude_none=True)- if request.ignore_invalid_urls is not None:- data["ignoreInvalidURLs"] = request.ignore_invalid_urls- data.pop("ignore_invalid_urls", None)if request.scrape_options is not None:validate_scrape_options(request.scrape_options)- scrape_data = prepare_scrape_options(request.scrape_options)++ return request++def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:+ """+ Prepare a search request payload.++ Args:+ request: Search request++ Returns:+ Request payload dictionary+ """+ validated_request = _validate_search_request(request)+ data = validated_request.model_dump(exclude_none=True, by_alias=True)++ if "limit" not in data and validated_request.limit is not None:+ data["limit"] = validated_request.limit+ if "timeout" not in data and validated_request.timeout is not None:+ data["timeout"] = validated_request.timeout++ if validated_request.ignore_invalid_urls is not None:+ data["ignoreInvalidURLs"] = validated_request.ignore_invalid_urls+ data.pop("ignore_invalid_urls", None)++ if validated_request.scrape_options is not None:+ scrape_data = prepare_scrape_options(validated_request.scrape_options)if scrape_data:data["scrapeOptions"] = scrape_datadata.pop("scrape_options", None)- return data---async def search(client: AsyncHttpClient, request: SearchRequest) -> SearchData:- payload = _prepare_search_request(request)- response = await client.post("/v2/search", payload)- if response.status_code >= 400:- handle_response_error(response, "search")- body = response.json()- if not body.get("success"):- raise Exception(body.get("error", "Unknown error occurred"))-- data = body.get("data", {})- search_data = SearchData()- for source_type, source_documents in data.items():- if isinstance(source_documents, list):- results = []- for doc_data in source_documents:- if isinstance(doc_data, dict):- if request.scrape_options is not None and any(- key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'summary', 'screenshot', 'changeTracking']- ):- normalized = normalize_document_input(doc_data)- results.append(Document(**normalized))- else:- results.append(SearchResult(- url=doc_data.get('url', ''),- title=doc_data.get('title'),- description=doc_data.get('description')- ))- elif isinstance(doc_data, str):- results.append(SearchResult(url=doc_data))- if hasattr(search_data, source_type):- setattr(search_data, source_type, results)- return search_data+ return datadiff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.pyindex b91683a5a..68d120092 100644--- a/apps/python-sdk/firecrawl/v2/methods/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/search.py@@ -3,11 +3,12 @@ Search functionality for Firecrawl v2 API."""import re-from typing import Optional, Dict, Any, Union-from ..types import SearchRequest, SearchData, SearchResult, Document+from typing import Dict, Any, Union, List, TypeVar, Type+from ..types import SearchRequest, SearchData, Document, SearchResultWeb, SearchResultNews, SearchResultImagesfrom ..utils.normalize import normalize_document_inputfrom ..utils import HttpClient, handle_response_error, validate_scrape_options, prepare_scrape_options+T = TypeVar("T")def search(client: HttpClient,@@ -27,48 +28,56 @@ def search(FirecrawlError: If the search operation fails"""request_data = _prepare_search_request(request)-- response = client.post("/v2/search", request_data)-- if not response.ok:- handle_response_error(response, "search")-- response_data = response.json()-- if not response_data.get("success"):- # Handle error case- error_msg = response_data.get("error", "Unknown error occurred")- raise Exception(f"Search failed: {error_msg}")-- data = response_data.get("data", {})- search_data = SearchData()-- for source_type, source_documents in data.items():- if isinstance(source_documents, list):- results = []- for doc_data in source_documents:- if isinstance(doc_data, dict):- # If page scraping options were provided, API returns full Document objects- if request.scrape_options is not None and any(- key in doc_data for key in ['markdown', 'html', 'rawHtml', 'links', 'summary', 'screenshot', 'changeTracking']- ):- normalized = normalize_document_input(doc_data)- results.append(Document(**normalized))- else:- # Minimal search result shape- results.append(SearchResult(- url=doc_data.get('url', ''),- title=doc_data.get('title'),- description=doc_data.get('description')- ))- elif isinstance(doc_data, str):- results.append(SearchResult(url=doc_data))-- if hasattr(search_data, source_type):- setattr(search_data, source_type, results)-- return search_data+ try:+ response = client.post("/v2/search", request_data)+ if response.status_code != 200:+ handle_response_error(response, "search")+ response_data = response.json()+ if not response_data.get("success"):+ handle_response_error(response, "search")+ data = response_data.get("data", {}) or {}+ out = SearchData()+ if "web" in data:+ out.web = _transform_array(data["web"], SearchResultWeb)+ if "news" in data:+ out.news = _transform_array(data["news"], SearchResultNews)+ if "images" in data:+ out.images = _transform_array(data["images"], SearchResultImages)+ return out+ except Exception as err:+ # If the error is an HTTP error from requests, handle it+ # (simulate isAxiosError by checking for requests' HTTPError or Response)+ if hasattr(err, "response"):+ handle_response_error(getattr(err, "response"), "search")+ raise err+def _transform_array(arr: List[Any], result_type: Type[T]) -> List[Union[T, 'Document']]:+ """+ Transforms an array of items into a list of result_type or Document.+ If the item dict contains any of the special keys, it is treated as a Document.+ Otherwise, it is treated as result_type.+ If the item is not a dict, it is wrapped as result_type with url=item.+ """+ results: List[Union[T, 'Document']] = []+ for item in arr:+ if item and isinstance(item, dict):+ if (+ "markdown" in item or+ "html" in item or+ "rawHtml" in item or+ "links" in item or+ "screenshot" in item or+ "changeTracking" in item or+ "summary" in item or+ "json" in item+ ):+ results.append(Document(**item))+ else:+ results.append(result_type(**item))+ else:+ # For non-dict items, assume it's a URL and wrap in result_type+ results.append(result_type(url=item))+ return resultsdef _validate_search_request(request: SearchRequest) -> SearchRequest:"""diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 6c3ba6366..159fa9cc6 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -327,11 +327,35 @@ class CrawlJob(BaseModel):next: Optional[str] = Nonedata: List[Document] = []-class SearchDocument(Document):- """A document from a search operation with URL and description."""+class SearchResultWeb(BaseModel):+ """A web search result with URL, title, and description."""url: strtitle: Optional[str] = None- description: Optional[str] = None+ description: Optional[str] = None++class SearchResultNews(BaseModel):+ """A news search result with URL, title, snippet, date, image URL, and position."""+ title: Optional[str] = None+ url: Optional[str] = None+ snippet: Optional[str] = None+ date: Optional[str] = None+ image_url: Optional[str] = None+ position: Optional[int] = None++class SearchResultImages(BaseModel):+ """An image search result with URL, title, image URL, image width, image height, and position."""+ title: Optional[str] = None+ image_url: Optional[str] = None+ image_width: Optional[int] = None+ image_height: Optional[int] = None+ url: Optional[str] = None+ position: Optional[int] = None++class SearchData(BaseModel):+ """Search results grouped by source type."""+ web: Optional[List[Union[SearchResultWeb, Document]]] = None+ news: Optional[List[Union[SearchResultNews, Document]]] = None+ images: Optional[List[Union[SearchResultImages, Document]]] = Noneclass MapDocument(Document):"""A document from a map operation with URL and description."""@@ -535,9 +559,9 @@ SearchResult = LinkResultclass SearchData(BaseModel):"""Search results grouped by source type."""- web: Optional[List[Union[LinkResult, SearchDocument]]] = None- news: Optional[List[Union[LinkResult, SearchDocument]]] = None- images: Optional[List[Union[LinkResult, SearchDocument]]] = None+ web: Optional[List[Union[SearchResultWeb, Document]]] = None+ news: Optional[List[Union[SearchResultNews, Document]]] = None+ images: Optional[List[Union[SearchResultImages, Document]]] = Noneclass SearchResponse(BaseResponse[SearchData]):"""Response from search operation."""diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.pyindex dc51ddcef..73e7e729f 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py@@ -15,6 +15,7 @@ class AsyncHttpClient:"Authorization": f"Bearer {api_key}","Content-Type": "application/json",},+ limits=httpx.Limits(max_keepalive_connections=0),)async def close(self) -> None: