Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update firecrawl.py
Nicolas0b62be58745d
diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex 117ca093c..5a3a80d08 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -3356,12 +3356,12 @@ class AsyncFirecrawlApp(FirecrawlApp):job_id (str): The ID of the extraction jobReturns:- ExtractResponse containing:- * success (bool): Whether extraction completed successfully- * data (Any): Extracted structured data- * error (str, optional): Error message if extraction failed- * warning (str, optional): Warning message if any- * sources (List[str], optional): Source URLs if requested+ ExtractResponse[Any] with:+ * success (bool): Whether request succeeded+ * data (Optional[Any]): Extracted data matching schema+ * error (Optional[str]): Error message if any+ * warning (Optional[str]): Warning message if any+ * sources (Optional[List[str]]): Source URLs if requestedRaises:ValueError: If status check fails@@ -3377,54 +3377,67 @@ class AsyncFirecrawlApp(FirecrawlApp):async def async_extract(self,- urls: List[str],- params: Optional[ExtractParams] = None,+ urls: Optional[List[str]] = None,+ *,+ prompt: Optional[str] = None,+ schema: Optional[Any] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = False,+ enable_web_search: Optional[bool] = False,+ show_sources: Optional[bool] = False,+ agent: Optional[Dict[str, Any]] = None,idempotency_key: Optional[str] = None) -> ExtractResponse[Any]:"""Initiate an asynchronous extraction job without waiting for completion.Args:- urls (List[str]): URLs to extract information from- params (Optional[ExtractParams]): See ExtractParams model:- Extraction Config:- * prompt - Custom extraction prompt- * schema - JSON schema/Pydantic model- * systemPrompt - System context-- Behavior Options:- * allowExternalLinks - Follow external links- * enableWebSearch - Enable web search- * includeSubdomains - Include subdomains- * showSources - Include source URLs-- Scraping Options:- * scrapeOptions - Page scraping config+ urls (Optional[List[str]]): URLs to extract from+ prompt (Optional[str]): Custom extraction prompt+ schema (Optional[Any]): JSON schema/Pydantic model+ system_prompt (Optional[str]): System context+ allow_external_links (Optional[bool]): Follow external links+ enable_web_search (Optional[bool]): Enable web search+ show_sources (Optional[bool]): Include source URLs+ agent (Optional[Dict[str, Any]]): Agent configurationidempotency_key (Optional[str]): Unique key to prevent duplicate requestsReturns:- ExtractResponse containing:- * success (bool): Whether job started successfully- * id (str): Unique identifier for the job- * error (str, optional): Error message if start failed+ ExtractResponse[Any] with:+ * success (bool): Whether request succeeded+ * data (Optional[Any]): Extracted data matching schema+ * error (Optional[str]): Error message if anyRaises:- ValueError: If job initiation fails+ ValueError: If job initiation fails"""headers = self._prepare_headers(idempotency_key)-- schema = params.get('schema') if params else None++ if not prompt and not schema:+ raise ValueError("Either prompt or schema is required")++ if not urls and not prompt:+ raise ValueError("Either urls or prompt is required")+if schema:if hasattr(schema, 'model_json_schema'):schema = schema.model_json_schema()- jsonData = {'urls': urls, **(params or {})}request_data = {- **jsonData,- 'allowExternalLinks': params.get('allow_external_links', False) if params else False,+ 'urls': urls or [],+ 'allowExternalLinks': allow_external_links,+ 'enableWebSearch': enable_web_search,+ 'showSources': show_sources,'schema': schema,'origin': f'python-sdk@{version}'}+ if prompt:+ request_data['prompt'] = prompt+ if system_prompt:+ request_data['systemPrompt'] = system_prompt+ if agent:+ request_data['agent'] = agent+try:return await self._async_post_request(f'{self.api_url}/v1/extract',@@ -3437,16 +3450,18 @@ class AsyncFirecrawlApp(FirecrawlApp):async def generate_llms_text(self,url: str,- params: Optional[Union[Dict[str, Any], GenerateLLMsTextParams]] = None) -> GenerateLLMsTextStatusResponse:+ *,+ max_urls: Optional[int] = None,+ show_full_text: Optional[bool] = None,+ experimental_stream: Optional[bool] = None) -> GenerateLLMsTextStatusResponse:"""Generate LLMs.txt for a given URL and monitor until completion.Args:url (str): Target URL to generate LLMs.txt from- params (Optional[Union[Dict[str, Any], GenerateLLMsTextParams]]): See GenerateLLMsTextParams model:- Generation Options:- * maxUrls - Maximum URLs to process (default: 10)- * showFullText - Include full text in output (default: False)+ max_urls (Optional[int]): Maximum URLs to process (default: 10)+ show_full_text (Optional[bool]): Include full text in output (default: False)+ experimental_stream (Optional[bool]): Enable experimental streamingReturns:GenerateLLMsTextStatusResponse containing:@@ -3461,15 +3476,15 @@ class AsyncFirecrawlApp(FirecrawlApp):Raises:Exception: If generation fails"""- if params is None:- params = {}-- if isinstance(params, dict):- generation_params = GenerateLLMsTextParams(**params)- else:- generation_params = params+ params = {}+ if max_urls is not None:+ params['maxUrls'] = max_urls+ if show_full_text is not None:+ params['showFullText'] = show_full_text+ if experimental_stream is not None:+ params['__experimental_stream'] = experimental_stream- response = await self.async_generate_llms_text(url, generation_params)+ response = await self.async_generate_llms_text(url, params)if not response.get('success') or 'id' not in response:return response@@ -3491,36 +3506,38 @@ class AsyncFirecrawlApp(FirecrawlApp):async def async_generate_llms_text(self,url: str,- params: Optional[Union[Dict[str, Any], GenerateLLMsTextParams]] = None) -> GenerateLLMsTextResponse:+ *,+ max_urls: Optional[int] = None,+ show_full_text: Optional[bool] = None,+ experimental_stream: Optional[bool] = None) -> GenerateLLMsTextResponse:"""Initiate an asynchronous LLMs.txt generation job without waiting for completion.Args:- url (str): Target URL to generate LLMs.txt from- params (Optional[Union[Dict[str, Any], GenerateLLMsTextParams]]): See GenerateLLMsTextParams model:- Generation Options:- * maxUrls - Maximum URLs to process (default: 10)- * showFullText - Include full text in output (default: False)+ url (str): Target URL to generate LLMs.txt from+ max_urls (Optional[int]): Maximum URLs to process (default: 10)+ show_full_text (Optional[bool]): Include full text in output (default: False)+ experimental_stream (Optional[bool]): Enable experimental streamingReturns:- GenerateLLMsTextResponse containing:- * success (bool): Whether job started successfully- * id (str): Unique identifier for the job- * error (str, optional): Error message if start failed+ GenerateLLMsTextResponse containing:+ * success (bool): Whether job started successfully+ * id (str): Unique identifier for the job+ * error (str, optional): Error message if start failedRaises:- ValueError: If job initiation fails+ ValueError: If job initiation fails"""- if params is None:- params = {}-- if isinstance(params, dict):- generation_params = GenerateLLMsTextParams(**params)- else:- generation_params = params+ params = {}+ if max_urls is not None:+ params['maxUrls'] = max_urls+ if show_full_text is not None:+ params['showFullText'] = show_full_text+ if experimental_stream is not None:+ params['__experimental_stream'] = experimental_streamheaders = self._prepare_headers()- json_data = {'url': url, **generation_params.dict(exclude_none=True)}+ json_data = {'url': url, **params.dict(exclude_none=True)}json_data['origin'] = f"python-sdk@{version}"try:@@ -3564,52 +3581,57 @@ class AsyncFirecrawlApp(FirecrawlApp):async def deep_research(self,query: str,- params: Optional[Union[Dict[str, Any], DeepResearchParams]] = None,+ *,+ max_depth: Optional[int] = None,+ time_limit: Optional[int] = None,+ max_urls: Optional[int] = None,+ analysis_prompt: Optional[str] = None,+ system_prompt: Optional[str] = None,+ __experimental_stream_steps: Optional[bool] = None,on_activity: Optional[Callable[[Dict[str, Any]], None]] = None,on_source: Optional[Callable[[Dict[str, Any]], None]] = None) -> DeepResearchStatusResponse:"""- Initiates a deep research operation on a given query and polls until completion, providing real-time updates via callbacks.+ Initiates a deep research operation on a given query and polls until completion.Args:- query: Research query or topic to investigate-- params: See DeepResearchParams model:- Research Settings:- * maxDepth - Maximum research depth (default: 7)- * timeLimit - Time limit in seconds (default: 270)- * maxUrls - Maximum URLs to process (default: 20)-- Callbacks:- * on_activity - Progress callback receiving:- {type, status, message, timestamp, depth}- * on_source - Source discovery callback receiving:- {url, title, description}+ query (str): Research query or topic to investigate+ max_depth (Optional[int]): Maximum depth of research exploration+ time_limit (Optional[int]): Time limit in seconds for research+ max_urls (Optional[int]): Maximum number of URLs to process+ analysis_prompt (Optional[str]): Custom prompt for analysis+ system_prompt (Optional[str]): Custom system prompt+ __experimental_stream_steps (Optional[bool]): Enable experimental streaming+ on_activity (Optional[Callable]): Progress callback receiving {type, status, message, timestamp, depth}+ on_source (Optional[Callable]): Source discovery callback receiving {url, title, description}Returns:- DeepResearchResponse containing:-- Status:- * success - Whether research completed successfully- * status - Current state (processing/completed/failed)- * error - Error message if failed-- Results:- * id - Unique identifier for the research job- * data - Research findings and analysis- * sources - List of discovered sources- * activities - Research progress log- * summaries - Generated research summaries+ DeepResearchStatusResponse containing:+ * success (bool): Whether research completed successfully+ * status (str): Current state (processing/completed/failed)+ * error (Optional[str]): Error message if failed+ * id (str): Unique identifier for the research job+ * data (Any): Research findings and analysis+ * sources (List[Dict]): List of discovered sources+ * activities (List[Dict]): Research progress log+ * summaries (List[str]): Generated research summariesRaises:- Exception: If research fails+ Exception: If research fails"""- if params is None:- params = {}-- if isinstance(params, dict):- research_params = DeepResearchParams(**params)- else:- research_params = params+ research_params = {}+ if max_depth is not None:+ research_params['maxDepth'] = max_depth+ if time_limit is not None:+ research_params['timeLimit'] = time_limit+ if max_urls is not None:+ research_params['maxUrls'] = max_urls+ if analysis_prompt is not None:+ research_params['analysisPrompt'] = analysis_prompt+ if system_prompt is not None:+ research_params['systemPrompt'] = system_prompt+ if __experimental_stream_steps is not None:+ research_params['__experimental_streamSteps'] = __experimental_stream_steps+ research_params = DeepResearchParams(**research_params)response = await self.async_deep_research(query, research_params)if not response.get('success') or 'id' not in response:@@ -3648,38 +3670,54 @@ class AsyncFirecrawlApp(FirecrawlApp):async def async_deep_research(self,query: str,- params: Optional[Union[Dict[str, Any], DeepResearchParams]] = None) -> DeepResearchResponse:+ *,+ max_depth: Optional[int] = None,+ time_limit: Optional[int] = None,+ max_urls: Optional[int] = None,+ analysis_prompt: Optional[str] = None,+ system_prompt: Optional[str] = None,+ __experimental_stream_steps: Optional[bool] = None) -> Dict[str, Any]:"""- Initiate an asynchronous deep research job without waiting for completion.+ Initiates an asynchronous deep research operation.Args:query (str): Research query or topic to investigate- params (Optional[Union[Dict[str, Any], DeepResearchParams]]): See DeepResearchParams model:- Research Settings:- * maxDepth - Maximum research depth (default: 7)- * timeLimit - Time limit in seconds (default: 270)- * maxUrls - Maximum URLs to process (default: 20)+ max_depth (Optional[int]): Maximum depth of research exploration+ time_limit (Optional[int]): Time limit in seconds for research+ max_urls (Optional[int]): Maximum number of URLs to process+ analysis_prompt (Optional[str]): Custom prompt for analysis+ system_prompt (Optional[str]): Custom system prompt+ __experimental_stream_steps (Optional[bool]): Enable experimental streamingReturns:- DeepResearchResponse containing:- * success (bool): Whether job started successfully- * id (str): Unique identifier for the job- * error (str, optional): Error message if start failed+ Dict[str, Any]: A response containing:+ * success (bool): Whether the research initiation was successful+ * id (str): The unique identifier for the research job+ * error (str, optional): Error message if initiation failedRaises:- ValueError: If job initiation fails+ Exception: If the research initiation fails."""- if params is None:- params = {}-- if isinstance(params, dict):- research_params = DeepResearchParams(**params)- else:- research_params = params+ research_params = {}+ if max_depth is not None:+ research_params['maxDepth'] = max_depth+ if time_limit is not None:+ research_params['timeLimit'] = time_limit+ if max_urls is not None:+ research_params['maxUrls'] = max_urls+ if analysis_prompt is not None:+ research_params['analysisPrompt'] = analysis_prompt+ if system_prompt is not None:+ research_params['systemPrompt'] = system_prompt+ if __experimental_stream_steps is not None:+ research_params['__experimental_streamSteps'] = __experimental_stream_steps+ research_params = DeepResearchParams(**research_params)headers = self._prepare_headers()+json_data = {'query': query, **research_params.dict(exclude_none=True)}json_data['origin'] = f"python-sdk@{version}"+try:return await self._async_post_request(f'{self.api_url}/v1/deep-research',@@ -3691,26 +3729,28 @@ class AsyncFirecrawlApp(FirecrawlApp):async def check_deep_research_status(self, id: str) -> DeepResearchStatusResponse:"""- Check the status of an asynchronous deep research job.+ Check the status of a deep research operation.Args:- id (str): The ID of the research job+ id (str): The ID of the deep research operation.Returns:- DeepResearchStatusResponse containing:- * success (bool): Whether research completed successfully- * status (str): Current state (processing/completed/failed)- * data (Dict[str, Any], optional): Research findings and analysis- * error (str, optional): Error message if failed- * expiresAt (str): When the research data expires- * currentDepth (int): Current research depth- * maxDepth (int): Maximum research depth- * activities (List[Dict[str, Any]]): Research progress log- * sources (List[Dict[str, Any]]): Discovered sources- * summaries (List[str]): Generated research summaries+ DeepResearchResponse containing:++ Status:+ * success - Whether research completed successfully+ * status - Current state (processing/completed/failed)+ * error - Error message if failed++ Results:+ * id - Unique identifier for the research job+ * data - Research findings and analysis+ * sources - List of discovered sources+ * activities - Research progress log+ * summaries - Generated research summariesRaises:- ValueError: If status check fails+ Exception: If the status check fails."""headers = self._prepare_headers()try:@@ -3724,52 +3764,80 @@ class AsyncFirecrawlApp(FirecrawlApp):async def search(self,query: str,- params: Optional[Union[Dict[str, Any], SearchParams]] = None) -> SearchResponse:+ *,+ limit: Optional[int] = None,+ tbs: Optional[str] = None,+ filter: Optional[str] = None,+ lang: Optional[str] = None,+ country: Optional[str] = None,+ location: Optional[str] = None,+ timeout: Optional[int] = None,+ scrape_options: Optional[CommonOptions] = None,+ params: Optional[Union[Dict[str, Any], SearchParams]] = None,+ **kwargs) -> SearchResponse:"""Asynchronously search for content using Firecrawl.Args:- query (str): Search query string- params (Optional[Union[Dict[str, Any], SearchParams]]): See SearchParams model:- Search Options:- * limit - Max results (default: 5)- * tbs - Time filter (e.g. "qdr:d")- * filter - Custom result filter-- Localization:- * lang - Language code (default: "en")- * country - Country code (default: "us")- * location - Geo-targeting-- Request Options:- * timeout - Request timeout (ms)- * scrapeOptions - Result scraping config+ query (str): Search query string+ limit (Optional[int]): Max results (default: 5)+ tbs (Optional[str]): Time filter (e.g. "qdr:d")+ filter (Optional[str]): Custom result filter+ lang (Optional[str]): Language code (default: "en")+ country (Optional[str]): Country code (default: "us")+ location (Optional[str]): Geo-targeting+ timeout (Optional[int]): Request timeout in milliseconds+ scrape_options (Optional[CommonOptions]): Result scraping configuration+ params (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters+ **kwargs: Additional keyword arguments for future compatibilityReturns:- SearchResponse containing:- * success (bool): Whether search completed successfully- * data (List[FirecrawlDocument]): Search results- * warning (str, optional): Warning message if any- * error (str, optional): Error message if search failed+ SearchResponse: Response containing:+ * success (bool): Whether request succeeded+ * data (List[FirecrawlDocument]): Search results+ * warning (Optional[str]): Warning message if any+ * error (Optional[str]): Error message if anyRaises:- Exception: If search fails+ Exception: If search fails or response cannot be parsed"""- if params is None:- params = {}+ # Build search parameters+ search_params = {}+ if params:+ if isinstance(params, dict):+ search_params.update(params)+ else:+ search_params.update(params.dict(exclude_none=True))- if isinstance(params, dict):- search_params = SearchParams(query=query, **params)- else:- search_params = params- search_params.query = query+ # Add individual parameters+ if limit is not None:+ search_params['limit'] = limit+ if tbs is not None:+ search_params['tbs'] = tbs+ if filter is not None:+ search_params['filter'] = filter+ if lang is not None:+ search_params['lang'] = lang+ if country is not None:+ search_params['country'] = country+ if location is not None:+ search_params['location'] = location+ if timeout is not None:+ search_params['timeout'] = timeout+ if scrape_options is not None:+ search_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)++ # Add any additional kwargs+ search_params.update(kwargs)- search_params_dict = search_params.dict(exclude_none=True)- search_params_dict['origin'] = f"python-sdk@{version}"+ # Create final params object+ final_params = SearchParams(query=query, **search_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['origin'] = f"python-sdk@{version}"return await self._async_post_request(f"{self.api_url}/v1/search",- search_params_dict,+ params_dict,{"Authorization": f"Bearer {self.api_key}"})