Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Python sdk/v2.1.0 (#1479)
Rafael Miller9e259571b1fc
diff --git a/apps/python-sdk/example.py b/apps/python-sdk/example.pyindex ded241cdf..9b28e17e7 100644--- a/apps/python-sdk/example.py+++ b/apps/python-sdk/example.py@@ -42,23 +42,7 @@ while attempts > 0 and crawl_status.status != 'completed':crawl_status = app.check_crawl_status(async_result.id)print(crawl_status)-# LLM Extraction:-# Define schema to extract contents into using pydantic-class ArticleSchema(BaseModel):- title: str- points: int- by: str- commentsURL: str--class TopArticlesSchema(BaseModel):- top: List[ArticleSchema] = Field(..., description="Top 5 stories")--extract_config = JsonConfig(schema=TopArticlesSchema.model_json_schema())--llm_extraction_result = app.scrape_url('https://news.ycombinator.com', formats=["extract"], extract=extract_config)--print(llm_extraction_result.extract)-+# JSON format:# Define schema to extract contents into using json schemajson_schema = {"type": "object",@@ -86,9 +70,6 @@ llm_extraction_result = app.scrape_url('https://news.ycombinator.com', formats=[print(llm_extraction_result.json)-print(llm_extraction_result['llm_extraction'])--# Map a website:map_result = app.map_url('https://firecrawl.dev', search="blog")print(map_result)diff --git a/apps/python-sdk/example_async.py b/apps/python-sdk/example_async.pyindex b4b76abf6..1ad022440 100644--- a/apps/python-sdk/example_async.py+++ b/apps/python-sdk/example_async.py@@ -2,7 +2,7 @@ import timeimport nest_asyncioimport uuidimport asyncio-from firecrawl.firecrawl import AsyncFirecrawlApp+from firecrawl.firecrawl import AsyncFirecrawlApp, ScrapeOptions, JsonConfigfrom pydantic import BaseModel, Fieldfrom typing import List@@ -84,6 +84,20 @@ async def example_map_and_extract():extract_result = await app.extract(['https://firecrawl.dev'], prompt="Extract the title, description, and links from the website", schema=extract_schema)print(extract_result)+async def example_deep_research():+ # Deep research example+ research_result = await app.deep_research(+ "What are the latest developments in large language models?",+ max_urls=4+ )+ print("Research Results:", research_result)++async def example_generate_llms_text():+ # Generate LLMs.txt example+ llms_result = await app.generate_llms_text(+ "https://firecrawl.dev")+ print("LLMs.txt Results:", llms_result)+# Define event handlers for websocketdef on_document(detail):print("DOC", detail)@@ -115,6 +129,8 @@ async def main():await example_llm_extraction()await example_map_and_extract()await example_websocket_crawl()+ await example_deep_research()+ await example_generate_llms_text()if __name__ == "__main__":asyncio.run(main())diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex b641b7cdd..214163e58 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -11,9 +11,9 @@ For more information visit https://github.com/firecrawl/import loggingimport os-from .firecrawl import FirecrawlApp, JsonConfig # noqa+from .firecrawl import FirecrawlApp, JsonConfig, ScrapeOptions # noqa-__version__ = "2.0.2"+__version__ = "2.1.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex 39323044d..8f12e781f 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -97,6 +97,16 @@ class ActionsResult(pydantic.BaseModel):"""Result of actions performed during scraping."""screenshots: List[str]+class ChangeTrackingData(pydantic.BaseModel):+ """+ Data for the change tracking format.+ """+ previousScrapeAt: Optional[str] = None+ changeStatus: str # "new" | "same" | "changed" | "removed"+ visibility: str # "visible" | "hidden"+ diff: Optional[Dict[str, Any]] = None+ json: Optional[Any] = None+class FirecrawlDocument(pydantic.BaseModel, Generic[T]):"""Document retrieved or processed by Firecrawl."""url: Optional[str] = None@@ -111,6 +121,7 @@ class FirecrawlDocument(pydantic.BaseModel, Generic[T]):actions: Optional[ActionsResult] = Nonetitle: Optional[str] = None # v1 search onlydescription: Optional[str] = None # v1 search only+ changeTracking: Optional[ChangeTrackingData] = Noneclass LocationConfig(pydantic.BaseModel):"""Location configuration for scraping."""@@ -124,9 +135,9 @@ class WebhookConfig(pydantic.BaseModel):metadata: Optional[Dict[str, str]] = Noneevents: Optional[List[Literal["completed", "failed", "page", "started"]]] = None-class CommonOptions(pydantic.BaseModel):+class ScrapeOptions(pydantic.BaseModel):"""Parameters for scraping operations."""- formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = Noneheaders: Optional[Dict[str, str]] = NoneincludeTags: Optional[List[str]] = NoneexcludeTags: Optional[List[str]] = None@@ -193,7 +204,7 @@ class JsonConfig(pydantic.BaseModel):systemPrompt: Optional[str] = Noneagent: Optional[ExtractAgent] = None-class ScrapeParams(CommonOptions):+class ScrapeParams(ScrapeOptions):"""Parameters for scraping operations."""extract: Optional[JsonConfig] = NonejsonOptions: Optional[JsonConfig] = None@@ -235,7 +246,7 @@ class CrawlParams(pydantic.BaseModel):allowBackwardLinks: Optional[bool] = NoneallowExternalLinks: Optional[bool] = NoneignoreSitemap: Optional[bool] = None- scrapeOptions: Optional[CommonOptions] = None+ scrapeOptions: Optional[ScrapeOptions] = Nonewebhook: Optional[Union[str, WebhookConfig]] = NonededuplicateSimilarURLs: Optional[bool] = NoneignoreQueryParameters: Optional[bool] = None@@ -289,7 +300,7 @@ class ExtractParams(pydantic.BaseModel):includeSubdomains: Optional[bool] = Noneorigin: Optional[str] = NoneshowSources: Optional[bool] = None- scrapeOptions: Optional[CommonOptions] = None+ scrapeOptions: Optional[ScrapeOptions] = Noneclass ExtractResponse(pydantic.BaseModel, Generic[T]):"""Response from extract operations."""@@ -309,7 +320,7 @@ class SearchParams(pydantic.BaseModel):location: Optional[str] = Noneorigin: Optional[str] = "api"timeout: Optional[int] = 60000- scrapeOptions: Optional[CommonOptions] = None+ scrapeOptions: Optional[ScrapeOptions] = Noneclass SearchResponse(pydantic.BaseModel):"""Response from search operations."""@@ -377,16 +388,6 @@ class GenerateLLMsTextStatusResponse(pydantic.BaseModel):status: Literal["processing", "completed", "failed"]error: Optional[str] = NoneexpiresAt: str--class ChangeTrackingData(pydantic.BaseModel):- """- Data for the change tracking format.- """- previousScrapeAt: Optional[str] = None- changeStatus: str # "new" | "same" | "changed" | "removed"- visibility: str # "visible" | "hidden"- diff: Optional[Dict[str, Any]] = None- json: Optional[Any] = Noneclass SearchResponse(pydantic.BaseModel):"""@@ -442,7 +443,7 @@ class FirecrawlApp:self,url: str,*,- formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json", "changeTracking"]]] = None,include_tags: Optional[List[str]] = None,exclude_tags: Optional[List[str]] = None,only_main_content: Optional[bool] = None,@@ -568,7 +569,7 @@ class FirecrawlApp:country: Optional[str] = None,location: Optional[str] = None,timeout: Optional[int] = None,- scrape_options: Optional[CommonOptions] = None,+ scrape_options: Optional[ScrapeOptions] = None,params: Optional[Union[Dict[str, Any], SearchParams]] = None,**kwargs) -> SearchResponse:"""@@ -583,7 +584,7 @@ class FirecrawlApp:country (Optional[str]): Country code (default: "us")location (Optional[str]): Geo-targetingtimeout (Optional[int]): Request timeout in milliseconds- scrape_options (Optional[CommonOptions]): Result scraping configuration+ scrape_options (Optional[ScrapeOptions]): Result scraping configurationparams (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters**kwargs: Additional keyword arguments for future compatibility@@ -664,7 +665,7 @@ class FirecrawlApp:allow_backward_links: Optional[bool] = None,allow_external_links: Optional[bool] = None,ignore_sitemap: Optional[bool] = None,- scrape_options: Optional[CommonOptions] = None,+ scrape_options: Optional[ScrapeOptions] = None,webhook: Optional[Union[str, WebhookConfig]] = None,deduplicate_similar_urls: Optional[bool] = None,ignore_query_parameters: Optional[bool] = None,@@ -686,7 +687,7 @@ class FirecrawlApp:allow_backward_links (Optional[bool]): Follow parent directory linksallow_external_links (Optional[bool]): Follow external domain linksignore_sitemap (Optional[bool]): Skip sitemap.xml processing- scrape_options (Optional[CommonOptions]): Page scraping configuration+ scrape_options (Optional[ScrapeOptions]): Page scraping configurationwebhook (Optional[Union[str, WebhookConfig]]): Notification webhook settingsdeduplicate_similar_urls (Optional[bool]): Remove similar URLsignore_query_parameters (Optional[bool]): Ignore URL parameters@@ -768,7 +769,7 @@ class FirecrawlApp:allow_backward_links: Optional[bool] = None,allow_external_links: Optional[bool] = None,ignore_sitemap: Optional[bool] = None,- scrape_options: Optional[CommonOptions] = None,+ scrape_options: Optional[ScrapeOptions] = None,webhook: Optional[Union[str, WebhookConfig]] = None,deduplicate_similar_urls: Optional[bool] = None,ignore_query_parameters: Optional[bool] = None,@@ -789,7 +790,7 @@ class FirecrawlApp:allow_backward_links (Optional[bool]): Follow parent directory linksallow_external_links (Optional[bool]): Follow external domain linksignore_sitemap (Optional[bool]): Skip sitemap.xml processing- scrape_options (Optional[CommonOptions]): Page scraping configuration+ scrape_options (Optional[ScrapeOptions]): Page scraping configurationwebhook (Optional[Union[str, WebhookConfig]]): Notification webhook settingsdeduplicate_similar_urls (Optional[bool]): Remove similar URLsignore_query_parameters (Optional[bool]): Ignore URL parameters@@ -1007,7 +1008,7 @@ class FirecrawlApp:allow_backward_links: Optional[bool] = None,allow_external_links: Optional[bool] = None,ignore_sitemap: Optional[bool] = None,- scrape_options: Optional[CommonOptions] = None,+ scrape_options: Optional[ScrapeOptions] = None,webhook: Optional[Union[str, WebhookConfig]] = None,deduplicate_similar_urls: Optional[bool] = None,ignore_query_parameters: Optional[bool] = None,@@ -1028,7 +1029,7 @@ class FirecrawlApp:allow_backward_links (Optional[bool]): Follow parent directory linksallow_external_links (Optional[bool]): Follow external domain linksignore_sitemap (Optional[bool]): Skip sitemap.xml processing- scrape_options (Optional[CommonOptions]): Page scraping configuration+ scrape_options (Optional[ScrapeOptions]): Page scraping configurationwebhook (Optional[Union[str, WebhookConfig]]): Notification webhook settingsdeduplicate_similar_urls (Optional[bool]): Remove similar URLsignore_query_parameters (Optional[bool]): Ignore URL parameters@@ -1741,7 +1742,7 @@ class FirecrawlApp:def async_extract(self,- urls: List[str],+ urls: Optional[List[str]] = None,*,prompt: Optional[str] = None,schema: Optional[Any] = None,@@ -1749,8 +1750,7 @@ class FirecrawlApp:allow_external_links: Optional[bool] = False,enable_web_search: Optional[bool] = False,show_sources: Optional[bool] = False,- agent: Optional[Dict[str, Any]] = None,- idempotency_key: Optional[str] = None) -> ExtractResponse[Any]:+ agent: Optional[Dict[str, Any]] = None) -> ExtractResponse[Any]:"""Initiate an asynchronous extract job.@@ -1774,7 +1774,7 @@ class FirecrawlApp:Raises:ValueError: If job initiation fails"""- headers = self._prepare_headers(idempotency_key)+ headers = self._prepare_headers()schema = schemaif schema:@@ -2922,9 +2922,9 @@ class AsyncFirecrawlApp(FirecrawlApp):headers)- if response.status_code == 200:+ if response.get('success'):try:- id = response.json().get('id')+ id = response.get('id')except:raise Exception(f'Failed to parse Firecrawl response as JSON.')return self._monitor_job_status(id, headers, poll_interval)@@ -3050,7 +3050,7 @@ class AsyncFirecrawlApp(FirecrawlApp):headers)- if response.status_code == 200:+ if response.get('status_code') == 200:try:return BatchScrapeResponse(**response.json())except:@@ -3059,7 +3059,7 @@ class AsyncFirecrawlApp(FirecrawlApp):self._handle_error(response, 'start batch scrape job')async def crawl_url(- self,+ self,url: str,*,include_paths: Optional[List[str]] = None,@@ -3070,7 +3070,7 @@ class AsyncFirecrawlApp(FirecrawlApp):allow_backward_links: Optional[bool] = None,allow_external_links: Optional[bool] = None,ignore_sitemap: Optional[bool] = None,- scrape_options: Optional[CommonOptions] = None,+ scrape_options: Optional[ScrapeOptions] = None,webhook: Optional[Union[str, WebhookConfig]] = None,deduplicate_similar_urls: Optional[bool] = None,ignore_query_parameters: Optional[bool] = None,@@ -3092,7 +3092,7 @@ class AsyncFirecrawlApp(FirecrawlApp):allow_backward_links (Optional[bool]): Follow parent directory linksallow_external_links (Optional[bool]): Follow external domain linksignore_sitemap (Optional[bool]): Skip sitemap.xml processing- scrape_options (Optional[CommonOptions]): Page scraping configuration+ scrape_options (Optional[ScrapeOptions]): Page scraping configurationwebhook (Optional[Union[str, WebhookConfig]]): Notification webhook settingsdeduplicate_similar_urls (Optional[bool]): Remove similar URLsignore_query_parameters (Optional[bool]): Ignore URL parameters@@ -3148,15 +3148,15 @@ class AsyncFirecrawlApp(FirecrawlApp):params_dict = final_params.dict(exclude_none=True)params_dict['url'] = urlparams_dict['origin'] = f"python-sdk@{version}"-# Make requestheaders = self._prepare_headers(idempotency_key)response = await self._async_post_request(f'{self.api_url}/v1/crawl', params_dict, headers)- if response.status_code == 200:+ print(response)+ if response.get('success'):try:- id = response.json().get('id')+ id = response.get('id')except:raise Exception(f'Failed to parse Firecrawl response as JSON.')return self._monitor_job_status(id, headers, poll_interval)@@ -3176,11 +3176,12 @@ class AsyncFirecrawlApp(FirecrawlApp):allow_backward_links: Optional[bool] = None,allow_external_links: Optional[bool] = None,ignore_sitemap: Optional[bool] = None,- scrape_options: Optional[CommonOptions] = None,+ scrape_options: Optional[ScrapeOptions] = None,webhook: Optional[Union[str, WebhookConfig]] = None,deduplicate_similar_urls: Optional[bool] = None,ignore_query_parameters: Optional[bool] = None,regex_on_full_url: Optional[bool] = None,+ poll_interval: Optional[int] = 2,idempotency_key: Optional[str] = None,**kwargs) -> CrawlResponse:@@ -3197,7 +3198,7 @@ class AsyncFirecrawlApp(FirecrawlApp):allow_backward_links (Optional[bool]): Follow parent directory linksallow_external_links (Optional[bool]): Follow external domain linksignore_sitemap (Optional[bool]): Skip sitemap.xml processing- scrape_options (Optional[CommonOptions]): Page scraping configuration+ scrape_options (Optional[ScrapeOptions]): Page scraping configurationwebhook (Optional[Union[str, WebhookConfig]]): Notification webhook settingsdeduplicate_similar_urls (Optional[bool]): Remove similar URLsignore_query_parameters (Optional[bool]): Ignore URL parameters@@ -3262,9 +3263,9 @@ class AsyncFirecrawlApp(FirecrawlApp):headers)- if response.status_code == 200:+ if response.get('success'):try:- return CrawlResponse(**response.json())+ return CrawlResponse(**response)except:raise Exception(f'Failed to parse Firecrawl response as JSON.')else:@@ -3303,7 +3304,7 @@ class AsyncFirecrawlApp(FirecrawlApp):headers)- if status_data['status'] == 'completed':+ if status_data.get('status') == 'completed':if 'data' in status_data:data = status_data['data']while 'next' in status_data:@@ -3317,26 +3318,24 @@ class AsyncFirecrawlApp(FirecrawlApp):data.extend(next_data.get('data', []))status_data = next_datastatus_data['data'] = data-- response = {- 'status': status_data.get('status'),- 'total': status_data.get('total'),- 'completed': status_data.get('completed'),- 'creditsUsed': status_data.get('creditsUsed'),- 'expiresAt': status_data.get('expiresAt'),- 'data': status_data.get('data')- }+ # Create CrawlStatusResponse object from status data+ response = CrawlStatusResponse(+ status=status_data.get('status'),+ total=status_data.get('total'),+ completed=status_data.get('completed'),+ creditsUsed=status_data.get('creditsUsed'),+ expiresAt=status_data.get('expiresAt'),+ data=status_data.get('data'),+ success=False if 'error' in status_data else True+ )if 'error' in status_data:- response['error'] = status_data['error']+ response.error = status_data.get('error')if 'next' in status_data:- response['next'] = status_data['next']+ response.next = status_data.get('next')- return {- 'success': False if 'error' in status_data else True,- **response- }+ return responseasync def _async_monitor_job_status(self, id: str, headers: Dict[str, str], poll_interval: int = 2) -> CrawlStatusResponse:"""@@ -3359,7 +3358,7 @@ class AsyncFirecrawlApp(FirecrawlApp):headers)- if status_data['status'] == 'completed':+ if status_data.get('status') == 'completed':if 'data' in status_data:data = status_data['data']while 'next' in status_data:@@ -3376,15 +3375,22 @@ class AsyncFirecrawlApp(FirecrawlApp):return status_dataelse:raise Exception('Job completed but no data was returned')- elif status_data['status'] in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:+ elif status_data.get('status') in ['active', 'paused', 'pending', 'queued', 'waiting', 'scraping']:await asyncio.sleep(max(poll_interval, 2))else:raise Exception(f'Job failed or was stopped. Status: {status_data["status"]}')async def map_url(- self,- url: str,- params: Optional[MapParams] = None) -> MapResponse:+ self,+ url: str,+ *,+ search: Optional[str] = None,+ ignore_sitemap: Optional[bool] = None,+ include_subdomains: Optional[bool] = None,+ sitemap_only: Optional[bool] = None,+ limit: Optional[int] = None,+ timeout: Optional[int] = None,+ params: Optional[MapParams] = None) -> MapResponse:"""Asynchronously map and discover links from a URL.@@ -3409,21 +3415,40 @@ class AsyncFirecrawlApp(FirecrawlApp):Raises:Exception: If mapping fails"""- headers = self._prepare_headers()- json_data = {'url': url}+ map_params = {}if params:- json_data.update(params)- json_data['origin'] = f"python-sdk@{version}"+ map_params.update(params.dict(exclude_none=True))+ # Add individual parameters+ if search is not None:+ map_params['search'] = search+ if ignore_sitemap is not None:+ map_params['ignoreSitemap'] = ignore_sitemap+ if include_subdomains is not None:+ map_params['includeSubdomains'] = include_subdomains+ if sitemap_only is not None:+ map_params['sitemapOnly'] = sitemap_only+ if limit is not None:+ map_params['limit'] = limit+ if timeout is not None:+ map_params['timeout'] = timeout++ # Create final params object+ final_params = MapParams(**map_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ # Make requestendpoint = f'/v1/map'response = await self._async_post_request(f'{self.api_url}{endpoint}',- json_data,- headers+ params_dict,+ headers={"Authorization": f"Bearer {self.api_key}"})if response.get('success') and 'links' in response:- return response+ return MapResponse(**response)elif 'error' in response:raise Exception(f'Failed to map URL. Error: {response["error"]}')else:@@ -3431,27 +3456,28 @@ class AsyncFirecrawlApp(FirecrawlApp):async def extract(self,- urls: List[str],- params: Optional[ExtractParams] = None) -> ExtractResponse[Any]:+ urls: Optional[List[str]] = None,+ *,+ prompt: Optional[str] = None,+ schema: Optional[Any] = None,+ system_prompt: Optional[str] = None,+ allow_external_links: Optional[bool] = False,+ enable_web_search: Optional[bool] = False,+ show_sources: Optional[bool] = False,+ agent: Optional[Dict[str, Any]] = None) -> ExtractResponse[Any]:+"""Asynchronously extract structured information from URLs.Args:- urls (List[str]): URLs to extract from- params (Optional[ExtractParams]): See ExtractParams model:- Extraction Config:- * prompt - Custom extraction prompt- * schema - JSON schema/Pydantic model- * systemPrompt - System context-- Behavior Options:- * allowExternalLinks - Follow external links- * enableWebSearch - Enable web search- * includeSubdomains - Include subdomains- * showSources - Include source URLs-- Scraping Options:- * scrapeOptions - Page scraping config+ urls (Optional[List[str]]): URLs to extract from+ prompt (Optional[str]): Custom extraction prompt+ schema (Optional[Any]): JSON schema/Pydantic model+ system_prompt (Optional[str]): System context+ allow_external_links (Optional[bool]): Follow external links+ enable_web_search (Optional[bool]): Enable web search+ show_sources (Optional[bool]): Include source URLs+ agent (Optional[Dict[str, Any]]): Agent configurationReturns:ExtractResponse with:@@ -3464,29 +3490,35 @@ class AsyncFirecrawlApp(FirecrawlApp):"""headers = self._prepare_headers()- if not params or (not params.get('prompt') and not params.get('schema')):+ if not prompt and not schema:raise ValueError("Either prompt or schema is required")- schema = params.get('schema')+ if not urls and not prompt:+ raise ValueError("Either urls or prompt is required")+if schema:if hasattr(schema, 'model_json_schema'):+ # Convert Pydantic model to JSON schemaschema = schema.model_json_schema()+ # Otherwise assume it's already a JSON schema dictrequest_data = {- 'urls': urls,- 'allowExternalLinks': params.get('allow_external_links', params.get('allowExternalLinks', False)),- 'enableWebSearch': params.get('enable_web_search', params.get('enableWebSearch', False)),- 'showSources': params.get('show_sources', params.get('showSources', False)),+ 'urls': urls or [],+ 'allowExternalLinks': allow_external_links,+ 'enableWebSearch': enable_web_search,+ 'showSources': show_sources,'schema': schema,- 'origin': f'python-sdk@{version}'+ 'origin': f'python-sdk@{get_version()}'}- if params.get('prompt'):- request_data['prompt'] = params['prompt']- if params.get('system_prompt'):- request_data['systemPrompt'] = params['system_prompt']- elif params.get('systemPrompt'):- request_data['systemPrompt'] = params['systemPrompt']+ # Only add prompt and systemPrompt if they exist+ if prompt:+ request_data['prompt'] = prompt+ if system_prompt:+ request_data['systemPrompt'] = system_prompt++ if agent:+ request_data['agent'] = agentresponse = await self._async_post_request(f'{self.api_url}/v1/extract',@@ -3506,7 +3538,7 @@ class AsyncFirecrawlApp(FirecrawlApp):)if status_data['status'] == 'completed':- return status_data+ return ExtractResponse(**status_data)elif status_data['status'] in ['failed', 'cancelled']:raise Exception(f'Extract job {status_data["status"]}. Error: {status_data["error"]}')@@ -3562,14 +3594,14 @@ class AsyncFirecrawlApp(FirecrawlApp):status_data = next_datastatus_data['data'] = data- response = {- 'status': status_data.get('status'),- 'total': status_data.get('total'),- 'completed': status_data.get('completed'),- 'creditsUsed': status_data.get('creditsUsed'),- 'expiresAt': status_data.get('expiresAt'),- 'data': status_data.get('data')- }+ response = BatchScrapeStatusResponse(+ status=status_data.get('status'),+ total=status_data.get('total'),+ completed=status_data.get('completed'),+ creditsUsed=status_data.get('creditsUsed'),+ expiresAt=status_data.get('expiresAt'),+ data=status_data.get('data')+ )if 'error' in status_data:response['error'] = status_data['error']@@ -3689,8 +3721,7 @@ class AsyncFirecrawlApp(FirecrawlApp):allow_external_links: Optional[bool] = False,enable_web_search: Optional[bool] = False,show_sources: Optional[bool] = False,- agent: Optional[Dict[str, Any]] = None,- idempotency_key: Optional[str] = None) -> ExtractResponse[Any]:+ agent: Optional[Dict[str, Any]] = None) -> ExtractResponse[Any]:"""Initiate an asynchronous extraction job without waiting for completion.@@ -3714,7 +3745,7 @@ class AsyncFirecrawlApp(FirecrawlApp):Raises:ValueError: If job initiation fails"""- headers = self._prepare_headers(idempotency_key)+ headers = self._prepare_headers()if not prompt and not schema:raise ValueError("Either prompt or schema is required")@@ -3726,14 +3757,14 @@ class AsyncFirecrawlApp(FirecrawlApp):if hasattr(schema, 'model_json_schema'):schema = schema.model_json_schema()- request_data = {- 'urls': urls or [],- 'allowExternalLinks': allow_external_links,- 'enableWebSearch': enable_web_search,- 'showSources': show_sources,- 'schema': schema,- 'origin': f'python-sdk@{version}'- }+ request_data = ExtractResponse(+ urls=urls or [],+ allowExternalLinks=allow_external_links,+ enableWebSearch=enable_web_search,+ showSources=show_sources,+ schema=schema,+ origin=f'python-sdk@{version}'+ )if prompt:request_data['prompt'] = prompt@@ -3810,7 +3841,7 @@ class AsyncFirecrawlApp(FirecrawlApp):await asyncio.sleep(2)- return {'success': False, 'error': 'LLMs.txt generation job terminated unexpectedly'}+ return GenerateLLMsTextStatusResponse(success=False, error='LLMs.txt generation job terminated unexpectedly')async def async_generate_llms_text(self,@@ -3845,6 +3876,12 @@ class AsyncFirecrawlApp(FirecrawlApp):if experimental_stream is not None:params['__experimental_stream'] = experimental_stream+ params = GenerateLLMsTextParams(+ maxUrls=max_urls,+ showFullText=show_full_text,+ __experimental_stream=experimental_stream+ )+headers = self._prepare_headers()json_data = {'url': url, **params.dict(exclude_none=True)}json_data['origin'] = f"python-sdk@{version}"@@ -3981,7 +4018,7 @@ class AsyncFirecrawlApp(FirecrawlApp):await asyncio.sleep(2)- return {'success': False, 'error': 'Deep research job terminated unexpectedly'}+ return DeepResearchStatusResponse(success=False, error='Deep research job terminated unexpectedly')async def async_deep_research(self,@@ -4088,7 +4125,7 @@ class AsyncFirecrawlApp(FirecrawlApp):country: Optional[str] = None,location: Optional[str] = None,timeout: Optional[int] = None,- scrape_options: Optional[CommonOptions] = None,+ scrape_options: Optional[ScrapeOptions] = None,params: Optional[Union[Dict[str, Any], SearchParams]] = None,**kwargs) -> SearchResponse:"""@@ -4103,7 +4140,7 @@ class AsyncFirecrawlApp(FirecrawlApp):country (Optional[str]): Country code (default: "us")location (Optional[str]): Geo-targetingtimeout (Optional[int]): Request timeout in milliseconds- scrape_options (Optional[CommonOptions]): Result scraping configuration+ scrape_options (Optional[ScrapeOptions]): Result scraping configurationparams (Optional[Union[Dict[str, Any], SearchParams]]): Additional search parameters**kwargs: Additional keyword arguments for future compatibility