Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update firecrawl.py
Nicolas5e6e41ab1752
diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex d56f951c4..1eb5f8e7d 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -441,6 +441,7 @@ class FirecrawlApp:def scrape_url(self,url: str,+ *,formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,include_tags: Optional[List[str]] = None,exclude_tags: Optional[List[str]] = None,@@ -455,7 +456,8 @@ class FirecrawlApp:proxy: Optional[Literal["basic", "stealth"]] = None,extract: Optional[ExtractConfig] = None,json_options: Optional[ExtractConfig] = None,- actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None) -> ScrapeResponse[Any]:+ actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,+ **kwargs) -> ScrapeResponse[Any]:"""Scrape and extract content from a URL.@@ -479,6 +481,7 @@ class FirecrawlApp:* jsonOptions - JSON extraction settings* actions - Actions to perform+Returns:ScrapeResponse with:* Requested content formats@@ -532,6 +535,7 @@ class FirecrawlApp:scrape_params['jsonOptions'] = json_options.dict(exclude_none=True)if actions:scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions]+ scrape_params.update(kwargs)# Make requestresponse = requests.post(@@ -558,6 +562,7 @@ class FirecrawlApp:def search(self,query: str,+ *,limit: Optional[int] = None,tbs: Optional[str] = None,filter: Optional[str] = None,@@ -649,97 +654,150 @@ class FirecrawlApp:else:self._handle_error(response, 'search')- def crawl_url(self, url: str,- params: Optional[CrawlParams] = None,- poll_interval: Optional[int] = 2,- idempotency_key: Optional[str] = None) -> CrawlStatusResponse:+ def crawl_url(+ self,+ url: str,+ *,+ include_paths: Optional[List[str]] = None,+ exclude_paths: Optional[List[str]] = None,+ max_depth: Optional[int] = None,+ max_discovery_depth: Optional[int] = None,+ limit: Optional[int] = None,+ allow_backward_links: Optional[bool] = None,+ allow_external_links: Optional[bool] = None,+ ignore_sitemap: Optional[bool] = None,+ scrape_options: Optional[CommonOptions] = None,+ webhook: Optional[Union[str, WebhookConfig]] = None,+ deduplicate_similar_urls: Optional[bool] = None,+ ignore_query_parameters: Optional[bool] = None,+ regex_on_full_url: Optional[bool] = None,+ poll_interval: Optional[int] = 2,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> CrawlStatusResponse:"""Crawl a website starting from a URL.Args:- url (str): Target URL to start crawling from- params (Optional[CrawlParams]): See CrawlParams model:- URL Discovery:- * includePaths - Patterns of URLs to include- * excludePaths - Patterns of URLs to exclude- * maxDepth - Maximum crawl depth- * maxDiscoveryDepth - Maximum depth for finding new URLs- * limit - Maximum pages to crawl-- Link Following:- * allowBackwardLinks - Follow parent directory links- * allowExternalLinks - Follow external domain links- * ignoreSitemap - Skip sitemap.xml processing-- Advanced:- * scrapeOptions - Page scraping configuration- * webhook - Notification webhook settings- * deduplicateSimilarURLs - Remove similar URLs- * ignoreQueryParameters - Ignore URL parameters- * regexOnFullURL - Apply regex to full URLs- poll_interval (int): Seconds between status checks (default: 2)- idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ url (str): Target URL to start crawling from+ include_paths (Optional[List[str]]): Patterns of URLs to include+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude+ max_depth (Optional[int]): Maximum crawl depth+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs+ limit (Optional[int]): Maximum pages to crawl+ allow_backward_links (Optional[bool]): Follow parent directory links+ allow_external_links (Optional[bool]): Follow external domain links+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ scrape_options (Optional[CommonOptions]): Page scraping configuration+ webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs+ ignore_query_parameters (Optional[bool]): Ignore URL parameters+ regex_on_full_url (Optional[bool]): Apply regex to full URLs+ poll_interval (Optional[int]): Seconds between status checks (default: 2)+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the APIReturns:- CrawlStatusResponse with:- * Crawling status and progress- * Crawled page contents- * Success/error information+ CrawlStatusResponse with:+ * Crawling status and progress+ * Crawled page contents+ * Success/error informationRaises:- Exception: If crawl fails+ Exception: If crawl fails"""- endpoint = f'/v1/crawl'+ crawl_params = {}++ # Add individual parameters+ if include_paths is not None:+ crawl_params['includePaths'] = include_paths+ if exclude_paths is not None:+ crawl_params['excludePaths'] = exclude_paths+ if max_depth is not None:+ crawl_params['maxDepth'] = max_depth+ if max_discovery_depth is not None:+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth+ if limit is not None:+ crawl_params['limit'] = limit+ if allow_backward_links is not None:+ crawl_params['allowBackwardLinks'] = allow_backward_links+ if allow_external_links is not None:+ crawl_params['allowExternalLinks'] = allow_external_links+ if ignore_sitemap is not None:+ crawl_params['ignoreSitemap'] = ignore_sitemap+ if scrape_options is not None:+ crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)+ if webhook is not None:+ crawl_params['webhook'] = webhook+ if deduplicate_similar_urls is not None:+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls+ if ignore_query_parameters is not None:+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters+ if regex_on_full_url is not None:+ crawl_params['regexOnFullURL'] = regex_on_full_url++ # Add any additional kwargs+ crawl_params.update(kwargs)++ # Create final params object+ final_params = CrawlParams(**crawl_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ # Make requestheaders = self._prepare_headers(idempotency_key)- json_data = {'url': url}- if params:- json_data.update(params)- json_data['origin'] = f"python-sdk@{version}"- response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers)+ response = self._post_request(f'{self.api_url}/v1/crawl', params_dict, headers)+if response.status_code == 200:try:id = response.json().get('id')except:raise Exception(f'Failed to parse Firecrawl response as JSON.')return self._monitor_job_status(id, headers, poll_interval)-else:self._handle_error(response, 'start crawl job')-def async_crawl_url(- self,- url: str,- params: Optional[CrawlParams] = None,- idempotency_key: Optional[str] = None) -> CrawlResponse:+ self,+ url: str,+ *,+ include_paths: Optional[List[str]] = None,+ exclude_paths: Optional[List[str]] = None,+ max_depth: Optional[int] = None,+ max_discovery_depth: Optional[int] = None,+ limit: Optional[int] = None,+ allow_backward_links: Optional[bool] = None,+ allow_external_links: Optional[bool] = None,+ ignore_sitemap: Optional[bool] = None,+ scrape_options: Optional[CommonOptions] = None,+ webhook: Optional[Union[str, WebhookConfig]] = None,+ deduplicate_similar_urls: Optional[bool] = None,+ ignore_query_parameters: Optional[bool] = None,+ regex_on_full_url: Optional[bool] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> CrawlResponse:"""Start an asynchronous crawl job.Args:url (str): Target URL to start crawling from-- params (Optional[CrawlParams]): See CrawlParams model:-- URL Discovery:- * includePaths - Patterns of URLs to include- * excludePaths - Patterns of URLs to exclude- * maxDepth - Maximum crawl depth- * maxDiscoveryDepth - Maximum depth for finding new URLs- * limit - Maximum pages to crawl-- Link Following:- * allowBackwardLinks - Follow parent directory links- * allowExternalLinks - Follow external domain links- * ignoreSitemap - Skip sitemap.xml processing-- Advanced:- * scrapeOptions - Page scraping configuration- * webhook - Notification webhook settings- * deduplicateSimilarURLs - Remove similar URLs- * ignoreQueryParameters - Ignore URL parameters- * regexOnFullURL - Apply regex to full URLs-- idempotency_key: Unique key to prevent duplicate requests+ include_paths (Optional[List[str]]): Patterns of URLs to include+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude+ max_depth (Optional[int]): Maximum crawl depth+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs+ limit (Optional[int]): Maximum pages to crawl+ allow_backward_links (Optional[bool]): Follow parent directory links+ allow_external_links (Optional[bool]): Follow external domain links+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ scrape_options (Optional[CommonOptions]): Page scraping configuration+ webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs+ ignore_query_parameters (Optional[bool]): Ignore URL parameters+ regex_on_full_url (Optional[bool]): Apply regex to full URLs+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the APIReturns:CrawlResponse with:@@ -751,16 +809,52 @@ class FirecrawlApp:Raises:Exception: If crawl initiation fails"""- endpoint = f'/v1/crawl'+ crawl_params = {}++ # Add individual parameters+ if include_paths is not None:+ crawl_params['includePaths'] = include_paths+ if exclude_paths is not None:+ crawl_params['excludePaths'] = exclude_paths+ if max_depth is not None:+ crawl_params['maxDepth'] = max_depth+ if max_discovery_depth is not None:+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth+ if limit is not None:+ crawl_params['limit'] = limit+ if allow_backward_links is not None:+ crawl_params['allowBackwardLinks'] = allow_backward_links+ if allow_external_links is not None:+ crawl_params['allowExternalLinks'] = allow_external_links+ if ignore_sitemap is not None:+ crawl_params['ignoreSitemap'] = ignore_sitemap+ if scrape_options is not None:+ crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)+ if webhook is not None:+ crawl_params['webhook'] = webhook+ if deduplicate_similar_urls is not None:+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls+ if ignore_query_parameters is not None:+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters+ if regex_on_full_url is not None:+ crawl_params['regexOnFullURL'] = regex_on_full_url++ # Add any additional kwargs+ crawl_params.update(kwargs)++ # Create final params object+ final_params = CrawlParams(**crawl_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ # Make requestheaders = self._prepare_headers(idempotency_key)- json_data = {'url': url}- if params:- json_data.update(params)- json_data['origin'] = f"python-sdk@{version}"- response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers)+ response = self._post_request(f'{self.api_url}/v1/crawl', params_dict, headers)+if response.status_code == 200:try:- return response.json()+ return CrawlResponse(**response.json())except:raise Exception(f'Failed to parse Firecrawl response as JSON.')else:@@ -842,10 +936,10 @@ class FirecrawlApp:if 'next' in status_data:response['next'] = status_data['next']- return {- 'success': False if 'error' in status_data else True,+ return CrawlStatusResponse(+ success=False if 'error' in status_data else True,**response- }+ )else:self._handle_error(response, 'check crawl status')@@ -872,7 +966,7 @@ class FirecrawlApp:response = self._get_request(f'{self.api_url}/v1/crawl/{id}/errors', headers)if response.status_code == 200:try:- return response.json()+ return CrawlErrorsResponse(**response.json())except:raise Exception(f'Failed to parse Firecrawl response as JSON.')else:@@ -906,254 +1000,519 @@ class FirecrawlApp:def crawl_url_and_watch(self,url: str,- params: Optional[CrawlParams] = None,- idempotency_key: Optional[str] = None) -> 'CrawlWatcher':+ *,+ include_paths: Optional[List[str]] = None,+ exclude_paths: Optional[List[str]] = None,+ max_depth: Optional[int] = None,+ max_discovery_depth: Optional[int] = None,+ limit: Optional[int] = None,+ allow_backward_links: Optional[bool] = None,+ allow_external_links: Optional[bool] = None,+ ignore_sitemap: Optional[bool] = None,+ scrape_options: Optional[CommonOptions] = None,+ webhook: Optional[Union[str, WebhookConfig]] = None,+ deduplicate_similar_urls: Optional[bool] = None,+ ignore_query_parameters: Optional[bool] = None,+ regex_on_full_url: Optional[bool] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> 'CrawlWatcher':"""Initiate a crawl job and return a CrawlWatcher to monitor the job via WebSocket.Args:- url (str): Target URL to start crawling from- params (Optional[CrawlParams]): See CrawlParams model for configuration:- URL Discovery:- * includePaths - Patterns of URLs to include- * excludePaths - Patterns of URLs to exclude- * maxDepth - Maximum crawl depth- * maxDiscoveryDepth - Maximum depth for finding new URLs- * limit - Maximum pages to crawl-- Link Following:- * allowBackwardLinks - Follow parent directory links- * allowExternalLinks - Follow external domain links- * ignoreSitemap - Skip sitemap.xml processing-- Advanced:- * scrapeOptions - Page scraping configuration- * webhook - Notification webhook settings- * deduplicateSimilarURLs - Remove similar URLs- * ignoreQueryParameters - Ignore URL parameters- * regexOnFullURL - Apply regex to full URLs- idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ url (str): Target URL to start crawling from+ include_paths (Optional[List[str]]): Patterns of URLs to include+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude+ max_depth (Optional[int]): Maximum crawl depth+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs+ limit (Optional[int]): Maximum pages to crawl+ allow_backward_links (Optional[bool]): Follow parent directory links+ allow_external_links (Optional[bool]): Follow external domain links+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ scrape_options (Optional[CommonOptions]): Page scraping configuration+ webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs+ ignore_query_parameters (Optional[bool]): Ignore URL parameters+ regex_on_full_url (Optional[bool]): Apply regex to full URLs+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the APIReturns:- AsyncCrawlWatcher: An instance to monitor the crawl job via WebSocket+ CrawlWatcher: An instance to monitor the crawl job via WebSocketRaises:- Exception: If crawl job fails to start- """- crawl_response = self.async_crawl_url(url, params, idempotency_key)- if crawl_response['success'] and 'id' in crawl_response:- return CrawlWatcher(crawl_response['id'], self)+ Exception: If crawl job fails to start+ """+ crawl_response = self.async_crawl_url(+ url,+ include_paths=include_paths,+ exclude_paths=exclude_paths,+ max_depth=max_depth,+ max_discovery_depth=max_discovery_depth,+ limit=limit,+ allow_backward_links=allow_backward_links,+ allow_external_links=allow_external_links,+ ignore_sitemap=ignore_sitemap,+ scrape_options=scrape_options,+ webhook=webhook,+ deduplicate_similar_urls=deduplicate_similar_urls,+ ignore_query_parameters=ignore_query_parameters,+ regex_on_full_url=regex_on_full_url,+ idempotency_key=idempotency_key,+ **kwargs+ )+ if crawl_response.success and crawl_response.id:+ return CrawlWatcher(crawl_response.id, self)else:raise Exception("Crawl job failed to start")def map_url(self,url: str,+ *,+ search: Optional[str] = None,+ ignore_sitemap: Optional[bool] = None,+ include_subdomains: Optional[bool] = None,+ sitemap_only: Optional[bool] = None,+ limit: Optional[int] = None,+ timeout: Optional[int] = None,params: Optional[MapParams] = None) -> MapResponse:"""Map and discover links from a URL.Args:- url: Target URL to map-- params: See MapParams model:-- Discovery Options:- * search - Filter pattern for URLs- * ignoreSitemap - Skip sitemap.xml- * includeSubdomains - Include subdomain links- * sitemapOnly - Only use sitemap.xml-- Limits:- * limit - Max URLs to return- * timeout - Request timeout (ms)+ url (str): Target URL to map+ search (Optional[str]): Filter pattern for URLs+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ include_subdomains (Optional[bool]): Include subdomain links+ sitemap_only (Optional[bool]): Only use sitemap.xml+ limit (Optional[int]): Maximum URLs to return+ timeout (Optional[int]): Request timeout in milliseconds+ params (Optional[MapParams]): Additional mapping parametersReturns:- MapResponse with:- * Discovered URLs- * Success/error status+ MapResponse: Response containing:+ * success (bool): Whether request succeeded+ * links (List[str]): Discovered URLs+ * error (Optional[str]): Error message if anyRaises:- Exception: If mapping fails+ Exception: If mapping fails or response cannot be parsed"""- endpoint = f'/v1/map'- headers = self._prepare_headers()-- # Prepare the base scrape parameters with the URL- json_data = {'url': url}+ # Build map parameters+ map_params = {}if params:- json_data.update(params)- json_data['origin'] = f"python-sdk@{version}"+ map_params.update(params.dict(exclude_none=True))- # Make the POST request with the prepared headers and JSON data+ # Add individual parameters+ if search is not None:+ map_params['search'] = search+ if ignore_sitemap is not None:+ map_params['ignoreSitemap'] = ignore_sitemap+ if include_subdomains is not None:+ map_params['includeSubdomains'] = include_subdomains+ if sitemap_only is not None:+ map_params['sitemapOnly'] = sitemap_only+ if limit is not None:+ map_params['limit'] = limit+ if timeout is not None:+ map_params['timeout'] = timeout++ # Create final params object+ final_params = MapParams(**map_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ # Make requestresponse = requests.post(- f'{self.api_url}{endpoint}',- headers=headers,- json=json_data,+ f"{self.api_url}/v1/map",+ headers={"Authorization": f"Bearer {self.api_key}"},+ json=params_dict)+if response.status_code == 200:try:- response = response.json()- except:- raise Exception(f'Failed to parse Firecrawl response as JSON.')- if response['success'] and 'links' in response:- return response- elif 'error' in response:- raise Exception(f'Failed to map URL. Error: {response["error"]}')- else:- raise Exception(f'Failed to map URL. Error: {response}')+ response_json = response.json()+ if response_json.get('success') and 'links' in response_json:+ return MapResponse(**response_json)+ elif "error" in response_json:+ raise Exception(f'Map failed. Error: {response_json["error"]}')+ else:+ raise Exception(f'Map failed. Error: {response_json}')+ except ValueError:+ raise Exception('Failed to parse Firecrawl response as JSON.')else:self._handle_error(response, 'map')- def batch_scrape_urls(self, urls: List[str],- params: Optional[ScrapeParams] = None,- poll_interval: Optional[int] = 2,- idempotency_key: Optional[str] = None) -> BatchScrapeStatusResponse:+ def batch_scrape_urls(+ self,+ urls: List[str],+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = None,+ location: Optional[LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth"]] = None,+ extract: Optional[ExtractConfig] = None,+ json_options: Optional[ExtractConfig] = None,+ actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,+ agent: Optional[AgentOptions] = None,+ poll_interval: Optional[int] = 2,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> BatchScrapeStatusResponse:"""Batch scrape multiple URLs and monitor until completion.Args:urls (List[str]): URLs to scrape- params (Optional[ScrapeParams]): See ScrapeParams model:- Content Options:- * formats - Content formats to retrieve- * includeTags - HTML tags to include- * excludeTags - HTML tags to exclude- * onlyMainContent - Extract main content only-- Request Options:- * headers - Custom HTTP headers- * timeout - Request timeout (ms)- * mobile - Use mobile user agent- * proxy - Proxy type-- Extraction Options:- * extract - Content extraction config- * jsonOptions - JSON extraction config- * actions - Actions to perform+ formats (Optional[List[Literal]]): Content formats to retrieve+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait time in milliseconds+ timeout (Optional[int]): Request timeout in milliseconds+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 encoded images+ block_ads (Optional[bool]): Block advertisements+ proxy (Optional[Literal]): Proxy type to use+ extract (Optional[ExtractConfig]): Content extraction config+ json_options (Optional[ExtractConfig]): JSON extraction config+ actions (Optional[List[Union]]): Actions to perform+ agent (Optional[AgentOptions]): Agent configuration+ poll_interval (Optional[int]): Seconds between status checks (default: 2)+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the APIReturns:- BatchScrapeStatusResponse with:- * Scraping status and progress- * Scraped content for each URL- * Success/error information+ BatchScrapeStatusResponse with:+ * Scraping status and progress+ * Scraped content for each URL+ * Success/error informationRaises:- Exception: If batch scrape fails+ Exception: If batch scrape fails"""- endpoint = f'/v1/batch/scrape'+ scrape_params = {}++ # Add individual parameters+ if formats is not None:+ scrape_params['formats'] = formats+ if headers is not None:+ scrape_params['headers'] = headers+ if include_tags is not None:+ scrape_params['includeTags'] = include_tags+ if exclude_tags is not None:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for is not None:+ scrape_params['waitFor'] = wait_for+ if timeout is not None:+ scrape_params['timeout'] = timeout+ if location is not None:+ scrape_params['location'] = location.dict(exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy is not None:+ scrape_params['proxy'] = proxy+ if extract is not None:+ if hasattr(extract.schema, 'schema'):+ extract.schema = extract.schema.schema()+ scrape_params['extract'] = extract.dict(exclude_none=True)+ if json_options is not None:+ if hasattr(json_options.schema, 'schema'):+ json_options.schema = json_options.schema.schema()+ scrape_params['jsonOptions'] = json_options.dict(exclude_none=True)+ if actions is not None:+ scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions]+ if agent is not None:+ scrape_params['agent'] = agent.dict(exclude_none=True)++ # Add any additional kwargs+ scrape_params.update(kwargs)++ # Create final params object+ final_params = ScrapeParams(**scrape_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['urls'] = urls+ params_dict['origin'] = f"python-sdk@{version}"++ # Make requestheaders = self._prepare_headers(idempotency_key)- json_data = {'urls': urls}- if params:- json_data.update(params)- json_data['origin'] = f"python-sdk@{version}"- response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers)+ response = self._post_request(f'{self.api_url}/v1/batch/scrape', params_dict, headers)+if response.status_code == 200:try:id = response.json().get('id')except:raise Exception(f'Failed to parse Firecrawl response as JSON.')return self._monitor_job_status(id, headers, poll_interval)-else:self._handle_error(response, 'start batch scrape job')-def async_batch_scrape_urls(- self,- urls: List[str],- params: Optional[ScrapeParams] = None,- idempotency_key: Optional[str] = None) -> BatchScrapeResponse:+ self,+ urls: List[str],+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = None,+ location: Optional[LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth"]] = None,+ extract: Optional[ExtractConfig] = None,+ json_options: Optional[ExtractConfig] = None,+ actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,+ agent: Optional[AgentOptions] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> BatchScrapeResponse:"""Initiate a batch scrape job asynchronously.Args:- urls (List[str]): List of URLs to scrape- params (Optional[ScrapeParams]): See ScrapeParams model for configuration:- Content Options:- * formats - Content formats to retrieve- * includeTags - HTML tags to include- * excludeTags - HTML tags to exclude- * onlyMainContent - Extract main content only-- Request Options:- * headers - Custom HTTP headers- * timeout - Request timeout (ms)- * mobile - Use mobile user agent- * proxy - Proxy type-- Extraction Options:- * extract - Content extraction config- * jsonOptions - JSON extraction config- * actions - Actions to perform- idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ urls (List[str]): URLs to scrape+ formats (Optional[List[Literal]]): Content formats to retrieve+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait time in milliseconds+ timeout (Optional[int]): Request timeout in milliseconds+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 encoded images+ block_ads (Optional[bool]): Block advertisements+ proxy (Optional[Literal]): Proxy type to use+ extract (Optional[ExtractConfig]): Content extraction config+ json_options (Optional[ExtractConfig]): JSON extraction config+ actions (Optional[List[Union]]): Actions to perform+ agent (Optional[AgentOptions]): Agent configuration+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the APIReturns:- BatchScrapeResponse with:- * success - Whether job started successfully- * id - Unique identifier for the job- * url - Status check URL- * error - Error message if start failed+ BatchScrapeResponse with:+ * success - Whether job started successfully+ * id - Unique identifier for the job+ * url - Status check URL+ * error - Error message if start failedRaises:- Exception: If job initiation fails+ Exception: If job initiation fails"""- endpoint = f'/v1/batch/scrape'+ scrape_params = {}++ # Add individual parameters+ if formats is not None:+ scrape_params['formats'] = formats+ if headers is not None:+ scrape_params['headers'] = headers+ if include_tags is not None:+ scrape_params['includeTags'] = include_tags+ if exclude_tags is not None:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for is not None:+ scrape_params['waitFor'] = wait_for+ if timeout is not None:+ scrape_params['timeout'] = timeout+ if location is not None:+ scrape_params['location'] = location.dict(exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy is not None:+ scrape_params['proxy'] = proxy+ if extract is not None:+ if hasattr(extract.schema, 'schema'):+ extract.schema = extract.schema.schema()+ scrape_params['extract'] = extract.dict(exclude_none=True)+ if json_options is not None:+ if hasattr(json_options.schema, 'schema'):+ json_options.schema = json_options.schema.schema()+ scrape_params['jsonOptions'] = json_options.dict(exclude_none=True)+ if actions is not None:+ scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions]+ if agent is not None:+ scrape_params['agent'] = agent.dict(exclude_none=True)++ # Add any additional kwargs+ scrape_params.update(kwargs)++ # Create final params object+ final_params = ScrapeParams(**scrape_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['urls'] = urls+ params_dict['origin'] = f"python-sdk@{version}"++ # Make requestheaders = self._prepare_headers(idempotency_key)- json_data = {'urls': urls}- if params:- json_data.update(params)- json_data['origin'] = f"python-sdk@{version}"- response = self._post_request(f'{self.api_url}{endpoint}', json_data, headers)+ response = self._post_request(f'{self.api_url}/v1/batch/scrape', params_dict, headers)+if response.status_code == 200:try:- return response.json()+ return BatchScrapeResponse(**response.json())except:raise Exception(f'Failed to parse Firecrawl response as JSON.')else:self._handle_error(response, 'start batch scrape job')def batch_scrape_urls_and_watch(- self,- urls: List[str],- params: Optional[ScrapeParams] = None,- idempotency_key: Optional[str] = None) -> 'CrawlWatcher':+ self,+ urls: List[str],+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = None,+ location: Optional[LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth"]] = None,+ extract: Optional[ExtractConfig] = None,+ json_options: Optional[ExtractConfig] = None,+ actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,+ agent: Optional[AgentOptions] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> 'CrawlWatcher':"""Initiate a batch scrape job and return a CrawlWatcher to monitor the job via WebSocket.Args:- urls (List[str]): List of URLs to scrape- params (Optional[ScrapeParams]): See ScrapeParams model for configuration:-- Content Options:- * formats - Content formats to retrieve- * includeTags - HTML tags to include- * excludeTags - HTML tags to exclude- * onlyMainContent - Extract main content only-- Request Options:- * headers - Custom HTTP headers- * timeout - Request timeout (ms)- * mobile - Use mobile user agent- * proxy - Proxy type-- Extraction Options:- * extract - Content extraction config- * jsonOptions - JSON extraction config- * actions - Actions to perform+ urls (List[str]): URLs to scrape+ formats (Optional[List[Literal]]): Content formats to retrieve+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait time in milliseconds+ timeout (Optional[int]): Request timeout in milliseconds+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 encoded images+ block_ads (Optional[bool]): Block advertisements+ proxy (Optional[Literal]): Proxy type to use+ extract (Optional[ExtractConfig]): Content extraction config+ json_options (Optional[ExtractConfig]): JSON extraction config+ actions (Optional[List[Union]]): Actions to perform+ agent (Optional[AgentOptions]): Agent configurationidempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the APIReturns:- AsyncCrawlWatcher: An instance to monitor the batch scrape job via WebSocket+ CrawlWatcher: An instance to monitor the batch scrape job via WebSocketRaises:Exception: If batch scrape job fails to start"""- crawl_response = self.async_batch_scrape_urls(urls, params, idempotency_key)- if crawl_response['success'] and 'id' in crawl_response:- return CrawlWatcher(crawl_response['id'], self)+ scrape_params = {}++ # Add individual parameters+ if formats is not None:+ scrape_params['formats'] = formats+ if headers is not None:+ scrape_params['headers'] = headers+ if include_tags is not None:+ scrape_params['includeTags'] = include_tags+ if exclude_tags is not None:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for is not None:+ scrape_params['waitFor'] = wait_for+ if timeout is not None:+ scrape_params['timeout'] = timeout+ if location is not None:+ scrape_params['location'] = location.dict(exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy is not None:+ scrape_params['proxy'] = proxy+ if extract is not None:+ if hasattr(extract.schema, 'schema'):+ extract.schema = extract.schema.schema()+ scrape_params['extract'] = extract.dict(exclude_none=True)+ if json_options is not None:+ if hasattr(json_options.schema, 'schema'):+ json_options.schema = json_options.schema.schema()+ scrape_params['jsonOptions'] = json_options.dict(exclude_none=True)+ if actions is not None:+ scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions]+ if agent is not None:+ scrape_params['agent'] = agent.dict(exclude_none=True)++ # Add any additional kwargs+ scrape_params.update(kwargs)++ # Create final params object+ final_params = ScrapeParams(**scrape_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['urls'] = urls+ params_dict['origin'] = f"python-sdk@{version}"++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = self._post_request(f'{self.api_url}/v1/batch/scrape', params_dict, headers)++ if response.status_code == 200:+ try:+ crawl_response = BatchScrapeResponse(**response.json())+ if crawl_response.success and crawl_response.id:+ return CrawlWatcher(crawl_response.id, self)+ else:+ raise Exception("Batch scrape job failed to start")+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')else:- raise Exception("Batch scrape job failed to start")+ self._handle_error(response, 'start batch scrape job')def check_batch_scrape_status(self, id: str) -> BatchScrapeStatusResponse:"""@@ -1203,25 +1562,17 @@ class FirecrawlApp:breakstatus_data['data'] = data- response = {+ return BatchScrapeStatusResponse(**{+ 'success': False if 'error' in status_data else True,'status': status_data.get('status'),'total': status_data.get('total'),'completed': status_data.get('completed'),'creditsUsed': status_data.get('creditsUsed'),'expiresAt': status_data.get('expiresAt'),- 'data': status_data.get('data')- }-- if 'error' in status_data:- response['error'] = status_data['error']-- if 'next' in status_data:- response['next'] = status_data['next']-- return {- 'success': False if 'error' in status_data else True,- **response- }+ 'data': status_data.get('data'),+ 'next': status_data.get('next'),+ 'error': status_data.get('error')+ })else:self._handle_error(response, 'check batch scrape status')@@ -1230,7 +1581,7 @@ class FirecrawlApp:Returns information about batch scrape errors.Args:- id (str): The ID of the crawl job.+ id (str): The ID of the crawl job.Returns:CrawlErrorsResponse: A response containing:@@ -1240,12 +1591,15 @@ class FirecrawlApp:* url (str): URL that caused the error* error (str): Error message* robotsBlocked (List[str]): List of URLs blocked by robots.txt++ Raises:+ Exception: If the error check request fails"""headers = self._prepare_headers()response = self._get_request(f'{self.api_url}/v1/batch/scrape/{id}/errors', headers)if response.status_code == 200:try:- return response.json()+ return CrawlErrorsResponse(**response.json())except:raise Exception(f'Failed to parse Firecrawl response as JSON.')else: