Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
async functions
rafaelmmiller0915db515c70
diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex 0f7964e1f..d622a7ceb 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -2792,225 +2792,472 @@ class AsyncFirecrawlApp(FirecrawlApp):raise Exception(f'Failed to scrape URL. Error: {error_content}')async def batch_scrape_urls(- self,- urls: List[str],- params: Optional[ScrapeParams] = None) -> BatchScrapeStatusResponse:+ self,+ urls: List[str],+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = None,+ location: Optional[LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth"]] = None,+ extract: Optional[ExtractConfig] = None,+ json_options: Optional[ExtractConfig] = None,+ actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,+ agent: Optional[AgentOptions] = None,+ poll_interval: Optional[int] = 2,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> BatchScrapeStatusResponse:"""Asynchronously scrape multiple URLs and monitor until completion.Args:urls (List[str]): URLs to scrape- params (Optional[ScrapeParams]): See ScrapeParams model:- Content Options:- * formats - Content formats to retrieve- * includeTags - HTML tags to include- * excludeTags - HTML tags to exclude- * onlyMainContent - Extract main content only-- Request Options:- * headers - Custom HTTP headers- * timeout - Request timeout (ms)- * mobile - Use mobile user agent- * proxy - Proxy type-- Extraction Options:- * extract - Content extraction config- * jsonOptions - JSON extraction config- * actions - Actions to perform+ formats (Optional[List[Literal]]): Content formats to retrieve+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait time in milliseconds+ timeout (Optional[int]): Request timeout in milliseconds+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 encoded images+ block_ads (Optional[bool]): Block advertisements+ proxy (Optional[Literal]): Proxy type to use+ extract (Optional[ExtractConfig]): Content extraction config+ json_options (Optional[ExtractConfig]): JSON extraction config+ actions (Optional[List[Union]]): Actions to perform+ agent (Optional[AgentOptions]): Agent configuration+ poll_interval (Optional[int]): Seconds between status checks (default: 2)+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the APIReturns:- BatchScrapeStatusResponse with:- * Scraping status and progress- * Scraped content for each URL- * Success/error information+ BatchScrapeStatusResponse with:+ * Scraping status and progress+ * Scraped content for each URL+ * Success/error informationRaises:- Exception: If batch scrape fails+ Exception: If batch scrape fails"""- headers = self._prepare_headers()- json_data = {'urls': urls}- if params:- json_data.update(params)- json_data['origin'] = f"python-sdk@{version}"+ scrape_params = {}++ # Add individual parameters+ if formats is not None:+ scrape_params['formats'] = formats+ if headers is not None:+ scrape_params['headers'] = headers+ if include_tags is not None:+ scrape_params['includeTags'] = include_tags+ if exclude_tags is not None:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for is not None:+ scrape_params['waitFor'] = wait_for+ if timeout is not None:+ scrape_params['timeout'] = timeout+ if location is not None:+ scrape_params['location'] = location.dict(exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy is not None:+ scrape_params['proxy'] = proxy+ if extract is not None:+ if hasattr(extract.schema, 'schema'):+ extract.schema = extract.schema.schema()+ scrape_params['extract'] = extract.dict(exclude_none=True)+ if json_options is not None:+ if hasattr(json_options.schema, 'schema'):+ json_options.schema = json_options.schema.schema()+ scrape_params['jsonOptions'] = json_options.dict(exclude_none=True)+ if actions is not None:+ scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions]+ if agent is not None:+ scrape_params['agent'] = agent.dict(exclude_none=True)++ # Add any additional kwargs+ scrape_params.update(kwargs)- endpoint = f'/v1/batch/scrape'+ # Create final params object+ final_params = ScrapeParams(**scrape_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['urls'] = urls+ params_dict['origin'] = f"python-sdk@{version}"++ # Make request+ headers = self._prepare_headers(idempotency_key)response = await self._async_post_request(- f'{self.api_url}{endpoint}',- json_data,+ f'{self.api_url}/v1/batch/scrape',+ params_dict,headers)- if response.get('success') and 'id' in response:- return await self._async_monitor_job_status(response['id'], headers)+ if response.status_code == 200:+ try:+ id = response.json().get('id')+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ return self._monitor_job_status(id, headers, poll_interval)else:- raise Exception(f'Failed to start batch scrape. Error: {response.get("error")}')+ self._handle_error(response, 'start batch scrape job')+async def async_batch_scrape_urls(- self,- urls: List[str],- params: Optional[ScrapeParams] = None,- idempotency_key: Optional[str] = None) -> BatchScrapeResponse:+ self,+ urls: List[str],+ *,+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ headers: Optional[Dict[str, str]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = None,+ location: Optional[LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth"]] = None,+ extract: Optional[ExtractConfig] = None,+ json_options: Optional[ExtractConfig] = None,+ actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None,+ agent: Optional[AgentOptions] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> BatchScrapeResponse:"""- Initiate an asynchronous batch scrape job without waiting for completion.+ Initiate a batch scrape job asynchronously.Args:- urls (List[str]): List of URLs to scrape- params (Optional[ScrapeParams]): See ScrapeParams model for configuration:- Content Options:- * formats - Content formats to retrieve- * includeTags - HTML tags to include- * excludeTags - HTML tags to exclude- * onlyMainContent - Extract main content only-- Request Options:- * headers - Custom HTTP headers- * timeout - Request timeout (ms)- * mobile - Use mobile user agent- * proxy - Proxy type-- Extraction Options:- * extract - Content extraction config- * jsonOptions - JSON extraction config- * actions - Actions to perform- idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ urls (List[str]): URLs to scrape+ formats (Optional[List[Literal]]): Content formats to retrieve+ headers (Optional[Dict[str, str]]): Custom HTTP headers+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait time in milliseconds+ timeout (Optional[int]): Request timeout in milliseconds+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 encoded images+ block_ads (Optional[bool]): Block advertisements+ proxy (Optional[Literal]): Proxy type to use+ extract (Optional[ExtractConfig]): Content extraction config+ json_options (Optional[ExtractConfig]): JSON extraction config+ actions (Optional[List[Union]]): Actions to perform+ agent (Optional[AgentOptions]): Agent configuration+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the APIReturns:- BatchScrapeResponse with:- * success - Whether job started successfully- * id - Unique identifier for the job- * url - Status check URL- * error - Error message if start failed+ BatchScrapeResponse with:+ * success - Whether job started successfully+ * id - Unique identifier for the job+ * url - Status check URL+ * error - Error message if start failedRaises:- Exception: If job initiation fails+ Exception: If job initiation fails"""- headers = self._prepare_headers(idempotency_key)- json_data = {'urls': urls}- if params:- json_data.update(params)- json_data['origin'] = f"python-sdk@{version}"+ scrape_params = {}- endpoint = f'/v1/batch/scrape'- return await self._async_post_request(- f'{self.api_url}{endpoint}',- json_data,+ # Add individual parameters+ if formats is not None:+ scrape_params['formats'] = formats+ if headers is not None:+ scrape_params['headers'] = headers+ if include_tags is not None:+ scrape_params['includeTags'] = include_tags+ if exclude_tags is not None:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for is not None:+ scrape_params['waitFor'] = wait_for+ if timeout is not None:+ scrape_params['timeout'] = timeout+ if location is not None:+ scrape_params['location'] = location.dict(exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy is not None:+ scrape_params['proxy'] = proxy+ if extract is not None:+ if hasattr(extract.schema, 'schema'):+ extract.schema = extract.schema.schema()+ scrape_params['extract'] = extract.dict(exclude_none=True)+ if json_options is not None:+ if hasattr(json_options.schema, 'schema'):+ json_options.schema = json_options.schema.schema()+ scrape_params['jsonOptions'] = json_options.dict(exclude_none=True)+ if actions is not None:+ scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions]+ if agent is not None:+ scrape_params['agent'] = agent.dict(exclude_none=True)++ # Add any additional kwargs+ scrape_params.update(kwargs)++ # Create final params object+ final_params = ScrapeParams(**scrape_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['urls'] = urls+ params_dict['origin'] = f"python-sdk@{version}"++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = await self._async_post_request(+ f'{self.api_url}/v1/batch/scrape',+ params_dict,headers)+ if response.status_code == 200:+ try:+ return BatchScrapeResponse(**response.json())+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, 'start batch scrape job')+async def crawl_url(- self,- url: str,- params: Optional[CrawlParams] = None,- poll_interval: int = 2,- idempotency_key: Optional[str] = None) -> CrawlStatusResponse:+ self,+ url: str,+ *,+ include_paths: Optional[List[str]] = None,+ exclude_paths: Optional[List[str]] = None,+ max_depth: Optional[int] = None,+ max_discovery_depth: Optional[int] = None,+ limit: Optional[int] = None,+ allow_backward_links: Optional[bool] = None,+ allow_external_links: Optional[bool] = None,+ ignore_sitemap: Optional[bool] = None,+ scrape_options: Optional[CommonOptions] = None,+ webhook: Optional[Union[str, WebhookConfig]] = None,+ deduplicate_similar_urls: Optional[bool] = None,+ ignore_query_parameters: Optional[bool] = None,+ regex_on_full_url: Optional[bool] = None,+ poll_interval: Optional[int] = 2,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> CrawlStatusResponse:"""- Asynchronously crawl a website starting from a URL and monitor until completion.+ Crawl a website starting from a URL.Args:- url (str): Target URL to start crawling from- params (Optional[CrawlParams]): See CrawlParams model:- URL Discovery:- * includePaths - Patterns of URLs to include- * excludePaths - Patterns of URLs to exclude- * maxDepth - Maximum crawl depth- * maxDiscoveryDepth - Maximum depth for finding new URLs- * limit - Maximum pages to crawl-- Link Following:- * allowBackwardLinks - Follow parent directory links- * allowExternalLinks - Follow external domain links- * ignoreSitemap - Skip sitemap.xml processing-- Advanced:- * scrapeOptions - Page scraping configuration- * webhook - Notification webhook settings- * deduplicateSimilarURLs - Remove similar URLs- * ignoreQueryParameters - Ignore URL parameters- * regexOnFullURL - Apply regex to full URLs- poll_interval (int): Seconds between status checks (default: 2)- idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ url (str): Target URL to start crawling from+ include_paths (Optional[List[str]]): Patterns of URLs to include+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude+ max_depth (Optional[int]): Maximum crawl depth+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs+ limit (Optional[int]): Maximum pages to crawl+ allow_backward_links (Optional[bool]): Follow parent directory links+ allow_external_links (Optional[bool]): Follow external domain links+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ scrape_options (Optional[CommonOptions]): Page scraping configuration+ webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs+ ignore_query_parameters (Optional[bool]): Ignore URL parameters+ regex_on_full_url (Optional[bool]): Apply regex to full URLs+ poll_interval (Optional[int]): Seconds between status checks (default: 2)+ idempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the APIReturns:- CrawlStatusResponse with:- * Crawling status and progress- * Crawled page contents- * Success/error information+ CrawlStatusResponse with:+ * Crawling status and progress+ * Crawled page contents+ * Success/error informationRaises:- Exception: If crawl fails+ Exception: If crawl fails"""- headers = self._prepare_headers(idempotency_key)- json_data = {'url': url}- if params:- json_data.update(params)- json_data['origin'] = f"python-sdk@{version}"+ crawl_params = {}++ # Add individual parameters+ if include_paths is not None:+ crawl_params['includePaths'] = include_paths+ if exclude_paths is not None:+ crawl_params['excludePaths'] = exclude_paths+ if max_depth is not None:+ crawl_params['maxDepth'] = max_depth+ if max_discovery_depth is not None:+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth+ if limit is not None:+ crawl_params['limit'] = limit+ if allow_backward_links is not None:+ crawl_params['allowBackwardLinks'] = allow_backward_links+ if allow_external_links is not None:+ crawl_params['allowExternalLinks'] = allow_external_links+ if ignore_sitemap is not None:+ crawl_params['ignoreSitemap'] = ignore_sitemap+ if scrape_options is not None:+ crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)+ if webhook is not None:+ crawl_params['webhook'] = webhook+ if deduplicate_similar_urls is not None:+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls+ if ignore_query_parameters is not None:+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters+ if regex_on_full_url is not None:+ crawl_params['regexOnFullURL'] = regex_on_full_url- endpoint = f'/v1/crawl'+ # Add any additional kwargs+ crawl_params.update(kwargs)++ # Create final params object+ final_params = CrawlParams(**crawl_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ # Make request+ headers = self._prepare_headers(idempotency_key)response = await self._async_post_request(- f'{self.api_url}{endpoint}',- json_data,- headers- )+ f'{self.api_url}/v1/crawl', params_dict, headers)- if response.get('success') and 'id' in response:- return await self._async_monitor_job_status(response['id'], headers, poll_interval)+ if response.status_code == 200:+ try:+ id = response.json().get('id')+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ return self._monitor_job_status(id, headers, poll_interval)else:- raise Exception(f'Failed to start crawl. Error: {response.get("error")}')+ self._handle_error(response, 'start crawl job')+async def async_crawl_url(- self,- url: str,- params: Optional[CrawlParams] = None,- idempotency_key: Optional[str] = None) -> CrawlResponse:+ self,+ url: str,+ *,+ include_paths: Optional[List[str]] = None,+ exclude_paths: Optional[List[str]] = None,+ max_depth: Optional[int] = None,+ max_discovery_depth: Optional[int] = None,+ limit: Optional[int] = None,+ allow_backward_links: Optional[bool] = None,+ allow_external_links: Optional[bool] = None,+ ignore_sitemap: Optional[bool] = None,+ scrape_options: Optional[CommonOptions] = None,+ webhook: Optional[Union[str, WebhookConfig]] = None,+ deduplicate_similar_urls: Optional[bool] = None,+ ignore_query_parameters: Optional[bool] = None,+ regex_on_full_url: Optional[bool] = None,+ idempotency_key: Optional[str] = None,+ **kwargs+ ) -> CrawlResponse:"""- Initiate an asynchronous crawl job without waiting for completion.+ Start an asynchronous crawl job.Args:url (str): Target URL to start crawling from- params (Optional[CrawlParams]): See CrawlParams model:- URL Discovery:- * includePaths - Patterns of URLs to include- * excludePaths - Patterns of URLs to exclude- * maxDepth - Maximum crawl depth- * maxDiscoveryDepth - Maximum depth for finding new URLs- * limit - Maximum pages to crawl-- Link Following:- * allowBackwardLinks - Follow parent directory links- * allowExternalLinks - Follow external domain links- * ignoreSitemap - Skip sitemap.xml processing-- Advanced:- * scrapeOptions - Page scraping configuration- * webhook - Notification webhook settings- * deduplicateSimilarURLs - Remove similar URLs- * ignoreQueryParameters - Ignore URL parameters- * regexOnFullURL - Apply regex to full URLs+ include_paths (Optional[List[str]]): Patterns of URLs to include+ exclude_paths (Optional[List[str]]): Patterns of URLs to exclude+ max_depth (Optional[int]): Maximum crawl depth+ max_discovery_depth (Optional[int]): Maximum depth for finding new URLs+ limit (Optional[int]): Maximum pages to crawl+ allow_backward_links (Optional[bool]): Follow parent directory links+ allow_external_links (Optional[bool]): Follow external domain links+ ignore_sitemap (Optional[bool]): Skip sitemap.xml processing+ scrape_options (Optional[CommonOptions]): Page scraping configuration+ webhook (Optional[Union[str, WebhookConfig]]): Notification webhook settings+ deduplicate_similar_urls (Optional[bool]): Remove similar URLs+ ignore_query_parameters (Optional[bool]): Ignore URL parameters+ regex_on_full_url (Optional[bool]): Apply regex to full URLsidempotency_key (Optional[str]): Unique key to prevent duplicate requests+ **kwargs: Additional parameters to pass to the APIReturns:- CrawlResponse with:- * success - Whether job started successfully- * id - Unique identifier for the job- * url - Status check URL- * error - Error message if start failed+ CrawlResponse with:+ * success - Whether crawl started successfully+ * id - Unique identifier for the crawl job+ * url - Status check URL for the crawl+ * error - Error message if start failedRaises:- Exception: If job initiation fails+ Exception: If crawl initiation fails"""- headers = self._prepare_headers(idempotency_key)- json_data = {'url': url}- if params:- json_data.update(params)- json_data['origin'] = f"python-sdk@{version}"+ crawl_params = {}- endpoint = f'/v1/crawl'- return await self._async_post_request(- f'{self.api_url}{endpoint}',- json_data,- headers+ # Add individual parameters+ if include_paths is not None:+ crawl_params['includePaths'] = include_paths+ if exclude_paths is not None:+ crawl_params['excludePaths'] = exclude_paths+ if max_depth is not None:+ crawl_params['maxDepth'] = max_depth+ if max_discovery_depth is not None:+ crawl_params['maxDiscoveryDepth'] = max_discovery_depth+ if limit is not None:+ crawl_params['limit'] = limit+ if allow_backward_links is not None:+ crawl_params['allowBackwardLinks'] = allow_backward_links+ if allow_external_links is not None:+ crawl_params['allowExternalLinks'] = allow_external_links+ if ignore_sitemap is not None:+ crawl_params['ignoreSitemap'] = ignore_sitemap+ if scrape_options is not None:+ crawl_params['scrapeOptions'] = scrape_options.dict(exclude_none=True)+ if webhook is not None:+ crawl_params['webhook'] = webhook+ if deduplicate_similar_urls is not None:+ crawl_params['deduplicateSimilarURLs'] = deduplicate_similar_urls+ if ignore_query_parameters is not None:+ crawl_params['ignoreQueryParameters'] = ignore_query_parameters+ if regex_on_full_url is not None:+ crawl_params['regexOnFullURL'] = regex_on_full_url++ # Add any additional kwargs+ crawl_params.update(kwargs)++ # Create final params object+ final_params = CrawlParams(**crawl_params)+ params_dict = final_params.dict(exclude_none=True)+ params_dict['url'] = url+ params_dict['origin'] = f"python-sdk@{version}"++ # Make request+ headers = self._prepare_headers(idempotency_key)+ response = await self._async_post_request(+ f'{self.api_url}/v1/crawl',+ params_dict,+ headers)+ if response.status_code == 200:+ try:+ return CrawlResponse(**response.json())+ except:+ raise Exception(f'Failed to parse Firecrawl response as JSON.')+ else:+ self._handle_error(response, 'start crawl job')+async def check_crawl_status(self, id: str) -> CrawlStatusResponse:"""Check the status and results of an asynchronous crawl job.