Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
async scrape
rafaelmmiller8cd82b560062
diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex ea99aaa73..f66b25d56 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -2308,29 +2308,41 @@ class AsyncFirecrawlApp(FirecrawlApp):async def scrape_url(self,url: str,- params: Optional[ScrapeParams] = None) -> ScrapeResponse[Any]:+ formats: Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]] = None,+ include_tags: Optional[List[str]] = None,+ exclude_tags: Optional[List[str]] = None,+ only_main_content: Optional[bool] = None,+ wait_for: Optional[int] = None,+ timeout: Optional[int] = None,+ location: Optional[LocationConfig] = None,+ mobile: Optional[bool] = None,+ skip_tls_verification: Optional[bool] = None,+ remove_base64_images: Optional[bool] = None,+ block_ads: Optional[bool] = None,+ proxy: Optional[Literal["basic", "stealth"]] = None,+ extract: Optional[ExtractConfig] = None,+ json_options: Optional[ExtractConfig] = None,+ actions: Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]] = None) -> ScrapeResponse[Any]:"""- Asynchronously scrape and extract content from a URL.+ Scrape and extract content from a URL asynchronously.Args:- url (str): Target URL to scrape- params (Optional[ScrapeParams]): See ScrapeParams model for configuration:- Content Options:- * formats - Content types to retrieve (markdown/html/etc)- * includeTags - HTML tags to include- * excludeTags - HTML tags to exclude- * onlyMainContent - Extract main content only-- Request Options:- * headers - Custom HTTP headers- * timeout - Request timeout (ms)- * mobile - Use mobile user agent- * proxy - Proxy type (basic/stealth)-- Extraction Options:- * extract - Content extraction settings- * jsonOptions - JSON extraction settings- * actions - Actions to perform+ url (str): Target URL to scrape+ formats (Optional[List[Literal["markdown", "html", "rawHtml", "content", "links", "screenshot", "screenshot@fullPage", "extract", "json"]]]): Content types to retrieve (markdown/html/etc)+ include_tags (Optional[List[str]]): HTML tags to include+ exclude_tags (Optional[List[str]]): HTML tags to exclude+ only_main_content (Optional[bool]): Extract main content only+ wait_for (Optional[int]): Wait for a specific element to appear+ timeout (Optional[int]): Request timeout (ms)+ location (Optional[LocationConfig]): Location configuration+ mobile (Optional[bool]): Use mobile user agent+ skip_tls_verification (Optional[bool]): Skip TLS verification+ remove_base64_images (Optional[bool]): Remove base64 images+ block_ads (Optional[bool]): Block ads+ proxy (Optional[Literal["basic", "stealth"]]): Proxy type (basic/stealth)+ extract (Optional[ExtractConfig]): Content extraction settings+ json_options (Optional[ExtractConfig]): JSON extraction settings+ actions (Optional[List[Union[WaitAction, ScreenshotAction, ClickAction, WriteAction, PressAction, ScrollAction, ScrapeAction, ExecuteJavascriptAction]]]): Actions to performReturns:ScrapeResponse with:@@ -2340,35 +2352,70 @@ class AsyncFirecrawlApp(FirecrawlApp):* Success/error statusRaises:- Exception: If scraping fails+ Exception: If scraping fails"""headers = self._prepare_headers()- scrape_params = {'url': url, 'origin': f'python-sdk@{version}'}- if params:- extract = params.get('extract', {})- if extract:- if 'schema' in extract and hasattr(extract['schema'], 'schema'):- extract['schema'] = extract['schema'].schema()- scrape_params['extract'] = extract+ # Build scrape parameters+ scrape_params = {+ 'url': url,+ 'origin': f"python-sdk@{version}"+ }- for key, value in params.items():- if key not in ['extract']:- scrape_params[key] = value+ # Add optional parameters if provided and not None+ if formats:+ scrape_params['formats'] = formats+ if include_tags:+ scrape_params['includeTags'] = include_tags+ if exclude_tags:+ scrape_params['excludeTags'] = exclude_tags+ if only_main_content is not None:+ scrape_params['onlyMainContent'] = only_main_content+ if wait_for:+ scrape_params['waitFor'] = wait_for+ if timeout:+ scrape_params['timeout'] = timeout+ if location:+ scrape_params['location'] = location.dict(exclude_none=True)+ if mobile is not None:+ scrape_params['mobile'] = mobile+ if skip_tls_verification is not None:+ scrape_params['skipTlsVerification'] = skip_tls_verification+ if remove_base64_images is not None:+ scrape_params['removeBase64Images'] = remove_base64_images+ if block_ads is not None:+ scrape_params['blockAds'] = block_ads+ if proxy:+ scrape_params['proxy'] = proxy+ if extract:+ extract_dict = extract.dict(exclude_none=True)+ if 'schema' in extract_dict and hasattr(extract.schema, 'schema'):+ extract_dict['schema'] = extract.schema.schema() # Ensure pydantic model schema is converted+ scrape_params['extract'] = extract_dict+ if json_options:+ json_options_dict = json_options.dict(exclude_none=True)+ if 'schema' in json_options_dict and hasattr(json_options.schema, 'schema'):+ json_options_dict['schema'] = json_options.schema.schema() # Ensure pydantic model schema is converted+ scrape_params['jsonOptions'] = json_options_dict+ if actions:+ scrape_params['actions'] = [action.dict(exclude_none=True) for action in actions]+ # Make async requestendpoint = f'/v1/scrape'response = await self._async_post_request(f'{self.api_url}{endpoint}',scrape_params,headers)-+if response.get('success') and 'data' in response:- return response['data']+ return ScrapeResponse(**response['data'])elif "error" in response:raise Exception(f'Failed to scrape URL. Error: {response["error"]}')else:- raise Exception(f'Failed to scrape URL. Error: {response}')+ # Use the response content directly if possible, otherwise a generic message+ error_content = response.get('error', str(response))+ raise Exception(f'Failed to scrape URL. Error: {error_content}')async def batch_scrape_urls(self,