🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat: Add an mcp server
Karim shoair172a5b4a0a06
diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 0049ff2..ee59a72 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -133,6 +133,13 @@ def install(force):print("The dependencies are already installed")+@command(help="Run Scrapling's MCP server (Check the docs for more info).")+def mcp():+ from scrapling.core.ai import ScraplingMCPServer++ ScraplingMCPServer().serve()++@command(help="Interactive scraping console")@option("-c",@@ -824,3 +831,4 @@ def main():main.add_command(install)main.add_command(shell)main.add_command(extract)+main.add_command(mcp)diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pynew file mode 100644index 0000000..07104f0--- /dev/null+++ b/scrapling/core/ai.py@@ -0,0 +1,613 @@+from asyncio import gather++from mcp.server.fastmcp import FastMCP+from pydantic import BaseModel, Field++from scrapling.core.shell import Convertor+from scrapling.engines.toolbelt import Response as _ScraplingResponse+from scrapling.fetchers import (+ Fetcher,+ FetcherSession,+ DynamicFetcher,+ AsyncDynamicSession,+ StealthyFetcher,+ AsyncStealthySession,+)+from scrapling.core._types import (+ Optional,+ Literal,+ Tuple,+ extraction_types,+ Union,+ Mapping,+ Dict,+ List,+ SelectorWaitStates,+ Generator,+)+from curl_cffi.requests import (+ BrowserTypeLiteral,+)+++class ResponseModel(BaseModel):+ """Request's response information structure."""++ status: int = Field(description="The status code returned by the website.")+ content: list[str] = Field(+ description="The content as Markdown/HTML or the text content of the page."+ )+ url: str = Field(+ description="The URL given by the user that resulted in this response."+ )+++def _ContentTranslator(+ content: Generator[str, None, None], page: _ScraplingResponse+) -> ResponseModel:+ """Convert a content generator to a list of ResponseModel objects."""+ return ResponseModel(+ status=page.status, content=[result for result in content], url=page.url+ )+++class ScraplingMCPServer:+ _server = FastMCP(name="Scrapling")++ @staticmethod+ @_server.tool()+ def get(+ url: str,+ impersonate: Optional[BrowserTypeLiteral] = "chrome",+ extraction_type: extraction_types = "markdown",+ css_selector: Optional[str] = None,+ main_content_only: bool = True,+ params: Optional[Union[Dict, List, Tuple]] = None,+ headers: Optional[Mapping[str, Optional[str]]] = None,+ cookies: Optional[Union[dict[str, str], list[tuple[str, str]]]] = None,+ timeout: Optional[Union[int, float]] = 30,+ follow_redirects: bool = True,+ max_redirects: int = 30,+ retries: Optional[int] = 3,+ retry_delay: Optional[int] = 1,+ proxy: Optional[str] = None,+ proxy_auth: Optional[Tuple[str, str]] = None,+ auth: Optional[Tuple[str, str]] = None,+ verify: Optional[bool] = True,+ http3: Optional[bool] = False,+ stealthy_headers: Optional[bool] = True,+ ) -> ResponseModel:+ """Make GET HTTP request to a URL and return a structured output of the result.+ Note: This is only suitable for low-mid protection levels. For high-protection levels or websites that require JS loading, use the other tools directly.+ Note: If the `css_selector` resolves to more than one element, all the elements will be returned.++ :param url: The URL to request.+ :param impersonate: Browser version to impersonate its fingerprint. It's using the latest chrome version by default.+ :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:+ - Markdown will convert the page content to Markdown format.+ - HTML will return the raw HTML content of the page.+ - Text will return the text content of the page.+ :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.+ :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.+ :param params: Query string parameters for the request.+ :param headers: Headers to include in the request.+ :param cookies: Cookies to use in the request.+ :param timeout: Number of seconds to wait before timing out.+ :param follow_redirects: Whether to follow redirects. Defaults to True.+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.+ :param retries: Number of retry attempts. Defaults to 3.+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".+ Cannot be used together with the `proxies` parameter.+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.+ :param verify: Whether to verify HTTPS certificates.+ :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.+ :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.+ """+ page = Fetcher.get(+ url,+ auth=auth,+ proxy=proxy,+ http3=http3,+ verify=verify,+ params=params,+ proxy_auth=proxy_auth,+ retry_delay=retry_delay,+ stealthy_headers=stealthy_headers,+ impersonate=impersonate,+ headers=headers,+ cookies=cookies,+ timeout=timeout,+ retries=retries,+ max_redirects=max_redirects,+ follow_redirects=follow_redirects,+ )+ return _ContentTranslator(+ Convertor._extract_content(+ page,+ css_selector=css_selector,+ extraction_type=extraction_type,+ main_content_only=main_content_only,+ ),+ page,+ )++ @staticmethod+ @_server.tool()+ async def bulk_get(+ urls: Tuple[str, ...],+ impersonate: Optional[BrowserTypeLiteral] = "chrome",+ extraction_type: extraction_types = "markdown",+ css_selector: Optional[str] = None,+ main_content_only: bool = True,+ params: Optional[Union[Dict, List, Tuple]] = None,+ headers: Optional[Mapping[str, Optional[str]]] = None,+ cookies: Optional[Union[dict[str, str], list[tuple[str, str]]]] = None,+ timeout: Optional[Union[int, float]] = 30,+ follow_redirects: bool = True,+ max_redirects: int = 30,+ retries: Optional[int] = 3,+ retry_delay: Optional[int] = 1,+ proxy: Optional[str] = None,+ proxy_auth: Optional[Tuple[str, str]] = None,+ auth: Optional[Tuple[str, str]] = None,+ verify: Optional[bool] = True,+ http3: Optional[bool] = False,+ stealthy_headers: Optional[bool] = True,+ ) -> List[ResponseModel]:+ """Make GET HTTP request to a group of URLs and for each URL, return a structured output of the result.+ Note: This is only suitable for low-mid protection levels. For high-protection levels or websites that require JS loading, use the other tools directly.+ Note: If the `css_selector` resolves to more than one element, all the elements will be returned.++ :param urls: A tuple of the URLs to request.+ :param impersonate: Browser version to impersonate its fingerprint. It's using the latest chrome version by default.+ :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:+ - Markdown will convert the page content to Markdown format.+ - HTML will return the raw HTML content of the page.+ - Text will return the text content of the page.+ :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.+ :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.+ :param params: Query string parameters for the request.+ :param headers: Headers to include in the request.+ :param cookies: Cookies to use in the request.+ :param timeout: Number of seconds to wait before timing out.+ :param follow_redirects: Whether to follow redirects. Defaults to True.+ :param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.+ :param retries: Number of retry attempts. Defaults to 3.+ :param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.+ :param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".+ Cannot be used together with the `proxies` parameter.+ :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).+ :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.+ :param verify: Whether to verify HTTPS certificates.+ :param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.+ :param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.+ """+ async with FetcherSession() as session:+ tasks = [+ session.get(+ url,+ auth=auth,+ proxy=proxy,+ http3=http3,+ verify=verify,+ params=params,+ headers=headers,+ cookies=cookies,+ timeout=timeout,+ retries=retries,+ proxy_auth=proxy_auth,+ retry_delay=retry_delay,+ impersonate=impersonate,+ max_redirects=max_redirects,+ follow_redirects=follow_redirects,+ stealthy_headers=stealthy_headers,+ )+ for url in urls+ ]+ responses = await gather(*tasks)+ return [+ _ContentTranslator(+ Convertor._extract_content(+ page,+ css_selector=css_selector,+ extraction_type=extraction_type,+ main_content_only=main_content_only,+ ),+ page,+ )+ for page in responses+ ]++ @staticmethod+ @_server.tool()+ async def fetch(+ url: str,+ extraction_type: extraction_types = "markdown",+ css_selector: Optional[str] = None,+ main_content_only: bool = True,+ headless: bool = False,+ google_search: bool = True,+ hide_canvas: bool = False,+ disable_webgl: bool = False,+ real_chrome: bool = False,+ stealth: bool = False,+ wait: Union[int, float] = 0,+ proxy: Optional[Union[str, Dict[str, str]]] = None,+ locale: str = "en-US",+ extra_headers: Optional[Dict[str, str]] = None,+ useragent: Optional[str] = None,+ cdp_url: Optional[str] = None,+ timeout: Union[int, float] = 30000,+ disable_resources: bool = False,+ wait_selector: Optional[str] = None,+ cookies: Optional[List[Dict]] = None,+ network_idle: bool = False,+ wait_selector_state: SelectorWaitStates = "attached",+ ) -> ResponseModel:+ """Use playwright to open a browser to fetch a URL and return a structured output of the result.+ Note: This is only suitable for low-mid protection levels.+ Note: If the `css_selector` resolves to more than one element, all the elements will be returned.++ :param url: The URL to request.+ :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:+ - Markdown will convert the page content to Markdown format.+ - HTML will return the raw HTML content of the page.+ - Text will return the text content of the page.+ :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.+ :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.+ :param headless: Run the browser in headless/hidden (default), or headful/visible mode.+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.+ :param cookies: Set cookies for the next request. It should be in a dictionary format that Playwright accepts.+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param locale: Set the locale for the browser if wanted. The default value is `en-US`.+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.+ :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.+ :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.+ :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.+ :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.+ """+ page = await DynamicFetcher.async_fetch(+ url,+ wait=wait,+ proxy=proxy,+ locale=locale,+ timeout=timeout,+ cookies=cookies,+ stealth=stealth,+ cdp_url=cdp_url,+ headless=headless,+ useragent=useragent,+ hide_canvas=hide_canvas,+ real_chrome=real_chrome,+ network_idle=network_idle,+ wait_selector=wait_selector,+ disable_webgl=disable_webgl,+ extra_headers=extra_headers,+ google_search=google_search,+ disable_resources=disable_resources,+ wait_selector_state=wait_selector_state,+ )+ return _ContentTranslator(+ Convertor._extract_content(+ page,+ css_selector=css_selector,+ extraction_type=extraction_type,+ main_content_only=main_content_only,+ ),+ page,+ )++ @staticmethod+ @_server.tool()+ async def bulk_fetch(+ urls: Tuple[str, ...],+ extraction_type: extraction_types = "markdown",+ css_selector: Optional[str] = None,+ main_content_only: bool = True,+ headless: bool = False,+ google_search: bool = True,+ hide_canvas: bool = False,+ disable_webgl: bool = False,+ real_chrome: bool = False,+ stealth: bool = False,+ wait: Union[int, float] = 0,+ proxy: Optional[Union[str, Dict[str, str]]] = None,+ locale: str = "en-US",+ extra_headers: Optional[Dict[str, str]] = None,+ useragent: Optional[str] = None,+ cdp_url: Optional[str] = None,+ timeout: Union[int, float] = 30000,+ disable_resources: bool = False,+ wait_selector: Optional[str] = None,+ cookies: Optional[List[Dict]] = None,+ network_idle: bool = False,+ wait_selector_state: SelectorWaitStates = "attached",+ ) -> List[ResponseModel]:+ """Use playwright to open a browser, then fetch a group of URLs at the same time, and for each page return a structured output of the result.+ Note: This is only suitable for low-mid protection levels.+ Note: If the `css_selector` resolves to more than one element, all the elements will be returned.++ :param urls: A tuple of the URLs to request.+ :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:+ - Markdown will convert the page content to Markdown format.+ - HTML will return the raw HTML content of the page.+ - Text will return the text content of the page.+ :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.+ :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.+ :param headless: Run the browser in headless/hidden (default), or headful/visible mode.+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ :param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.+ :param cookies: Set cookies for the next request. It should be in a dictionary format that Playwright accepts.+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param locale: Set the locale for the browser if wanted. The default value is `en-US`.+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ :param stealth: Enables stealth mode, check the documentation to see what stealth mode does currently.+ :param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.+ :param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.+ :param disable_webgl: Disables WebGL and WebGL 2.0 support entirely.+ :param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers/NSTBrowser through CDP.+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.+ """+ async with AsyncDynamicSession(+ wait=wait,+ proxy=proxy,+ locale=locale,+ timeout=timeout,+ cookies=cookies,+ stealth=stealth,+ cdp_url=cdp_url,+ headless=headless,+ max_pages=len(urls),+ useragent=useragent,+ hide_canvas=hide_canvas,+ real_chrome=real_chrome,+ network_idle=network_idle,+ wait_selector=wait_selector,+ google_search=google_search,+ disable_webgl=disable_webgl,+ extra_headers=extra_headers,+ disable_resources=disable_resources,+ wait_selector_state=wait_selector_state,+ ) as session:+ tasks = [session.fetch(url) for url in urls]+ responses = await gather(*tasks)+ return [+ _ContentTranslator(+ Convertor._extract_content(+ page,+ css_selector=css_selector,+ extraction_type=extraction_type,+ main_content_only=main_content_only,+ ),+ page,+ )+ for page in responses+ ]++ @staticmethod+ @_server.tool()+ async def stealthy_fetch(+ url: str,+ extraction_type: extraction_types = "markdown",+ css_selector: Optional[str] = None,+ main_content_only: bool = True,+ headless: Union[bool, Literal["virtual"]] = True, # noqa: F821+ block_images: bool = False,+ disable_resources: bool = False,+ block_webrtc: bool = False,+ allow_webgl: bool = True,+ network_idle: bool = False,+ humanize: Union[bool, float] = True,+ solve_cloudflare: bool = False,+ wait: Union[int, float] = 0,+ timeout: Union[int, float] = 30000,+ wait_selector: Optional[str] = None,+ addons: Optional[List[str]] = None,+ wait_selector_state: SelectorWaitStates = "attached",+ cookies: Optional[List[Dict]] = None,+ google_search: bool = True,+ extra_headers: Optional[Dict[str, str]] = None,+ proxy: Optional[Union[str, Dict[str, str]]] = None,+ os_randomize: bool = False,+ disable_ads: bool = False,+ geoip: bool = False,+ additional_arguments: Optional[Dict] = None,+ ) -> ResponseModel:+ """Use Scrapling's version of the Camoufox browser to fetch a URL and return a structured output of the result.+ Note: This is best suitable for high protection levels. It's slower than the other tools.+ Note: If the `css_selector` resolves to more than one element, all the elements will be returned.++ :param url: The URL to request.+ :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:+ - Markdown will convert the page content to Markdown format.+ - HTML will return the raw HTML content of the page.+ - Text will return the text content of the page.+ :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.+ :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.+ :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.+ :param block_images: Prevent the loading of images through Firefox preferences.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ :param block_webrtc: Blocks WebRTC entirely.+ :param cookies: Set cookies for the next request.+ :param addons: List of Firefox addons to use. Must be paths to extracted addons.+ :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.+ :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.+ :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.+ :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.+ It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.+ :param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.+ """+ page = await StealthyFetcher.async_fetch(+ url,+ wait=wait,+ proxy=proxy,+ geoip=geoip,+ addons=addons,+ timeout=timeout,+ cookies=cookies,+ headless=headless,+ humanize=humanize,+ allow_webgl=allow_webgl,+ disable_ads=disable_ads,+ network_idle=network_idle,+ block_images=block_images,+ block_webrtc=block_webrtc,+ os_randomize=os_randomize,+ wait_selector=wait_selector,+ google_search=google_search,+ extra_headers=extra_headers,+ solve_cloudflare=solve_cloudflare,+ disable_resources=disable_resources,+ wait_selector_state=wait_selector_state,+ additional_arguments=additional_arguments,+ )+ return _ContentTranslator(+ Convertor._extract_content(+ page,+ css_selector=css_selector,+ extraction_type=extraction_type,+ main_content_only=main_content_only,+ ),+ page,+ )++ @staticmethod+ @_server.tool()+ async def bulk_stealthy_fetch(+ urls: Tuple[str, ...],+ extraction_type: extraction_types = "markdown",+ css_selector: Optional[str] = None,+ main_content_only: bool = True,+ headless: Union[bool, Literal["virtual"]] = True, # noqa: F821+ block_images: bool = False,+ disable_resources: bool = False,+ block_webrtc: bool = False,+ allow_webgl: bool = True,+ network_idle: bool = False,+ humanize: Union[bool, float] = True,+ solve_cloudflare: bool = False,+ wait: Union[int, float] = 0,+ timeout: Union[int, float] = 30000,+ wait_selector: Optional[str] = None,+ addons: Optional[List[str]] = None,+ wait_selector_state: SelectorWaitStates = "attached",+ cookies: Optional[List[Dict]] = None,+ google_search: bool = True,+ extra_headers: Optional[Dict[str, str]] = None,+ proxy: Optional[Union[str, Dict[str, str]]] = None,+ os_randomize: bool = False,+ disable_ads: bool = False,+ geoip: bool = False,+ additional_arguments: Optional[Dict] = None,+ ) -> List[ResponseModel]:+ """Use Scrapling's version of the Camoufox browser to fetch a group of URLs at the same time, and for each page return a structured output of the result.+ Note: This is best suitable for high protection levels. It's slower than the other tools.+ Note: If the `css_selector` resolves to more than one element, all the elements will be returned.++ :param urls: A tuple of the URLs to request.+ :param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:+ - Markdown will convert the page content to Markdown format.+ - HTML will return the raw HTML content of the page.+ - Text will return the text content of the page.+ :param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.+ :param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.+ :param headless: Run the browser in headless/hidden (default), virtual screen mode, or headful/visible mode.+ :param block_images: Prevent the loading of images through Firefox preferences.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ :param disable_resources: Drop requests of unnecessary resources for a speed boost. It depends, but it made requests ~25% faster in my tests for some websites.+ Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.+ This can help save your proxy usage but be careful with this option as it makes some websites never finish loading.+ :param block_webrtc: Blocks WebRTC entirely.+ :param cookies: Set cookies for the next request.+ :param addons: List of Firefox addons to use. Must be paths to extracted addons.+ :param humanize: Humanize the cursor movement. Takes either True or the MAX duration in seconds of the cursor movement. The cursor typically takes up to 1.5 seconds to move across the window.+ :param solve_cloudflare: Solves all 3 types of the Cloudflare's Turnstile wait page before returning the response to you.+ :param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param disable_ads: Disabled by default, this installs the `uBlock Origin` addon on the browser if enabled.+ :param os_randomize: If enabled, Scrapling will randomize the OS fingerprints used. The default is Scrapling matching the fingerprints with the current OS.+ :param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.+ :param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000+ :param wait_selector: Wait for a specific CSS selector to be in a specific state.+ :param geoip: Recommended to use with proxies; Automatically use IP's longitude, latitude, timezone, country, locale, and spoof the WebRTC IP address.+ It will also calculate and spoof the browser's language based on the distribution of language speakers in the target region.+ :param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.+ :param google_search: Enabled by default, Scrapling will set the referer header to be as if this request came from a Google search of this website's domain name.+ :param extra_headers: A dictionary of extra headers to add to the request. _The referer set by the `google_search` argument takes priority over the referer set here if used together._+ :param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.+ :param additional_arguments: Additional arguments to be passed to Camoufox as additional settings, and it takes higher priority than Scrapling's settings.+ """+ async with AsyncStealthySession(+ wait=wait,+ proxy=proxy,+ geoip=geoip,+ addons=addons,+ timeout=timeout,+ cookies=cookies,+ headless=headless,+ humanize=humanize,+ max_pages=len(urls),+ allow_webgl=allow_webgl,+ disable_ads=disable_ads,+ block_images=block_images,+ block_webrtc=block_webrtc,+ network_idle=network_idle,+ os_randomize=os_randomize,+ wait_selector=wait_selector,+ google_search=google_search,+ extra_headers=extra_headers,+ solve_cloudflare=solve_cloudflare,+ disable_resources=disable_resources,+ wait_selector_state=wait_selector_state,+ additional_arguments=additional_arguments,+ ) as session:+ tasks = [session.fetch(url) for url in urls]+ responses = await gather(*tasks)+ return [+ _ContentTranslator(+ Convertor._extract_content(+ page,+ css_selector=css_selector,+ extraction_type=extraction_type,+ main_content_only=main_content_only,+ ),+ page,+ )+ for page in responses+ ]++ def serve(self):+ """Serve the MCP server."""+ self._server.run(transport="stdio")diff --git a/scrapling/core/shell.py b/scrapling/core/shell.pyindex 80b168c..1f4808b 100644--- a/scrapling/core/shell.py+++ b/scrapling/core/shell.py@@ -36,6 +36,7 @@ from scrapling.core._types import (Any,Union,extraction_types,+ Generator,)from scrapling.fetchers import (Fetcher,@@ -589,7 +590,7 @@ class Convertor:extraction_type: extraction_types = "markdown",css_selector: Optional[str] = None,main_content_only: bool = False,- ) -> str:+ ) -> Generator[str, None, None]:"""Extract the content of an Adaptor"""if not page or not isinstance(page, Adaptor):raise TypeError("Input must be of type `Adaptor`")@@ -599,24 +600,25 @@ class Convertor:if main_content_only:page = page.css_first("body") or page- page = page if not css_selector else page.css_first(css_selector)- match extraction_type:- case "markdown":- return cls._convert_to_markdown(page.body)- case "html":- return page.body- case "text":- txt_content = page.get_all_text(strip=True)- for s in (- "\n",- "\r",- "\t",- " ",- ):- # Remove consecutive white-spaces- txt_content = re_sub(f"[{s}]+", s, txt_content)- return txt_content- return ""+ pages = [page] if not css_selector else page.css(css_selector)+ for page in pages:+ match extraction_type:+ case "markdown":+ yield cls._convert_to_markdown(page.body)+ case "html":+ yield page.body+ case "text":+ txt_content = page.get_all_text(strip=True)+ for s in (+ "\n",+ "\r",+ "\t",+ " ",+ ):+ # Remove consecutive white-spaces+ txt_content = re_sub(f"[{s}]+", s, txt_content)+ yield txt_content+ yield ""@classmethoddef write_content_to_file(@@ -635,7 +637,11 @@ class Convertor:with open(filename, "w", encoding="utf-8") as f:extension = filename.split(".")[-1]f.write(- cls._extract_content(- page, cls._extension_map[extension], css_selector=css_selector+ "".join(+ cls._extract_content(+ page,+ cls._extension_map[extension],+ css_selector=css_selector,+ )))