🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: complete MCP schema validation for remaining bare array types
Karim shoair921314c3144c
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex ad575f9..f637437 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -37,12 +37,12 @@ class ResponseModel(BaseModel):url: str = Field(description="The URL given by the user that resulted in this response.")-def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel:+def _content_translator(content: Generator[str, None, None], page: _ScraplingResponse) -> ResponseModel:"""Convert a content generator to a list of ResponseModel objects."""return ResponseModel(status=page.status, content=[result for result in content], url=page.url)-def _NormalizeCredentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuple[str, str]]:+def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuple[str, str]]:"""Convert a credentials dictionary to a tuple accepted by fetchers."""if not credentials:return None@@ -51,7 +51,7 @@ def _NormalizeCredentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuppassword = credentials.get("password")if username is None or password is None:- return None+ raise ValueError("Credentials dictionary must contain both 'username' and 'password' keys")return username, password@@ -64,7 +64,7 @@ class ScraplingMCPServer:extraction_type: extraction_types = "markdown",css_selector: Optional[str] = None,main_content_only: bool = True,- params: Optional[Dict | List | Tuple] = None,+ params: Optional[Dict] = None,headers: Optional[Mapping[str, Optional[str]]] = None,cookies: Optional[Dict[str, str]] = None,timeout: Optional[int | float] = 30,@@ -107,8 +107,8 @@ class ScraplingMCPServer::param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain."""- normalized_proxy_auth = _NormalizeCredentials(proxy_auth)- normalized_auth = _NormalizeCredentials(auth)+ normalized_proxy_auth = _normalize_credentials(proxy_auth)+ normalized_auth = _normalize_credentials(auth)page = Fetcher.get(url,@@ -128,7 +128,7 @@ class ScraplingMCPServer:max_redirects=max_redirects,follow_redirects=follow_redirects,)- return _ContentTranslator(+ return _content_translator(Convertor._extract_content(page,css_selector=css_selector,@@ -140,12 +140,12 @@ class ScraplingMCPServer:@staticmethodasync def bulk_get(- urls: Tuple[str, ...],+ urls: List[str],impersonate: ImpersonateType = "chrome",extraction_type: extraction_types = "markdown",css_selector: Optional[str] = None,main_content_only: bool = True,- params: Optional[Dict | List | Tuple] = None,+ params: Optional[Dict] = None,headers: Optional[Mapping[str, Optional[str]]] = None,cookies: Optional[Dict[str, str]] = None,timeout: Optional[int | float] = 30,@@ -164,7 +164,7 @@ class ScraplingMCPServer:Note: This is only suitable for low-mid protection levels. For high-protection levels or websites that require JS loading, use the other tools directly.Note: If the `css_selector` resolves to more than one element, all the elements will be returned.- :param urls: A tuple of the URLs to request.+ :param urls: A list of the URLs to request.:param impersonate: Browser version to impersonate its fingerprint. It's using the latest chrome version by default.:param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:- Markdown will convert the page content to Markdown format.@@ -188,8 +188,8 @@ class ScraplingMCPServer::param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain."""- normalized_proxy_auth = _NormalizeCredentials(proxy_auth)- normalized_auth = _NormalizeCredentials(auth)+ normalized_proxy_auth = _normalize_credentials(proxy_auth)+ normalized_auth = _normalize_credentials(auth)async with FetcherSession() as session:tasks: List[Any] = [@@ -215,7 +215,7 @@ class ScraplingMCPServer:]responses = await gather(*tasks)return [- _ContentTranslator(+ _content_translator(Convertor._extract_content(page,css_selector=css_selector,@@ -299,7 +299,7 @@ class ScraplingMCPServer:disable_resources=disable_resources,wait_selector_state=wait_selector_state,)- return _ContentTranslator(+ return _content_translator(Convertor._extract_content(page,css_selector=css_selector,@@ -311,7 +311,7 @@ class ScraplingMCPServer:@staticmethodasync def bulk_fetch(- urls: Tuple[str, ...],+ urls: List[str],extraction_type: extraction_types = "markdown",css_selector: Optional[str] = None,main_content_only: bool = True,@@ -336,7 +336,7 @@ class ScraplingMCPServer:Note: This is only suitable for low-mid protection levels.Note: If the `css_selector` resolves to more than one element, all the elements will be returned.- :param urls: A tuple of the URLs to request.+ :param urls: A list of the URLs to request.:param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:- Markdown will convert the page content to Markdown format.- HTML will return the raw HTML content of the page.@@ -384,7 +384,7 @@ class ScraplingMCPServer:tasks = [session.fetch(url) for url in urls]responses = await gather(*tasks)return [- _ContentTranslator(+ _content_translator(Convertor._extract_content(page,css_selector=css_selector,@@ -483,7 +483,7 @@ class ScraplingMCPServer:disable_resources=disable_resources,wait_selector_state=wait_selector_state,)- return _ContentTranslator(+ return _content_translator(Convertor._extract_content(page,css_selector=css_selector,@@ -495,7 +495,7 @@ class ScraplingMCPServer:@staticmethodasync def bulk_stealthy_fetch(- urls: Tuple[str, ...],+ urls: List[str],extraction_type: extraction_types = "markdown",css_selector: Optional[str] = None,main_content_only: bool = True,@@ -525,7 +525,7 @@ class ScraplingMCPServer:Note: This is the only suitable fetcher for high protection levels.Note: If the `css_selector` resolves to more than one element, all the elements will be returned.- :param urls: A tuple of the URLs to request.+ :param urls: A list of the URLs to request.:param extraction_type: The type of content to extract from the page. Defaults to "markdown". Options are:- Markdown will convert the page content to Markdown format.- HTML will return the raw HTML content of the page.@@ -582,7 +582,7 @@ class ScraplingMCPServer:tasks = [session.fetch(url) for url in urls]responses = await gather(*tasks)return [- _ContentTranslator(+ _content_translator(Convertor._extract_content(page,css_selector=css_selector,