Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: make MCP get schemas validator-safe

Robin Edeb77a1b9419eb
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index 171359a..ad575f9 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -42,6 +42,20 @@ def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResp
return ResponseModel(status=page.status, content=[result for result in content], url=page.url)
+def _NormalizeCredentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuple[str, str]]:
+ """Convert a credentials dictionary to a tuple accepted by fetchers."""
+ if not credentials:
+ return None
+
+ username = credentials.get("username")
+ password = credentials.get("password")
+
+ if username is None or password is None:
+ return None
+
+ return username, password
+
+
class ScraplingMCPServer:
@staticmethod
def get(
@@ -52,15 +66,15 @@ class ScraplingMCPServer:
main_content_only: bool = True,
params: Optional[Dict | List | Tuple] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
- cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None,
+ cookies: Optional[Dict[str, str]] = None,
timeout: Optional[int | float] = 30,
follow_redirects: bool = True,
max_redirects: int = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
proxy: Optional[str] = None,
- proxy_auth: Optional[Tuple[str, str]] = None,
- auth: Optional[Tuple[str, str]] = None,
+ proxy_auth: Optional[Dict[str, str]] = None,
+ auth: Optional[Dict[str, str]] = None,
verify: Optional[bool] = True,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
@@ -87,20 +101,23 @@ class ScraplingMCPServer:
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
- :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
- :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param proxy_auth: HTTP basic auth for proxy in dictionary format with `username` and `password` keys.
+ :param auth: HTTP basic auth in dictionary format with `username` and `password` keys.
:param verify: Whether to verify HTTPS certificates.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
"""
+ normalized_proxy_auth = _NormalizeCredentials(proxy_auth)
+ normalized_auth = _NormalizeCredentials(auth)
+
page = Fetcher.get(
url,
- auth=auth,
+ auth=normalized_auth,
proxy=proxy,
http3=http3,
verify=verify,
params=params,
- proxy_auth=proxy_auth,
+ proxy_auth=normalized_proxy_auth,
retry_delay=retry_delay,
stealthy_headers=stealthy_headers,
impersonate=impersonate,
@@ -130,15 +147,15 @@ class ScraplingMCPServer:
main_content_only: bool = True,
params: Optional[Dict | List | Tuple] = None,
headers: Optional[Mapping[str, Optional[str]]] = None,
- cookies: Optional[Dict[str, str] | list[tuple[str, str]]] = None,
+ cookies: Optional[Dict[str, str]] = None,
timeout: Optional[int | float] = 30,
follow_redirects: bool = True,
max_redirects: int = 30,
retries: Optional[int] = 3,
retry_delay: Optional[int] = 1,
proxy: Optional[str] = None,
- proxy_auth: Optional[Tuple[str, str]] = None,
- auth: Optional[Tuple[str, str]] = None,
+ proxy_auth: Optional[Dict[str, str]] = None,
+ auth: Optional[Dict[str, str]] = None,
verify: Optional[bool] = True,
http3: Optional[bool] = False,
stealthy_headers: Optional[bool] = True,
@@ -165,17 +182,20 @@ class ScraplingMCPServer:
:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.
:param proxy: Proxy URL to use. Format: "http://username:password@localhost:8030".
Cannot be used together with the `proxies` parameter.
- :param proxy_auth: HTTP basic auth for proxy, tuple of (username, password).
- :param auth: HTTP basic auth tuple of (username, password). Only basic auth is supported.
+ :param proxy_auth: HTTP basic auth for proxy in dictionary format with `username` and `password` keys.
+ :param auth: HTTP basic auth in dictionary format with `username` and `password` keys.
:param verify: Whether to verify HTTPS certificates.
:param http3: Whether to use HTTP3. Defaults to False. It might be problematic if used it with `impersonate`.
:param stealthy_headers: If enabled (default), it creates and adds real browser headers. It also sets the referer header as if this request came from a Google search of URL's domain.
"""
+ normalized_proxy_auth = _NormalizeCredentials(proxy_auth)
+ normalized_auth = _NormalizeCredentials(auth)
+
async with FetcherSession() as session:
tasks: List[Any] = [
session.get(
url,
- auth=auth,
+ auth=normalized_auth,
proxy=proxy,
http3=http3,
verify=verify,
@@ -184,7 +204,7 @@ class ScraplingMCPServer:
cookies=cookies,
timeout=timeout,
retries=retries,
- proxy_auth=proxy_auth,
+ proxy_auth=normalized_proxy_auth,
retry_delay=retry_delay,
impersonate=impersonate,
max_redirects=max_redirects,