Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(mcp): remove unneeded code and fix type hint for mypy

Karim shoair0f6dcccf5f81
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index 85ceefd..2fabf43 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -17,6 +17,7 @@ from scrapling.fetchers import (
from scrapling.core._types import (
Optional,
Literal,
+ Union,
Tuple,
Mapping,
Dict,
@@ -100,28 +101,6 @@ def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tu
return username, password
-def _build_fetch_kwargs(**params) -> Dict[str, Any]:
- """Build kwargs dict for session.fetch() from request-level parameters."""
- kwargs: Dict[str, Any] = {}
- # These are the params that session.fetch() accepts as per-request overrides
- request_level_keys = (
- "wait",
- "timeout",
- "google_search",
- "extra_headers",
- "disable_resources",
- "wait_selector",
- "wait_selector_state",
- "network_idle",
- "proxy",
- "solve_cloudflare",
- )
- for key in request_level_keys:
- if key in params and params[key] is not None:
- kwargs[key] = params[key]
- return kwargs
-
-
class ScraplingMCPServer:
def __init__(self):
self._sessions: Dict[str, _SessionEntry] = {}
@@ -215,6 +194,7 @@ class ScraplingMCPServer:
wait_selector_state=wait_selector_state,
)
+ session: Union[AsyncDynamicSession, AsyncStealthySession]
if session_type == "stealthy":
session = AsyncStealthySession(
**common_kwargs,
@@ -563,18 +543,21 @@ class ScraplingMCPServer:
"""
if session_id:
entry = self._get_session(session_id, "dynamic")
- fetch_kwargs = _build_fetch_kwargs(
- wait=wait,
- timeout=timeout,
- google_search=google_search,
- extra_headers=extra_headers,
- disable_resources=disable_resources,
- wait_selector=wait_selector,
- wait_selector_state=wait_selector_state,
- network_idle=network_idle,
- proxy=proxy,
- )
- tasks = [entry.session.fetch(url, **fetch_kwargs) for url in urls]
+ tasks = [
+ entry.session.fetch(
+ url,
+ wait=wait,
+ timeout=timeout,
+ google_search=google_search,
+ extra_headers=extra_headers,
+ disable_resources=disable_resources,
+ wait_selector=wait_selector,
+ wait_selector_state=wait_selector_state,
+ network_idle=network_idle,
+ proxy=proxy,
+ )
+ for url in urls
+ ]
responses = await gather(*tasks)
else:
async with AsyncDynamicSession(
@@ -767,19 +750,22 @@ class ScraplingMCPServer:
"""
if session_id:
entry = self._get_session(session_id, "stealthy")
- fetch_kwargs = _build_fetch_kwargs(
- wait=wait,
- timeout=timeout,
- google_search=google_search,
- extra_headers=extra_headers,
- disable_resources=disable_resources,
- wait_selector=wait_selector,
- wait_selector_state=wait_selector_state,
- network_idle=network_idle,
- proxy=proxy,
- solve_cloudflare=solve_cloudflare,
- )
- tasks = [entry.session.fetch(url, **fetch_kwargs) for url in urls]
+ tasks = [
+ entry.session.fetch(
+ url,
+ wait=wait,
+ timeout=timeout,
+ google_search=google_search,
+ extra_headers=extra_headers,
+ disable_resources=disable_resources,
+ wait_selector=wait_selector,
+ wait_selector_state=wait_selector_state,
+ network_idle=network_idle,
+ proxy=proxy,
+ solve_cloudflare=solve_cloudflare,
+ )
+ for url in urls
+ ]
responses = await gather(*tasks)
else:
async with AsyncStealthySession(