Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(ai): size the MCP bulk page pools within the validator bounds (#393)

Yiğit ERDOĞANf8893d9b58de
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index 985714b..4f58bb1 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -44,6 +44,14 @@ SessionType = Literal["dynamic", "stealthy"]
ScreenshotType = Literal["png", "jpeg"]
MCP_EXECUTABLE_PATH_ENV = "SCRAPLING_EXECUTABLE_PATH"
MCP_AUTH_TOKEN_ENV = "SCRAPLING_MCP_AUTH_TOKEN" # nosec B105 - the name of the variable, not a token
+
+_MAX_POOL_PAGES = 50 # Upper bound of `PagesCount` in scrapling/engines/_browsers/_validators.py
+
+
+def _page_pool_size(urls: Sequence[str]) -> int:
+ """Return a page pool size that covers the batch without leaving the validator's bounds."""
+ return min(max(len(urls), 1), _MAX_POOL_PAGES)
+
_FETCH_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=True, open_world_hint=True)
_SESSION_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=False, destructive_hint=False, open_world_hint=True)
_LIST_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=True, open_world_hint=False)
@@ -620,7 +628,7 @@ class ScraplingMCPServer:
"""Use playwright to open a browser, then fetch a group of URLs at the same time, and for each page return a structured output of the result.
Only suitable for low-mid protection levels.
- :param urls: A list of the URLs to request.
+ :param urls: A list of the URLs to request. Batches bigger than 50 URLs are fetched through a pool of 50 concurrent pages.
:param extraction_type: The type of content to extract from the page: "markdown" (default), "html", or "text".
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
@@ -673,7 +681,7 @@ class ScraplingMCPServer:
cdp_url=cdp_url,
headless=headless,
block_ads=True,
- max_pages=len(urls),
+ max_pages=_page_pool_size(urls),
useragent=useragent,
timezone_id=timezone_id,
real_chrome=real_chrome,
@@ -817,7 +825,7 @@ class ScraplingMCPServer:
"""Use the stealthy fetcher to fetch a group of URLs at the same time, and for each page return a structured output of the result.
The only fetcher suitable for high-protection websites.
- :param urls: A list of the URLs to request.
+ :param urls: A list of the URLs to request. Batches bigger than 50 URLs are fetched through a pool of 50 concurrent pages.
:param extraction_type: The type of content to extract from the page: "markdown" (default), "html", or "text".
:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.
:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.
@@ -876,6 +884,7 @@ class ScraplingMCPServer:
cookies=cookies,
headless=headless,
block_ads=True,
+ max_pages=_page_pool_size(urls),
useragent=useragent,
timezone_id=timezone_id,
real_chrome=real_chrome,
diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py
index 68b4894..572ec94 100644
--- a/tests/ai/test_ai_mcp.py
+++ b/tests/ai/test_ai_mcp.py
@@ -20,9 +20,11 @@ from scrapling.core.ai import (
SessionCreatedModel,
SessionClosedModel,
_normalize_credentials,
+ _page_pool_size,
_StaticTokenVerifier,
_translate_response,
)
+from scrapling.fetchers import AsyncDynamicSession
def test_translate_response_strips_control_characters():
@@ -326,6 +328,54 @@ class TestExecutablePath:
assert _FakeStealthySession.instances[0].kwargs["executable_path"] == "/opt/default-chromium"
+class TestBulkPagePool:
+ """Test the page pool sizing of the bulk browser tools"""
+
+ @pytest.fixture(autouse=True)
+ def reset_fakes(self):
+ _FakeDynamicSession.instances = []
+ _FakeStealthySession.instances = []
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("url_count,expected_pages", [(3, 3), (60, 50), (0, 1)])
+ async def test_bulk_fetch_sizes_pool_within_validator_bounds(self, monkeypatch, url_count, expected_pages):
+ """bulk_fetch opens a pool that covers the batch but stays inside the 1..50 `PagesCount` range"""
+ monkeypatch.setattr("scrapling.core.ai.AsyncDynamicSession", _FakeDynamicSession)
+ server = ScraplingMCPServer()
+ urls = [f"https://example.com/{index}" for index in range(url_count)]
+
+ results = await server.bulk_fetch(urls=urls)
+
+ max_pages = _FakeDynamicSession.instances[0].kwargs["max_pages"]
+ assert max_pages == expected_pages, f"Expected max_pages {expected_pages} for {url_count} URLs, got {max_pages}"
+ assert len(results) == url_count, f"Expected {url_count} responses, got {len(results)}"
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("url_count,expected_pages", [(4, 4), (60, 50), (0, 1)])
+ async def test_bulk_stealthy_fetch_sizes_pool_within_validator_bounds(self, monkeypatch, url_count, expected_pages):
+ """bulk_stealthy_fetch sizes its pool to the batch instead of leaving it at the default of 1"""
+ monkeypatch.setattr("scrapling.core.ai.AsyncStealthySession", _FakeStealthySession)
+ server = ScraplingMCPServer()
+ urls = [f"https://example.com/{index}" for index in range(url_count)]
+
+ results = await server.bulk_stealthy_fetch(urls=urls)
+
+ max_pages = _FakeStealthySession.instances[0].kwargs["max_pages"]
+ assert max_pages == expected_pages, f"Expected max_pages {expected_pages} for {url_count} URLs, got {max_pages}"
+ assert len(results) == url_count, f"Expected {url_count} responses, got {len(results)}"
+
+ @pytest.mark.parametrize("url_count", [0, 1, 50, 60, 500])
+ def test_page_pool_size_is_accepted_by_session_validation(self, url_count):
+ """The computed pool size always passes the real session validation without launching a browser"""
+ urls = [f"https://example.com/{index}" for index in range(url_count)]
+
+ session = AsyncDynamicSession(max_pages=_page_pool_size(urls))
+
+ assert session.max_pages == _page_pool_size(urls), (
+ f"Expected the session to keep max_pages {_page_pool_size(urls)}, got {session.max_pages}"
+ )
+
+
def _png_height(data: bytes) -> int:
"""Read the height field from a PNG IHDR chunk."""
return struct.unpack(">I", data[20:24])[0]