🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(ai): size the MCP bulk page pools within the validator bounds (#393)
Yiğit ERDOĞANf8893d9b58de
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex 985714b..4f58bb1 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -44,6 +44,14 @@ SessionType = Literal["dynamic", "stealthy"]ScreenshotType = Literal["png", "jpeg"]MCP_EXECUTABLE_PATH_ENV = "SCRAPLING_EXECUTABLE_PATH"MCP_AUTH_TOKEN_ENV = "SCRAPLING_MCP_AUTH_TOKEN" # nosec B105 - the name of the variable, not a token++_MAX_POOL_PAGES = 50 # Upper bound of `PagesCount` in scrapling/engines/_browsers/_validators.py+++def _page_pool_size(urls: Sequence[str]) -> int:+ """Return a page pool size that covers the batch without leaving the validator's bounds."""+ return min(max(len(urls), 1), _MAX_POOL_PAGES)+_FETCH_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=True, open_world_hint=True)_SESSION_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=False, destructive_hint=False, open_world_hint=True)_LIST_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=True, open_world_hint=False)@@ -620,7 +628,7 @@ class ScraplingMCPServer:"""Use playwright to open a browser, then fetch a group of URLs at the same time, and for each page return a structured output of the result.Only suitable for low-mid protection levels.- :param urls: A list of the URLs to request.+ :param urls: A list of the URLs to request. Batches bigger than 50 URLs are fetched through a pool of 50 concurrent pages.:param extraction_type: The type of content to extract from the page: "markdown" (default), "html", or "text".:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.@@ -673,7 +681,7 @@ class ScraplingMCPServer:cdp_url=cdp_url,headless=headless,block_ads=True,- max_pages=len(urls),+ max_pages=_page_pool_size(urls),useragent=useragent,timezone_id=timezone_id,real_chrome=real_chrome,@@ -817,7 +825,7 @@ class ScraplingMCPServer:"""Use the stealthy fetcher to fetch a group of URLs at the same time, and for each page return a structured output of the result.The only fetcher suitable for high-protection websites.- :param urls: A list of the URLs to request.+ :param urls: A list of the URLs to request. Batches bigger than 50 URLs are fetched through a pool of 50 concurrent pages.:param extraction_type: The type of content to extract from the page: "markdown" (default), "html", or "text".:param css_selector: CSS selector to extract the content from the page. If main_content_only is True, then it will be executed on the main content of the page. Defaults to None.:param main_content_only: Whether to extract only the main content of the page. Defaults to True. The main content here is the data inside the `<body>` tag.@@ -876,6 +884,7 @@ class ScraplingMCPServer:cookies=cookies,headless=headless,block_ads=True,+ max_pages=_page_pool_size(urls),useragent=useragent,timezone_id=timezone_id,real_chrome=real_chrome,diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pyindex 68b4894..572ec94 100644--- a/tests/ai/test_ai_mcp.py+++ b/tests/ai/test_ai_mcp.py@@ -20,9 +20,11 @@ from scrapling.core.ai import (SessionCreatedModel,SessionClosedModel,_normalize_credentials,+ _page_pool_size,_StaticTokenVerifier,_translate_response,)+from scrapling.fetchers import AsyncDynamicSessiondef test_translate_response_strips_control_characters():@@ -326,6 +328,54 @@ class TestExecutablePath:assert _FakeStealthySession.instances[0].kwargs["executable_path"] == "/opt/default-chromium"+class TestBulkPagePool:+ """Test the page pool sizing of the bulk browser tools"""++ @pytest.fixture(autouse=True)+ def reset_fakes(self):+ _FakeDynamicSession.instances = []+ _FakeStealthySession.instances = []++ @pytest.mark.asyncio+ @pytest.mark.parametrize("url_count,expected_pages", [(3, 3), (60, 50), (0, 1)])+ async def test_bulk_fetch_sizes_pool_within_validator_bounds(self, monkeypatch, url_count, expected_pages):+ """bulk_fetch opens a pool that covers the batch but stays inside the 1..50 `PagesCount` range"""+ monkeypatch.setattr("scrapling.core.ai.AsyncDynamicSession", _FakeDynamicSession)+ server = ScraplingMCPServer()+ urls = [f"https://example.com/{index}" for index in range(url_count)]++ results = await server.bulk_fetch(urls=urls)++ max_pages = _FakeDynamicSession.instances[0].kwargs["max_pages"]+ assert max_pages == expected_pages, f"Expected max_pages {expected_pages} for {url_count} URLs, got {max_pages}"+ assert len(results) == url_count, f"Expected {url_count} responses, got {len(results)}"++ @pytest.mark.asyncio+ @pytest.mark.parametrize("url_count,expected_pages", [(4, 4), (60, 50), (0, 1)])+ async def test_bulk_stealthy_fetch_sizes_pool_within_validator_bounds(self, monkeypatch, url_count, expected_pages):+ """bulk_stealthy_fetch sizes its pool to the batch instead of leaving it at the default of 1"""+ monkeypatch.setattr("scrapling.core.ai.AsyncStealthySession", _FakeStealthySession)+ server = ScraplingMCPServer()+ urls = [f"https://example.com/{index}" for index in range(url_count)]++ results = await server.bulk_stealthy_fetch(urls=urls)++ max_pages = _FakeStealthySession.instances[0].kwargs["max_pages"]+ assert max_pages == expected_pages, f"Expected max_pages {expected_pages} for {url_count} URLs, got {max_pages}"+ assert len(results) == url_count, f"Expected {url_count} responses, got {len(results)}"++ @pytest.mark.parametrize("url_count", [0, 1, 50, 60, 500])+ def test_page_pool_size_is_accepted_by_session_validation(self, url_count):+ """The computed pool size always passes the real session validation without launching a browser"""+ urls = [f"https://example.com/{index}" for index in range(url_count)]++ session = AsyncDynamicSession(max_pages=_page_pool_size(urls))++ assert session.max_pages == _page_pool_size(urls), (+ f"Expected the session to keep max_pages {_page_pool_size(urls)}, got {session.max_pages}"+ )++def _png_height(data: bytes) -> int:"""Read the height field from a PNG IHDR chunk."""return struct.unpack(">I", data[20:24])[0]