🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(ai)!: keep session settings on MCP session fetches (#418)
Yiğit ERDOĞANd4e121ce7ce8
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex fa6376b..9354f64 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -53,6 +53,11 @@ def _page_pool_size(urls: Sequence[str]) -> int:return min(max(len(urls), 1), _MAX_POOL_PAGES)+def _supplied_params(**params: Any) -> Dict[str, Any]:+ """Keep only the parameters the caller set, so the unset ones fall back to the session/fetcher defaults."""+ return {name: value for name, value in params.items() if value is not None}++_FETCH_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=True, open_world_hint=True)_SESSION_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=False, destructive_hint=False, open_world_hint=True)_LIST_TOOL_ANNOTATIONS = ToolAnnotations(read_only_hint=True, open_world_hint=False)@@ -321,11 +326,11 @@ class ScraplingMCPServer:image_type: ScreenshotType = "png",full_page: bool = False,quality: Optional[int] = None,- wait: int | float = 0,+ wait: int | float | None = None,wait_selector: Optional[str] = None,- wait_selector_state: SelectorWaitStates = "attached",- network_idle: bool = False,- timeout: int | float = 30000,+ wait_selector_state: Optional[SelectorWaitStates] = None,+ network_idle: bool | None = None,+ timeout: int | float | None = None,) -> List[ImageContent | TextContent]:"""Capture a screenshot of a web page using an existing browser session and return it as an image.A browser session must be opened first with `open_session` (either `dynamic` or `stealthy`); the session ID is then passed here.@@ -335,11 +340,11 @@ class ScraplingMCPServer::param image_type: Image format. Defaults to "png". Use "jpeg" for smaller file sizes.:param full_page: When True, captures the full scrollable page instead of just the viewport. Defaults to False.:param quality: Image quality (0-100) for JPEG only. Raises if passed with `image_type="png"`.- :param wait: Time in milliseconds to wait after page load before capturing. Defaults to 0.- :param wait_selector: Optional CSS selector to wait for before capturing.- :param wait_selector_state: State to wait for the selector. Defaults to "attached".- :param network_idle: Wait for the page until there are no network connections for at least 500 ms.- :param timeout: Timeout in milliseconds for page operations. Defaults to 30,000.+ :param wait: Time in milliseconds to wait after page load before capturing. Uses the session's setting when omitted.+ :param wait_selector: Optional CSS selector to wait for before capturing. Uses the session's setting when omitted.+ :param wait_selector_state: State to wait for the selector. Uses the session's setting when omitted.+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms. Uses the session's setting when omitted.+ :param timeout: Timeout in milliseconds for page operations. Uses the session's setting when omitted."""if quality is not None and image_type != "jpeg":raise ValueError("'quality' is only valid when 'image_type' is 'jpeg'.")@@ -361,12 +366,14 @@ class ScraplingMCPServer:await entry.session.fetch(url,- wait=wait,- timeout=timeout,- network_idle=network_idle,- wait_selector=wait_selector,- wait_selector_state=wait_selector_state,page_action=_capture,+ **_supplied_params(+ wait=wait,+ timeout=timeout,+ network_idle=network_idle,+ wait_selector=wait_selector,+ wait_selector_state=wait_selector_state,+ ),)if "error" in captured:@@ -529,9 +536,9 @@ class ScraplingMCPServer:css_selector: Optional[str] = None,main_content_only: bool = True,headless: bool = True, # noqa: F821- google_search: bool = True,+ google_search: bool | None = None,real_chrome: bool = False,- wait: int | float = 0,+ wait: int | float | None = None,proxy: Optional[str | Dict[str, str]] = None,timezone_id: str | None = None,locale: str | None = None,@@ -539,12 +546,12 @@ class ScraplingMCPServer:useragent: Optional[str] = None,cdp_url: Optional[str] = None,executable_path: Optional[str] = None,- timeout: int | float = 30000,- disable_resources: bool = False,+ timeout: int | float | None = None,+ disable_resources: bool | None = None,wait_selector: Optional[str] = None,cookies: Sequence[SetCookieParam] | None = None,- network_idle: bool = False,- wait_selector_state: SelectorWaitStates = "attached",+ network_idle: bool | None = None,+ wait_selector_state: Optional[SelectorWaitStates] = None,session_id: Optional[str] = None,) -> ResponseModel:"""Use playwright to open a browser to fetch a URL and return a structured output of the result.@@ -573,7 +580,7 @@ class ScraplingMCPServer::param google_search: Enabled by default, Scrapling will set a Google referer header.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.- :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one.+ :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one, and every option you leave out here keeps the value the session was opened with."""results = await self.bulk_fetch(urls=[url],@@ -608,9 +615,9 @@ class ScraplingMCPServer:css_selector: Optional[str] = None,main_content_only: bool = True,headless: bool = True, # noqa: F821- google_search: bool = True,+ google_search: bool | None = None,real_chrome: bool = False,- wait: int | float = 0,+ wait: int | float | None = None,proxy: Optional[str | Dict[str, str]] = None,timezone_id: str | None = None,locale: str | None = None,@@ -618,12 +625,12 @@ class ScraplingMCPServer:useragent: Optional[str] = None,cdp_url: Optional[str] = None,executable_path: Optional[str] = None,- timeout: int | float = 30000,- disable_resources: bool = False,+ timeout: int | float | None = None,+ disable_resources: bool | None = None,wait_selector: Optional[str] = None,cookies: Sequence[SetCookieParam] | None = None,- network_idle: bool = False,- wait_selector_state: SelectorWaitStates = "attached",+ network_idle: bool | None = None,+ wait_selector_state: Optional[SelectorWaitStates] = None,session_id: Optional[str] = None,) -> List[ResponseModel]:"""Use playwright to open a browser, then fetch a group of URLs at the same time, and for each page return a structured output of the result.@@ -652,32 +659,26 @@ class ScraplingMCPServer::param google_search: Enabled by default, Scrapling will set a Google referer header.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.- :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one.+ :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one, and every option you leave out here keeps the value the session was opened with."""+ fetch_params = _supplied_params(+ wait=wait,+ timeout=timeout,+ google_search=google_search,+ extra_headers=extra_headers,+ disable_resources=disable_resources,+ wait_selector=wait_selector,+ wait_selector_state=wait_selector_state,+ network_idle=network_idle,+ )if session_id:entry = self._get_session(session_id, "dynamic")- tasks = [- entry.session.fetch(- url,- wait=wait,- timeout=timeout,- google_search=google_search,- extra_headers=extra_headers,- disable_resources=disable_resources,- wait_selector=wait_selector,- wait_selector_state=wait_selector_state,- network_idle=network_idle,- proxy=proxy,- )- for url in urls- ]+ tasks = [entry.session.fetch(url, proxy=proxy, **fetch_params) for url in urls]responses = await gather(*tasks)else:async with AsyncDynamicSession(- wait=wait,proxy=proxy,locale=locale,- timeout=timeout,cookies=cookies,cdp_url=cdp_url,headless=headless,@@ -686,13 +687,8 @@ class ScraplingMCPServer:useragent=useragent,timezone_id=timezone_id,real_chrome=real_chrome,- network_idle=network_idle,- wait_selector=wait_selector,- google_search=google_search,- extra_headers=extra_headers,executable_path=self._resolve_executable_path(executable_path),- disable_resources=disable_resources,- wait_selector_state=wait_selector_state,+ **fetch_params,) as session:tasks = [session.fetch(url) for url in urls]responses = await gather(*tasks)@@ -706,9 +702,9 @@ class ScraplingMCPServer:css_selector: Optional[str] = None,main_content_only: bool = True,headless: bool = True, # noqa: F821- google_search: bool = True,+ google_search: bool | None = None,real_chrome: bool = False,- wait: int | float = 0,+ wait: int | float | None = None,proxy: Optional[str | Dict[str, str]] = None,timezone_id: str | None = None,locale: str | None = None,@@ -717,15 +713,15 @@ class ScraplingMCPServer:hide_canvas: bool = False,cdp_url: Optional[str] = None,executable_path: Optional[str] = None,- timeout: int | float = 30000,- disable_resources: bool = False,+ timeout: int | float | None = None,+ disable_resources: bool | None = None,wait_selector: Optional[str] = None,cookies: Sequence[SetCookieParam] | None = None,- network_idle: bool = False,- wait_selector_state: SelectorWaitStates = "attached",+ network_idle: bool | None = None,+ wait_selector_state: Optional[SelectorWaitStates] = None,block_webrtc: bool = False,allow_webgl: bool = True,- solve_cloudflare: bool = False,+ solve_cloudflare: bool | None = None,additional_args: Optional[Dict] = None,session_id: Optional[str] = None,) -> ResponseModel:@@ -760,7 +756,7 @@ class ScraplingMCPServer::param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.- :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one.+ :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one, and every option you leave out here keeps the value the session was opened with."""results = await self.bulk_stealthy_fetch(urls=[url],@@ -800,9 +796,9 @@ class ScraplingMCPServer:css_selector: Optional[str] = None,main_content_only: bool = True,headless: bool = True, # noqa: F821- google_search: bool = True,+ google_search: bool | None = None,real_chrome: bool = False,- wait: int | float = 0,+ wait: int | float | None = None,proxy: Optional[str | Dict[str, str]] = None,timezone_id: str | None = None,locale: str | None = None,@@ -811,15 +807,15 @@ class ScraplingMCPServer:hide_canvas: bool = False,cdp_url: Optional[str] = None,executable_path: Optional[str] = None,- timeout: int | float = 30000,- disable_resources: bool = False,+ timeout: int | float | None = None,+ disable_resources: bool | None = None,wait_selector: Optional[str] = None,cookies: Sequence[SetCookieParam] | None = None,- network_idle: bool = False,- wait_selector_state: SelectorWaitStates = "attached",+ network_idle: bool | None = None,+ wait_selector_state: Optional[SelectorWaitStates] = None,block_webrtc: bool = False,allow_webgl: bool = True,- solve_cloudflare: bool = False,+ solve_cloudflare: bool | None = None,additional_args: Optional[Dict] = None,session_id: Optional[str] = None,) -> List[ResponseModel]:@@ -854,34 +850,28 @@ class ScraplingMCPServer::param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.:param additional_args: Additional arguments to be passed to Playwright's context as additional settings, and it takes higher priority than Scrapling's settings.- :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one.+ :param session_id: Optional session ID from open_session. If provided, reuses the existing browser session instead of creating a new one, and every option you leave out here keeps the value the session was opened with."""+ fetch_params = _supplied_params(+ wait=wait,+ timeout=timeout,+ google_search=google_search,+ extra_headers=extra_headers,+ disable_resources=disable_resources,+ wait_selector=wait_selector,+ wait_selector_state=wait_selector_state,+ network_idle=network_idle,+ solve_cloudflare=solve_cloudflare,+ )if session_id:entry = self._get_session(session_id, "stealthy")- tasks = [- entry.session.fetch(- url,- wait=wait,- timeout=timeout,- google_search=google_search,- extra_headers=extra_headers,- disable_resources=disable_resources,- wait_selector=wait_selector,- wait_selector_state=wait_selector_state,- network_idle=network_idle,- proxy=proxy,- solve_cloudflare=solve_cloudflare,- )- for url in urls- ]+ tasks = [entry.session.fetch(url, proxy=proxy, **fetch_params) for url in urls]responses = await gather(*tasks)else:async with AsyncStealthySession(- wait=wait,proxy=proxy,locale=locale,cdp_url=cdp_url,- timeout=timeout,cookies=cookies,headless=headless,block_ads=True,@@ -891,16 +881,10 @@ class ScraplingMCPServer:real_chrome=real_chrome,hide_canvas=hide_canvas,allow_webgl=allow_webgl,- network_idle=network_idle,block_webrtc=block_webrtc,- wait_selector=wait_selector,- google_search=google_search,- extra_headers=extra_headers,executable_path=self._resolve_executable_path(executable_path),additional_args=additional_args,- solve_cloudflare=solve_cloudflare,- disable_resources=disable_resources,- wait_selector_state=wait_selector_state,+ **fetch_params,) as session:tasks = [session.fetch(url) for url in urls]responses = await gather(*tasks)@@ -940,7 +924,7 @@ class ScraplingMCPServer:5. For all fetch tools, `main_content_only` is enabled by default and returns only the content inside the page's `<body>` tag. Pass `main_content_only=False` when you need the full page instead.6. If the task consists of multiple requests to the same website, open a session to be more efficient.7. For browser-based tools, if a `session_id` is provided (from open_session), the browser session will be reused instead of creating a new one.- When using a session, browser-level params (headless, proxy, locale, etc.) are ignored since they were set at session creation time.+ When using a session, browser-level params (headless, locale, useragent, etc.) are ignored since they were set at session creation time, and any per-request param you leave out keeps the value the session was opened with.8. If you are making multiple requests, use the bulk version of the tool to be more efficient.9. If you are crawling/browsing a website, be more efficient by using the `css_selector` parameter to only access the parts you are interested in and save money/time. Example: use the `a` selector to extract the urls right away.10. The user can pass a CDP URL to connect to a remote browser session through the `open_session` tool, then use it in the rest of the tools.diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pyindex 7ba645f..3e07491 100644--- a/tests/ai/test_ai_mcp.py+++ b/tests/ai/test_ai_mcp.py@@ -24,9 +24,11 @@ from scrapling.core.ai import (_normalize_credentials,_page_pool_size,_StaticTokenVerifier,+ _supplied_params,_translate_response,)-from scrapling.fetchers import AsyncDynamicSession+from scrapling.engines._browsers._validators import PlaywrightConfig, StealthConfig, validate_fetch+from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySessiondef test_translate_response_strips_control_characters():@@ -49,11 +51,21 @@ def test_translate_response_strips_control_characters():assert not any(ord(c) < 0x20 and c not in "\t\n\r" for c in joined)+class _FakePage:+ """The page object a fake session hands to a `page_action`."""++ url = "https://example.com/captured"++ async def screenshot(self, **kwargs: Any) -> bytes:+ return b"fake-png-bytes"++class _FakeAsyncBrowserSession:instances: list["_FakeAsyncBrowserSession"] = []def __init__(self, **kwargs: Any) -> None:self.kwargs = kwargs+ self.fetch_calls: list[dict[str, Any]] = []self._is_alive = Falsetype(self).instances.append(self)@@ -71,6 +83,9 @@ class _FakeAsyncBrowserSession:self._is_alive = Falseasync def fetch(self, url: str, **kwargs: Any) -> Response:+ self.fetch_calls.append(kwargs)+ if kwargs.get("page_action") is not None:+ await kwargs["page_action"](_FakePage())return Response(url=url,content="<html><body>ok</body></html>",@@ -378,6 +393,257 @@ class TestBulkPagePool:)+async def _noop_page_action(page: Any) -> None:+ """Stand-in for the `page_action` the screenshot tool always sends."""+++class TestSessionSettingsSurviveFetchKwargs:+ """The kwargs the tools send on the session path must not overwrite the settings of the session"""++ def test_dynamic_session_settings_survive_bulk_fetch_kwargs(self):+ """Options left out of a `bulk_fetch` call keep the values `open_session` was given"""+ session = AsyncDynamicSession(+ wait=1500,+ timeout=45000,+ network_idle=True,+ disable_resources=True,+ google_search=False,+ wait_selector="#main",+ wait_selector_state="visible",+ extra_headers={"x-test": "1"},+ )++ params = validate_fetch(+ _supplied_params(+ wait=None,+ timeout=None,+ google_search=None,+ extra_headers=None,+ disable_resources=None,+ wait_selector=None,+ wait_selector_state=None,+ network_idle=None,+ ),+ session,+ PlaywrightConfig,+ )++ assert params.wait == 1500, f"Expected the session's wait 1500, got {params.wait}"+ assert params.timeout == 45000, f"Expected the session's timeout 45000, got {params.timeout}"+ assert params.network_idle is True, f"Expected the session's network_idle True, got {params.network_idle}"+ assert params.disable_resources is True, (+ f"Expected the session's disable_resources True, got {params.disable_resources}"+ )+ assert params.google_search is False, f"Expected the session's google_search False, got {params.google_search}"+ assert params.wait_selector == "#main", (+ f"Expected the session's wait_selector '#main', got {params.wait_selector}"+ )+ assert params.wait_selector_state == "visible", (+ f"Expected the session's wait_selector_state 'visible', got {params.wait_selector_state}"+ )+ assert params.extra_headers == {"x-test": "1"}, (+ f"Expected the session's extra_headers, got {params.extra_headers}"+ )++ def test_stealthy_session_keeps_solving_cloudflare(self):+ """A session opened with `solve_cloudflare` keeps solving, and keeps the timeout that comes with it"""+ session = AsyncStealthySession(solve_cloudflare=True)++ assert session._config.timeout == 60_000, (+ f"Expected solve_cloudflare to raise the session timeout to 60,000, got {session._config.timeout}"+ )++ params = validate_fetch(+ _supplied_params(+ wait=None,+ timeout=None,+ google_search=None,+ extra_headers=None,+ disable_resources=None,+ wait_selector=None,+ wait_selector_state=None,+ network_idle=None,+ solve_cloudflare=None,+ ),+ session,+ StealthConfig,+ )++ assert params.solve_cloudflare is True, (+ f"Expected the session's solve_cloudflare True, got {params.solve_cloudflare}"+ )+ assert params.timeout == 60_000, f"Expected the session's timeout 60,000, got {params.timeout}"++ def test_screenshot_kwargs_keep_session_settings(self):+ """The screenshot tool only has a session path, so its unset options must defer to the session too"""+ session = AsyncDynamicSession(wait=1500, timeout=45000, network_idle=True, wait_selector="#main")++ params = validate_fetch(+ {+ "page_action": _noop_page_action,+ **_supplied_params(+ wait=None,+ timeout=None,+ network_idle=None,+ wait_selector=None,+ wait_selector_state=None,+ ),+ },+ session,+ PlaywrightConfig,+ )++ assert params.page_action is _noop_page_action, "Expected the capture callback to stay on the fetch params"+ assert params.wait == 1500, f"Expected the session's wait 1500, got {params.wait}"+ assert params.timeout == 45000, f"Expected the session's timeout 45000, got {params.timeout}"+ assert params.network_idle is True, f"Expected the session's network_idle True, got {params.network_idle}"+ assert params.wait_selector == "#main", (+ f"Expected the session's wait_selector '#main', got {params.wait_selector}"+ )++ def test_supplied_values_still_override_the_session(self):+ """Deferring to the session must not cost the caller the ability to override it per request"""+ session = AsyncDynamicSession(wait=1500, timeout=45000, network_idle=True)++ params = validate_fetch(+ _supplied_params(wait=None, timeout=15000, network_idle=False),+ session,+ PlaywrightConfig,+ )++ assert params.timeout == 15000, f"Expected the supplied timeout 15000, got {params.timeout}"+ assert params.network_idle is False, f"Expected the supplied network_idle False, got {params.network_idle}"+ assert params.wait == 1500, f"Expected the untouched wait to stay 1500, got {params.wait}"+++class TestSessionParameterForwarding:+ """What the tools actually hand to a session's `fetch`, and what they hand to a fresh session"""++ @pytest.fixture(autouse=True)+ def reset_fakes(self):+ _FakeDynamicSession.instances = []+ _FakeStealthySession.instances = []++ @staticmethod+ def _session_fetch_calls(fake: type[_FakeAsyncBrowserSession]) -> list[dict[str, Any]]:+ return fake.instances[0].fetch_calls++ @pytest.mark.asyncio+ async def test_bulk_fetch_forwards_nothing_it_was_not_given(self, monkeypatch):+ """An option the caller didn't pass must not reach the session as the tool's own default"""+ monkeypatch.setattr("scrapling.core.ai.AsyncDynamicSession", _FakeDynamicSession)+ server = ScraplingMCPServer()+ opened = await server.open_session(session_type="dynamic")++ await server.bulk_fetch(urls=["https://example.com/1"], session_id=opened.session_id)++ forwarded = self._session_fetch_calls(_FakeDynamicSession)[0]+ assert forwarded == {"proxy": None}, f"Expected only the proxy override to be forwarded, got {forwarded}"++ @pytest.mark.asyncio+ async def test_bulk_fetch_forwards_what_it_was_given(self, monkeypatch):+ """Options the caller did pass still reach the session"""+ monkeypatch.setattr("scrapling.core.ai.AsyncDynamicSession", _FakeDynamicSession)+ server = ScraplingMCPServer()+ opened = await server.open_session(session_type="dynamic")++ await server.bulk_fetch(+ urls=["https://example.com/1"], session_id=opened.session_id, network_idle=True, timeout=45000+ )++ forwarded = self._session_fetch_calls(_FakeDynamicSession)[0]+ assert forwarded == {"proxy": None, "network_idle": True, "timeout": 45000}, (+ f"Expected only the supplied options to be forwarded, got {forwarded}"+ )++ @pytest.mark.asyncio+ async def test_fetch_forwards_nothing_it_was_not_given(self, monkeypatch):+ """The single-URL wrapper passes the unset options through to `bulk_fetch` untouched"""+ monkeypatch.setattr("scrapling.core.ai.AsyncDynamicSession", _FakeDynamicSession)+ server = ScraplingMCPServer()+ opened = await server.open_session(session_type="dynamic")++ await server.fetch(url="https://example.com/1", session_id=opened.session_id)++ forwarded = self._session_fetch_calls(_FakeDynamicSession)[0]+ assert forwarded == {"proxy": None}, f"Expected only the proxy override to be forwarded, got {forwarded}"++ @pytest.mark.asyncio+ async def test_bulk_stealthy_fetch_leaves_solve_cloudflare_to_the_session(self, monkeypatch):+ """`solve_cloudflare` is kept on the fetch params even when it is False, so it must not be sent unasked"""+ monkeypatch.setattr("scrapling.core.ai.AsyncStealthySession", _FakeStealthySession)+ server = ScraplingMCPServer()+ opened = await server.open_session(session_type="stealthy", solve_cloudflare=True)++ await server.bulk_stealthy_fetch(urls=["https://example.com/1"], session_id=opened.session_id)++ forwarded = self._session_fetch_calls(_FakeStealthySession)[0]+ assert "solve_cloudflare" not in forwarded, (+ f"Expected solve_cloudflare to be left to the session, got {forwarded}"+ )++ @pytest.mark.asyncio+ async def test_bulk_stealthy_fetch_forwards_an_explicit_solve_cloudflare(self, monkeypatch):+ """A caller can still turn Cloudflare solving on for a single request"""+ monkeypatch.setattr("scrapling.core.ai.AsyncStealthySession", _FakeStealthySession)+ server = ScraplingMCPServer()+ opened = await server.open_session(session_type="stealthy")++ await server.bulk_stealthy_fetch(+ urls=["https://example.com/1"], session_id=opened.session_id, solve_cloudflare=True+ )++ forwarded = self._session_fetch_calls(_FakeStealthySession)[0]+ assert forwarded.get("solve_cloudflare") is True, (+ f"Expected the supplied solve_cloudflare to be forwarded, got {forwarded}"+ )++ @pytest.mark.asyncio+ async def test_screenshot_forwards_only_the_capture_callback(self, monkeypatch):+ """The screenshot tool needs the capture callback, and nothing else the caller didn't ask for"""+ monkeypatch.setattr("scrapling.core.ai.AsyncDynamicSession", _FakeDynamicSession)+ server = ScraplingMCPServer()+ opened = await server.open_session(session_type="dynamic")++ result = await server.screenshot(url="https://example.com/1", session_id=opened.session_id)++ forwarded = self._session_fetch_calls(_FakeDynamicSession)[0]+ assert set(forwarded) == {"page_action"}, f"Expected only the capture callback to be forwarded, got {forwarded}"+ assert isinstance(result[0], ImageContent), f"Expected an image content block, got {type(result[0]).__name__}"++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "tool,fake,fake_name",+ [+ ("bulk_fetch", _FakeDynamicSession, "AsyncDynamicSession"),+ ("bulk_stealthy_fetch", _FakeStealthySession, "AsyncStealthySession"),+ ],+ )+ async def test_a_fresh_session_is_built_with_the_same_settings_as_before(self, monkeypatch, tool, fake, fake_name):+ """Without a session ID the tools still build a session with the fetcher defaults they always used"""+ monkeypatch.setattr(f"scrapling.core.ai.{fake_name}", fake)+ server = ScraplingMCPServer()++ await getattr(server, tool)(urls=["https://example.com/1"])++ real_session_type = AsyncDynamicSession if fake is _FakeDynamicSession else AsyncStealthySession+ config = real_session_type(**fake.instances[0].kwargs)._config+ expected = {+ "wait": 0,+ "timeout": 30000,+ "google_search": True,+ "extra_headers": None,+ "disable_resources": False,+ "wait_selector": None,+ "wait_selector_state": "attached",+ "network_idle": False,+ }+ for name, value in expected.items():+ assert getattr(config, name) == value, (+ f"Expected {tool} to keep building a session with {name}={value}, got {getattr(config, name)}"+ )++def _png_height(data: bytes) -> int:"""Read the height field from a PNG IHDR chunk."""return struct.unpack(">I", data[20:24])[0]