🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat: add new mcp tool to screenshot pages
Karim shoair78e388f75cfe
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex 315733f..464d35b 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -3,7 +3,8 @@ from asyncio import gatherfrom datetime import datetime, timezonefrom dataclasses import dataclass, field-from mcp.server.fastmcp import FastMCP+from mcp.server.fastmcp import FastMCP, Image+from mcp.types import ImageContent, TextContentfrom pydantic import BaseModel, Fieldfrom scrapling.core.shell import Convertor@@ -31,6 +32,7 @@ from scrapling.core._types import ()SessionType = Literal["dynamic", "stealthy"]+ScreenshotType = Literal["png", "jpeg"]class ResponseModel(BaseModel):@@ -106,14 +108,14 @@ class ScraplingMCPServer:def __init__(self):self._sessions: Dict[str, _SessionEntry] = {}- def _get_session(self, session_id: str, expected_type: SessionType) -> _SessionEntry:- """Look up a session by ID and validate its type."""+ def _get_session(self, session_id: str, expected_type: Optional[SessionType]) -> _SessionEntry:+ """Look up a session by ID, optionally validating its type. Pass `None` to skip the type check."""entry = self._sessions.get(session_id)if entry is None:raise ValueError(f"Session '{session_id}' not found. Use list_sessions to see active sessions.")if not entry.session._is_alive:raise ValueError(f"Session '{session_id}' is no longer alive. Open a new session.")- if entry.session_type != expected_type:+ if expected_type is not None and entry.session_type != expected_type:raise ValueError(f"Session '{session_id}' is a '{entry.session_type}' session, but this tool requires a "f"'{expected_type}' session. Use the matching fetch tool for your session type."@@ -260,6 +262,69 @@ class ScraplingMCPServer:for sid, entry in self._sessions.items()]+ async def screenshot(+ self,+ url: str,+ session_id: str,+ image_type: ScreenshotType = "png",+ full_page: bool = False,+ quality: Optional[int] = None,+ wait: int | float = 0,+ wait_selector: Optional[str] = None,+ wait_selector_state: SelectorWaitStates = "attached",+ network_idle: bool = False,+ timeout: int | float = 30000,+ ) -> List[ImageContent | TextContent]:+ """Capture a screenshot of a web page using an existing browser session and return it as an image.+ A browser session must be opened first with `open_session` (either `dynamic` or `stealthy`); the session ID is then passed here.++ :param url: The URL to navigate to and capture.+ :param session_id: ID of an open browser session created with `open_session`.+ :param image_type: Image format. Defaults to "png". Use "jpeg" for smaller file sizes.+ :param full_page: When True, captures the full scrollable page instead of just the viewport. Defaults to False.+ :param quality: Image quality (0-100) for JPEG only. Raises if passed with `image_type="png"`.+ :param wait: Time in milliseconds to wait after page load before capturing. Defaults to 0.+ :param wait_selector: Optional CSS selector to wait for before capturing.+ :param wait_selector_state: State to wait for the selector. Defaults to "attached".+ :param network_idle: Wait for the page until there are no network connections for at least 500 ms.+ :param timeout: Timeout in milliseconds for page operations. Defaults to 30,000.+ """+ if quality is not None and image_type != "jpeg":+ raise ValueError("'quality' is only valid when 'image_type' is 'jpeg'.")++ entry = self._get_session(session_id, expected_type=None)++ screenshot_kwargs: Dict[str, Any] = {"type": image_type, "full_page": full_page}+ if quality is not None:+ screenshot_kwargs["quality"] = quality++ captured: Dict[str, Any] = {}++ async def _capture(page: Any) -> None:+ try:+ captured["bytes"] = await page.screenshot(**screenshot_kwargs)+ captured["url"] = page.url+ except Exception as exc:+ captured["error"] = exc++ await entry.session.fetch(+ url,+ wait=wait,+ timeout=timeout,+ network_idle=network_idle,+ wait_selector=wait_selector,+ wait_selector_state=wait_selector_state,+ page_action=_capture,+ )++ if "error" in captured:+ raise captured["error"]+ if "bytes" not in captured:+ raise RuntimeError(f"Failed to capture screenshot for {url}")++ image = Image(data=captured["bytes"], format=image_type).to_image_content()+ return [image, TextContent(type="text", text=captured["url"])]+@staticmethodasync def get(url: str,@@ -298,7 +363,8 @@ class ScraplingMCPServer::param headers: Headers to include in the request.:param cookies: Cookies to use in the request.:param timeout: Number of seconds to wait before timing out.- :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.+ :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection).+ Pass True to follow all redirects without restriction.:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.:param retries: Number of retry attempts. Defaults to 3.:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -371,7 +437,8 @@ class ScraplingMCPServer::param headers: Headers to include in the request.:param cookies: Cookies to use in the request.:param timeout: Number of seconds to wait before timing out.- :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection). Pass True to follow all redirects without restriction.+ :param follow_redirects: Whether to follow redirects. Defaults to "safe", which follows redirects but rejects those targeting internal/private IPs (SSRF protection).+ Pass True to follow all redirects without restriction.:param max_redirects: Maximum number of redirects. Default 30, use -1 for unlimited.:param retries: Number of retry attempts. Defaults to 3.:param retry_delay: Number of seconds to wait between retry attempts. Defaults to 1 second.@@ -835,4 +902,6 @@ class ScraplingMCPServer:description=self.bulk_stealthy_fetch.__doc__,structured_output=True,)+ # Screenshot tool (returns image + url content blocks, not structured JSON)+ server.add_tool(self.screenshot, title="screenshot", description=self.screenshot.__doc__)server.run(transport="stdio" if not http else "streamable-http")diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pyindex 4806088..19e2f02 100644--- a/tests/ai/test_ai_mcp.py+++ b/tests/ai/test_ai_mcp.py@@ -1,5 +1,12 @@+import base64+import struct+from contextlib import contextmanager+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from threading import Thread+import pytestimport pytest_httpbin+from mcp.types import ImageContent, TextContentfrom scrapling.core.ai import (ScraplingMCPServer,@@ -197,6 +204,118 @@ class TestSessionManagement:await server.close_session("dupe")+def _png_height(data: bytes) -> int:+ """Read the height field from a PNG IHDR chunk."""+ return struct.unpack(">I", data[20:24])[0]+++@contextmanager+def _serve_html(body: bytes):+ """Serve a fixed HTML body on localhost, yielding its URL."""++ class _Handler(BaseHTTPRequestHandler):+ def do_GET(self):+ self.send_response(200)+ self.send_header("Content-Type", "text/html; charset=utf-8")+ self.send_header("Content-Length", str(len(body)))+ self.end_headers()+ self.wfile.write(body)++ def log_message(self, *args, **kwargs):+ pass++ server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler)+ thread = Thread(target=server.serve_forever, daemon=True)+ thread.start()+ try:+ yield f"http://127.0.0.1:{server.server_address[1]}/"+ finally:+ server.shutdown()+ server.server_close()+++@pytest_httpbin.use_class_based_httpbin+class TestScreenshot:+ """Test the screenshot tool"""++ @pytest.fixture(scope="class")+ def test_url(self, httpbin):+ return f"{httpbin.url}/html"++ @pytest.fixture+ def server(self):+ return ScraplingMCPServer()++ @pytest.mark.asyncio+ async def test_screenshot_png_with_dynamic_session(self, server, test_url):+ """PNG screenshot via a dynamic session returns image and url content blocks"""+ opened = await server.open_session(session_type="dynamic", headless=True)+ try:+ result = await server.screenshot(url=test_url, session_id=opened.session_id)+ assert isinstance(result, list) and len(result) == 2+ assert isinstance(result[0], ImageContent)+ assert result[0].mimeType == "image/png"+ assert isinstance(result[1], TextContent)+ assert result[1].text == test_url+ finally:+ await server.close_session(opened.session_id)++ @pytest.mark.asyncio+ async def test_screenshot_jpeg_with_quality(self, server, test_url):+ """JPEG screenshot with quality parameter via a dynamic session"""+ opened = await server.open_session(session_type="dynamic", headless=True)+ try:+ result = await server.screenshot(url=test_url, session_id=opened.session_id, image_type="jpeg", quality=80)+ assert isinstance(result[0], ImageContent)+ assert result[0].mimeType == "image/jpeg"+ finally:+ await server.close_session(opened.session_id)++ @pytest.mark.asyncio+ async def test_screenshot_with_stealthy_session(self, server, test_url):+ """PNG screenshot via a stealthy session"""+ opened = await server.open_session(session_type="stealthy", headless=True)+ try:+ result = await server.screenshot(url=test_url, session_id=opened.session_id)+ assert isinstance(result[0], ImageContent)+ assert result[0].mimeType == "image/png"+ finally:+ await server.close_session(opened.session_id)++ @pytest.mark.asyncio+ async def test_screenshot_full_page_taller_than_viewport(self, server):+ """full_page=True produces an image taller than the viewport-only capture"""+ tall_html = b"<html><body><div style='height:5000px;background:#abc'></div></body></html>"+ with _serve_html(tall_html) as tall_url:+ opened = await server.open_session(session_type="dynamic", headless=True)+ try:+ viewport_result = await server.screenshot(url=tall_url, session_id=opened.session_id, full_page=False)+ full_result = await server.screenshot(url=tall_url, session_id=opened.session_id, full_page=True)++ viewport_png = base64.b64decode(viewport_result[0].data)+ full_png = base64.b64decode(full_result[0].data)++ assert _png_height(full_png) > _png_height(viewport_png)+ finally:+ await server.close_session(opened.session_id)++ @pytest.mark.asyncio+ async def test_screenshot_invalid_session_id_raises(self, server, test_url):+ """Unknown session_id raises ValueError"""+ with pytest.raises(ValueError, match="not found"):+ await server.screenshot(url=test_url, session_id="does-not-exist")++ @pytest.mark.asyncio+ async def test_screenshot_quality_with_png_raises(self, server, test_url):+ """quality is rejected when image_type is png"""+ opened = await server.open_session(session_type="dynamic", headless=True)+ try:+ with pytest.raises(ValueError, match="quality"):+ await server.screenshot(url=test_url, session_id=opened.session_id, image_type="png", quality=90)+ finally:+ await server.close_session(opened.session_id)++class TestNormalizeCredentials:"""Test the _normalize_credentials helper"""