🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(mcp/cli): Include open Shadow DOM content in the response HTML.
Karim shoair6bbc6e01c56f
diff --git a/docs/ai/mcp-server.md b/docs/ai/mcp-server.mdindex aa83feb..a0abcae 100644--- a/docs/ai/mcp-server.md+++ b/docs/ai/mcp-server.md@@ -37,6 +37,10 @@ The Scrapling MCP Server provides thirteen powerful tools for web scraping, spli#### 📸 Screenshots- **`screenshot`**: Capture a PNG or JPEG screenshot of a page using an open browser session, returned as an image content block the model can actually see (not a base64 string blob). Supports full-page captures, JPEG quality, and the usual readiness controls (`wait`, `wait_selector`, `network_idle`).+### Shadow DOM++Set `pierce_shadow=true` on `fetch`, `bulk_fetch`, `stealthy_fetch`, `bulk_stealthy_fetch`, or `session_fetch` to include open Shadow DOM content. It defaults to `false`. For persistent sessions, pass it on each `session_fetch` call. See [Shadow DOM](../fetching/dynamic.md#shadow-dom) for selector examples and limits.+### Key Capabilities- **Smart Content Extraction**: Convert web pages/elements to Markdown, HTML, or extract a clean version of the text content- **CSS Selector Support**: Use the Scrapling engine to target specific elements with precision before handing the content to the AIdiff --git a/docs/cli/extract-commands.md b/docs/cli/extract-commands.mdindex 05b0997..9f48a23 100644--- a/docs/cli/extract-commands.md+++ b/docs/cli/extract-commands.md@@ -252,6 +252,13 @@ We will go through each command in detail below.### Browsers fetching+Both `fetch` and `stealthy-fetch` support `--pierce-shadow` to include open Shadow DOM content. It is off by default; use `--no-pierce-shadow` to disable it explicitly. See [Shadow DOM](../fetching/dynamic.md#shadow-dom) for selector examples and limits.++```bash+scrapling extract fetch "https://example.com" content.md --pierce-shadow+scrapling extract stealthy-fetch "https://example.com" content.md --pierce-shadow+```+1. **fetch - Handle Dynamic Content**For websites that load content with dynamic content or have slight protection@@ -282,6 +289,7 @@ We will go through each command in detail below.Options:--headless / --no-headless Run browser in headless mode (default: True)--disable-resources / --enable-resources Drop unnecessary resources for speed boost (default: False)+ --pierce-shadow / --no-pierce-shadow Include open Shadow DOM content (default: False)--network-idle / --no-network-idle Wait for network idle (default: False)--timeout INTEGER Timeout in milliseconds (default: 30000)--wait INTEGER Additional wait time in milliseconds after page load (default: 0)@@ -331,6 +339,7 @@ We will go through each command in detail below.--block-webrtc / --allow-webrtc Block WebRTC entirely (default: False)--solve-cloudflare / --no-solve-cloudflare Solve Cloudflare challenges (default: False)--allow-webgl / --block-webgl Allow WebGL (default: True)+ --pierce-shadow / --no-pierce-shadow Include open Shadow DOM content (default: False)--network-idle / --no-network-idle Wait for network idle (default: False)--real-chrome/--no-real-chrome If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)--timeout INTEGER Timeout in milliseconds (default: 30000)diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 8a4d905..6bbb485 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -336,6 +336,11 @@ def _common_browser_options(f):default=30000,help="Timeout in milliseconds (default: 30000)",),+ option(+ "--pierce-shadow/--no-pierce-shadow",+ default=False,+ help="Include open Shadow DOM content (default: False)",+ ),option("--network-idle/--no-network-idle",default=False,@@ -543,6 +548,7 @@ def __build_browser_kwargs(headless,disable_resources,network_idle,+ pierce_shadow,timeout,wait,wait_selector,@@ -559,6 +565,7 @@ def __build_browser_kwargs("headless": headless,"disable_resources": disable_resources,"network_idle": network_idle,+ "pierce_shadow": pierce_shadow,"timeout": timeout,"locale": locale,"real_chrome": real_chrome,@@ -589,6 +596,7 @@ def fetch(headless,disable_resources,network_idle,+ pierce_shadow,timeout,wait,css_selector,@@ -608,6 +616,7 @@ def fetch(headless,disable_resources,network_idle,+ pierce_shadow,timeout,wait,wait_selector,@@ -650,6 +659,7 @@ def stealthy_fetch(headless,disable_resources,network_idle,+ pierce_shadow,timeout,wait,css_selector,@@ -673,6 +683,7 @@ def stealthy_fetch(headless,disable_resources,network_idle,+ pierce_shadow,timeout,wait,wait_selector,diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex f986108..cd12284 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -601,6 +601,7 @@ class ScraplingMCPServer:cookies: Sequence[SetCookieParam] | None = None,network_idle: bool = False,wait_selector_state: SelectorWaitStates = "attached",+ pierce_shadow: bool = False,) -> ResponseModel:"""Use playwright to open a browser to fetch a URL and return a structured output of the result.Only suitable for low-mid protection levels.@@ -614,6 +615,7 @@ class ScraplingMCPServer:Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param cookies: Set cookies for the next request. It should be in a dictionary format that Playwright accepts.+ :param pierce_shadow: Include open Shadow DOM content in the response. Defaults to False.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.@@ -650,6 +652,7 @@ class ScraplingMCPServer:wait_selector=wait_selector,cookies=cookies,network_idle=network_idle,+ pierce_shadow=pierce_shadow,wait_selector_state=wait_selector_state,)return results[0]@@ -677,6 +680,7 @@ class ScraplingMCPServer:cookies: Sequence[SetCookieParam] | None = None,network_idle: bool = False,wait_selector_state: SelectorWaitStates = "attached",+ pierce_shadow: bool = False,) -> List[ResponseModel]:"""Use playwright to open a browser, then fetch a group of URLs at the same time, and for each page return a structured output of the result.Only suitable for low-mid protection levels.@@ -690,6 +694,7 @@ class ScraplingMCPServer:Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param cookies: Set cookies for the next request. It should be in a dictionary format that Playwright accepts.+ :param pierce_shadow: Include open Shadow DOM content in the response. Defaults to False.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.@@ -719,6 +724,7 @@ class ScraplingMCPServer:timezone_id=timezone_id,real_chrome=real_chrome,network_idle=network_idle,+ pierce_shadow=pierce_shadow,wait_selector=wait_selector,google_search=google_search,extra_headers=extra_headers,@@ -759,6 +765,7 @@ class ScraplingMCPServer:allow_webgl: bool = True,solve_cloudflare: bool = False,additional_args: Optional[Dict] = None,+ pierce_shadow: bool = False,) -> ResponseModel:"""Use the stealthy fetcher to fetch a URL and return a structured output of the result.The only fetcher suitable for high-protection websites.@@ -774,6 +781,7 @@ class ScraplingMCPServer::param cookies: Set cookies for the next request.:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.+ :param pierce_shadow: Include open Shadow DOM content in the response. Defaults to False.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.@@ -814,6 +822,7 @@ class ScraplingMCPServer:wait_selector=wait_selector,cookies=cookies,network_idle=network_idle,+ pierce_shadow=pierce_shadow,wait_selector_state=wait_selector_state,block_webrtc=block_webrtc,allow_webgl=allow_webgl,@@ -850,6 +859,7 @@ class ScraplingMCPServer:allow_webgl: bool = True,solve_cloudflare: bool = False,additional_args: Optional[Dict] = None,+ pierce_shadow: bool = False,) -> List[ResponseModel]:"""Use the stealthy fetcher to fetch a group of URLs at the same time, and for each page return a structured output of the result.The only fetcher suitable for high-protection websites.@@ -865,6 +875,7 @@ class ScraplingMCPServer::param cookies: Set cookies for the next request.:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.+ :param pierce_shadow: Include open Shadow DOM content in the response. Defaults to False.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.@@ -899,6 +910,7 @@ class ScraplingMCPServer:hide_canvas=hide_canvas,allow_webgl=allow_webgl,network_idle=network_idle,+ pierce_shadow=pierce_shadow,block_webrtc=block_webrtc,wait_selector=wait_selector,google_search=google_search,@@ -932,6 +944,7 @@ class ScraplingMCPServer:extra_headers: Optional[Dict[str, str]] = None,blocked_domains: Optional[Set[str]] = None,solve_cloudflare: bool = False,+ pierce_shadow: bool = False,) -> ResponseModel:"""Fetch a URL through a browser session previously opened with `open_session` and return a structured output of the result.The session (dynamic or stealthy) holds the browser-level configuration; every option here applies to this request only, with the defaults shown.@@ -944,6 +957,7 @@ class ScraplingMCPServer::param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.:param google_search: Enabled by default, Scrapling will set a Google referer header.+ :param pierce_shadow: Include open Shadow DOM content in the response. Defaults to False.:param network_idle: Wait for the page until there are no network connections for at least 500 ms.:param load_dom: Enabled by default, wait for all JavaScript on the page to fully load and execute.:param disable_resources: Drop requests for unnecessary resources for a speed boost.@@ -971,6 +985,7 @@ class ScraplingMCPServer:timeout=timeout,google_search=google_search,network_idle=network_idle,+ pierce_shadow=pierce_shadow,load_dom=load_dom,disable_resources=disable_resources,wait_selector=wait_selector,diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pyindex adc3c99..709ae31 100644--- a/tests/ai/test_ai_mcp.py+++ b/tests/ai/test_ai_mcp.py@@ -505,6 +505,7 @@ class TestSessionFetchForwarding:"google_search": True,"network_idle": False,"load_dom": True,+ "pierce_shadow": False,"disable_resources": False,"wait_selector": None,"wait_selector_state": "attached",@@ -934,6 +935,8 @@ class TestServerToolRegistration:assert "session_id" not in props, f"{name} still exposes session_id"assert props["timeout"]["default"] == 30000, f"{name} hides the real timeout default"assert props["google_search"]["default"] is True+ assert props["pierce_shadow"]["type"] == "boolean"+ assert props["pierce_shadow"]["default"] is Falserequest_props = tools["make_request"].input_schema["properties"]assert request_props["method"]["default"] == "GET"@@ -948,6 +951,8 @@ class TestServerToolRegistration:session_props = tools["session_fetch"].input_schema["properties"]assert session_props["timeout"]["default"] == 30000+ assert session_props["pierce_shadow"]["type"] == "boolean"+ assert session_props["pierce_shadow"]["default"] is Falseassert "solve_cloudflare" in session_propsassert set(tools["session_fetch"].input_schema["required"]) >= {"url", "session_id"}@@ -991,3 +996,42 @@ class TestServerToolRegistration:assert annotations[name].open_world_hint is Trueassert annotations["list_sessions"].read_only_hint is Trueassert annotations["list_sessions"].open_world_hint is False++[email protected]("enabled", [False, True])[email protected]("tool", ["fetch", "bulk_fetch", "stealthy_fetch", "bulk_stealthy_fetch"])+async def test_shadow_option_reaches_browser_session(monkeypatch, tool, enabled):+ fake = _FakeStealthySession if "stealthy" in tool else _FakeDynamicSession+ target = "AsyncStealthySession" if "stealthy" in tool else "AsyncDynamicSession"+ monkeypatch.setattr("scrapling.core.ai." + target, fake)+ server = ScraplingMCPServer()._build_server("127.0.0.1", 8000)+ args: dict[str, Any] = (+ {"urls": ["https://example.com"]} if tool.startswith("bulk_") else {"url": "https://example.com"}+ )+ async with Client(server) as client:+ result = await client.call_tool(tool, {**args, "pierce_shadow": enabled})+ assert not result.is_error+ assert fake.instances[-1].kwargs["pierce_shadow"] is enabled+ assert not fake.instances[-1]._is_alive++[email protected]("session_type", ["dynamic", "stealthy"])+async def test_shadow_option_changes_per_session_request(monkeypatch, session_type):+ fake = _FakeStealthySession if session_type == "stealthy" else _FakeDynamicSession+ target = "AsyncStealthySession" if session_type == "stealthy" else "AsyncDynamicSession"+ monkeypatch.setattr("scrapling.core.ai." + target, fake)+ server = ScraplingMCPServer()._build_server("127.0.0.1", 8000)+ async with Client(server) as client:+ await client.call_tool("open_session", {"session_type": session_type, "session_id": "shadow-test"})+ try:+ for args, expected in [({"pierce_shadow": True}, True), ({"pierce_shadow": False}, False), ({}, False)]:+ result = await client.call_tool(+ "session_fetch", {"url": "https://example.com", "session_id": "shadow-test", **args}+ )+ assert not result.is_error+ assert fake.instances[-1].fetch_calls[-1]["pierce_shadow"] is expected+ finally:+ await client.call_tool("close_session", {"session_id": "shadow-test"})+ assert not fake.instances[-1]._is_alivediff --git a/tests/cli/test_shadow_cli.py b/tests/cli/test_shadow_cli.pynew file mode 100644index 0000000..097a5e2--- /dev/null+++ b/tests/cli/test_shadow_cli.py@@ -0,0 +1,37 @@+from pathlib import Path+from unittest.mock import patch++import pytest+from click.testing import CliRunner++from scrapling.cli import main+from scrapling.parser import Selector++[email protected](("command", "fetcher"), [("fetch", "DynamicFetcher"), ("stealthy-fetch", "StealthyFetcher")])+ ("flags", "enabled"), [([], False), (["--pierce-shadow"], True), (["--no-pierce-shadow"], False)]+)+def test_shadow_option_reaches_fetcher(+ tmp_path: Path, command: str, fetcher: str, flags: list[str], enabled: bool+) -> None:+ output = tmp_path / "page.txt"+ with patch(f"scrapling.fetchers.{fetcher}.fetch", return_value=Selector("<p>Content</p>")) as fetch:+ result = CliRunner().invoke(main, ["extract", command, "https://example.com", str(output), *flags])+ assert result.exit_code == 0, result.output+ assert fetch.call_args.kwargs["pierce_shadow"] is enabled+ assert output.read_text() == "Content"++[email protected]("command", ["fetch", "stealthy-fetch"])+def test_shadow_option_in_browser_help(command: str) -> None:+ result = CliRunner().invoke(main, ["extract", command, "--help"])+ assert result.exit_code == 0+ assert "--pierce-shadow / --no-pierce-shadow" in result.output++[email protected]("command", ["get", "post", "put", "delete"])+def test_shadow_option_not_in_http_help(command: str) -> None:+ result = CliRunner().invoke(main, ["extract", command, "--help"])+ assert result.exit_code == 0+ assert "pierce-shadow" not in result.output