Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(mcp/cli): Include open Shadow DOM content in the response HTML.

Karim shoair6bbc6e01c56f
diff --git a/docs/ai/mcp-server.md b/docs/ai/mcp-server.md
index aa83feb..a0abcae 100644
--- a/docs/ai/mcp-server.md
+++ b/docs/ai/mcp-server.md
@@ -37,6 +37,10 @@ The Scrapling MCP Server provides thirteen powerful tools for web scraping, spli
#### 📸 Screenshots
- **`screenshot`**: Capture a PNG or JPEG screenshot of a page using an open browser session, returned as an image content block the model can actually see (not a base64 string blob). Supports full-page captures, JPEG quality, and the usual readiness controls (`wait`, `wait_selector`, `network_idle`).
+### Shadow DOM
+
+Set `pierce_shadow=true` on `fetch`, `bulk_fetch`, `stealthy_fetch`, `bulk_stealthy_fetch`, or `session_fetch` to include open Shadow DOM content. It defaults to `false`. For persistent sessions, pass it on each `session_fetch` call. See [Shadow DOM](../fetching/dynamic.md#shadow-dom) for selector examples and limits.
+
### Key Capabilities
- **Smart Content Extraction**: Convert web pages/elements to Markdown, HTML, or extract a clean version of the text content
- **CSS Selector Support**: Use the Scrapling engine to target specific elements with precision before handing the content to the AI
diff --git a/docs/cli/extract-commands.md b/docs/cli/extract-commands.md
index 05b0997..9f48a23 100644
--- a/docs/cli/extract-commands.md
+++ b/docs/cli/extract-commands.md
@@ -252,6 +252,13 @@ We will go through each command in detail below.
### Browsers fetching
+Both `fetch` and `stealthy-fetch` support `--pierce-shadow` to include open Shadow DOM content. It is off by default; use `--no-pierce-shadow` to disable it explicitly. See [Shadow DOM](../fetching/dynamic.md#shadow-dom) for selector examples and limits.
+
+```bash
+scrapling extract fetch "https://example.com" content.md --pierce-shadow
+scrapling extract stealthy-fetch "https://example.com" content.md --pierce-shadow
+```
+
1. **fetch - Handle Dynamic Content**
For websites that load content with dynamic content or have slight protection
@@ -282,6 +289,7 @@ We will go through each command in detail below.
Options:
--headless / --no-headless Run browser in headless mode (default: True)
--disable-resources / --enable-resources Drop unnecessary resources for speed boost (default: False)
+ --pierce-shadow / --no-pierce-shadow Include open Shadow DOM content (default: False)
--network-idle / --no-network-idle Wait for network idle (default: False)
--timeout INTEGER Timeout in milliseconds (default: 30000)
--wait INTEGER Additional wait time in milliseconds after page load (default: 0)
@@ -331,6 +339,7 @@ We will go through each command in detail below.
--block-webrtc / --allow-webrtc Block WebRTC entirely (default: False)
--solve-cloudflare / --no-solve-cloudflare Solve Cloudflare challenges (default: False)
--allow-webgl / --block-webgl Allow WebGL (default: True)
+ --pierce-shadow / --no-pierce-shadow Include open Shadow DOM content (default: False)
--network-idle / --no-network-idle Wait for network idle (default: False)
--real-chrome/--no-real-chrome If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it. (default: False)
--timeout INTEGER Timeout in milliseconds (default: 30000)
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 8a4d905..6bbb485 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -336,6 +336,11 @@ def _common_browser_options(f):
default=30000,
help="Timeout in milliseconds (default: 30000)",
),
+ option(
+ "--pierce-shadow/--no-pierce-shadow",
+ default=False,
+ help="Include open Shadow DOM content (default: False)",
+ ),
option(
"--network-idle/--no-network-idle",
default=False,
@@ -543,6 +548,7 @@ def __build_browser_kwargs(
headless,
disable_resources,
network_idle,
+ pierce_shadow,
timeout,
wait,
wait_selector,
@@ -559,6 +565,7 @@ def __build_browser_kwargs(
"headless": headless,
"disable_resources": disable_resources,
"network_idle": network_idle,
+ "pierce_shadow": pierce_shadow,
"timeout": timeout,
"locale": locale,
"real_chrome": real_chrome,
@@ -589,6 +596,7 @@ def fetch(
headless,
disable_resources,
network_idle,
+ pierce_shadow,
timeout,
wait,
css_selector,
@@ -608,6 +616,7 @@ def fetch(
headless,
disable_resources,
network_idle,
+ pierce_shadow,
timeout,
wait,
wait_selector,
@@ -650,6 +659,7 @@ def stealthy_fetch(
headless,
disable_resources,
network_idle,
+ pierce_shadow,
timeout,
wait,
css_selector,
@@ -673,6 +683,7 @@ def stealthy_fetch(
headless,
disable_resources,
network_idle,
+ pierce_shadow,
timeout,
wait,
wait_selector,
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index f986108..cd12284 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -601,6 +601,7 @@ class ScraplingMCPServer:
cookies: Sequence[SetCookieParam] | None = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
+ pierce_shadow: bool = False,
) -> ResponseModel:
"""Use playwright to open a browser to fetch a URL and return a structured output of the result.
Only suitable for low-mid protection levels.
@@ -614,6 +615,7 @@ class ScraplingMCPServer:
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request. It should be in a dictionary format that Playwright accepts.
+ :param pierce_shadow: Include open Shadow DOM content in the response. Defaults to False.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
@@ -650,6 +652,7 @@ class ScraplingMCPServer:
wait_selector=wait_selector,
cookies=cookies,
network_idle=network_idle,
+ pierce_shadow=pierce_shadow,
wait_selector_state=wait_selector_state,
)
return results[0]
@@ -677,6 +680,7 @@ class ScraplingMCPServer:
cookies: Sequence[SetCookieParam] | None = None,
network_idle: bool = False,
wait_selector_state: SelectorWaitStates = "attached",
+ pierce_shadow: bool = False,
) -> List[ResponseModel]:
"""Use playwright to open a browser, then fetch a group of URLs at the same time, and for each page return a structured output of the result.
Only suitable for low-mid protection levels.
@@ -690,6 +694,7 @@ class ScraplingMCPServer:
Requests dropped are of type `font`, `image`, `media`, `beacon`, `object`, `imageset`, `texttrack`, `websocket`, `csp_report`, and `stylesheet`.
:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.
:param cookies: Set cookies for the next request. It should be in a dictionary format that Playwright accepts.
+ :param pierce_shadow: Include open Shadow DOM content in the response. Defaults to False.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
@@ -719,6 +724,7 @@ class ScraplingMCPServer:
timezone_id=timezone_id,
real_chrome=real_chrome,
network_idle=network_idle,
+ pierce_shadow=pierce_shadow,
wait_selector=wait_selector,
google_search=google_search,
extra_headers=extra_headers,
@@ -759,6 +765,7 @@ class ScraplingMCPServer:
allow_webgl: bool = True,
solve_cloudflare: bool = False,
additional_args: Optional[Dict] = None,
+ pierce_shadow: bool = False,
) -> ResponseModel:
"""Use the stealthy fetcher to fetch a URL and return a structured output of the result.
The only fetcher suitable for high-protection websites.
@@ -774,6 +781,7 @@ class ScraplingMCPServer:
:param cookies: Set cookies for the next request.
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
+ :param pierce_shadow: Include open Shadow DOM content in the response. Defaults to False.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
@@ -814,6 +822,7 @@ class ScraplingMCPServer:
wait_selector=wait_selector,
cookies=cookies,
network_idle=network_idle,
+ pierce_shadow=pierce_shadow,
wait_selector_state=wait_selector_state,
block_webrtc=block_webrtc,
allow_webgl=allow_webgl,
@@ -850,6 +859,7 @@ class ScraplingMCPServer:
allow_webgl: bool = True,
solve_cloudflare: bool = False,
additional_args: Optional[Dict] = None,
+ pierce_shadow: bool = False,
) -> List[ResponseModel]:
"""Use the stealthy fetcher to fetch a group of URLs at the same time, and for each page return a structured output of the result.
The only fetcher suitable for high-protection websites.
@@ -865,6 +875,7 @@ class ScraplingMCPServer:
:param cookies: Set cookies for the next request.
:param solve_cloudflare: Solves all types of the Cloudflare's Turnstile/Interstitial challenges before returning the response to you.
:param allow_webgl: Enabled by default. Disabling WebGL is not recommended as many WAFs now check if WebGL is enabled.
+ :param pierce_shadow: Include open Shadow DOM content in the response. Defaults to False.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the ` Response ` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
@@ -899,6 +910,7 @@ class ScraplingMCPServer:
hide_canvas=hide_canvas,
allow_webgl=allow_webgl,
network_idle=network_idle,
+ pierce_shadow=pierce_shadow,
block_webrtc=block_webrtc,
wait_selector=wait_selector,
google_search=google_search,
@@ -932,6 +944,7 @@ class ScraplingMCPServer:
extra_headers: Optional[Dict[str, str]] = None,
blocked_domains: Optional[Set[str]] = None,
solve_cloudflare: bool = False,
+ pierce_shadow: bool = False,
) -> ResponseModel:
"""Fetch a URL through a browser session previously opened with `open_session` and return a structured output of the result.
The session (dynamic or stealthy) holds the browser-level configuration; every option here applies to this request only, with the defaults shown.
@@ -944,6 +957,7 @@ class ScraplingMCPServer:
:param wait: The time (milliseconds) the fetcher will wait after everything finishes before closing the page and returning the `Response` object.
:param timeout: The timeout in milliseconds that is used in all operations and waits through the page.
:param google_search: Enabled by default, Scrapling will set a Google referer header.
+ :param pierce_shadow: Include open Shadow DOM content in the response. Defaults to False.
:param network_idle: Wait for the page until there are no network connections for at least 500 ms.
:param load_dom: Enabled by default, wait for all JavaScript on the page to fully load and execute.
:param disable_resources: Drop requests for unnecessary resources for a speed boost.
@@ -971,6 +985,7 @@ class ScraplingMCPServer:
timeout=timeout,
google_search=google_search,
network_idle=network_idle,
+ pierce_shadow=pierce_shadow,
load_dom=load_dom,
disable_resources=disable_resources,
wait_selector=wait_selector,
diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py
index adc3c99..709ae31 100644
--- a/tests/ai/test_ai_mcp.py
+++ b/tests/ai/test_ai_mcp.py
@@ -505,6 +505,7 @@ class TestSessionFetchForwarding:
"google_search": True,
"network_idle": False,
"load_dom": True,
+ "pierce_shadow": False,
"disable_resources": False,
"wait_selector": None,
"wait_selector_state": "attached",
@@ -934,6 +935,8 @@ class TestServerToolRegistration:
assert "session_id" not in props, f"{name} still exposes session_id"
assert props["timeout"]["default"] == 30000, f"{name} hides the real timeout default"
assert props["google_search"]["default"] is True
+ assert props["pierce_shadow"]["type"] == "boolean"
+ assert props["pierce_shadow"]["default"] is False
request_props = tools["make_request"].input_schema["properties"]
assert request_props["method"]["default"] == "GET"
@@ -948,6 +951,8 @@ class TestServerToolRegistration:
session_props = tools["session_fetch"].input_schema["properties"]
assert session_props["timeout"]["default"] == 30000
+ assert session_props["pierce_shadow"]["type"] == "boolean"
+ assert session_props["pierce_shadow"]["default"] is False
assert "solve_cloudflare" in session_props
assert set(tools["session_fetch"].input_schema["required"]) >= {"url", "session_id"}
@@ -991,3 +996,42 @@ class TestServerToolRegistration:
assert annotations[name].open_world_hint is True
assert annotations["list_sessions"].read_only_hint is True
assert annotations["list_sessions"].open_world_hint is False
+
+
[email protected]("enabled", [False, True])
[email protected]("tool", ["fetch", "bulk_fetch", "stealthy_fetch", "bulk_stealthy_fetch"])
+async def test_shadow_option_reaches_browser_session(monkeypatch, tool, enabled):
+ fake = _FakeStealthySession if "stealthy" in tool else _FakeDynamicSession
+ target = "AsyncStealthySession" if "stealthy" in tool else "AsyncDynamicSession"
+ monkeypatch.setattr("scrapling.core.ai." + target, fake)
+ server = ScraplingMCPServer()._build_server("127.0.0.1", 8000)
+ args: dict[str, Any] = (
+ {"urls": ["https://example.com"]} if tool.startswith("bulk_") else {"url": "https://example.com"}
+ )
+ async with Client(server) as client:
+ result = await client.call_tool(tool, {**args, "pierce_shadow": enabled})
+ assert not result.is_error
+ assert fake.instances[-1].kwargs["pierce_shadow"] is enabled
+ assert not fake.instances[-1]._is_alive
+
+
[email protected]("session_type", ["dynamic", "stealthy"])
+async def test_shadow_option_changes_per_session_request(monkeypatch, session_type):
+ fake = _FakeStealthySession if session_type == "stealthy" else _FakeDynamicSession
+ target = "AsyncStealthySession" if session_type == "stealthy" else "AsyncDynamicSession"
+ monkeypatch.setattr("scrapling.core.ai." + target, fake)
+ server = ScraplingMCPServer()._build_server("127.0.0.1", 8000)
+ async with Client(server) as client:
+ await client.call_tool("open_session", {"session_type": session_type, "session_id": "shadow-test"})
+ try:
+ for args, expected in [({"pierce_shadow": True}, True), ({"pierce_shadow": False}, False), ({}, False)]:
+ result = await client.call_tool(
+ "session_fetch", {"url": "https://example.com", "session_id": "shadow-test", **args}
+ )
+ assert not result.is_error
+ assert fake.instances[-1].fetch_calls[-1]["pierce_shadow"] is expected
+ finally:
+ await client.call_tool("close_session", {"session_id": "shadow-test"})
+ assert not fake.instances[-1]._is_alive
diff --git a/tests/cli/test_shadow_cli.py b/tests/cli/test_shadow_cli.py
new file mode 100644
index 0000000..097a5e2
--- /dev/null
+++ b/tests/cli/test_shadow_cli.py
@@ -0,0 +1,37 @@
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+from click.testing import CliRunner
+
+from scrapling.cli import main
+from scrapling.parser import Selector
+
+
[email protected](("command", "fetcher"), [("fetch", "DynamicFetcher"), ("stealthy-fetch", "StealthyFetcher")])
+ ("flags", "enabled"), [([], False), (["--pierce-shadow"], True), (["--no-pierce-shadow"], False)]
+)
+def test_shadow_option_reaches_fetcher(
+ tmp_path: Path, command: str, fetcher: str, flags: list[str], enabled: bool
+) -> None:
+ output = tmp_path / "page.txt"
+ with patch(f"scrapling.fetchers.{fetcher}.fetch", return_value=Selector("<p>Content</p>")) as fetch:
+ result = CliRunner().invoke(main, ["extract", command, "https://example.com", str(output), *flags])
+ assert result.exit_code == 0, result.output
+ assert fetch.call_args.kwargs["pierce_shadow"] is enabled
+ assert output.read_text() == "Content"
+
+
[email protected]("command", ["fetch", "stealthy-fetch"])
+def test_shadow_option_in_browser_help(command: str) -> None:
+ result = CliRunner().invoke(main, ["extract", command, "--help"])
+ assert result.exit_code == 0
+ assert "--pierce-shadow / --no-pierce-shadow" in result.output
+
+
[email protected]("command", ["get", "post", "put", "delete"])
+def test_shadow_option_not_in_http_help(command: str) -> None:
+ result = CliRunner().invoke(main, ["extract", command, "--help"])
+ assert result.exit_code == 0
+ assert "pierce-shadow" not in result.output