Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(cdp): Make browsers accept CDP urls with http protocol

Karim shoair9ecffdb42f40
diff --git a/docs/ai/mcp-server.md b/docs/ai/mcp-server.md
index a21e884..a9810bd 100644
--- a/docs/ai/mcp-server.md
+++ b/docs/ai/mcp-server.md
@@ -183,8 +183,27 @@ In a Claude Desktop configuration, add the option to the server arguments:
You can also set the `SCRAPLING_EXECUTABLE_PATH` environment variable before starting the server. Tool calls can still pass `executable_path` directly when a single request or session needs a different browser executable.
+### Connecting to Remote Browsers
+
+`open_session` doesn't have to launch a browser locally. Pass a CDP url and it will connect to an already-running browser through the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/), whether that browser is on the same machine, another host, or a managed browser provider:
+```
+Open a stealthy browser session on wss://cdp.provider.example/session/abc123, then use it to scrape the product details from https://shop.example.com. Close the session when you're done.
+```
+Both session types (`dynamic` and `stealthy`) accept it, and the `session_id` you get back is used with the fetch and screenshot tools as usual.
+
+The URL can be a WebSocket endpoint (`ws://`/`wss://`), which is what managed browser providers hand out, or the HTTP endpoint of a browser you started yourself with the remote debugging port enabled:
+```commandline
+chrome --remote-debugging-port=9222
+```
+That one is reached with `cdp_url="http://localhost:9222"`, or with the host's address if the browser is running on another machine.
+
+!!! note "Notes:"
+
+ * The browser is already running, so options that only apply while launching one are ignored for CDP sessions: `headless`, `real_chrome`, and `executable_path` (including the server-wide default above).<br/>
+ * Everything else still applies (`locale`, `useragent`, `proxy`, `cookies`, `timezone_id`, and so on), as each session creates its own browser context on the remote browser.
+
### Streamable HTTP
-As per version 0.3.6, we have added the ability to make the MCP server use the 'Streamable HTTP' transport mode instead of the traditional 'stdio' transport.
+Since version 0.3.6, we have added the ability to make the MCP server use the 'Streamable HTTP' transport mode instead of the traditional 'stdio' transport.
So instead of using the following command (the 'stdio' one):
```bash
diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md
index 49b0877..1c78a77 100644
--- a/docs/fetching/dynamic.md
+++ b/docs/fetching/dynamic.md
@@ -52,6 +52,8 @@ DynamicFetcher.fetch('https://example.com', cdp_url='ws://localhost:9222')
```
Instead of launching a browser locally (Chromium/Google Chrome), you can connect to a remote browser through the [Chrome DevTools Protocol](https://chromedevtools.github.io/devtools-protocol/).
+The URL can be a WebSocket endpoint (`ws://`/`wss://`), which is what managed browser providers hand out, or the HTTP endpoint of a browser (ex: `http://localhost:9222`).
+
!!! note "Notes:"
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index 76ce660..7f7e0bd 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -41,8 +41,8 @@ def _is_invalid_file_path(value: str, label: str = "Init script") -> bool | str:
@lru_cache(2)
def _is_invalid_cdp_url(cdp_url: str) -> bool | str:
"""Fast CDP URL validation"""
- if not cdp_url.startswith(("ws://", "wss://")):
- return "CDP URL must use 'ws://' or 'wss://' scheme"
+ if not cdp_url.startswith(("ws://", "wss://", "http://", "https://")):
+ return "CDP URL must use 'ws://', 'wss://', 'http://', or 'https://' scheme"
netloc = urlparse(cdp_url).netloc
if not netloc: # pragma: no cover
diff --git a/tests/fetchers/test_validator.py b/tests/fetchers/test_validator.py
index a682f9e..86ecf45 100644
--- a/tests/fetchers/test_validator.py
+++ b/tests/fetchers/test_validator.py
@@ -11,12 +11,7 @@ class TestValidators:
def test_playwright_config_valid(self):
"""Test valid PlaywrightConfig"""
- params = {
- "max_pages": 2,
- "headless": True,
- "timeout": 30000,
- "proxy": "http://proxy.example.com:8080"
- }
+ params = {"max_pages": 2, "headless": True, "timeout": 30000, "proxy": "http://proxy.example.com:8080"}
config = validate(params, PlaywrightConfig)
@@ -51,14 +46,23 @@ class TestValidators:
with pytest.raises(TypeError):
validate(params, PlaywrightConfig)
+ for cdp_url in ("ftp://localhost:9222", "localhost:9222", "http://"):
+ with pytest.raises(TypeError):
+ validate({"cdp_url": cdp_url}, PlaywrightConfig)
+
+ def test_playwright_config_cdp_url_schemes(self):
+ """Test PlaywrightConfig accepts both WebSocket and HTTP CDP endpoints"""
+ for cdp_url in (
+ "ws://localhost:9222",
+ "wss://cdp.example.com/session",
+ "http://localhost:9222",
+ "https://cdp.example.com",
+ ):
+ assert validate({"cdp_url": cdp_url}, PlaywrightConfig).cdp_url == cdp_url
+
def test_stealth_config_valid(self):
"""Test valid StealthConfig"""
- params = {
- "max_pages": 1,
- "headless": True,
- "solve_cloudflare": False,
- "timeout": 30000
- }
+ params = {"max_pages": 1, "headless": True, "solve_cloudflare": False, "timeout": 30000}
config = validate(params, StealthConfig)
@@ -71,7 +75,7 @@ class TestValidators:
"""Test StealthConfig timeout adjustment for Cloudflare"""
params = {
"solve_cloudflare": True,
- "timeout": 10000 # Less than the required 60,000
+ "timeout": 10000, # Less than the required 60,000
}
config = validate(params, StealthConfig)