🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat: add MCP executable path support
Sami Rusanid933c947c687
diff --git a/docs/ai/mcp-server.md b/docs/ai/mcp-server.mdindex 0ab72b5..a21e884 100644--- a/docs/ai/mcp-server.md+++ b/docs/ai/mcp-server.md@@ -154,6 +154,35 @@ Here's the main article from Anthropic on [how to add MCP servers to Claude codeThen, after you've added the server, you need to completely quit and restart the app you used above. In Claude Desktop, you should see an MCP server indicator (🔧) in the bottom-right corner of the chat input or see `ScraplingServer` in the `Search and tools` dropdown in the chat input box.+### Custom Browser Executable++Browser-based tools (`fetch`, `bulk_fetch`, `stealthy_fetch`, `bulk_stealthy_fetch`, and `open_session`) can use a custom Chromium-compatible browser executable instead of the bundled Chromium. This is useful for custom browser builds or lightweight browser engines.++To configure it once for the whole MCP server, pass the executable path when starting the server:++```bash+scrapling mcp --executable-path "/path/to/chromium"+```++In a Claude Desktop configuration, add the option to the server arguments:++```json+{+ "mcpServers": {+ "ScraplingServer": {+ "command": "/Users/<MyUsername>/.venv/bin/scrapling",+ "args": [+ "mcp",+ "--executable-path",+ "/path/to/chromium"+ ]+ }+ }+}+```++You can also set the `SCRAPLING_EXECUTABLE_PATH` environment variable before starting the server. Tool calls can still pass `executable_path` directly when a single request or session needs a different browser executable.+### Streamable HTTPAs per version 0.3.6, we have added the ability to make the MCP server use the 'Streamable HTTP' transport mode instead of the traditional 'stdio' transport.@@ -356,4 +385,4 @@ This protection runs automatically on all MCP tool responses. Keep `main_content----*Built with ❤️ by the Scrapling team. Happy scraping!*\ No newline at end of file+*Built with ❤️ by the Scrapling team. Happy scraping!*diff --git a/docs/api-reference/mcp-server.md b/docs/api-reference/mcp-server.mdindex 045018a..952f15c 100644--- a/docs/api-reference/mcp-server.md+++ b/docs/api-reference/mcp-server.md@@ -22,6 +22,12 @@ server = ScraplingMCPServer()server.serve(http=False, host="0.0.0.0", port=8000)```+To set a custom Chromium-compatible browser executable for browser-based MCP tools, pass `executable_path`:++```python+server = ScraplingMCPServer(executable_path="/path/to/chromium")+```+## Response ModelThe standardized response structure that's returned by all MCP server tools:@@ -52,4 +58,4 @@ The main MCP server class that provides all web scraping tools:## ::: scrapling.core.ai.ScraplingMCPServerhandler: python- :docstring:\ No newline at end of file+ :docstring:diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 4b9eb86..942848f 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -157,10 +157,16 @@ def install(force): # pragma: no cover@option("--port", type=int, default=8000, help="The port to use if streamable-http transport is enabled (Default: 8000)")-def mcp(http, host, port):+@option(+ "--executable-path",+ type=str,+ default=None,+ help="Path to a custom Chromium-compatible browser executable for browser-based MCP tools",+)+def mcp(http, host, port, executable_path):from scrapling.core.ai import ScraplingMCPServer- server = ScraplingMCPServer()+ server = ScraplingMCPServer(executable_path=executable_path)server.serve(http, host, port)diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex 464d35b..82d9113 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -1,4 +1,5 @@from uuid import uuid4+from os import environfrom asyncio import gatherfrom datetime import datetime, timezonefrom dataclasses import dataclass, field@@ -33,6 +34,7 @@ from scrapling.core._types import (SessionType = Literal["dynamic", "stealthy"]ScreenshotType = Literal["png", "jpeg"]+MCP_EXECUTABLE_PATH_ENV = "SCRAPLING_EXECUTABLE_PATH"class ResponseModel(BaseModel):@@ -105,8 +107,18 @@ def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tuclass ScraplingMCPServer:- def __init__(self):+ def __init__(self, executable_path: Optional[str] = None):+ """Create a Scrapling MCP server.++ :param executable_path: Optional global Chromium-compatible browser executable path for browser tools.+ If omitted, the SCRAPLING_EXECUTABLE_PATH environment variable is used when set.+ """self._sessions: Dict[str, _SessionEntry] = {}+ self._executable_path = executable_path or environ.get(MCP_EXECUTABLE_PATH_ENV) or None++ def _resolve_executable_path(self, executable_path: Optional[str]) -> Optional[str]:+ """Return a per-call executable path or the server-wide default."""+ return executable_path or self._executable_pathdef _get_session(self, session_id: str, expected_type: Optional[SessionType]) -> _SessionEntry:"""Look up a session by ID, optionally validating its type. Pass `None` to skip the type check."""@@ -136,6 +148,7 @@ class ScraplingMCPServer:extra_headers: Optional[Dict[str, str]] = None,useragent: Optional[str] = None,cdp_url: Optional[str] = None,+ executable_path: Optional[str] = None,timeout: int | float = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,@@ -166,6 +179,7 @@ class ScraplingMCPServer::param extra_headers: A dictionary of extra headers to add to the request.:param useragent: Pass a useragent string to be used. Otherwise the fetcher will generate a real Useragent of the same browser and use it.:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.+ :param executable_path: Absolute path to a custom Chromium-compatible browser executable. Overrides the server-wide default for this session.:param timeout: The timeout in milliseconds that is used in all operations and waits through the page. The default is 30,000.:param disable_resources: Drop requests for unnecessary resources for a speed boost.:param wait_selector: Wait for a specific CSS selector to be in a specific state.@@ -202,6 +216,7 @@ class ScraplingMCPServer:wait_selector=wait_selector,google_search=google_search,extra_headers=extra_headers,+ executable_path=self._resolve_executable_path(executable_path),disable_resources=disable_resources,wait_selector_state=wait_selector_state,)@@ -494,6 +509,7 @@ class ScraplingMCPServer:extra_headers: Optional[Dict[str, str]] = None,useragent: Optional[str] = None,cdp_url: Optional[str] = None,+ executable_path: Optional[str] = None,timeout: int | float = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,@@ -530,6 +546,7 @@ class ScraplingMCPServer::param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.+ :param executable_path: Absolute path to a custom Chromium-compatible browser executable. Overrides the server-wide default for this request.:param google_search: Enabled by default, Scrapling will set a Google referer header.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.@@ -550,6 +567,7 @@ class ScraplingMCPServer:extra_headers=extra_headers,useragent=useragent,cdp_url=cdp_url,+ executable_path=executable_path,timeout=timeout,disable_resources=disable_resources,wait_selector=wait_selector,@@ -576,6 +594,7 @@ class ScraplingMCPServer:extra_headers: Optional[Dict[str, str]] = None,useragent: Optional[str] = None,cdp_url: Optional[str] = None,+ executable_path: Optional[str] = None,timeout: int | float = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,@@ -612,6 +631,7 @@ class ScraplingMCPServer::param wait_selector_state: The state to wait for the selector given with `wait_selector`. The default state is `attached`.:param real_chrome: If you have a Chrome browser installed on your device, enable this, and the Fetcher will launch an instance of your browser and use it.:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.+ :param executable_path: Absolute path to a custom Chromium-compatible browser executable. Overrides the server-wide default for this request.:param google_search: Enabled by default, Scrapling will set a Google referer header.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.@@ -653,6 +673,7 @@ class ScraplingMCPServer:wait_selector=wait_selector,google_search=google_search,extra_headers=extra_headers,+ executable_path=self._resolve_executable_path(executable_path),disable_resources=disable_resources,wait_selector_state=wait_selector_state,) as session:@@ -678,6 +699,7 @@ class ScraplingMCPServer:useragent: Optional[str] = None,hide_canvas: bool = False,cdp_url: Optional[str] = None,+ executable_path: Optional[str] = None,timeout: int | float = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,@@ -722,6 +744,7 @@ class ScraplingMCPServer::param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.:param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak.:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.+ :param executable_path: Absolute path to a custom Chromium-compatible browser executable. Overrides the server-wide default for this request.:param google_search: Enabled by default, Scrapling will set a Google referer header.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.@@ -744,6 +767,7 @@ class ScraplingMCPServer:useragent=useragent,hide_canvas=hide_canvas,cdp_url=cdp_url,+ executable_path=executable_path,timeout=timeout,disable_resources=disable_resources,wait_selector=wait_selector,@@ -775,6 +799,7 @@ class ScraplingMCPServer:useragent: Optional[str] = None,hide_canvas: bool = False,cdp_url: Optional[str] = None,+ executable_path: Optional[str] = None,timeout: int | float = 30000,disable_resources: bool = False,wait_selector: Optional[str] = None,@@ -819,6 +844,7 @@ class ScraplingMCPServer::param hide_canvas: Add random noise to canvas operations to prevent fingerprinting.:param block_webrtc: Forces WebRTC to respect proxy settings to prevent local IP address leak.:param cdp_url: Instead of launching a new browser instance, connect to this CDP URL to control real browsers through CDP.+ :param executable_path: Absolute path to a custom Chromium-compatible browser executable. Overrides the server-wide default for this request.:param google_search: Enabled by default, Scrapling will set a Google referer header.:param extra_headers: A dictionary of extra headers to add to the request. _The referer set by `google_search` takes priority over the referer set here if used together._:param proxy: The proxy to be used with requests, it can be a string or a dictionary with the keys 'server', 'username', and 'password' only.@@ -864,6 +890,7 @@ class ScraplingMCPServer:wait_selector=wait_selector,google_search=google_search,extra_headers=extra_headers,+ executable_path=self._resolve_executable_path(executable_path),additional_args=additional_args,solve_cloudflare=solve_cloudflare,disable_resources=disable_resources,diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.pyindex 19e2f02..3043a76 100644--- a/tests/ai/test_ai_mcp.py+++ b/tests/ai/test_ai_mcp.py@@ -1,5 +1,6 @@import base64import struct+from typing import Anyfrom contextlib import contextmanagerfrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom threading import Thread@@ -16,6 +17,48 @@ from scrapling.core.ai import (SessionClosedModel,_normalize_credentials,)+from scrapling.engines.toolbelt.custom import Response+++class _FakeAsyncBrowserSession:+ instances: list["_FakeAsyncBrowserSession"] = []++ def __init__(self, **kwargs: Any) -> None:+ self.kwargs = kwargs+ self._is_alive = False+ type(self).instances.append(self)++ async def __aenter__(self):+ self._is_alive = True+ return self++ async def __aexit__(self, exc_type: Any, exc: Any, tb: Any) -> None:+ self._is_alive = False++ async def start(self) -> None:+ self._is_alive = True++ async def close(self) -> None:+ self._is_alive = False++ async def fetch(self, url: str, **kwargs: Any) -> Response:+ return Response(+ url=url,+ content="<html><body>ok</body></html>",+ status=200,+ reason="OK",+ cookies={},+ headers={},+ request_headers={},+ )+++class _FakeDynamicSession(_FakeAsyncBrowserSession):+ instances = []+++class _FakeStealthySession(_FakeAsyncBrowserSession):+ instances = []@pytest_httpbin.use_class_based_httpbin@@ -204,6 +247,60 @@ class TestSessionManagement:await server.close_session("dupe")+class TestExecutablePath:+ """Test custom browser executable path plumbing in the MCP browser tools"""++ @pytest.fixture(autouse=True)+ def reset_fakes(self):+ _FakeDynamicSession.instances = []+ _FakeStealthySession.instances = []++ @pytest.mark.asyncio+ async def test_open_session_passes_executable_path(self, monkeypatch):+ """open_session forwards per-session executable_path to the dynamic session"""+ monkeypatch.setattr("scrapling.core.ai.AsyncDynamicSession", _FakeDynamicSession)+ server = ScraplingMCPServer()++ created = await server.open_session(session_type="dynamic", executable_path="/tmp/chrome")++ assert _FakeDynamicSession.instances[0].kwargs["executable_path"] == "/tmp/chrome"+ await server.close_session(created.session_id)++ @pytest.mark.asyncio+ async def test_open_session_uses_environment_default(self, monkeypatch):+ """open_session uses SCRAPLING_EXECUTABLE_PATH when no per-call value is provided"""+ monkeypatch.setenv("SCRAPLING_EXECUTABLE_PATH", "/opt/custom-chromium")+ monkeypatch.setattr("scrapling.core.ai.AsyncStealthySession", _FakeStealthySession)+ server = ScraplingMCPServer()++ created = await server.open_session(session_type="stealthy")++ assert _FakeStealthySession.instances[0].kwargs["executable_path"] == "/opt/custom-chromium"+ await server.close_session(created.session_id)++ @pytest.mark.asyncio+ async def test_fetch_overrides_global_executable_path(self, monkeypatch):+ """fetch forwards a per-call executable_path instead of the server default"""+ monkeypatch.setattr("scrapling.core.ai.AsyncDynamicSession", _FakeDynamicSession)+ server = ScraplingMCPServer(executable_path="/opt/default-chromium")++ result = await server.fetch(url="https://example.com", executable_path="/opt/request-chromium")++ assert isinstance(result, ResponseModel)+ assert _FakeDynamicSession.instances[0].kwargs["executable_path"] == "/opt/request-chromium"++ @pytest.mark.asyncio+ async def test_stealthy_fetch_uses_global_executable_path(self, monkeypatch):+ """stealthy_fetch forwards the server executable_path default"""+ monkeypatch.setattr("scrapling.core.ai.AsyncStealthySession", _FakeStealthySession)+ server = ScraplingMCPServer(executable_path="/opt/default-chromium")++ result = await server.stealthy_fetch(url="https://example.com")++ assert isinstance(result, ResponseModel)+ assert _FakeStealthySession.instances[0].kwargs["executable_path"] == "/opt/default-chromium"++def _png_height(data: bytes) -> int:"""Read the height field from a PNG IHDR chunk."""return struct.unpack(">I", data[20:24])[0]diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.pyindex 642699d..ae60e53 100644--- a/tests/cli/test_cli.py+++ b/tests/cli/test_cli.py@@ -57,7 +57,19 @@ class TestCLI:result = runner.invoke(mcp)assert result.exit_code == 0- mock_instance.serve.assert_called_once()+ mock_server.assert_called_once_with(executable_path=None)+ mock_instance.serve.assert_called_once_with(False, "0.0.0.0", 8000)++ def test_mcp_command_with_executable_path(self, runner):+ """Test MCP command with a custom browser executable"""+ with patch('scrapling.core.ai.ScraplingMCPServer') as mock_server:+ mock_instance = MagicMock()+ mock_server.return_value = mock_instance++ result = runner.invoke(mcp, ['--executable-path', '/opt/custom-chromium'])+ assert result.exit_code == 0+ mock_server.assert_called_once_with(executable_path='/opt/custom-chromium')+ mock_instance.serve.assert_called_once_with(False, "0.0.0.0", 8000)def test_extract_get_command(self, runner, tmp_path, html_url):"""Test extract `get` command"""