Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(mcp): Add auth to the mcp server and the option to set allowed host

Karim shoair563b134f9eac
diff --git a/docs/ai/mcp-server.md b/docs/ai/mcp-server.md
index a9810bd..ef23bde 100644
--- a/docs/ai/mcp-server.md
+++ b/docs/ai/mcp-server.md
@@ -218,6 +218,42 @@ Hence, the default value for the host the server is listening to is '0.0.0.0' an
scrapling mcp --http --host '127.0.0.1' --port 8000
```
+### Authentication
+
+The 'stdio' transport is only reachable by the program that started it, but the moment you switch to 'Streamable HTTP', anyone who can reach the port can call every tool, and that includes fetching any URL from the machine running the server. So if the server is listening on anything other than localhost, give it a token:
+```bash
+scrapling mcp --http --auth-token "$(openssl rand -hex 32)"
+```
+Clients then have to send that token in an `Authorization` header, and any request without it is rejected with a `401`:
+```json
+{
+ "mcpServers": {
+ "ScraplingServer": {
+ "url": "https://your-server.example.com/mcp",
+ "headers": {
+ "Authorization": "Bearer <your-token>"
+ }
+ }
+ }
+}
+```
+Passing the token on the command line leaves it in your shell history and in the process list, so prefer the `SCRAPLING_MCP_AUTH_TOKEN` environment variable:
+```bash
+export SCRAPLING_MCP_AUTH_TOKEN="<your-token>"
+scrapling mcp --http
+```
+When the server listens on a public address, you should also tell it which host names to accept, which turns on protection against DNS-rebinding attacks (a website your browser visits trying to talk to your server). The option can be repeated:
+```bash
+scrapling mcp --http --allowed-host 'your-server.example.com:8000'
+```
+
+!!! note "Notes:"
+
+ * Authentication applies to the 'Streamable HTTP' transport only. It's ignored with 'stdio', and the server logs a warning to tell you so.<br/>
+ * Plain HTTP sends the token in cleartext, so put the server behind a reverse proxy that terminates TLS before exposing it to the internet.<br/>
+ * This is a single shared key, not per-client credentials, so every client uses the same token, and rotating it means restarting the server.<br/>
+ * Starting the server with `--http` and no token still works for local use, but it logs a warning telling you that it's unauthenticated.
+
## Examples
Now we will show you some examples of prompts we used while testing the MCP server, but you are probably more creative than we are and better at prompt engineering than we are :)
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 0721d21..32a8243 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -164,11 +164,25 @@ def install(force): # pragma: no cover
default=None,
help="Path to a custom Chromium-compatible browser executable for browser-based MCP tools",
)
-def mcp(http, host, port, executable_path):
+@option(
+ "--auth-token",
+ type=str,
+ default=None,
+ help="Require clients to send this token as `Authorization: Bearer <token>` (streamable-http only). "
+ "Setting the SCRAPLING_MCP_AUTH_TOKEN environment variable instead keeps it out of the process list",
+)
+@option(
+ "--allowed-host",
+ type=str,
+ multiple=True,
+ help="Enable DNS-rebinding protection and accept requests for this host only, like 'mcp.example.com:8000' "
+ "(repeatable). Recommended whenever the server listens on a public address",
+)
+def mcp(http, host, port, executable_path, auth_token, allowed_host):
from scrapling.core.ai import ScraplingMCPServer
- server = ScraplingMCPServer(executable_path=executable_path)
- server.serve(http, host, port)
+ server = ScraplingMCPServer(executable_path=executable_path, auth_token=auth_token)
+ server.serve(http, host, port, allowed_hosts=allowed_host)
@command(help="Interactive scraping console")
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index 7f87a2c..81d2b32 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -1,13 +1,18 @@
from uuid import uuid4
from os import environ
+from hmac import compare_digest
from asyncio import gather
from datetime import datetime, timezone
from dataclasses import dataclass, field
from mcp.server.fastmcp import FastMCP, Image
+from mcp.server.auth.provider import AccessToken, TokenVerifier
+from mcp.server.auth.settings import AuthSettings
+from mcp.server.transport_security import TransportSecuritySettings
from mcp.types import ImageContent, TextContent
-from pydantic import BaseModel, Field
+from pydantic import AnyHttpUrl, BaseModel, Field
+from scrapling.core.utils import log
from scrapling.core.shell import Convertor, _CONTROL_CHARS_PATTERN
from scrapling.engines.toolbelt.custom import Response as _ScraplingResponse
from scrapling.engines.static import ImpersonateType
@@ -35,6 +40,7 @@ from scrapling.core._types import (
SessionType = Literal["dynamic", "stealthy"]
ScreenshotType = Literal["png", "jpeg"]
MCP_EXECUTABLE_PATH_ENV = "SCRAPLING_EXECUTABLE_PATH"
+MCP_AUTH_TOKEN_ENV = "SCRAPLING_MCP_AUTH_TOKEN" # nosec B105 - the name of the variable, not a token
class ResponseModel(BaseModel):
@@ -107,15 +113,31 @@ def _normalize_credentials(credentials: Optional[Dict[str, str]]) -> Optional[Tu
return username, password
+class _StaticTokenVerifier(TokenVerifier):
+ """Verifies requests against a single shared bearer token."""
+
+ def __init__(self, token: str):
+ self._token = token.encode()
+
+ async def verify_token(self, token: str) -> Optional[AccessToken]:
+ if compare_digest(token.encode(), self._token):
+ return AccessToken(token=token, client_id="scrapling-mcp", scopes=[])
+ return None
+
+
class ScraplingMCPServer:
- def __init__(self, executable_path: Optional[str] = None):
+ def __init__(self, executable_path: Optional[str] = None, auth_token: Optional[str] = None):
"""Create a Scrapling MCP server.
:param executable_path: Optional global Chromium-compatible browser executable path for browser tools.
If omitted, the SCRAPLING_EXECUTABLE_PATH environment variable is used when set.
+ :param auth_token: Optional shared token that clients must send as `Authorization: Bearer <token>`.
+ If omitted, the SCRAPLING_MCP_AUTH_TOKEN environment variable is used when set. It only applies
+ to the streamable-http transport.
"""
self._sessions: Dict[str, _SessionEntry] = {}
self._executable_path = executable_path or environ.get(MCP_EXECUTABLE_PATH_ENV) or None
+ self._auth_token = auth_token or environ.get(MCP_AUTH_TOKEN_ENV) or None
def _resolve_executable_path(self, executable_path: Optional[str]) -> Optional[str]:
"""Return a per-call executable path or the server-wide default."""
@@ -902,9 +924,22 @@ class ScraplingMCPServer:
return [_translate_response(page, extraction_type, css_selector, main_content_only) for page in responses]
- def serve(self, http: bool, host: str, port: int):
- """Serve the MCP server."""
- server = FastMCP(name="Scrapling", host=host, port=port)
+ def _build_server(self, host: str, port: int, allowed_hosts: Sequence[str] = ()) -> FastMCP:
+ """Build the FastMCP server with all tools registered and the optional security settings applied."""
+ settings: Dict[str, Any] = {}
+ if self._auth_token:
+ base_url = AnyHttpUrl(f"http://{host}:{port}")
+ settings["token_verifier"] = _StaticTokenVerifier(self._auth_token)
+ settings["auth"] = AuthSettings(issuer_url=base_url, resource_server_url=base_url)
+
+ if allowed_hosts:
+ settings["transport_security"] = TransportSecuritySettings(
+ enable_dns_rebinding_protection=True,
+ allowed_hosts=list(allowed_hosts),
+ allowed_origins=[f"{scheme}://{host_}" for host_ in allowed_hosts for scheme in ("http", "https")],
+ )
+
+ server = FastMCP(name="Scrapling", host=host, port=port, **settings)
# Session management tools
server.add_tool(self.open_session, title="open_session", structured_output=True)
server.add_tool(self.close_session, title="close_session", structured_output=True)
@@ -932,4 +967,19 @@ class ScraplingMCPServer:
)
# Screenshot tool (returns image + url content blocks, not structured JSON)
server.add_tool(self.screenshot, title="screenshot", description=self.screenshot.__doc__)
- server.run(transport="stdio" if not http else "streamable-http")
+ return server
+
+ def serve(self, http: bool, host: str, port: int, allowed_hosts: Sequence[str] = ()):
+ """Serve the MCP server."""
+ if http and not self._auth_token:
+ log.warning(
+ f"The MCP server is running over HTTP without authentication, so anyone who can reach "
+ f"{host}:{port} can use every tool, including fetching arbitrary URLs from this machine. "
+ f"Pass `--auth-token` (or set the {MCP_AUTH_TOKEN_ENV} environment variable) to require a bearer token."
+ )
+ elif self._auth_token and not http:
+ log.warning(
+ "The authentication token only applies to the streamable-http transport, so it's ignored with stdio."
+ )
+
+ self._build_server(host, port, allowed_hosts).run(transport="stdio" if not http else "streamable-http")
diff --git a/tests/ai/test_ai_mcp.py b/tests/ai/test_ai_mcp.py
index 03395c5..afc9d36 100644
--- a/tests/ai/test_ai_mcp.py
+++ b/tests/ai/test_ai_mcp.py
@@ -11,12 +11,14 @@ from mcp.types import ImageContent, TextContent
from scrapling.parser import Selector
from scrapling.core.ai import (
+ MCP_AUTH_TOKEN_ENV,
ScraplingMCPServer,
ResponseModel,
SessionInfo,
SessionCreatedModel,
SessionClosedModel,
_normalize_credentials,
+ _StaticTokenVerifier,
_translate_response,
)
@@ -448,3 +450,80 @@ class TestNormalizeCredentials:
def test_missing_username_raises(self):
with pytest.raises(ValueError, match="username"):
_normalize_credentials({"password": "pass"})
+
+
+SHARED_KEY = "s3cret"
+UNICODE_KEY = "ünïcode-tökén"
+
+
+class TestStaticTokenVerifier:
+ """Test the shared bearer token verifier"""
+
+ @pytest.mark.asyncio
+ async def test_correct_token_is_accepted(self):
+ result = await _StaticTokenVerifier(SHARED_KEY).verify_token(SHARED_KEY)
+
+ assert result is not None
+ assert result.token == SHARED_KEY
+ assert result.scopes == []
+ assert result.expires_at is None
+
+ @pytest.mark.asyncio
+ async def test_wrong_tokens_are_rejected(self):
+ verifier = _StaticTokenVerifier(SHARED_KEY)
+
+ for token in ("", "wrong", "s3cre", "s3cret ", "S3CRET"):
+ assert await verifier.verify_token(token) is None
+
+ @pytest.mark.asyncio
+ async def test_non_ascii_token(self):
+ """Tokens are compared as bytes, so non-ASCII characters must not raise"""
+ verifier = _StaticTokenVerifier(UNICODE_KEY)
+
+ assert await verifier.verify_token(UNICODE_KEY) is not None
+ assert await verifier.verify_token("unicode-token") is None
+
+
+class TestMCPServerAuthentication:
+ """Test how the authentication token and transport security reach the FastMCP server"""
+
+ def test_no_token_leaves_auth_disabled(self, monkeypatch):
+ monkeypatch.delenv(MCP_AUTH_TOKEN_ENV, raising=False)
+ server = ScraplingMCPServer()
+
+ assert server._auth_token is None
+ assert server._build_server("127.0.0.1", 8000).settings.auth is None
+
+ def test_token_enables_auth(self, monkeypatch):
+ monkeypatch.delenv(MCP_AUTH_TOKEN_ENV, raising=False)
+ built = ScraplingMCPServer(auth_token=SHARED_KEY)._build_server("127.0.0.1", 8000)
+
+ assert built.settings.auth is not None
+ assert str(built.settings.auth.issuer_url) == "http://127.0.0.1:8000/"
+ assert str(built.settings.auth.resource_server_url) == "http://127.0.0.1:8000/"
+
+ def test_token_read_from_environment(self, monkeypatch):
+ env_key, explicit_key = "from-env", "explicit"
+ monkeypatch.setenv(MCP_AUTH_TOKEN_ENV, env_key)
+
+ assert ScraplingMCPServer()._auth_token == env_key
+ assert ScraplingMCPServer(auth_token=explicit_key)._auth_token == explicit_key
+
+ def test_all_tools_are_registered_with_auth_enabled(self, monkeypatch):
+ """FastMCP raises when `auth` and `token_verifier` are mismatched, so building must stay valid"""
+ monkeypatch.delenv(MCP_AUTH_TOKEN_ENV, raising=False)
+ built = ScraplingMCPServer(auth_token=SHARED_KEY)._build_server("0.0.0.0", 8000)
+
+ assert len(built._tool_manager.list_tools()) == 10
+
+ def test_allowed_hosts_enable_dns_rebinding_protection(self, monkeypatch):
+ monkeypatch.delenv(MCP_AUTH_TOKEN_ENV, raising=False)
+ server = ScraplingMCPServer()
+
+ assert server._build_server("0.0.0.0", 8000).settings.transport_security is None
+
+ security = server._build_server("0.0.0.0", 8000, ("mcp.example.com:8000",)).settings.transport_security
+ assert security is not None
+ assert security.enable_dns_rebinding_protection is True
+ assert security.allowed_hosts == ["mcp.example.com:8000"]
+ assert security.allowed_origins == ["http://mcp.example.com:8000", "https://mcp.example.com:8000"]
diff --git a/tests/cli/test_cli.py b/tests/cli/test_cli.py
index 42d0d4d..f9bdad4 100644
--- a/tests/cli/test_cli.py
+++ b/tests/cli/test_cli.py
@@ -55,8 +55,8 @@ class TestCLI:
result = runner.invoke(mcp)
assert result.exit_code == 0
- mock_server.assert_called_once_with(executable_path=None)
- mock_instance.serve.assert_called_once_with(False, "0.0.0.0", 8000)
+ mock_server.assert_called_once_with(executable_path=None, auth_token=None)
+ mock_instance.serve.assert_called_once_with(False, "0.0.0.0", 8000, allowed_hosts=())
def test_mcp_command_with_executable_path(self, runner):
"""Test MCP command with a custom browser executable"""
@@ -66,8 +66,34 @@ class TestCLI:
result = runner.invoke(mcp, ["--executable-path", "/opt/custom-chromium"])
assert result.exit_code == 0
- mock_server.assert_called_once_with(executable_path="/opt/custom-chromium")
- mock_instance.serve.assert_called_once_with(False, "0.0.0.0", 8000)
+ mock_server.assert_called_once_with(executable_path="/opt/custom-chromium", auth_token=None)
+ mock_instance.serve.assert_called_once_with(False, "0.0.0.0", 8000, allowed_hosts=())
+
+ def test_mcp_command_with_auth_token(self, runner):
+ """Test MCP command with a shared authentication token"""
+ with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
+ mock_instance = MagicMock()
+ mock_server.return_value = mock_instance
+ shared_key = "s3cret"
+
+ result = runner.invoke(mcp, ["--http", "--auth-token", shared_key])
+ assert result.exit_code == 0
+ mock_server.assert_called_once_with(executable_path=None, auth_token=shared_key)
+ mock_instance.serve.assert_called_once_with(True, "0.0.0.0", 8000, allowed_hosts=())
+
+ def test_mcp_command_with_allowed_hosts(self, runner):
+ """Test MCP command with repeated allowed hosts"""
+ with patch("scrapling.core.ai.ScraplingMCPServer") as mock_server:
+ mock_instance = MagicMock()
+ mock_server.return_value = mock_instance
+
+ result = runner.invoke(
+ mcp, ["--http", "--allowed-host", "mcp.example.com:8000", "--allowed-host", "127.0.0.1:8000"]
+ )
+ assert result.exit_code == 0
+ mock_instance.serve.assert_called_once_with(
+ True, "0.0.0.0", 8000, allowed_hosts=("mcp.example.com:8000", "127.0.0.1:8000")
+ )
def test_extract_get_command(self, runner, tmp_path, html_url):
"""Test extract `get` command"""