Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat: Make mcp able to use http transport

Karim shoair4a661b487535
diff --git a/scrapling/cli.py b/scrapling/cli.py
index 6f9487d..44fcd33 100644
--- a/scrapling/cli.py
+++ b/scrapling/cli.py
@@ -136,10 +136,26 @@ def install(force): # pragma: no cover
@command(help="Run Scrapling's MCP server (Check the docs for more info).")
-def mcp():
+@option(
+ "--http",
+ type=bool,
+ default=False,
+ help="Whether to run the MCP server in streamable-http transport or leave it as stdio (Default: False)",
+)
+@option(
+ "--host",
+ type=str,
+ default="0.0.0.0",
+ help="The host to use if streamable-http transport is enabled (Default: '0.0.0.0')",
+)
+@option(
+ "--port", type=int, default=8000, help="The port to use if streamable-http transport is enabled (Default: 8000)"
+)
+def mcp(http, host, port):
from scrapling.core.ai import ScraplingMCPServer
- ScraplingMCPServer().serve()
+ server = ScraplingMCPServer(host, port)
+ server.run(transport="stdio" if not http else "streamable-http")
@command(help="Interactive scraping console")
diff --git a/scrapling/core/ai.py b/scrapling/core/ai.py
index ae517fa..26eb564 100644
--- a/scrapling/core/ai.py
+++ b/scrapling/core/ai.py
@@ -41,10 +41,9 @@ def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingResp
return ResponseModel(status=page.status, content=[result for result in content], url=page.url)
-class ScraplingMCPServer:
- _server = FastMCP(name="Scrapling")
+def ScraplingMCPServer(host: str, port: int) -> FastMCP:
+ _server = FastMCP(name="Scrapling", host=host, port=port)
- @staticmethod
@_server.tool()
def get(
url: str,
@@ -123,7 +122,6 @@ class ScraplingMCPServer:
page,
)
- @staticmethod
@_server.tool()
async def bulk_get(
urls: Tuple[str, ...],
@@ -210,7 +208,6 @@ class ScraplingMCPServer:
for page in responses
]
- @staticmethod
@_server.tool()
async def fetch(
url: str,
@@ -299,7 +296,6 @@ class ScraplingMCPServer:
page,
)
- @staticmethod
@_server.tool()
async def bulk_fetch(
urls: Tuple[str, ...],
@@ -393,7 +389,6 @@ class ScraplingMCPServer:
for page in responses
]
- @staticmethod
@_server.tool()
async def stealthy_fetch(
url: str,
@@ -493,7 +488,6 @@ class ScraplingMCPServer:
page,
)
- @staticmethod
@_server.tool()
async def bulk_stealthy_fetch(
urls: Tuple[str, ...],
@@ -598,6 +592,4 @@ class ScraplingMCPServer:
for page in responses
]
- def serve(self):
- """Serve the MCP server."""
- self._server.run(transport="stdio")
+ return _server