🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat: Make mcp able to use http transport
Karim shoair4a661b487535
diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 6f9487d..44fcd33 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -136,10 +136,26 @@ def install(force): # pragma: no cover@command(help="Run Scrapling's MCP server (Check the docs for more info).")-def mcp():+@option(+ "--http",+ type=bool,+ default=False,+ help="Whether to run the MCP server in streamable-http transport or leave it as stdio (Default: False)",+)+@option(+ "--host",+ type=str,+ default="0.0.0.0",+ help="The host to use if streamable-http transport is enabled (Default: '0.0.0.0')",+)+@option(+ "--port", type=int, default=8000, help="The port to use if streamable-http transport is enabled (Default: 8000)"+)+def mcp(http, host, port):from scrapling.core.ai import ScraplingMCPServer- ScraplingMCPServer().serve()+ server = ScraplingMCPServer(host, port)+ server.run(transport="stdio" if not http else "streamable-http")@command(help="Interactive scraping console")diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex ae517fa..26eb564 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -41,10 +41,9 @@ def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingRespreturn ResponseModel(status=page.status, content=[result for result in content], url=page.url)-class ScraplingMCPServer:- _server = FastMCP(name="Scrapling")+def ScraplingMCPServer(host: str, port: int) -> FastMCP:+ _server = FastMCP(name="Scrapling", host=host, port=port)- @staticmethod@_server.tool()def get(url: str,@@ -123,7 +122,6 @@ class ScraplingMCPServer:page,)- @staticmethod@_server.tool()async def bulk_get(urls: Tuple[str, ...],@@ -210,7 +208,6 @@ class ScraplingMCPServer:for page in responses]- @staticmethod@_server.tool()async def fetch(url: str,@@ -299,7 +296,6 @@ class ScraplingMCPServer:page,)- @staticmethod@_server.tool()async def bulk_fetch(urls: Tuple[str, ...],@@ -393,7 +389,6 @@ class ScraplingMCPServer:for page in responses]- @staticmethod@_server.tool()async def stealthy_fetch(url: str,@@ -493,7 +488,6 @@ class ScraplingMCPServer:page,)- @staticmethod@_server.tool()async def bulk_stealthy_fetch(urls: Tuple[str, ...],@@ -598,6 +592,4 @@ class ScraplingMCPServer:for page in responses]- def serve(self):- """Serve the MCP server."""- self._server.run(transport="stdio")+ return _server