🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor: better implementation for the mcp mode
Karim shoair74f20d2a0e7f
diff --git a/scrapling/cli.py b/scrapling/cli.pyindex 44fcd33..04938d0 100644--- a/scrapling/cli.py+++ b/scrapling/cli.py@@ -154,8 +154,8 @@ def install(force): # pragma: no coverdef mcp(http, host, port):from scrapling.core.ai import ScraplingMCPServer- server = ScraplingMCPServer(host, port)- server.run(transport="stdio" if not http else "streamable-http")+ server = ScraplingMCPServer()+ server.serve(http, host, port)@command(help="Interactive scraping console")diff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex 26eb564..b9503a3 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -41,10 +41,8 @@ def _ContentTranslator(content: Generator[str, None, None], page: _ScraplingRespreturn ResponseModel(status=page.status, content=[result for result in content], url=page.url)-def ScraplingMCPServer(host: str, port: int) -> FastMCP:- _server = FastMCP(name="Scrapling", host=host, port=port)-- @_server.tool()+class ScraplingMCPServer:+ @staticmethoddef get(url: str,impersonate: Optional[BrowserTypeLiteral] = "chrome",@@ -122,7 +120,7 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:page,)- @_server.tool()+ @staticmethodasync def bulk_get(urls: Tuple[str, ...],impersonate: Optional[BrowserTypeLiteral] = "chrome",@@ -208,7 +206,7 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:for page in responses]- @_server.tool()+ @staticmethodasync def fetch(url: str,extraction_type: extraction_types = "markdown",@@ -296,7 +294,7 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:page,)- @_server.tool()+ @staticmethodasync def bulk_fetch(urls: Tuple[str, ...],extraction_type: extraction_types = "markdown",@@ -389,7 +387,7 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:for page in responses]- @_server.tool()+ @staticmethodasync def stealthy_fetch(url: str,extraction_type: extraction_types = "markdown",@@ -488,7 +486,7 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:page,)- @_server.tool()+ @staticmethodasync def bulk_stealthy_fetch(urls: Tuple[str, ...],extraction_type: extraction_types = "markdown",@@ -592,4 +590,22 @@ def ScraplingMCPServer(host: str, port: int) -> FastMCP:for page in responses]- return _server+ def serve(self, http: bool, host: str, port: int):+ """Serve the MCP server."""+ server = FastMCP(name="Scrapling", host=host, port=port)+ server.add_tool(self.get, title="get", description=self.get.__doc__, structured_output=True)+ server.add_tool(self.bulk_get, title="bulk_get", description=self.bulk_get.__doc__, structured_output=True)+ server.add_tool(self.fetch, title="fetch", description=self.fetch.__doc__, structured_output=True)+ server.add_tool(+ self.bulk_fetch, title="bulk_fetch", description=self.bulk_fetch.__doc__, structured_output=True+ )+ server.add_tool(+ self.stealthy_fetch, title="stealthy_fetch", description=self.stealthy_fetch.__doc__, structured_output=True+ )+ server.add_tool(+ self.bulk_stealthy_fetch,+ title="bulk_stealthy_fetch",+ description=self.bulk_stealthy_fetch.__doc__,+ structured_output=True,+ )+ server.run(transport="stdio" if not http else "streamable-http")