🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(mcp): Add new tools to list and view browser background requests
Karim shoairbe1874eb9e6e
diff --git a/scrapling/core/_network_formatting.py b/scrapling/core/_network_formatting.pynew file mode 100644index 0000000..d2455b8--- /dev/null+++ b/scrapling/core/_network_formatting.py@@ -0,0 +1,62 @@+from pydantic import BaseModel++from scrapling.core._types import Dict, Literal, Optional+from scrapling.engines._browsers._network import NetworkRequest+from scrapling.engines.toolbelt.convertor import ResponseFactory++NetworkPart = Literal["summary", "request_headers", "request_body", "response_headers", "response_body"]+++class NetworkRequestInfo(BaseModel):+ """Summary of a recorded browser request."""++ id: int+ url: str+ method: str+ resource_type: str+ status: int+++class NetworkRequestModel(BaseModel):+ """One complete saved request part, with an optional note."""++ request_id: int+ part: NetworkPart+ data: Dict[str, str] | NetworkRequestInfo | str | None = None+ note: Optional[str] = None+++def _decode(body: bytes, encoding: str) -> str:+ try:+ return body.decode(encoding, errors="replace")+ except LookupError:+ return body.decode("utf-8", errors="replace")+++def _request_details(record: NetworkRequest, part: NetworkPart) -> NetworkRequestModel:+ """Read one saved request part without browser access."""+ result = NetworkRequestModel(request_id=record.id, part=part)+ if part == "summary":+ result.data = NetworkRequestInfo.model_validate(record, from_attributes=True)+ return result+ headers = record.request_headers if part.startswith("request_") else record.response_headers+ if part.endswith("headers"):+ result.data = headers.copy()+ if record.response.meta.get("request_headers_partial" if part == "request_headers" else "headers_partial"):+ result.note = "Headers are partial; full headers unavailable."+ return result+ body = record.request_body if part == "request_body" else record.response.body+ if part == "response_body" and record.response.meta.get("body_note"):+ result.note = record.response.meta["body_note"]+ elif body is None:+ result.note = "Body is absent."+ elif body and not ResponseFactory._text_content(headers):+ result.note = "Only text bodies can be displayed."+ else:+ encoding = (+ ResponseFactory._extract_encoding(headers.get("content-type"))+ if part == "request_body"+ else record.response.encoding+ )+ result.data = _decode(body, encoding)+ return resultdiff --git a/scrapling/core/ai.py b/scrapling/core/ai.pyindex e05c1c3..2a09900 100644--- a/scrapling/core/ai.py+++ b/scrapling/core/ai.py@@ -27,6 +27,7 @@ from scrapling.core._browser_actions import (_run_actions,_validate_actions,)+from scrapling.core._network_formatting import NetworkPart, NetworkRequestInfo, NetworkRequestModel, _request_detailsfrom scrapling.core.shell import Convertor, _CONTROL_CHARS_PATTERNfrom scrapling.engines.toolbelt.custom import Response as _ScraplingResponsefrom scrapling.engines.static import ImpersonateType@@ -111,6 +112,15 @@ class ResponseModel(BaseModel):url: str = Field(description="The URL given by the user that resulted in this response.")+class NetworkRequestsModel(BaseModel):+ """Recorded request summaries and pagination details."""++ requests: List[NetworkRequestInfo]+ next_cursor: int+ has_more: bool+ dropped_count: int++class SessionInfo(BaseModel):"""Information about an open browser session."""@@ -285,6 +295,7 @@ class ScraplingMCPServer:cdp_url=cdp_url,headless=headless,block_ads=True,+ record_requests=True,useragent=useragent,timezone_id=timezone_id,real_chrome=real_chrome,@@ -349,6 +360,54 @@ class ScraplingMCPServer:for sid, entry in self._sessions.items()]+ async def browser_network_requests(+ self,+ session_id: str,+ url_pattern: Optional[str] = None,+ include_static: bool = False,+ after_id: Annotated[int, Field(ge=0)] = 0,+ limit: Annotated[int, Field(ge=1, le=200)] = 50,+ ) -> NetworkRequestsModel:+ """List completed requests across page changes, with IDs and pagination. Recording starts when the browser session opens.++ :param session_id: Browser session ID.+ :param url_pattern: Filter URLs by regular expression.+ :param include_static: Include successful non-API requests too.+ :param after_id: Return requests with a higher ID; use next_cursor for more.+ :param limit: Maximum requests to return.+ """+ network = self._get_session(session_id, ["stealthy"]).session.network+ records = network.search(+ url_pattern=url_pattern, include_static=include_static, after_id=after_id, limit=limit + 1+ )+ shown = records[:limit]+ return NetworkRequestsModel(+ requests=[NetworkRequestInfo.model_validate(record, from_attributes=True) for record in shown],+ next_cursor=shown[-1].id if shown else max(after_id, network.last_id),+ has_more=len(records) > limit,+ dropped_count=network.dropped_count,+ )++ async def browser_network_request(+ self,+ session_id: str,+ request_id: Annotated[int, Field(ge=1)],+ part: NetworkPart = "summary",+ ) -> NetworkRequestModel:+ """Read a complete saved request part without resending it, with structured data and availability notes.++ :param session_id: Browser session ID.+ :param request_id: ID from `browser_network_requests`.+ :param part: Request summary, headers, or text body; incomplete headers are marked.+ """+ network = self._get_session(session_id, ["stealthy"]).session.network+ record = network.get(request_id)+ if record is None:+ raise ValueError(+ f"Request {request_id} was not found or is no longer retained. Use browser_network_requests."+ )+ return _request_details(record, part)+async def browser_snapshot(self,session_id: str,@@ -994,6 +1053,7 @@ class ScraplingMCPServer:11. Set `extraction_type="snapshot"` on `browser_fetch` to get an AI ARIA snapshot with element references and bounding boxes in its content field. Use `css_selector` to snapshot one element, or omit it for the whole page. `main_content_only` and `pierce_shadow` do not filter snapshots. Use `browser_snapshot` to read the current page without navigating; it returns plain text with boxes by default. Set `depth` to limit the tree or `boxes=False` to omit boxes.12. Use `browser_actions` for ordered mouse, field, key, and wait actions on the current page. Move/click targets use exactly one selector, current snapshot ref, or viewport (x, y) pair in CSS pixels; fields use one selector/ref. Wheel acts at the current pointer and does not wait for scrolling; coordinate clicks do not wait for navigation. Load waits observe the current document, so prefer a result-specific element for delayed navigation.13. Use `browser_snapshot` to inspect results before repeating failed actions or choosing targets that depend on page changes. Actions may partly complete before an error; completed effects are not undone or retried. To submit with Enter, focus the intended control and use a press_key action with key="Enter".+14. Browser sessions save completed requests and responses across page changes. Failed and unfinished requests are omitted; closing does not wait for captures still running. Use `browser_network_requests` to find IDs, then `browser_network_request` to read one part. Successful non-API traffic is hidden unless include_static=True. Keep up to 1000 requests and 20 MiB of saved response bodies; oldest records are removed when either limit is reached. Only response bodies with a text Content-Type are read and saved; binary or unknown types keep headers and a skip note. Bodies over 1 MiB are skipped; unavailable bodies are marked. `browser_network_request` returns summary and headers as objects, and the full saved body as text. Closing an MCP session removes access to its history. Use next_cursor as after_id to continue the request list.""",}if self._auth_token:@@ -1083,6 +1143,16 @@ class ScraplingMCPServer:structured_output=False,annotations=_INPUT_TOOL_ANNOTATIONS,)+ server.add_tool(+ self.browser_network_requests,+ title="List network requests",+ annotations=_LIST_TOOL_ANNOTATIONS,+ )+ server.add_tool(+ self.browser_network_request,+ title="Read network request",+ annotations=_LIST_TOOL_ANNOTATIONS,+ )# Screenshot tool (returns image + url content blocks, not structured JSON)server.add_tool(self.browser_screenshot,