🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(fetchers): add early validation for the `capture_xhr`'s value before starting the browser
Karim shoair1d0d62cfeeb5
diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.mdindex 33f16f9..d51f333 100644--- a/docs/fetching/dynamic.md+++ b/docs/fetching/dynamic.md@@ -355,6 +355,8 @@ with DynamicSession(capture_xhr=r"https://api\.example\.com/.*", headless=True)Each item in `captured_xhr` is a full `Response` object with the same properties (`.url`, `.status`, `.headers`, `.body`, etc.). When `capture_xhr` is not set or is `None`, `captured_xhr` is an empty list.+Use `capture_xhr=".*"` to capture all XHR/fetch URLs. Invalid regular expressions raise an error before the browser starts.+### Network HistoryEnable `record_requests` on a browser session to inspect completed requests across page loads, actions, and tabs. The history covers the session's contexts, including temporary proxy contexts, and saves responses as Scrapling `Response` objects. Find API calls, HTTP errors, and redirects without sending requests again, then read saved headers and bodies even after the session closes. Failed and unfinished requests are omitted; completed HTTP 4xx and 5xx responses are included.diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.pyindex ced5159..a349a77 100644--- a/scrapling/engines/_browsers/_validators.py+++ b/scrapling/engines/_browsers/_validators.py@@ -1,4 +1,5 @@from pathlib import Path+from re import compile as re_compile, error as RegexErrorfrom typing import Annotatedfrom functools import lru_cachefrom urllib.parse import urlparse@@ -124,6 +125,11 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):self.additional_args = {}if not self.capture_xhr:self.capture_xhr = None+ else:+ try:+ re_compile(self.capture_xhr)+ except RegexError as error:+ raise ValueError(f"Invalid capture_xhr regex: {error}. Use '.*' to match all URLs.") from errorif self.init_script is not None:validation_msg = _is_invalid_file_path(self.init_script)diff --git a/tests/fetchers/test_network_browser.py b/tests/fetchers/test_network_browser.pyindex 45dca2a..db77925 100644--- a/tests/fetchers/test_network_browser.py+++ b/tests/fetchers/test_network_browser.py@@ -322,10 +322,13 @@ def _check_later(network: Any, after_id: int) -> tuple[Any, Any, Any]:@pytest.mark.parametrize("session_type", [DynamicSession, StealthySession])-def test_sync_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:[email protected]("capture_xhr", [".*", r"/api/"])+def test_sync_network_browser_history(+ session_type: Any, network_url: str, release_pending: Event, capture_xhr: str+) -> None:with session_type(**_options(- capture_xhr=r"/api/",+ capture_xhr=capture_xhr,page_action=lambda page: page.locator("body[data-initial=ready]").wait_for(state="attached"),)) as session:@@ -393,11 +396,14 @@ def test_sync_network_browser_history(session_type: Any, network_url: str, relea@pytest.mark.asyncio@pytest.mark.parametrize("session_type", [AsyncDynamicSession, AsyncStealthySession])-async def test_async_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:[email protected]("capture_xhr", [".*", r"/api/"])+async def test_async_network_browser_history(+ session_type: Any, network_url: str, release_pending: Event, capture_xhr: str+) -> None:async def initial_page(page: Any) -> None:await page.locator("body[data-initial=ready]").wait_for(state="attached")- async with session_type(**_options(capture_xhr=r"/api/", page_action=initial_page)) as session:+ async with session_type(**_options(capture_xhr=capture_xhr, page_action=initial_page)) as session:response = await session.fetch(network_url + "/")page = session.page_pool.pages[0].pageawait _wait_saved_async(diff --git a/tests/fetchers/test_validator.py b/tests/fetchers/test_validator.pyindex 86ecf45..d753d38 100644--- a/tests/fetchers/test_validator.py+++ b/tests/fetchers/test_validator.py@@ -1,4 +1,5 @@import pytest+from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySessionfrom scrapling.engines._browsers._validators import (validate,StealthConfig,@@ -9,6 +10,19 @@ from scrapling.engines._browsers._validators import (class TestValidators:"""Test configuration validators"""+ @pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])+ @pytest.mark.parametrize("pattern", [None, "", ".*", r"/api/", r"(?i)/api/\d+$"])+ def test_capture_xhr_valid(self, model, pattern):+ assert validate({"capture_xhr": pattern}, model).capture_xhr == (pattern or None)++ @pytest.mark.parametrize(+ "session_type", [DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession]+ )+ @pytest.mark.parametrize("pattern", ["*/*", "[", "("])+ def test_capture_xhr_invalid(self, session_type, pattern):+ with pytest.raises(TypeError, match=r"Invalid capture_xhr regex: .*Use '\.\*' to match all URLs"):+ session_type(capture_xhr=pattern)+def test_playwright_config_valid(self):"""Test valid PlaywrightConfig"""params = {"max_pages": 2, "headless": True, "timeout": 30000, "proxy": "http://proxy.example.com:8080"}