Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(fetchers): add early validation for the `capture_xhr`'s value before starting the browser

Karim shoair1d0d62cfeeb5
diff --git a/docs/fetching/dynamic.md b/docs/fetching/dynamic.md
index 33f16f9..d51f333 100644
--- a/docs/fetching/dynamic.md
+++ b/docs/fetching/dynamic.md
@@ -355,6 +355,8 @@ with DynamicSession(capture_xhr=r"https://api\.example\.com/.*", headless=True)
Each item in `captured_xhr` is a full `Response` object with the same properties (`.url`, `.status`, `.headers`, `.body`, etc.). When `capture_xhr` is not set or is `None`, `captured_xhr` is an empty list.
+Use `capture_xhr=".*"` to capture all XHR/fetch URLs. Invalid regular expressions raise an error before the browser starts.
+
### Network History
Enable `record_requests` on a browser session to inspect completed requests across page loads, actions, and tabs. The history covers the session's contexts, including temporary proxy contexts, and saves responses as Scrapling `Response` objects. Find API calls, HTTP errors, and redirects without sending requests again, then read saved headers and bodies even after the session closes. Failed and unfinished requests are omitted; completed HTTP 4xx and 5xx responses are included.
diff --git a/scrapling/engines/_browsers/_validators.py b/scrapling/engines/_browsers/_validators.py
index ced5159..a349a77 100644
--- a/scrapling/engines/_browsers/_validators.py
+++ b/scrapling/engines/_browsers/_validators.py
@@ -1,4 +1,5 @@
from pathlib import Path
+from re import compile as re_compile, error as RegexError
from typing import Annotated
from functools import lru_cache
from urllib.parse import urlparse
@@ -124,6 +125,11 @@ class PlaywrightConfig(Struct, kw_only=True, frozen=False, weakref=True):
self.additional_args = {}
if not self.capture_xhr:
self.capture_xhr = None
+ else:
+ try:
+ re_compile(self.capture_xhr)
+ except RegexError as error:
+ raise ValueError(f"Invalid capture_xhr regex: {error}. Use '.*' to match all URLs.") from error
if self.init_script is not None:
validation_msg = _is_invalid_file_path(self.init_script)
diff --git a/tests/fetchers/test_network_browser.py b/tests/fetchers/test_network_browser.py
index 45dca2a..db77925 100644
--- a/tests/fetchers/test_network_browser.py
+++ b/tests/fetchers/test_network_browser.py
@@ -322,10 +322,13 @@ def _check_later(network: Any, after_id: int) -> tuple[Any, Any, Any]:
@pytest.mark.parametrize("session_type", [DynamicSession, StealthySession])
-def test_sync_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:
[email protected]("capture_xhr", [".*", r"/api/"])
+def test_sync_network_browser_history(
+ session_type: Any, network_url: str, release_pending: Event, capture_xhr: str
+) -> None:
with session_type(
**_options(
- capture_xhr=r"/api/",
+ capture_xhr=capture_xhr,
page_action=lambda page: page.locator("body[data-initial=ready]").wait_for(state="attached"),
)
) as session:
@@ -393,11 +396,14 @@ def test_sync_network_browser_history(session_type: Any, network_url: str, relea
@pytest.mark.asyncio
@pytest.mark.parametrize("session_type", [AsyncDynamicSession, AsyncStealthySession])
-async def test_async_network_browser_history(session_type: Any, network_url: str, release_pending: Event) -> None:
[email protected]("capture_xhr", [".*", r"/api/"])
+async def test_async_network_browser_history(
+ session_type: Any, network_url: str, release_pending: Event, capture_xhr: str
+) -> None:
async def initial_page(page: Any) -> None:
await page.locator("body[data-initial=ready]").wait_for(state="attached")
- async with session_type(**_options(capture_xhr=r"/api/", page_action=initial_page)) as session:
+ async with session_type(**_options(capture_xhr=capture_xhr, page_action=initial_page)) as session:
response = await session.fetch(network_url + "/")
page = session.page_pool.pages[0].page
await _wait_saved_async(
diff --git a/tests/fetchers/test_validator.py b/tests/fetchers/test_validator.py
index 86ecf45..d753d38 100644
--- a/tests/fetchers/test_validator.py
+++ b/tests/fetchers/test_validator.py
@@ -1,4 +1,5 @@
import pytest
+from scrapling.fetchers import AsyncDynamicSession, AsyncStealthySession, DynamicSession, StealthySession
from scrapling.engines._browsers._validators import (
validate,
StealthConfig,
@@ -9,6 +10,19 @@ from scrapling.engines._browsers._validators import (
class TestValidators:
"""Test configuration validators"""
+ @pytest.mark.parametrize("model", [PlaywrightConfig, StealthConfig])
+ @pytest.mark.parametrize("pattern", [None, "", ".*", r"/api/", r"(?i)/api/\d+$"])
+ def test_capture_xhr_valid(self, model, pattern):
+ assert validate({"capture_xhr": pattern}, model).capture_xhr == (pattern or None)
+
+ @pytest.mark.parametrize(
+ "session_type", [DynamicSession, StealthySession, AsyncDynamicSession, AsyncStealthySession]
+ )
+ @pytest.mark.parametrize("pattern", ["*/*", "[", "("])
+ def test_capture_xhr_invalid(self, session_type, pattern):
+ with pytest.raises(TypeError, match=r"Invalid capture_xhr regex: .*Use '\.\*' to match all URLs"):
+ session_type(capture_xhr=pattern)
+
def test_playwright_config_valid(self):
"""Test valid PlaywrightConfig"""
params = {"max_pages": 2, "headless": True, "timeout": 30000, "proxy": "http://proxy.example.com:8080"}