Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

test: adding new tests for the `configure` function

Karim shoairb626b4d585c1
diff --git a/tests/fetchers/async/test_requests.py b/tests/fetchers/async/test_requests.py
index 51417cd..f0bccd1 100644
--- a/tests/fetchers/async/test_requests.py
+++ b/tests/fetchers/async/test_requests.py
@@ -6,6 +6,17 @@ from scrapling.fetchers import AsyncFetcher
AsyncFetcher.adaptive = True
+def _reset_async_fetcher_config():
+ """Snapshot and restore the mutable class-level parser config around a test."""
+ snapshot = {k: getattr(AsyncFetcher, k) for k in AsyncFetcher.parser_keywords}
+ try:
+ yield
+ finally:
+ for k, v in snapshot.items():
+ setattr(AsyncFetcher, k, v)
+
+
@pytest_httpbin.use_class_based_httpbin
@pytest.mark.asyncio
class TestAsyncFetcher:
@@ -124,3 +135,28 @@ class TestAsyncFetcher:
timeout=None,
)
).status == 200
+
+ async def test_configure_propagates_to_response(
+ self, fetcher, urls, _reset_async_fetcher_config
+ ):
+ """`AsyncFetcher.configure()` must reach the Response's Selector on the HTTP path."""
+ AsyncFetcher.configure(adaptive=False, adaptive_domain="")
+ baseline = await fetcher.get(urls["html_url"])
+ assert baseline._storage is None
+
+ AsyncFetcher.configure(adaptive=True, adaptive_domain="configured.test")
+ configured = await fetcher.get(urls["html_url"])
+ assert configured._storage is not None
+ assert configured.url == "configured.test"
+
+ async def test_selector_config_overrides_configure(
+ self, fetcher, urls, _reset_async_fetcher_config
+ ):
+ """A per-request ``selector_config`` overrides the class-level configure()."""
+ AsyncFetcher.configure(adaptive=True, adaptive_domain="from-configure.test")
+ response = await fetcher.get(
+ urls["html_url"],
+ selector_config={"adaptive_domain": "from-request.test"},
+ )
+ assert response._storage is not None
+ assert response.url == "from-request.test"
diff --git a/tests/fetchers/sync/test_requests.py b/tests/fetchers/sync/test_requests.py
index 225932e..20682a1 100644
--- a/tests/fetchers/sync/test_requests.py
+++ b/tests/fetchers/sync/test_requests.py
@@ -6,6 +6,17 @@ from scrapling import Fetcher
Fetcher.adaptive = True
+def _reset_fetcher_config():
+ """Snapshot and restore the mutable class-level parser config around a test."""
+ snapshot = {k: getattr(Fetcher, k) for k in Fetcher.parser_keywords}
+ try:
+ yield
+ finally:
+ for k, v in snapshot.items():
+ setattr(Fetcher, k, v)
+
+
@pytest_httpbin.use_class_based_httpbin
class TestFetcher:
@pytest.fixture(scope="class")
@@ -119,3 +130,24 @@ class TestFetcher:
).status
== 200
)
+
+ def test_configure_propagates_to_response(self, fetcher, _reset_fetcher_config):
+ """`Fetcher.configure()` must reach the Response's Selector on the HTTP path."""
+ Fetcher.configure(adaptive=False, adaptive_domain="")
+ baseline = fetcher.get(self.html_url)
+ assert baseline._storage is None
+
+ Fetcher.configure(adaptive=True, adaptive_domain="configured.test")
+ configured = fetcher.get(self.html_url)
+ assert configured._storage is not None
+ assert configured.url == "configured.test"
+
+ def test_selector_config_overrides_configure(self, fetcher, _reset_fetcher_config):
+ """A per-request ``selector_config`` overrides the class-level configure()."""
+ Fetcher.configure(adaptive=True, adaptive_domain="from-configure.test")
+ response = fetcher.get(
+ self.html_url,
+ selector_config={"adaptive_domain": "from-request.test"},
+ )
+ assert response._storage is not None
+ assert response.url == "from-request.test"