🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
test: adding new tests for the `configure` function
Karim shoairb626b4d585c1
diff --git a/tests/fetchers/async/test_requests.py b/tests/fetchers/async/test_requests.pyindex 51417cd..f0bccd1 100644--- a/tests/fetchers/async/test_requests.py+++ b/tests/fetchers/async/test_requests.py@@ -6,6 +6,17 @@ from scrapling.fetchers import AsyncFetcherAsyncFetcher.adaptive = True+def _reset_async_fetcher_config():+ """Snapshot and restore the mutable class-level parser config around a test."""+ snapshot = {k: getattr(AsyncFetcher, k) for k in AsyncFetcher.parser_keywords}+ try:+ yield+ finally:+ for k, v in snapshot.items():+ setattr(AsyncFetcher, k, v)++@pytest_httpbin.use_class_based_httpbin@pytest.mark.asyncioclass TestAsyncFetcher:@@ -124,3 +135,28 @@ class TestAsyncFetcher:timeout=None,)).status == 200++ async def test_configure_propagates_to_response(+ self, fetcher, urls, _reset_async_fetcher_config+ ):+ """`AsyncFetcher.configure()` must reach the Response's Selector on the HTTP path."""+ AsyncFetcher.configure(adaptive=False, adaptive_domain="")+ baseline = await fetcher.get(urls["html_url"])+ assert baseline._storage is None++ AsyncFetcher.configure(adaptive=True, adaptive_domain="configured.test")+ configured = await fetcher.get(urls["html_url"])+ assert configured._storage is not None+ assert configured.url == "configured.test"++ async def test_selector_config_overrides_configure(+ self, fetcher, urls, _reset_async_fetcher_config+ ):+ """A per-request ``selector_config`` overrides the class-level configure()."""+ AsyncFetcher.configure(adaptive=True, adaptive_domain="from-configure.test")+ response = await fetcher.get(+ urls["html_url"],+ selector_config={"adaptive_domain": "from-request.test"},+ )+ assert response._storage is not None+ assert response.url == "from-request.test"diff --git a/tests/fetchers/sync/test_requests.py b/tests/fetchers/sync/test_requests.pyindex 225932e..20682a1 100644--- a/tests/fetchers/sync/test_requests.py+++ b/tests/fetchers/sync/test_requests.py@@ -6,6 +6,17 @@ from scrapling import FetcherFetcher.adaptive = True+def _reset_fetcher_config():+ """Snapshot and restore the mutable class-level parser config around a test."""+ snapshot = {k: getattr(Fetcher, k) for k in Fetcher.parser_keywords}+ try:+ yield+ finally:+ for k, v in snapshot.items():+ setattr(Fetcher, k, v)++@pytest_httpbin.use_class_based_httpbinclass TestFetcher:@pytest.fixture(scope="class")@@ -119,3 +130,24 @@ class TestFetcher:).status== 200)++ def test_configure_propagates_to_response(self, fetcher, _reset_fetcher_config):+ """`Fetcher.configure()` must reach the Response's Selector on the HTTP path."""+ Fetcher.configure(adaptive=False, adaptive_domain="")+ baseline = fetcher.get(self.html_url)+ assert baseline._storage is None++ Fetcher.configure(adaptive=True, adaptive_domain="configured.test")+ configured = fetcher.get(self.html_url)+ assert configured._storage is not None+ assert configured.url == "configured.test"++ def test_selector_config_overrides_configure(self, fetcher, _reset_fetcher_config):+ """A per-request ``selector_config`` overrides the class-level configure()."""+ Fetcher.configure(adaptive=True, adaptive_domain="from-configure.test")+ response = fetcher.get(+ self.html_url,+ selector_config={"adaptive_domain": "from-request.test"},+ )+ assert response._storage is not None+ assert response.url == "from-request.test"