🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix: apply the session-level proxy when no per-request proxy is given
Karim shoair74c5848060ca
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 9b3e951..6f18b2a 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -19,6 +19,7 @@ from scrapling.core._types import (Unpack,Optional,Awaitable,+ ProxyType,SUPPORTED_HTTP_METHODS,FollowRedirects,)@@ -244,10 +245,11 @@ class _SyncSessionLogic(_ConfigurationLogic):try:for attempt in range(max_retries):+ proxy: Optional[ProxyType]if self._proxy_rotator and static_proxy is None:proxy = self._proxy_rotator.get_proxy()else:- proxy = static_proxy+ proxy = static_proxy or self._default_proxyrequest_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)try:@@ -461,10 +463,11 @@ class _ASyncSessionLogic(_ConfigurationLogic):try:# Determine if we should use proxy rotationfor attempt in range(max_retries):+ proxy: Optional[ProxyType]if self._proxy_rotator and static_proxy is None:proxy = self._proxy_rotator.get_proxy()else:- proxy = static_proxy+ proxy = static_proxy or self._default_proxyrequest_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)try:diff --git a/tests/fetchers/async/test_requests_session.py b/tests/fetchers/async/test_requests_session.pyindex 3846e69..442a11f 100644--- a/tests/fetchers/async/test_requests_session.py+++ b/tests/fetchers/async/test_requests_session.py@@ -1,6 +1,9 @@+import pytest+from unittest.mock import patch, MagicMock, AsyncMock+from curl_cffi.curl import CurlError--from scrapling.engines.static import AsyncFetcherClient+from scrapling.engines.static import _ASyncSessionLogic as AsyncFetcherSession, AsyncFetcherClient+from scrapling.engines.toolbelt import ProxyRotatorclass TestFetcherSession:@@ -13,3 +16,47 @@ class TestFetcherSession:# Should not have context manager methodsassert client.__aenter__ is Noneassert client.__aexit__ is None++ @pytest.mark.asyncio+ async def test_session_level_proxy_is_applied(self):+ """Session-level proxy must reach the request, not be silently dropped (#295)"""+ proxy = "http://10.255.255.1:9999"++ async with AsyncFetcherSession(proxy=proxy) as session:+ with (+ patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),+ ):+ await session.get("http://example.com")++ assert mocked_request.call_args.kwargs["proxy"] == proxy++ @pytest.mark.asyncio+ async def test_per_request_proxy_overrides_session_proxy(self):+ """A per-request proxy must take precedence over the session-level proxy"""+ request_proxy = "http://10.255.255.2:9999"++ async with AsyncFetcherSession(proxy="http://10.255.255.1:9999") as session:+ with (+ patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),+ ):+ await session.get("http://example.com", proxy=request_proxy)++ assert mocked_request.call_args.kwargs["proxy"] == request_proxy++ @pytest.mark.asyncio+ async def test_proxy_rotates_per_retry_attempt(self):+ """With a rotator, every retry attempt must pull a fresh proxy"""+ rotator = ProxyRotator(["http://p1:8080", "http://p2:8080"])++ async with AsyncFetcherSession(proxy_rotator=rotator, retries=2, retry_delay=0) as session:+ with (+ patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),+ ):+ mocked_request.side_effect = [CurlError("transient"), MagicMock()]+ await session.get("http://example.com")++ proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list]+ assert proxies_used == ["http://p1:8080", "http://p2:8080"]diff --git a/tests/fetchers/sync/test_requests_session.py b/tests/fetchers/sync/test_requests_session.pyindex 152fbc4..2620e37 100644--- a/tests/fetchers/sync/test_requests_session.py+++ b/tests/fetchers/sync/test_requests_session.py@@ -1,7 +1,9 @@import pytest-+from unittest.mock import patch, MagicMock+from curl_cffi.curl import CurlErrorfrom scrapling.engines.static import _SyncSessionLogic as FetcherSession, FetcherClient+from scrapling.engines.toolbelt import ProxyRotatorclass TestFetcherSession:@@ -9,11 +11,7 @@ class TestFetcherSession:def test_fetcher_session_creation(self):"""Test FetcherSession creation"""- session = FetcherSession(- timeout=30,- retries=3,- stealthy_headers=True- )+ session = FetcherSession(timeout=30, retries=3, stealthy_headers=True)assert session._default_timeout == 30assert session._default_retries == 3@@ -43,3 +41,44 @@ class TestFetcherSession:# Should not have context manager methodsassert client.__enter__ is Noneassert client.__exit__ is None++ def test_session_level_proxy_is_applied(self):+ """Session-level proxy must reach the request, not be silently dropped (#295)"""+ proxy = "http://10.255.255.1:9999"++ with FetcherSession(proxy=proxy) as session:+ with (+ patch.object(session._curl_session, "request") as mocked_request,+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),+ ):+ session.get("http://example.com")++ assert mocked_request.call_args.kwargs["proxy"] == proxy++ def test_per_request_proxy_overrides_session_proxy(self):+ """A per-request proxy must take precedence over the session-level proxy"""+ request_proxy = "http://10.255.255.2:9999"++ with FetcherSession(proxy="http://10.255.255.1:9999") as session:+ with (+ patch.object(session._curl_session, "request") as mocked_request,+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),+ ):+ session.get("http://example.com", proxy=request_proxy)++ assert mocked_request.call_args.kwargs["proxy"] == request_proxy++ def test_proxy_rotates_per_retry_attempt(self):+ """With a rotator, every retry attempt must pull a fresh proxy"""+ rotator = ProxyRotator(["http://p1:8080", "http://p2:8080"])++ with FetcherSession(proxy_rotator=rotator, retries=2, retry_delay=0) as session:+ with (+ patch.object(session._curl_session, "request") as mocked_request,+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),+ ):+ mocked_request.side_effect = [CurlError("transient"), MagicMock()]+ session.get("http://example.com")++ proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list]+ assert proxies_used == ["http://p1:8080", "http://p2:8080"]