Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix: apply the session-level proxy when no per-request proxy is given

Karim shoair74c5848060ca
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 9b3e951..6f18b2a 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -19,6 +19,7 @@ from scrapling.core._types import (
Unpack,
Optional,
Awaitable,
+ ProxyType,
SUPPORTED_HTTP_METHODS,
FollowRedirects,
)
@@ -244,10 +245,11 @@ class _SyncSessionLogic(_ConfigurationLogic):
try:
for attempt in range(max_retries):
+ proxy: Optional[ProxyType]
if self._proxy_rotator and static_proxy is None:
proxy = self._proxy_rotator.get_proxy()
else:
- proxy = static_proxy
+ proxy = static_proxy or self._default_proxy
request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)
try:
@@ -461,10 +463,11 @@ class _ASyncSessionLogic(_ConfigurationLogic):
try:
# Determine if we should use proxy rotation
for attempt in range(max_retries):
+ proxy: Optional[ProxyType]
if self._proxy_rotator and static_proxy is None:
proxy = self._proxy_rotator.get_proxy()
else:
- proxy = static_proxy
+ proxy = static_proxy or self._default_proxy
request_args = self._merge_request_args(stealth=stealth, proxy=proxy, **kwargs)
try:
diff --git a/tests/fetchers/async/test_requests_session.py b/tests/fetchers/async/test_requests_session.py
index 3846e69..442a11f 100644
--- a/tests/fetchers/async/test_requests_session.py
+++ b/tests/fetchers/async/test_requests_session.py
@@ -1,6 +1,9 @@
+import pytest
+from unittest.mock import patch, MagicMock, AsyncMock
+from curl_cffi.curl import CurlError
-
-from scrapling.engines.static import AsyncFetcherClient
+from scrapling.engines.static import _ASyncSessionLogic as AsyncFetcherSession, AsyncFetcherClient
+from scrapling.engines.toolbelt import ProxyRotator
class TestFetcherSession:
@@ -13,3 +16,47 @@ class TestFetcherSession:
# Should not have context manager methods
assert client.__aenter__ is None
assert client.__aexit__ is None
+
+ @pytest.mark.asyncio
+ async def test_session_level_proxy_is_applied(self):
+ """Session-level proxy must reach the request, not be silently dropped (#295)"""
+ proxy = "http://10.255.255.1:9999"
+
+ async with AsyncFetcherSession(proxy=proxy) as session:
+ with (
+ patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,
+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
+ ):
+ await session.get("http://example.com")
+
+ assert mocked_request.call_args.kwargs["proxy"] == proxy
+
+ @pytest.mark.asyncio
+ async def test_per_request_proxy_overrides_session_proxy(self):
+ """A per-request proxy must take precedence over the session-level proxy"""
+ request_proxy = "http://10.255.255.2:9999"
+
+ async with AsyncFetcherSession(proxy="http://10.255.255.1:9999") as session:
+ with (
+ patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,
+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
+ ):
+ await session.get("http://example.com", proxy=request_proxy)
+
+ assert mocked_request.call_args.kwargs["proxy"] == request_proxy
+
+ @pytest.mark.asyncio
+ async def test_proxy_rotates_per_retry_attempt(self):
+ """With a rotator, every retry attempt must pull a fresh proxy"""
+ rotator = ProxyRotator(["http://p1:8080", "http://p2:8080"])
+
+ async with AsyncFetcherSession(proxy_rotator=rotator, retries=2, retry_delay=0) as session:
+ with (
+ patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,
+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
+ ):
+ mocked_request.side_effect = [CurlError("transient"), MagicMock()]
+ await session.get("http://example.com")
+
+ proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list]
+ assert proxies_used == ["http://p1:8080", "http://p2:8080"]
diff --git a/tests/fetchers/sync/test_requests_session.py b/tests/fetchers/sync/test_requests_session.py
index 152fbc4..2620e37 100644
--- a/tests/fetchers/sync/test_requests_session.py
+++ b/tests/fetchers/sync/test_requests_session.py
@@ -1,7 +1,9 @@
import pytest
-
+from unittest.mock import patch, MagicMock
+from curl_cffi.curl import CurlError
from scrapling.engines.static import _SyncSessionLogic as FetcherSession, FetcherClient
+from scrapling.engines.toolbelt import ProxyRotator
class TestFetcherSession:
@@ -9,11 +11,7 @@ class TestFetcherSession:
def test_fetcher_session_creation(self):
"""Test FetcherSession creation"""
- session = FetcherSession(
- timeout=30,
- retries=3,
- stealthy_headers=True
- )
+ session = FetcherSession(timeout=30, retries=3, stealthy_headers=True)
assert session._default_timeout == 30
assert session._default_retries == 3
@@ -43,3 +41,44 @@ class TestFetcherSession:
# Should not have context manager methods
assert client.__enter__ is None
assert client.__exit__ is None
+
+ def test_session_level_proxy_is_applied(self):
+ """Session-level proxy must reach the request, not be silently dropped (#295)"""
+ proxy = "http://10.255.255.1:9999"
+
+ with FetcherSession(proxy=proxy) as session:
+ with (
+ patch.object(session._curl_session, "request") as mocked_request,
+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
+ ):
+ session.get("http://example.com")
+
+ assert mocked_request.call_args.kwargs["proxy"] == proxy
+
+ def test_per_request_proxy_overrides_session_proxy(self):
+ """A per-request proxy must take precedence over the session-level proxy"""
+ request_proxy = "http://10.255.255.2:9999"
+
+ with FetcherSession(proxy="http://10.255.255.1:9999") as session:
+ with (
+ patch.object(session._curl_session, "request") as mocked_request,
+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
+ ):
+ session.get("http://example.com", proxy=request_proxy)
+
+ assert mocked_request.call_args.kwargs["proxy"] == request_proxy
+
+ def test_proxy_rotates_per_retry_attempt(self):
+ """With a rotator, every retry attempt must pull a fresh proxy"""
+ rotator = ProxyRotator(["http://p1:8080", "http://p2:8080"])
+
+ with FetcherSession(proxy_rotator=rotator, retries=2, retry_delay=0) as session:
+ with (
+ patch.object(session._curl_session, "request") as mocked_request,
+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
+ ):
+ mocked_request.side_effect = [CurlError("transient"), MagicMock()]
+ session.get("http://example.com")
+
+ proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list]
+ assert proxies_used == ["http://p1:8080", "http://p2:8080"]