Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): pass stealthy_headers to the session instead of curl_cffi (#475)

Yiğit ERDOĞAN06ae92f17f95
diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.py
index 5799e8c..513b423 100644
--- a/scrapling/spiders/session.py
+++ b/scrapling/spiders/session.py
@@ -116,6 +116,9 @@ class SessionManager:
if isinstance(client, _ASyncSessionLogic):
kwargs = request._session_kwargs.copy()
method = cast(SUPPORTED_HTTP_METHODS, kwargs.pop("method", "GET"))
+ if "stealthy_headers" in kwargs:
+ # Same mapping as the public session methods (`get`, `post`, ...), so it doesn't reach curl_cffi
+ kwargs["stealth"] = kwargs.pop("stealthy_headers")
response = await client._make_request(
method=method,
url=request.url,
diff --git a/tests/spiders/test_session.py b/tests/spiders/test_session.py
index 0c422ef..f7728c3 100644
--- a/tests/spiders/test_session.py
+++ b/tests/spiders/test_session.py
@@ -407,3 +407,68 @@ class TestSessionManagerFetch:
mock_client._make_request.assert_called_once()
call_kwargs = mock_client._make_request.call_args
assert call_kwargs.kwargs["method"] == "POST"
+
+ @staticmethod
+ def _manager_with_mock_client():
+ """Build a SessionManager whose default FetcherSession records `_make_request` calls."""
+ from scrapling.engines.static import _ASyncSessionLogic
+ from scrapling.fetchers import FetcherSession
+ from scrapling.engines.toolbelt.custom import Response
+
+ mock_response = Response(
+ url="https://example.com",
+ content=b"ok",
+ status=200,
+ reason="OK",
+ cookies={},
+ headers={"content-type": "text/html"},
+ request_headers={},
+ )
+ mock_response.meta = {}
+
+ mock_client = AsyncMock(spec=_ASyncSessionLogic)
+ mock_client._make_request = AsyncMock(return_value=mock_response)
+
+ mock_session = AsyncMock(spec=FetcherSession)
+ mock_session._client = mock_client
+ mock_session._is_alive = True
+
+ manager = SessionManager()
+ manager._sessions["default"] = mock_session
+ manager._default_session_id = "default"
+ manager._started = True
+ return manager, mock_client
+
+ @pytest.mark.asyncio
+ async def test_fetch_passes_stealthy_headers_as_stealth(self):
+ """`stealthy_headers` is mapped to `stealth` like the public session methods do, not sent to curl_cffi."""
+ manager, mock_client = self._manager_with_mock_client()
+ request = Request("https://example.com", stealthy_headers=False)
+
+ await manager.fetch(request)
+
+ call_kwargs = mock_client._make_request.call_args.kwargs
+ assert call_kwargs["stealth"] is False
+ assert "stealthy_headers" not in call_kwargs
+ # The request keeps its kwargs, so retries and copies still carry the setting
+ assert request._session_kwargs["stealthy_headers"] is False
+
+ @pytest.mark.asyncio
+ async def test_fetch_still_accepts_stealth(self):
+ """Passing the internal `stealth` name directly keeps working."""
+ manager, mock_client = self._manager_with_mock_client()
+
+ await manager.fetch(Request("https://example.com", stealth=False))
+
+ assert mock_client._make_request.call_args.kwargs["stealth"] is False
+
+ @pytest.mark.asyncio
+ async def test_fetch_without_stealthy_headers_uses_session_default(self):
+ """Without either name, nothing is passed and the session falls back to its own setting."""
+ manager, mock_client = self._manager_with_mock_client()
+
+ await manager.fetch(Request("https://example.com"))
+
+ call_kwargs = mock_client._make_request.call_args.kwargs
+ assert "stealth" not in call_kwargs
+ assert "stealthy_headers" not in call_kwargs