🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): pass stealthy_headers to the session instead of curl_cffi (#475)
Yiğit ERDOĞAN06ae92f17f95
diff --git a/scrapling/spiders/session.py b/scrapling/spiders/session.pyindex 5799e8c..513b423 100644--- a/scrapling/spiders/session.py+++ b/scrapling/spiders/session.py@@ -116,6 +116,9 @@ class SessionManager:if isinstance(client, _ASyncSessionLogic):kwargs = request._session_kwargs.copy()method = cast(SUPPORTED_HTTP_METHODS, kwargs.pop("method", "GET"))+ if "stealthy_headers" in kwargs:+ # Same mapping as the public session methods (`get`, `post`, ...), so it doesn't reach curl_cffi+ kwargs["stealth"] = kwargs.pop("stealthy_headers")response = await client._make_request(method=method,url=request.url,diff --git a/tests/spiders/test_session.py b/tests/spiders/test_session.pyindex 0c422ef..f7728c3 100644--- a/tests/spiders/test_session.py+++ b/tests/spiders/test_session.py@@ -407,3 +407,68 @@ class TestSessionManagerFetch:mock_client._make_request.assert_called_once()call_kwargs = mock_client._make_request.call_argsassert call_kwargs.kwargs["method"] == "POST"++ @staticmethod+ def _manager_with_mock_client():+ """Build a SessionManager whose default FetcherSession records `_make_request` calls."""+ from scrapling.engines.static import _ASyncSessionLogic+ from scrapling.fetchers import FetcherSession+ from scrapling.engines.toolbelt.custom import Response++ mock_response = Response(+ url="https://example.com",+ content=b"ok",+ status=200,+ reason="OK",+ cookies={},+ headers={"content-type": "text/html"},+ request_headers={},+ )+ mock_response.meta = {}++ mock_client = AsyncMock(spec=_ASyncSessionLogic)+ mock_client._make_request = AsyncMock(return_value=mock_response)++ mock_session = AsyncMock(spec=FetcherSession)+ mock_session._client = mock_client+ mock_session._is_alive = True++ manager = SessionManager()+ manager._sessions["default"] = mock_session+ manager._default_session_id = "default"+ manager._started = True+ return manager, mock_client++ @pytest.mark.asyncio+ async def test_fetch_passes_stealthy_headers_as_stealth(self):+ """`stealthy_headers` is mapped to `stealth` like the public session methods do, not sent to curl_cffi."""+ manager, mock_client = self._manager_with_mock_client()+ request = Request("https://example.com", stealthy_headers=False)++ await manager.fetch(request)++ call_kwargs = mock_client._make_request.call_args.kwargs+ assert call_kwargs["stealth"] is False+ assert "stealthy_headers" not in call_kwargs+ # The request keeps its kwargs, so retries and copies still carry the setting+ assert request._session_kwargs["stealthy_headers"] is False++ @pytest.mark.asyncio+ async def test_fetch_still_accepts_stealth(self):+ """Passing the internal `stealth` name directly keeps working."""+ manager, mock_client = self._manager_with_mock_client()++ await manager.fetch(Request("https://example.com", stealth=False))++ assert mock_client._make_request.call_args.kwargs["stealth"] is False++ @pytest.mark.asyncio+ async def test_fetch_without_stealthy_headers_uses_session_default(self):+ """Without either name, nothing is passed and the session falls back to its own setting."""+ manager, mock_client = self._manager_with_mock_client()++ await manager.fetch(Request("https://example.com"))++ call_kwargs = mock_client._make_request.call_args.kwargs+ assert "stealth" not in call_kwargs+ assert "stealthy_headers" not in call_kwargs