🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(static): send the request once when retries is below 1 (#420)
Yiğit ERDOĞANac18622ea2eb
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.pyindex 6f18b2a..91b81ef 100644--- a/scrapling/engines/static.py+++ b/scrapling/engines/static.py@@ -228,7 +228,8 @@ class _SyncSessionLogic(_ConfigurationLogic):stealth = self._stealth if stealth is None else stealthselector_config = self._get_param(kwargs, "selector_config", self.selector_config) or self.selector_config- max_retries = self._get_param(kwargs, "retries", self._default_retries)+ # Always attempt the request once; `retries` below 1 (or `None`) means "send it, but don't retry"+ max_retries = max(1, self._get_param(kwargs, "retries", self._default_retries) or 1)retry_delay = self._get_param(kwargs, "retry_delay", self._default_retry_delay)static_proxy = kwargs.pop("proxy", None)@@ -443,7 +444,8 @@ class _ASyncSessionLogic(_ConfigurationLogic):stealth = self._stealth if stealth is None else stealthselector_config = self._get_param(kwargs, "selector_config", self.selector_config) or self.selector_config- max_retries = self._get_param(kwargs, "retries", self._default_retries)+ # Always attempt the request once; `retries` below 1 (or `None`) means "send it, but don't retry"+ max_retries = max(1, self._get_param(kwargs, "retries", self._default_retries) or 1)retry_delay = self._get_param(kwargs, "retry_delay", self._default_retry_delay)static_proxy = kwargs.pop("proxy", None)diff --git a/tests/fetchers/async/test_requests.py b/tests/fetchers/async/test_requests.pyindex f0bccd1..f9df33a 100644--- a/tests/fetchers/async/test_requests.py+++ b/tests/fetchers/async/test_requests.py@@ -160,3 +160,8 @@ class TestAsyncFetcher:)assert response._storage is not Noneassert response.url == "from-request.test"++ async def test_retries_below_one_still_performs_the_request(self, fetcher, urls):+ """``retries`` below 1 means "send the request once", not "send nothing"."""+ assert (await fetcher.get(urls["status_200"], retries=0)).status == 200+ assert (await fetcher.get(urls["status_200"], retries=-1)).status == 200diff --git a/tests/fetchers/async/test_requests_session.py b/tests/fetchers/async/test_requests_session.pyindex 442a11f..05d891c 100644--- a/tests/fetchers/async/test_requests_session.py+++ b/tests/fetchers/async/test_requests_session.py@@ -60,3 +60,28 @@ class TestFetcherSession:proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list]assert proxies_used == ["http://p1:8080", "http://p2:8080"]++ @pytest.mark.asyncio+ @pytest.mark.parametrize("retries", [0, -1, None])+ async def test_retries_below_one_still_sends_the_request(self, retries):+ """A session-level retries below 1 must still send the request once instead of skipping it"""+ async with AsyncFetcherSession(retries=retries, retry_delay=0) as session:+ with (+ patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),+ ):+ await session.get("http://example.com")++ assert mocked_request.call_count == 1++ @pytest.mark.asyncio+ async def test_per_request_retries_below_one_still_sends_the_request(self):+ """A per-request retries of 0 must override the session default without skipping the request"""+ async with AsyncFetcherSession(retries=3, retry_delay=0) as session:+ with (+ patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),+ ):+ await session.get("http://example.com", retries=0)++ assert mocked_request.call_count == 1diff --git a/tests/fetchers/sync/test_requests.py b/tests/fetchers/sync/test_requests.pyindex 20682a1..5f9c03b 100644--- a/tests/fetchers/sync/test_requests.py+++ b/tests/fetchers/sync/test_requests.py@@ -151,3 +151,8 @@ class TestFetcher:)assert response._storage is not Noneassert response.url == "from-request.test"++ def test_retries_below_one_still_performs_the_request(self, fetcher):+ """``retries`` below 1 means "send the request once", not "send nothing"."""+ assert fetcher.get(self.status_200, retries=0).status == 200+ assert fetcher.get(self.status_200, retries=-1).status == 200diff --git a/tests/fetchers/sync/test_requests_session.py b/tests/fetchers/sync/test_requests_session.pyindex 2620e37..fde37a3 100644--- a/tests/fetchers/sync/test_requests_session.py+++ b/tests/fetchers/sync/test_requests_session.py@@ -82,3 +82,26 @@ class TestFetcherSession:proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list]assert proxies_used == ["http://p1:8080", "http://p2:8080"]++ @pytest.mark.parametrize("retries", [0, -1, None])+ def test_retries_below_one_still_sends_the_request(self, retries):+ """A session-level retries below 1 must still send the request once instead of skipping it"""+ with FetcherSession(retries=retries, retry_delay=0) as session:+ with (+ patch.object(session._curl_session, "request") as mocked_request,+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),+ ):+ session.get("http://example.com")++ assert mocked_request.call_count == 1++ def test_per_request_retries_below_one_still_sends_the_request(self):+ """A per-request retries of 0 must override the session default without skipping the request"""+ with FetcherSession(retries=3, retry_delay=0) as session:+ with (+ patch.object(session._curl_session, "request") as mocked_request,+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),+ ):+ session.get("http://example.com", retries=0)++ assert mocked_request.call_count == 1