Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(static): send the request once when retries is below 1 (#420)

Yiğit ERDOĞANac18622ea2eb
diff --git a/scrapling/engines/static.py b/scrapling/engines/static.py
index 6f18b2a..91b81ef 100644
--- a/scrapling/engines/static.py
+++ b/scrapling/engines/static.py
@@ -228,7 +228,8 @@ class _SyncSessionLogic(_ConfigurationLogic):
stealth = self._stealth if stealth is None else stealth
selector_config = self._get_param(kwargs, "selector_config", self.selector_config) or self.selector_config
- max_retries = self._get_param(kwargs, "retries", self._default_retries)
+ # Always attempt the request once; `retries` below 1 (or `None`) means "send it, but don't retry"
+ max_retries = max(1, self._get_param(kwargs, "retries", self._default_retries) or 1)
retry_delay = self._get_param(kwargs, "retry_delay", self._default_retry_delay)
static_proxy = kwargs.pop("proxy", None)
@@ -443,7 +444,8 @@ class _ASyncSessionLogic(_ConfigurationLogic):
stealth = self._stealth if stealth is None else stealth
selector_config = self._get_param(kwargs, "selector_config", self.selector_config) or self.selector_config
- max_retries = self._get_param(kwargs, "retries", self._default_retries)
+ # Always attempt the request once; `retries` below 1 (or `None`) means "send it, but don't retry"
+ max_retries = max(1, self._get_param(kwargs, "retries", self._default_retries) or 1)
retry_delay = self._get_param(kwargs, "retry_delay", self._default_retry_delay)
static_proxy = kwargs.pop("proxy", None)
diff --git a/tests/fetchers/async/test_requests.py b/tests/fetchers/async/test_requests.py
index f0bccd1..f9df33a 100644
--- a/tests/fetchers/async/test_requests.py
+++ b/tests/fetchers/async/test_requests.py
@@ -160,3 +160,8 @@ class TestAsyncFetcher:
)
assert response._storage is not None
assert response.url == "from-request.test"
+
+ async def test_retries_below_one_still_performs_the_request(self, fetcher, urls):
+ """``retries`` below 1 means "send the request once", not "send nothing"."""
+ assert (await fetcher.get(urls["status_200"], retries=0)).status == 200
+ assert (await fetcher.get(urls["status_200"], retries=-1)).status == 200
diff --git a/tests/fetchers/async/test_requests_session.py b/tests/fetchers/async/test_requests_session.py
index 442a11f..05d891c 100644
--- a/tests/fetchers/async/test_requests_session.py
+++ b/tests/fetchers/async/test_requests_session.py
@@ -60,3 +60,28 @@ class TestFetcherSession:
proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list]
assert proxies_used == ["http://p1:8080", "http://p2:8080"]
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("retries", [0, -1, None])
+ async def test_retries_below_one_still_sends_the_request(self, retries):
+ """A session-level retries below 1 must still send the request once instead of skipping it"""
+ async with AsyncFetcherSession(retries=retries, retry_delay=0) as session:
+ with (
+ patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,
+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
+ ):
+ await session.get("http://example.com")
+
+ assert mocked_request.call_count == 1
+
+ @pytest.mark.asyncio
+ async def test_per_request_retries_below_one_still_sends_the_request(self):
+ """A per-request retries of 0 must override the session default without skipping the request"""
+ async with AsyncFetcherSession(retries=3, retry_delay=0) as session:
+ with (
+ patch.object(session._async_curl_session, "request", new=AsyncMock()) as mocked_request,
+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
+ ):
+ await session.get("http://example.com", retries=0)
+
+ assert mocked_request.call_count == 1
diff --git a/tests/fetchers/sync/test_requests.py b/tests/fetchers/sync/test_requests.py
index 20682a1..5f9c03b 100644
--- a/tests/fetchers/sync/test_requests.py
+++ b/tests/fetchers/sync/test_requests.py
@@ -151,3 +151,8 @@ class TestFetcher:
)
assert response._storage is not None
assert response.url == "from-request.test"
+
+ def test_retries_below_one_still_performs_the_request(self, fetcher):
+ """``retries`` below 1 means "send the request once", not "send nothing"."""
+ assert fetcher.get(self.status_200, retries=0).status == 200
+ assert fetcher.get(self.status_200, retries=-1).status == 200
diff --git a/tests/fetchers/sync/test_requests_session.py b/tests/fetchers/sync/test_requests_session.py
index 2620e37..fde37a3 100644
--- a/tests/fetchers/sync/test_requests_session.py
+++ b/tests/fetchers/sync/test_requests_session.py
@@ -82,3 +82,26 @@ class TestFetcherSession:
proxies_used = [call.kwargs["proxy"] for call in mocked_request.call_args_list]
assert proxies_used == ["http://p1:8080", "http://p2:8080"]
+
+ @pytest.mark.parametrize("retries", [0, -1, None])
+ def test_retries_below_one_still_sends_the_request(self, retries):
+ """A session-level retries below 1 must still send the request once instead of skipping it"""
+ with FetcherSession(retries=retries, retry_delay=0) as session:
+ with (
+ patch.object(session._curl_session, "request") as mocked_request,
+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
+ ):
+ session.get("http://example.com")
+
+ assert mocked_request.call_count == 1
+
+ def test_per_request_retries_below_one_still_sends_the_request(self):
+ """A per-request retries of 0 must override the session default without skipping the request"""
+ with FetcherSession(retries=3, retry_delay=0) as session:
+ with (
+ patch.object(session._curl_session, "request") as mocked_request,
+ patch("scrapling.engines.static.ResponseFactory.from_http_request", return_value=MagicMock()),
+ ):
+ session.get("http://example.com", retries=0)
+
+ assert mocked_request.call_count == 1