Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support CONCURRENT_REQUESTS = 0 for unlimited concurrency (#7840)

Adrian14478e3f2425
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index b07dff180..e287c3bd5 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -575,7 +575,7 @@ CONCURRENT_REQUESTS
Default: ``16``
The maximum number of concurrent (i.e. simultaneous) requests that will be
-performed by the Scrapy downloader.
+performed by the Scrapy downloader. Use ``0`` for no limit.
.. setting:: CONCURRENT_REQUESTS_PER_DOMAIN
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 7c0ee0eec..eb2079d0c 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -138,7 +138,8 @@ class Downloader:
self.active.remove(request)
def needs_backout(self) -> bool:
- return len(self.active) >= self.total_concurrency
+ # A total concurrency of 0 means no limit.
+ return 0 < self.total_concurrency <= len(self.active)
@_warn_spider_arg
def _get_slot(
diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.py
index d5a4e9fcd..8bbffb233 100644
--- a/scrapy/core/downloader/handlers/_httpx.py
+++ b/scrapy/core/downloader/handlers/_httpx.py
@@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base):
self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)
self._bind_host: str | None = self._get_bind_address_host()
self._limits: httpx.Limits = httpx.Limits(
- # hard limit on simultaneous connections
- max_connections=self._pool_size_total,
+ # hard limit on simultaneous connections (None for no limit, which
+ # is what a CONCURRENT_REQUESTS of 0 means)
+ max_connections=self._pool_size_total or None,
# total number of idle connections in the pool (extra ones are closed)
- max_keepalive_connections=self._pool_size_total,
+ max_keepalive_connections=self._pool_size_total or None,
)
self._default_client: httpx.AsyncClient = self._make_client()
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index fdd5edc27..3e4139b3e 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -14,6 +14,7 @@ from twisted.web import server, static
from twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBody
from twisted.web.client import Response as TxResponse
+from scrapy import Request
from scrapy.core.downloader import Downloader, Slot, tls
from scrapy.core.downloader.contextfactory import (
_load_context_factory_from_settings,
@@ -296,6 +297,23 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):
await self._assert_factory_works(server_url, client_context_factory)
+ ("concurrency", "active", "expected"),
+ [
+ (2, 1, False),
+ (2, 2, True),
+ (0, 0, False),
+ (0, 2, False),
+ ],
+)
+def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:
+ crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
+ downloader = Downloader(crawler)
+ downloader.active = {Request(f"https://example.com/{i}") for i in range(active)}
+ assert downloader.needs_backout() is expected
+ downloader.close()
+
+
@coroutine_test
async def test_fetch_deprecated_spider_arg():
class CustomDownloader(Downloader):
diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.py
index 5ceb93382..976daacaf 100644
--- a/tests/test_downloader_handler_httpx.py
+++ b/tests/test_downloader_handler_httpx.py
@@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import (
HttpxDownloadHandler,
)
from scrapy.exceptions import DownloadFailedError
+from scrapy.utils.misc import build_from_crawler
+from scrapy.utils.test import get_crawler
from tests.utils.bases.download_handlers_http import (
TestHttpBase,
TestHttpProxyBase,
@@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):
@pytest.mark.requires_internet
class TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):
pass
+
+
[email protected](("concurrency", "expected"), [(16, 16), (0, None)])
+@coroutine_test
+async def test_pool_limits(concurrency: int, expected: int | None) -> None:
+ crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})
+ handler = build_from_crawler(HttpxDownloadHandler, crawler)
+ try:
+ assert handler._limits.max_connections == expected
+ assert handler._limits.max_keepalive_connections == expected
+ finally:
+ await handler.close()