Scrapy, a fast high-level web crawling & scraping framework for Python.
Support CONCURRENT_REQUESTS = 0 for unlimited concurrency (#7840)
Adrian14478e3f2425
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex b07dff180..e287c3bd5 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -575,7 +575,7 @@ CONCURRENT_REQUESTSDefault: ``16``The maximum number of concurrent (i.e. simultaneous) requests that will be-performed by the Scrapy downloader.+performed by the Scrapy downloader. Use ``0`` for no limit... setting:: CONCURRENT_REQUESTS_PER_DOMAINdiff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 7c0ee0eec..eb2079d0c 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -138,7 +138,8 @@ class Downloader:self.active.remove(request)def needs_backout(self) -> bool:- return len(self.active) >= self.total_concurrency+ # A total concurrency of 0 means no limit.+ return 0 < self.total_concurrency <= len(self.active)@_warn_spider_argdef _get_slot(diff --git a/scrapy/core/downloader/handlers/_httpx.py b/scrapy/core/downloader/handlers/_httpx.pyindex d5a4e9fcd..8bbffb233 100644--- a/scrapy/core/downloader/handlers/_httpx.py+++ b/scrapy/core/downloader/handlers/_httpx.py@@ -92,10 +92,11 @@ class HttpxDownloadHandler(_Base):self._ssl_context: ssl.SSLContext = _make_ssl_context(crawler.settings)self._bind_host: str | None = self._get_bind_address_host()self._limits: httpx.Limits = httpx.Limits(- # hard limit on simultaneous connections- max_connections=self._pool_size_total,+ # hard limit on simultaneous connections (None for no limit, which+ # is what a CONCURRENT_REQUESTS of 0 means)+ max_connections=self._pool_size_total or None,# total number of idle connections in the pool (extra ones are closed)- max_keepalive_connections=self._pool_size_total,+ max_keepalive_connections=self._pool_size_total or None,)self._default_client: httpx.AsyncClient = self._make_client()diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex fdd5edc27..3e4139b3e 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -14,6 +14,7 @@ from twisted.web import server, staticfrom twisted.web.client import Agent, BrowserLikePolicyForHTTPS, readBodyfrom twisted.web.client import Response as TxResponse+from scrapy import Requestfrom scrapy.core.downloader import Downloader, Slot, tlsfrom scrapy.core.downloader.contextfactory import (_load_context_factory_from_settings,@@ -296,6 +297,23 @@ class TestContextFactoryTLSMethod(TestContextFactoryBase):await self._assert_factory_works(server_url, client_context_factory)+ ("concurrency", "active", "expected"),+ [+ (2, 1, False),+ (2, 2, True),+ (0, 0, False),+ (0, 2, False),+ ],+)+def test_needs_backout(concurrency: int, active: int, expected: bool) -> None:+ crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})+ downloader = Downloader(crawler)+ downloader.active = {Request(f"https://example.com/{i}") for i in range(active)}+ assert downloader.needs_backout() is expected+ downloader.close()++@coroutine_testasync def test_fetch_deprecated_spider_arg():class CustomDownloader(Downloader):diff --git a/tests/test_downloader_handler_httpx.py b/tests/test_downloader_handler_httpx.pyindex 5ceb93382..976daacaf 100644--- a/tests/test_downloader_handler_httpx.py+++ b/tests/test_downloader_handler_httpx.py@@ -15,6 +15,8 @@ from scrapy.core.downloader.handlers._httpx import (HttpxDownloadHandler,)from scrapy.exceptions import DownloadFailedError+from scrapy.utils.misc import build_from_crawler+from scrapy.utils.test import get_crawlerfrom tests.utils.bases.download_handlers_http import (TestHttpBase,TestHttpProxyBase,@@ -161,3 +163,15 @@ class TestMitmProxy(HttpxDownloadHandlerMixin, TestMitmProxyBase):@pytest.mark.requires_internetclass TestRealWebsite(HttpxDownloadHandlerMixin, TestRealWebsiteBase):pass++[email protected](("concurrency", "expected"), [(16, 16), (0, None)])+@coroutine_test+async def test_pool_limits(concurrency: int, expected: int | None) -> None:+ crawler = get_crawler(settings_dict={"CONCURRENT_REQUESTS": concurrency})+ handler = build_from_crawler(HttpxDownloadHandler, crawler)+ try:+ assert handler._limits.max_connections == expected+ assert handler._limits.max_keepalive_connections == expected+ finally:+ await handler.close()