Scrapy, a fast high-level web crawling & scraping framework for Python.
Warn only once per request about the download warn size (#7963)
Adrian95fe7acc86f8
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 17dca5acb..3b5c08666 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -548,7 +548,8 @@ class _ScrapyAgent:txresponse._transport._producer.abortConnection()raise DownloadCancelledError(warning_msg)- if warnsize and expected_size > warnsize:+ reached_warnsize = bool(warnsize and expected_size > warnsize)+ if reached_warnsize:logger.warning(get_warnsize_msg(expected_size, warnsize, request, expected=True))@@ -561,6 +562,7 @@ class _ScrapyAgent:request=request,maxsize=maxsize,warnsize=warnsize,+ reached_warnsize=reached_warnsize,fail_on_dataloss=fail_on_dataloss,crawler=self._crawler,tls_verbose_logging=self._tls_verbose_logging,@@ -625,6 +627,7 @@ class _ResponseReader(Protocol):fail_on_dataloss: bool,crawler: Crawler,*,+ reached_warnsize: bool = False,tls_verbose_logging: bool = False,):self._finished: Deferred[_ResultT] = finished@@ -634,7 +637,7 @@ class _ResponseReader(Protocol):self._maxsize: int = maxsizeself._warnsize: int = warnsizeself._fail_on_dataloss: bool = fail_on_dataloss- self._reached_warnsize: bool = False+ self._reached_warnsize: bool = reached_warnsizeself._bytes_received: int = 0self._certificate: ssl.Certificate | None = Noneself._ip_address: ipaddress.IPv4Address | ipaddress.IPv6Address | None = Nonediff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex e44f9bcb8..65244b938 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -628,6 +628,7 @@ class TestHttpBase(ABC):"Expected to receive 5 bytes which is larger than download warn size (4)"in caplog.text)+ assert caplog.text.count("download warn size (4)") == 1@coroutine_testasync def test_download_with_warnsize_no_content_length(