Scrapy, a fast high-level web crawling & scraping framework for Python.
Change DOWNLOAD_MAXSIZE logger level from Error to Warning
WinterComes2d8ec9d44fa2
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex fb04d1fb7..a78b19318 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -394,13 +394,14 @@ class ScrapyAgent:fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)if maxsize and expected_size > maxsize:- error_msg = ("Cancelling download of %(url)s: expected response "- "size (%(size)s) larger than download max size (%(maxsize)s).")- error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}+ warning_msg = ("Expected response size (%(size)s) larger than "+ "download max size (%(maxsize)s) in request %(request)s.")+ warning_args = {'request': request, 'size': expected_size, 'maxsize': maxsize}++ logger.warning(warning_msg, warning_args)- logger.error(error_msg, error_args)txresponse._transport._producer.loseConnection()- raise defer.CancelledError(error_msg % error_args)+ raise defer.CancelledError(warning_msg % warning_args)if warnsize and expected_size > warnsize:logger.warning("Expected response size (%(size)s) larger than "@@ -523,11 +524,11 @@ class _ResponseReader(protocol.Protocol):self._finish_response(flags=["download_stopped"], failure=failure)if self._maxsize and self._bytes_received > self._maxsize:- logger.error("Received (%(bytes)s) bytes larger than download "- "max size (%(maxsize)s) in request %(request)s.",- {'bytes': self._bytes_received,- 'maxsize': self._maxsize,- 'request': self._request})+ logger.warning("Received (%(bytes)s) bytes larger than download "+ "max size (%(maxsize)s) in request %(request)s.",+ {'bytes': self._bytes_received,+ 'maxsize': self._maxsize,+ 'request': self._request})# Clear buffer earlier to avoid keeping data in memory for a long time.self._bodybuf.truncate(0)self._finished.cancel()diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 13063d106..7059f0892 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -410,7 +410,7 @@ class Http11TestCase(HttpTestCase):request = Request(self.getURL('largechunkedfile'))def check(logger):- logger.error.assert_called_once_with(mock.ANY, mock.ANY)+ logger.warning.assert_called_once_with(mock.ANY, mock.ANY)d = self.download_request(request, Spider('foo', download_maxsize=1500))yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)