Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Change DOWNLOAD_MAXSIZE logger level from Error to Warning

WinterComes2d8ec9d44fa2
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index fb04d1fb7..a78b19318 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -394,13 +394,14 @@ class ScrapyAgent:
fail_on_dataloss = request.meta.get('download_fail_on_dataloss', self._fail_on_dataloss)
if maxsize and expected_size > maxsize:
- error_msg = ("Cancelling download of %(url)s: expected response "
- "size (%(size)s) larger than download max size (%(maxsize)s).")
- error_args = {'url': request.url, 'size': expected_size, 'maxsize': maxsize}
+ warning_msg = ("Expected response size (%(size)s) larger than "
+ "download max size (%(maxsize)s) in request %(request)s.")
+ warning_args = {'request': request, 'size': expected_size, 'maxsize': maxsize}
+
+ logger.warning(warning_msg, warning_args)
- logger.error(error_msg, error_args)
txresponse._transport._producer.loseConnection()
- raise defer.CancelledError(error_msg % error_args)
+ raise defer.CancelledError(warning_msg % warning_args)
if warnsize and expected_size > warnsize:
logger.warning("Expected response size (%(size)s) larger than "
@@ -523,11 +524,11 @@ class _ResponseReader(protocol.Protocol):
self._finish_response(flags=["download_stopped"], failure=failure)
if self._maxsize and self._bytes_received > self._maxsize:
- logger.error("Received (%(bytes)s) bytes larger than download "
- "max size (%(maxsize)s) in request %(request)s.",
- {'bytes': self._bytes_received,
- 'maxsize': self._maxsize,
- 'request': self._request})
+ logger.warning("Received (%(bytes)s) bytes larger than download "
+ "max size (%(maxsize)s) in request %(request)s.",
+ {'bytes': self._bytes_received,
+ 'maxsize': self._maxsize,
+ 'request': self._request})
# Clear buffer earlier to avoid keeping data in memory for a long time.
self._bodybuf.truncate(0)
self._finished.cancel()
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 13063d106..7059f0892 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -410,7 +410,7 @@ class Http11TestCase(HttpTestCase):
request = Request(self.getURL('largechunkedfile'))
def check(logger):
- logger.error.assert_called_once_with(mock.ANY, mock.ANY)
+ logger.warning.assert_called_once_with(mock.ANY, mock.ANY)
d = self.download_request(request, Spider('foo', download_maxsize=1500))
yield self.assertFailure(d, defer.CancelledError, error.ConnectionAborted)