Scrapy, a fast high-level web crawling & scraping framework for Python.
retry stats
Mikhail Korobove285b1d6c2aa
diff --git a/scrapy/downloadermiddlewares/retry.py b/scrapy/downloadermiddlewares/retry.pyindex c9c512be8..d84697b14 100644--- a/scrapy/downloadermiddlewares/retry.py+++ b/scrapy/downloadermiddlewares/retry.py@@ -22,6 +22,7 @@ from twisted.web.client import ResponseFailedfrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.response import response_status_messagefrom scrapy.core.downloader.handlers.http11 import TunnelError+from scrapy.utils.python import global_object_namelogger = logging.getLogger(__name__)@@ -35,16 +36,18 @@ class RetryMiddleware(object):ConnectionLost, TCPTimedOutError, ResponseFailed,IOError, TunnelError)- def __init__(self, settings):+ def __init__(self, crawler):+ settings = crawler.settingsif not settings.getbool('RETRY_ENABLED'):raise NotConfiguredself.max_retry_times = settings.getint('RETRY_TIMES')self.retry_http_codes = set(int(x) for x in settings.getlist('RETRY_HTTP_CODES'))self.priority_adjust = settings.getint('RETRY_PRIORITY_ADJUST')+ self.stats = crawler.stats@classmethoddef from_crawler(cls, crawler):- return cls(crawler.settings)+ return cls(crawler)def process_response(self, request, response, spider):if request.meta.get('dont_retry', False):@@ -70,8 +73,15 @@ class RetryMiddleware(object):retryreq.meta['retry_times'] = retriesretryreq.dont_filter = Trueretryreq.priority = request.priority + self.priority_adjust++ if isinstance(reason, Exception):+ reason = global_object_name(reason.__class__)++ self.stats.inc_value('retry/count')+ self.stats.inc_value('retry/reason_count/%s' % reason)return retryreqelse:+ self.stats.inc_value('retry/max_reached')logger.debug("Gave up retrying %(request)s (failed %(retries)d times): %(reason)s",{'request': request, 'retries': retries, 'reason': reason},extra={'spider': spider})diff --git a/scrapy/downloadermiddlewares/stats.py b/scrapy/downloadermiddlewares/stats.pyindex 9c0ad90a5..ef0aafce0 100644--- a/scrapy/downloadermiddlewares/stats.py+++ b/scrapy/downloadermiddlewares/stats.py@@ -1,6 +1,8 @@from scrapy.exceptions import NotConfiguredfrom scrapy.utils.request import request_httpreprfrom scrapy.utils.response import response_httprepr+from scrapy.utils.python import global_object_name+class DownloaderStats(object):@@ -27,6 +29,6 @@ class DownloaderStats(object):return responsedef process_exception(self, request, exception, spider):- ex_class = "%s.%s" % (exception.__class__.__module__, exception.__class__.__name__)+ ex_class = global_object_name(exception.__class__)self.stats.inc_value('downloader/exception_count', spider=spider)self.stats.inc_value('downloader/exception_type_count/%s' % ex_class, spider=spider)diff --git a/scrapy/utils/misc.py b/scrapy/utils/misc.pyindex 30c9e5058..35f855007 100644--- a/scrapy/utils/misc.py+++ b/scrapy/utils/misc.py@@ -113,7 +113,7 @@ def md5sum(file):m.update(d)return m.hexdigest()+def rel_has_nofollow(rel):"""Return True if link rel attribute has nofollow type"""return True if rel is not None and 'nofollow' in rel.split() else False-diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 42fbbda7f..4c500abf4 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -344,3 +344,14 @@ def without_none_values(iterable):return {k: v for k, v in six.iteritems(iterable) if v is not None}except AttributeError:return type(iterable)((v for v in iterable if v is not None))+++def global_object_name(obj):+ """+ Return full name of a global object.++ >>> from scrapy import Request+ >>> global_object_name(Request)+ 'scrapy.http.request.Request'+ """+ return "%s.%s" % (obj.__module__, obj.__name__)diff --git a/scrapy/utils/response.py b/scrapy/utils/response.pyindex deb5741be..bf276b5ca 100644--- a/scrapy/utils/response.py+++ b/scrapy/utils/response.py@@ -43,7 +43,8 @@ def get_meta_refresh(response):def response_status_message(status):"""Return status code plus status text descriptive message"""- return '%s %s' % (status, to_native_str(http.RESPONSES.get(int(status), "Unknown Status")))+ message = http.RESPONSES.get(int(status), "Unknown Status")+ return '%s %s' % (status, to_native_str(message))def response_httprepr(response):diff --git a/tests/test_downloadermiddleware_retry.py b/tests/test_downloadermiddleware_retry.pyindex e129b71f8..b833cb448 100644--- a/tests/test_downloadermiddleware_retry.py+++ b/tests/test_downloadermiddleware_retry.py@@ -13,9 +13,9 @@ from scrapy.utils.test import get_crawlerclass RetryTest(unittest.TestCase):def setUp(self):- crawler = get_crawler(Spider)- self.spider = crawler._create_spider('foo')- self.mw = RetryMiddleware.from_crawler(crawler)+ self.crawler = get_crawler(Spider)+ self.spider = self.crawler._create_spider('foo')+ self.mw = RetryMiddleware.from_crawler(self.crawler)self.mw.max_retry_times = 2def test_priority_adjust(self):@@ -70,6 +70,10 @@ class RetryTest(unittest.TestCase):# discard itassert self.mw.process_response(req, rsp, self.spider) is rsp+ assert self.crawler.stats.get_value('retry/max_reached') == 1+ assert self.crawler.stats.get_value('retry/reason_count/503 Service Unavailable') == 2+ assert self.crawler.stats.get_value('retry/count') == 2+def test_twistederrors(self):exceptions = [defer.TimeoutError, TCPTimedOutError, TimeoutError,DNSLookupError, ConnectionRefusedError, ConnectionDone,@@ -79,6 +83,11 @@ class RetryTest(unittest.TestCase):req = Request('http://www.scrapytest.org/%s' % exc.__name__)self._test_retry_exception(req, exc('foo'))+ stats = self.crawler.stats+ assert stats.get_value('retry/max_reached') == len(exceptions)+ assert stats.get_value('retry/count') == len(exceptions) * 2+ assert stats.get_value('retry/reason_count/twisted.internet.defer.TimeoutError') == 2+def _test_retry_exception(self, req, exception):# first retryreq = self.mw.process_exception(req, exception, self.spider)diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex 0f06fd53d..6213a51e8 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -101,7 +101,9 @@ class ProxyConnectTestCase(TestCase):self._assert_got_response_code(407, l)def _assert_got_response_code(self, code, log):+ print(log)self.assertEqual(str(log).count('Crawled (%d)' % code), 1)def _assert_got_tunnel_error(self, log):- self.assertEqual(str(log).count('TunnelError'), 1)+ print(log)+ self.assertIn('TunnelError', str(log))