Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Log skipped urls by length to INFO, add skipped stats

Dmitriy Pomazunovskiy6e5ea7924c7e
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index 5be1f80cb..ee3cb9fd6 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -14,22 +14,27 @@ logger = logging.getLogger(__name__)
class UrlLengthMiddleware:
- def __init__(self, maxlength):
+ def __init__(self, maxlength, stats):
self.maxlength = maxlength
+ self.stats = stats
@classmethod
- def from_settings(cls, settings):
+ def from_crawler(cls, crawler):
+ settings = crawler.settings
maxlength = settings.getint('URLLENGTH_LIMIT')
if not maxlength:
raise NotConfigured
- return cls(maxlength)
+ return cls(maxlength, crawler.stats)
def process_spider_output(self, response, result, spider):
def _filter(request):
if isinstance(request, Request) and len(request.url) > self.maxlength:
- logger.debug("Ignoring link (url length > %(maxlength)d): %(url)s ",
- {'maxlength': self.maxlength, 'url': request.url},
- extra={'spider': spider})
+ logger.info(
+ "Ignoring link (url length > %(maxlength)d): %(url)s ",
+ {'maxlength': self.maxlength, 'url': request.url},
+ extra={'spider': spider}
+ )
+ self.stats.inc_value('urllength/request_ignored_count', spider=spider)
return False
else:
return True
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 5ef2b23fd..33c524627 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -1,20 +1,45 @@
from unittest import TestCase
+from testfixtures import LogCapture
+
from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
from scrapy.http import Response, Request
from scrapy.spiders import Spider
+from scrapy.statscollectors import StatsCollector
+from scrapy.utils.test import get_crawler
class TestUrlLengthMiddleware(TestCase):
- def test_process_spider_output(self):
- res = Response('http://scrapytest.org')
+ def setUp(self):
+ crawler = get_crawler(Spider)
+ self.spider = crawler._create_spider('foo')
+
+ self.stats = StatsCollector(crawler)
+ self.stats.open_spider(self.spider)
+
+ self.maxlength = 25
+ self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats)
+
+ self.response = Response('http://scrapytest.org')
+ self.short_url_req = Request('http://scrapytest.org/')
+ self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
+ self.reqs = [self.short_url_req, self.long_url_req]
+
+ def tearDown(self):
+ self.stats.close_spider(self.spider, '')
+
+ def process_spider_output(self):
+ return list(self.mw.process_spider_output(self.response, self.reqs, self.spider))
+
+ def test_middleware_works(self):
+ self.assertEqual(self.process_spider_output(), [self.short_url_req])
+
+ def test_logging(self):
+ with LogCapture() as log:
+ self.process_spider_output()
- short_url_req = Request('http://scrapytest.org/')
- long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
- reqs = [short_url_req, long_url_req]
+ ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider)
+ self.assertEqual(ric, 1)
- mw = UrlLengthMiddleware(maxlength=25)
- spider = Spider('foo')
- out = list(mw.process_spider_output(res, reqs, spider))
- self.assertEqual(out, [short_url_req])
+ self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log))