Scrapy, a fast high-level web crawling & scraping framework for Python.
Log skipped urls by length to INFO, add skipped stats
Dmitriy Pomazunovskiy6e5ea7924c7e
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.pyindex 5be1f80cb..ee3cb9fd6 100644--- a/scrapy/spidermiddlewares/urllength.py+++ b/scrapy/spidermiddlewares/urllength.py@@ -14,22 +14,27 @@ logger = logging.getLogger(__name__)class UrlLengthMiddleware:- def __init__(self, maxlength):+ def __init__(self, maxlength, stats):self.maxlength = maxlength+ self.stats = stats@classmethod- def from_settings(cls, settings):+ def from_crawler(cls, crawler):+ settings = crawler.settingsmaxlength = settings.getint('URLLENGTH_LIMIT')if not maxlength:raise NotConfigured- return cls(maxlength)+ return cls(maxlength, crawler.stats)def process_spider_output(self, response, result, spider):def _filter(request):if isinstance(request, Request) and len(request.url) > self.maxlength:- logger.debug("Ignoring link (url length > %(maxlength)d): %(url)s ",- {'maxlength': self.maxlength, 'url': request.url},- extra={'spider': spider})+ logger.info(+ "Ignoring link (url length > %(maxlength)d): %(url)s ",+ {'maxlength': self.maxlength, 'url': request.url},+ extra={'spider': spider}+ )+ self.stats.inc_value('urllength/request_ignored_count', spider=spider)return Falseelse:return Truediff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.pyindex 5ef2b23fd..33c524627 100644--- a/tests/test_spidermiddleware_urllength.py+++ b/tests/test_spidermiddleware_urllength.py@@ -1,20 +1,45 @@from unittest import TestCase+from testfixtures import LogCapture+from scrapy.spidermiddlewares.urllength import UrlLengthMiddlewarefrom scrapy.http import Response, Requestfrom scrapy.spiders import Spider+from scrapy.statscollectors import StatsCollector+from scrapy.utils.test import get_crawlerclass TestUrlLengthMiddleware(TestCase):- def test_process_spider_output(self):- res = Response('http://scrapytest.org')+ def setUp(self):+ crawler = get_crawler(Spider)+ self.spider = crawler._create_spider('foo')++ self.stats = StatsCollector(crawler)+ self.stats.open_spider(self.spider)++ self.maxlength = 25+ self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats)++ self.response = Response('http://scrapytest.org')+ self.short_url_req = Request('http://scrapytest.org/')+ self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url')+ self.reqs = [self.short_url_req, self.long_url_req]++ def tearDown(self):+ self.stats.close_spider(self.spider, '')++ def process_spider_output(self):+ return list(self.mw.process_spider_output(self.response, self.reqs, self.spider))++ def test_middleware_works(self):+ self.assertEqual(self.process_spider_output(), [self.short_url_req])++ def test_logging(self):+ with LogCapture() as log:+ self.process_spider_output()- short_url_req = Request('http://scrapytest.org/')- long_url_req = Request('http://scrapytest.org/this_is_a_long_url')- reqs = [short_url_req, long_url_req]+ ric = self.stats.get_value('urllength/request_ignored_count', spider=self.spider)+ self.assertEqual(ric, 1)- mw = UrlLengthMiddleware(maxlength=25)- spider = Spider('foo')- out = list(mw.process_spider_output(res, reqs, spider))- self.assertEqual(out, [short_url_req])+ self.assertIn(f'Ignoring link (url length > {self.maxlength})', str(log))