Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use from_settings for backward compatibility

Dmitriy Pomazunovskiyd4b2b6125519
diff --git a/scrapy/spidermiddlewares/urllength.py b/scrapy/spidermiddlewares/urllength.py
index ee3cb9fd6..450d4ff40 100644
--- a/scrapy/spidermiddlewares/urllength.py
+++ b/scrapy/spidermiddlewares/urllength.py
@@ -14,17 +14,15 @@ logger = logging.getLogger(__name__)
class UrlLengthMiddleware:
- def __init__(self, maxlength, stats):
+ def __init__(self, maxlength):
self.maxlength = maxlength
- self.stats = stats
@classmethod
- def from_crawler(cls, crawler):
- settings = crawler.settings
+ def from_settings(cls, settings):
maxlength = settings.getint('URLLENGTH_LIMIT')
if not maxlength:
raise NotConfigured
- return cls(maxlength, crawler.stats)
+ return cls(maxlength)
def process_spider_output(self, response, result, spider):
def _filter(request):
@@ -34,7 +32,7 @@ class UrlLengthMiddleware:
{'maxlength': self.maxlength, 'url': request.url},
extra={'spider': spider}
)
- self.stats.inc_value('urllength/request_ignored_count', spider=spider)
+ spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
return False
else:
return True
diff --git a/tests/test_spidermiddleware_urllength.py b/tests/test_spidermiddleware_urllength.py
index 33c524627..6a72d2a8d 100644
--- a/tests/test_spidermiddleware_urllength.py
+++ b/tests/test_spidermiddleware_urllength.py
@@ -5,7 +5,6 @@ from testfixtures import LogCapture
from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
from scrapy.http import Response, Request
from scrapy.spiders import Spider
-from scrapy.statscollectors import StatsCollector
from scrapy.utils.test import get_crawler
@@ -14,21 +13,16 @@ class TestUrlLengthMiddleware(TestCase):
def setUp(self):
crawler = get_crawler(Spider)
self.spider = crawler._create_spider('foo')
-
- self.stats = StatsCollector(crawler)
- self.stats.open_spider(self.spider)
+ self.stats = self.spider.crawler.stats
self.maxlength = 25
- self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats)
+ self.mw = UrlLengthMiddleware(maxlength=self.maxlength)
self.response = Response('http://scrapytest.org')
self.short_url_req = Request('http://scrapytest.org/')
self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
self.reqs = [self.short_url_req, self.long_url_req]
- def tearDown(self):
- self.stats.close_spider(self.spider, '')
-
def process_spider_output(self):
return list(self.mw.process_spider_output(self.response, self.reqs, self.spider))