Scrapy, a fast high-level web crawling & scraping framework for Python.
Add from_crawler support to dupefilters
Eugenio Lacuesta701cd2ff9d4b
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex a54b4daf0..faed27fd1 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -26,7 +26,12 @@ class Scheduler(object):def from_crawler(cls, crawler):settings = crawler.settingsdupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])- dupefilter = dupefilter_cls.from_settings(settings)+ if hasattr(dupefilter_cls, 'from_crawler'):+ dupefilter = dupefilter_cls.from_crawler(crawler)+ elif hasattr(dupefilter_cls, 'from_settings'):+ dupefilter = dupefilter_cls.from_settings(crawler.settings)+ else:+ dupefilter = dupefilter_cls()pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex 2d1a4bfff..81524fddd 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -6,10 +6,31 @@ import shutilfrom scrapy.dupefilters import RFPDupeFilterfrom scrapy.http import Requestfrom scrapy.utils.python import to_bytes+from scrapy.utils.job import job_dir+from scrapy.utils.test import get_crawlerclass RFPDupeFilterTest(unittest.TestCase):+ def test_dupefilter_from_crawler(self):++ class FromCrawlerRFPDupeFilter(RFPDupeFilter):++ @classmethod+ def from_crawler(cls, crawler):+ debug = crawler.settings.getbool('DUPEFILTER_DEBUG')+ df = cls(job_dir(crawler.settings), debug)+ df.user_agent = crawler.settings.get('USER_AGENT')+ return df++ crawler = get_crawler(settings_dict={'DUPEFILTER_DEBUG': True, 'USER_AGENT': 'test ua'})+ dupefilter = FromCrawlerRFPDupeFilter.from_crawler(crawler)++ self.assertTrue(dupefilter.debug)+ self.assertEqual(dupefilter.user_agent, 'test ua')++ dupefilter.close('finished')+def test_filter(self):dupefilter = RFPDupeFilter()dupefilter.open()