Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add from_crawler support to dupefilters

Eugenio Lacuesta701cd2ff9d4b
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index a54b4daf0..faed27fd1 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -26,7 +26,12 @@ class Scheduler(object):
def from_crawler(cls, crawler):
settings = crawler.settings
dupefilter_cls = load_object(settings['DUPEFILTER_CLASS'])
- dupefilter = dupefilter_cls.from_settings(settings)
+ if hasattr(dupefilter_cls, 'from_crawler'):
+ dupefilter = dupefilter_cls.from_crawler(crawler)
+ elif hasattr(dupefilter_cls, 'from_settings'):
+ dupefilter = dupefilter_cls.from_settings(crawler.settings)
+ else:
+ dupefilter = dupefilter_cls()
pqclass = load_object(settings['SCHEDULER_PRIORITY_QUEUE'])
dqclass = load_object(settings['SCHEDULER_DISK_QUEUE'])
mqclass = load_object(settings['SCHEDULER_MEMORY_QUEUE'])
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index 2d1a4bfff..81524fddd 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -6,10 +6,31 @@ import shutil
from scrapy.dupefilters import RFPDupeFilter
from scrapy.http import Request
from scrapy.utils.python import to_bytes
+from scrapy.utils.job import job_dir
+from scrapy.utils.test import get_crawler
class RFPDupeFilterTest(unittest.TestCase):
+ def test_dupefilter_from_crawler(self):
+
+ class FromCrawlerRFPDupeFilter(RFPDupeFilter):
+
+ @classmethod
+ def from_crawler(cls, crawler):
+ debug = crawler.settings.getbool('DUPEFILTER_DEBUG')
+ df = cls(job_dir(crawler.settings), debug)
+ df.user_agent = crawler.settings.get('USER_AGENT')
+ return df
+
+ crawler = get_crawler(settings_dict={'DUPEFILTER_DEBUG': True, 'USER_AGENT': 'test ua'})
+ dupefilter = FromCrawlerRFPDupeFilter.from_crawler(crawler)
+
+ self.assertTrue(dupefilter.debug)
+ self.assertEqual(dupefilter.user_agent, 'test ua')
+
+ dupefilter.close('finished')
+
def test_filter(self):
dupefilter = RFPDupeFilter()
dupefilter.open()