Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Testing stats and log messages from RFPDupeFilter

Júlio César Batista8eade7d8640e
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.py
index db69597a2..d7eb98c97 100644
--- a/tests/test_dupefilters.py
+++ b/tests/test_dupefilters.py
@@ -2,6 +2,7 @@ import hashlib
import tempfile
import unittest
import shutil
+from testfixtures import LogCapture
from scrapy.dupefilters import RFPDupeFilter
from scrapy.http import Request
@@ -9,7 +10,7 @@ from scrapy.core.scheduler import Scheduler
from scrapy.utils.python import to_bytes
from scrapy.utils.job import job_dir
from scrapy.utils.test import get_crawler
-
+from tests.spiders import SimpleSpider
class FromCrawlerRFPDupeFilter(RFPDupeFilter):
@@ -126,3 +127,57 @@ class RFPDupeFilterTest(unittest.TestCase):
assert case_insensitive_dupefilter.request_seen(r2)
case_insensitive_dupefilter.close('finished')
+
+ def test_log(self):
+ with LogCapture() as l:
+ settings = {'DUPEFILTER_DEBUG': False,
+ 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
+ crawler = get_crawler(SimpleSpider, settings_dict=settings)
+ scheduler = Scheduler.from_crawler(crawler)
+ spider = SimpleSpider.from_crawler(crawler)
+
+ dupefilter = scheduler.df
+ dupefilter.open()
+
+ r1 = Request('http://scrapytest.org/index.html')
+ r2 = Request('http://scrapytest.org/index.html')
+
+ dupefilter.log(r1, spider)
+ dupefilter.log(r2, spider)
+
+ assert crawler.stats.get_value('dupefilter/filtered') == 2
+ l.check_present(('scrapy.dupefilters', 'DEBUG',
+ ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
+ ' - no more duplicates will be shown'
+ ' (see DUPEFILTER_DEBUG to show all duplicates)')))
+
+ dupefilter.close('finished')
+
+ def test_log_debug(self):
+ with LogCapture() as l:
+ settings = {'DUPEFILTER_DEBUG': True,
+ 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}
+ crawler = get_crawler(SimpleSpider, settings_dict=settings)
+ scheduler = Scheduler.from_crawler(crawler)
+ spider = SimpleSpider.from_crawler(crawler)
+
+ dupefilter = scheduler.df
+ dupefilter.open()
+
+ r1 = Request('http://scrapytest.org/index.html')
+ r2 = Request('http://scrapytest.org/index.html',
+ headers={'Referer': 'http://scrapytest.org/INDEX.html'}
+ )
+
+ dupefilter.log(r1, spider)
+ dupefilter.log(r2, spider)
+
+ assert crawler.stats.get_value('dupefilter/filtered') == 2
+ l.check_present(('scrapy.dupefilters', 'DEBUG',
+ ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
+ ' (referer: None)')))
+ l.check_present(('scrapy.dupefilters', 'DEBUG',
+ ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'
+ ' (referer: http://scrapytest.org/INDEX.html)')))
+
+ dupefilter.close('finished')