Scrapy, a fast high-level web crawling & scraping framework for Python.
Testing stats and log messages from RFPDupeFilter
Júlio César Batista8eade7d8640e
diff --git a/tests/test_dupefilters.py b/tests/test_dupefilters.pyindex db69597a2..d7eb98c97 100644--- a/tests/test_dupefilters.py+++ b/tests/test_dupefilters.py@@ -2,6 +2,7 @@ import hashlibimport tempfileimport unittestimport shutil+from testfixtures import LogCapturefrom scrapy.dupefilters import RFPDupeFilterfrom scrapy.http import Request@@ -9,7 +10,7 @@ from scrapy.core.scheduler import Schedulerfrom scrapy.utils.python import to_bytesfrom scrapy.utils.job import job_dirfrom scrapy.utils.test import get_crawler-+from tests.spiders import SimpleSpiderclass FromCrawlerRFPDupeFilter(RFPDupeFilter):@@ -126,3 +127,57 @@ class RFPDupeFilterTest(unittest.TestCase):assert case_insensitive_dupefilter.request_seen(r2)case_insensitive_dupefilter.close('finished')++ def test_log(self):+ with LogCapture() as l:+ settings = {'DUPEFILTER_DEBUG': False,+ 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}+ crawler = get_crawler(SimpleSpider, settings_dict=settings)+ scheduler = Scheduler.from_crawler(crawler)+ spider = SimpleSpider.from_crawler(crawler)++ dupefilter = scheduler.df+ dupefilter.open()++ r1 = Request('http://scrapytest.org/index.html')+ r2 = Request('http://scrapytest.org/index.html')++ dupefilter.log(r1, spider)+ dupefilter.log(r2, spider)++ assert crawler.stats.get_value('dupefilter/filtered') == 2+ l.check_present(('scrapy.dupefilters', 'DEBUG',+ ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'+ ' - no more duplicates will be shown'+ ' (see DUPEFILTER_DEBUG to show all duplicates)')))++ dupefilter.close('finished')++ def test_log_debug(self):+ with LogCapture() as l:+ settings = {'DUPEFILTER_DEBUG': True,+ 'DUPEFILTER_CLASS': __name__ + '.FromCrawlerRFPDupeFilter'}+ crawler = get_crawler(SimpleSpider, settings_dict=settings)+ scheduler = Scheduler.from_crawler(crawler)+ spider = SimpleSpider.from_crawler(crawler)++ dupefilter = scheduler.df+ dupefilter.open()++ r1 = Request('http://scrapytest.org/index.html')+ r2 = Request('http://scrapytest.org/index.html',+ headers={'Referer': 'http://scrapytest.org/INDEX.html'}+ )++ dupefilter.log(r1, spider)+ dupefilter.log(r2, spider)++ assert crawler.stats.get_value('dupefilter/filtered') == 2+ l.check_present(('scrapy.dupefilters', 'DEBUG',+ ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'+ ' (referer: None)')))+ l.check_present(('scrapy.dupefilters', 'DEBUG',+ ('Filtered duplicate request: <GET http://scrapytest.org/index.html>'+ ' (referer: http://scrapytest.org/INDEX.html)')))++ dupefilter.close('finished')