Scrapy, a fast high-level web crawling & scraping framework for Python.
Integration test integration testing only everything is working, not logic of PQ use method create slot attribute in constructor corect class for test case stop crawler in teardown method use class correct entity naming python 2 adaptation integration test with crawler and spider
Vostretsov Nikitad970be64cc47
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.pyindex e1cf5842d..9bdc82b30 100644--- a/tests/test_scheduler.py+++ b/tests/test_scheduler.py@@ -2,12 +2,17 @@ import shutilimport tempfileimport unittest+from twisted.internet import defer+from twisted.trial.unittest import TestCase+from scrapy.crawler import Crawlerfrom scrapy.core.scheduler import Schedulerfrom scrapy.http import Requestfrom scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_writefrom scrapy.signals import request_reached_downloader, response_downloadedfrom scrapy.spiders import Spider+from scrapy.utils.test import get_crawler+from tests.mockserver import MockServerclass MockCrawler(Crawler):@@ -223,6 +228,13 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester,self.assertEqual(len(part), len(set(part)))+def _is_slots_unique(base_slots, result_slots):+ unique_slots = len(set(s for _, s in base_slots))+ for i in range(0, len(result_slots), unique_slots):+ part = result_slots[i:i + unique_slots]+ assert len(part) == len(set(part))++class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,unittest.TestCase):priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'@@ -259,7 +271,33 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,spider=self.spider)- unique_slots = len(set(s for _, s in _SLOTS))- for i in range(0, len(_SLOTS), unique_slots):- part = slots[i:i + unique_slots]- self.assertEqual(len(part), len(set(part)))+ _is_slots_unique(_SLOTS, slots)+++class StartUrlsSpider(Spider):++ def __init__(self, start_urls):+ self.start_urls = start_urls+++class TestIntegrationWithDownloaderAwareOnDisk(TestCase):+ def setUp(self):+ self.crawler = get_crawler(+ StartUrlsSpider,+ {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',+ 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}+ )++ @defer.inlineCallbacks+ def tearDown(self):+ yield self.crawler.stop()++ @defer.inlineCallbacks+ def test_integration_downloader_aware_priority_queue(self):+ with MockServer() as mockserver:++ url = mockserver.url("/status?n=200", is_secure=False)+ slots = [url] * 6+ yield self.crawler.crawl(slots)+ self.assertEqual(self.crawler.stats.get_value('downloader/response_count'),+ len(slots))