Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Integration test integration testing only everything is working, not logic of PQ use method create slot attribute in constructor corect class for test case stop crawler in teardown method use class correct entity naming python 2 adaptation integration test with crawler and spider

Vostretsov Nikitad970be64cc47
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
index e1cf5842d..9bdc82b30 100644
--- a/tests/test_scheduler.py
+++ b/tests/test_scheduler.py
@@ -2,12 +2,17 @@ import shutil
import tempfile
import unittest
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
from scrapy.crawler import Crawler
from scrapy.core.scheduler import Scheduler
from scrapy.http import Request
from scrapy.pqueues import _scheduler_slot_read, _scheduler_slot_write
from scrapy.signals import request_reached_downloader, response_downloaded
from scrapy.spiders import Spider
+from scrapy.utils.test import get_crawler
+from tests.mockserver import MockServer
class MockCrawler(Crawler):
@@ -223,6 +228,13 @@ class TestSchedulerWithDownloaderAwareInMemory(BaseSchedulerInMemoryTester,
self.assertEqual(len(part), len(set(part)))
+def _is_slots_unique(base_slots, result_slots):
+ unique_slots = len(set(s for _, s in base_slots))
+ for i in range(0, len(result_slots), unique_slots):
+ part = result_slots[i:i + unique_slots]
+ assert len(part) == len(set(part))
+
+
class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
unittest.TestCase):
priority_queue_cls = 'scrapy.pqueues.DownloaderAwarePriorityQueue'
@@ -259,7 +271,33 @@ class TestSchedulerWithDownloaderAwareOnDisk(BaseSchedulerOnDiskTester,
spider=self.spider
)
- unique_slots = len(set(s for _, s in _SLOTS))
- for i in range(0, len(_SLOTS), unique_slots):
- part = slots[i:i + unique_slots]
- self.assertEqual(len(part), len(set(part)))
+ _is_slots_unique(_SLOTS, slots)
+
+
+class StartUrlsSpider(Spider):
+
+ def __init__(self, start_urls):
+ self.start_urls = start_urls
+
+
+class TestIntegrationWithDownloaderAwareOnDisk(TestCase):
+ def setUp(self):
+ self.crawler = get_crawler(
+ StartUrlsSpider,
+ {'SCHEDULER_PRIORITY_QUEUE': 'scrapy.pqueues.DownloaderAwarePriorityQueue',
+ 'DUPEFILTER_CLASS': 'scrapy.dupefilters.BaseDupeFilter'}
+ )
+
+ @defer.inlineCallbacks
+ def tearDown(self):
+ yield self.crawler.stop()
+
+ @defer.inlineCallbacks
+ def test_integration_downloader_aware_priority_queue(self):
+ with MockServer() as mockserver:
+
+ url = mockserver.url("/status?n=200", is_secure=False)
+ slots = [url] * 6
+ yield self.crawler.crawl(slots)
+ self.assertEqual(self.crawler.stats.get_value('downloader/response_count'),
+ len(slots))