Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

per slot settings: test added (delays for each download slots)

GeorgeA920ffc52a491e6
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
new file mode 100644
index 000000000..6ea03fb78
--- /dev/null
+++ b/tests/test_downloaderslotssettings.py
@@ -0,0 +1,75 @@
+import time
+
+from scrapy.crawler import CrawlerRunner
+from scrapy.http import Request
+
+from tests.mockserver import MockServer
+from tests.spiders import MetaSpider
+
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
+
+class DownloaderSlotsSettingsTestSpider(MetaSpider):
+
+ name = 'downloader_slots'
+
+ custom_settings = {
+ "DOWNLOAD_DELAY": 1,
+ "RANDOMIZE_DOWNLOAD_DELAY": False,
+ "DOWNLOAD_SLOTS": {
+ 'quotes.toscrape.com': {
+ 'concurrency': 1,
+ 'delay': 1.5,
+ 'randomize_delay': False
+ },
+ 'books.toscrape.com': {
+ 'delay': 2,
+ 'randomize_delay': False
+ }
+ }
+ }
+
+ def start_requests(self):
+ self.times = {None: []}
+
+ slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None]
+
+ for slot in slots:
+ url = self.mockserver.url(f"/?downloader_slot={slot}")
+ self.times[slot] = []
+ yield Request(url, callback=self.parse, meta={'download_slot': slot})
+
+ def parse(self, response):
+ slot = response.meta.get('download_slot', None)
+ self.times[slot].append(time.time())
+ url = self.mockserver.url(f"/?downloader_slot={slot}&req=2")
+ yield Request(url, callback=self.not_parse, meta={'download_slot': slot})
+
+ def not_parse(self, response):
+ slot = response.meta.get('download_slot', None)
+ self.times[slot].append(time.time())
+
+
+class CrawlTestCase(TestCase):
+
+ def setUp(self):
+ self.mockserver = MockServer()
+ self.mockserver.__enter__()
+ self.runner = CrawlerRunner()
+
+ def tearDown(self):
+ self.mockserver.__exit__(None, None, None)
+
+ @defer.inlineCallbacks
+ def test_delay(self):
+ crawler = CrawlerRunner().create_crawler(DownloaderSlotsSettingsTestSpider)
+ yield crawler.crawl(mockserver=self.mockserver)
+ slots = crawler.engine.downloader.slots
+ times = crawler.spider.times
+ tolerance = 0.3
+
+ delays_real = {k: v[1] - v[0] for k, v in times.items()}
+ error_delta = {k: 1 - delays_real[k] / v.delay for k, v in slots.items()}
+
+ self.assertTrue(max(list(error_delta.values())) < tolerance)