Scrapy, a fast high-level web crawling & scraping framework for Python.
per slot settings: creation of Downloader.Slot objects from per slot settings added
GeorgeA92eea56c4912eb
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 4f7ab594f..f680b9082 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -83,6 +83,7 @@ class Downloader:self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)self._slot_gc_loop = task.LoopingCall(self._slot_gc)self._slot_gc_loop.start(60)+ self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {})def fetch(self, request, spider):def _deactivate(response):@@ -99,9 +100,13 @@ class Downloader:def _get_slot(self, request, spider):key = self._get_slot_key(request, spider)if key not in self.slots:- conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency+ conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency)conc, delay = _get_concurrency_delay(conc, spider, self.settings)- self.slots[key] = Slot(conc, delay, self.randomize_delay)+ delay = self.per_slot_settings.get(key,{}).get('delay', delay)+ randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay)+ new_slot = Slot(conc, delay, randomize_delay)+ self.slots[key] = new_slot+return key, self.slots[key]