Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

per slot settings: creation of Downloader.Slot objects from per slot settings added

GeorgeA92eea56c4912eb
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 4f7ab594f..f680b9082 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -83,6 +83,7 @@ class Downloader:
self.middleware = DownloaderMiddlewareManager.from_crawler(crawler)
self._slot_gc_loop = task.LoopingCall(self._slot_gc)
self._slot_gc_loop.start(60)
+ self.per_slot_settings = self.settings.getdict('PER_SLOT_SETTINGS', {})
def fetch(self, request, spider):
def _deactivate(response):
@@ -99,9 +100,13 @@ class Downloader:
def _get_slot(self, request, spider):
key = self._get_slot_key(request, spider)
if key not in self.slots:
- conc = self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
+ conc = self.per_slot_settings.get(key,{}).get('concurrency', self.ip_concurrency if self.ip_concurrency else self.domain_concurrency)
conc, delay = _get_concurrency_delay(conc, spider, self.settings)
- self.slots[key] = Slot(conc, delay, self.randomize_delay)
+ delay = self.per_slot_settings.get(key,{}).get('delay', delay)
+ randomize_delay = self.per_slot_settings.get(key, {}).get('randomize_delay', self.randomize_delay)
+ new_slot = Slot(conc, delay, randomize_delay)
+ self.slots[key] = new_slot
+
return key, self.slots[key]