Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

check CONCURRENT_REQUESTS_PER_IP is not set

Vostretsov Nikita8e8ce301b1a5
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.py
index 75fc198d0..d9effc9d1 100644
--- a/scrapy/pqueues.py
+++ b/scrapy/pqueues.py
@@ -171,6 +171,14 @@ class DownloaderAwarePriorityQueue(SlotBasedPriorityQueue):
return cls(crawler, qfactory, startprios)
def __init__(self, crawler, qfactory, startprios={}):
+ ip_concurrency_key = 'CONCURRENT_REQUESTS_PER_IP'
+ ip_concurrency = crawler.settings.getint(ip_concurrency_key, 0)
+
+ if ip_concurrency > 0:
+ raise ValueError('"%s" does not support %s=%d' % (self.__class__,
+ ip_concurrency_key,
+ ip_concurrency))
+
super(DownloaderAwarePriorityQueue, self).__init__(qfactory,
startprios)
self._slots = {slot: 0 for slot in self.pqueues}