Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

per_slot_settings(docs): code style fix (from previous pre-commit check)

Georgiy Zatserklianyi045092e8d753
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 43069c50d..175e8f7f0 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -783,15 +783,8 @@ Allows to define concurrency/delay parameters on per slot(domain) basis:
.. code-block:: python
DOWNLOAD_SLOTS = {
- 'quotes.toscrape.com': {
- 'concurrency': 1,
- 'delay': 2,
- 'randomize_delay': False
- },
- 'books.toscrape.com': {
- 'delay': 3,
- 'randomize_delay': False
- }
+ "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
+ "books.toscrape.com": {"delay": 3, "randomize_delay": False},
}
.. note::
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 3691df48a..ace483a23 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -103,13 +103,14 @@ class Downloader:
if key not in self.slots:
slot_settings = self.per_slot_settings.get(key, {})
conc = (
- self.ip_concurrency if self.ip_concurrency else self.domain_concurrency)
+ self.ip_concurrency if self.ip_concurrency else self.domain_concurrency
+ )
conc, delay = _get_concurrency_delay(conc, spider, self.settings)
conc, delay = (
- slot_settings.get('concurrency', conc),
- slot_settings.get('delay', delay)
+ slot_settings.get("concurrency", conc),
+ slot_settings.get("delay", delay),
)
- randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay)
+ randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
new_slot = Slot(conc, delay, randomize_delay)
self.slots[key] = new_slot
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 5ee06d5bc..a167161a1 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -1,31 +1,31 @@
import time
+from twisted.internet import defer
+from twisted.trial.unittest import TestCase
+
from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
from tests.mockserver import MockServer
from tests.spiders import MetaSpider
-from twisted.internet import defer
-from twisted.trial.unittest import TestCase
-
class DownloaderSlotsSettingsTestSpider(MetaSpider):
- name = 'downloader_slots'
+ name = "downloader_slots"
custom_settings = {
"DOWNLOAD_DELAY": 1,
"RANDOMIZE_DOWNLOAD_DELAY": False,
"DOWNLOAD_SLOTS": {
- 'quotes.toscrape.com': {
- 'concurrency': 1,
- 'delay': 2,
- 'randomize_delay': False
+ "quotes.toscrape.com": {
+ "concurrency": 1,
+ "delay": 2,
+ "randomize_delay": False
},
- 'books.toscrape.com': {
- 'delay': 3,
- 'randomize_delay': False
+ "books.toscrape.com": {
+ "delay": 3,
+ "randomize_delay": False
}
}
}
@@ -33,21 +33,21 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):
def start_requests(self):
self.times = {None: []}
- slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None]
+ slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None]
for slot in slots:
url = self.mockserver.url(f"/?downloader_slot={slot}")
self.times[slot] = []
- yield Request(url, callback=self.parse, meta={'download_slot': slot})
+ yield Request(url, callback=self.parse, meta={"download_slot": slot})
def parse(self, response):
- slot = response.meta.get('download_slot', None)
+ slot = response.meta.get("download_slot", None)
self.times[slot].append(time.time())
url = self.mockserver.url(f"/?downloader_slot={slot}&req=2")
- yield Request(url, callback=self.not_parse, meta={'download_slot': slot})
+ yield Request(url, callback=self.not_parse, meta={"download_slot": slot})
def not_parse(self, response):
- slot = response.meta.get('download_slot', None)
+ slot = response.meta.get("download_slot", None)
self.times[slot].append(time.time())
@@ -70,6 +70,9 @@ class CrawlTestCase(TestCase):
tolerance = 0.3
delays_real = {k: v[1] - v[0] for k, v in times.items()}
- error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()}
+ error_delta = {
+ k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay)
+ for k, v in slots.items()
+ }
self.assertTrue(max(list(error_delta.values())) < tolerance)