Scrapy, a fast high-level web crawling & scraping framework for Python.
per_slot_settings(docs): code style fix (from previous pre-commit check)
Georgiy Zatserklianyi045092e8d753
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 43069c50d..175e8f7f0 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -783,15 +783,8 @@ Allows to define concurrency/delay parameters on per slot(domain) basis:.. code-block:: pythonDOWNLOAD_SLOTS = {- 'quotes.toscrape.com': {- 'concurrency': 1,- 'delay': 2,- 'randomize_delay': False- },- 'books.toscrape.com': {- 'delay': 3,- 'randomize_delay': False- }+ "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},+ "books.toscrape.com": {"delay": 3, "randomize_delay": False},}.. note::diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 3691df48a..ace483a23 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -103,13 +103,14 @@ class Downloader:if key not in self.slots:slot_settings = self.per_slot_settings.get(key, {})conc = (- self.ip_concurrency if self.ip_concurrency else self.domain_concurrency)+ self.ip_concurrency if self.ip_concurrency else self.domain_concurrency+ )conc, delay = _get_concurrency_delay(conc, spider, self.settings)conc, delay = (- slot_settings.get('concurrency', conc),- slot_settings.get('delay', delay)+ slot_settings.get("concurrency", conc),+ slot_settings.get("delay", delay),)- randomize_delay = slot_settings.get('randomize_delay', self.randomize_delay)+ randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)new_slot = Slot(conc, delay, randomize_delay)self.slots[key] = new_slotdiff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 5ee06d5bc..a167161a1 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -1,31 +1,31 @@import time+from twisted.internet import defer+from twisted.trial.unittest import TestCase+from scrapy.crawler import CrawlerRunnerfrom scrapy.http import Requestfrom tests.mockserver import MockServerfrom tests.spiders import MetaSpider-from twisted.internet import defer-from twisted.trial.unittest import TestCase-class DownloaderSlotsSettingsTestSpider(MetaSpider):- name = 'downloader_slots'+ name = "downloader_slots"custom_settings = {"DOWNLOAD_DELAY": 1,"RANDOMIZE_DOWNLOAD_DELAY": False,"DOWNLOAD_SLOTS": {- 'quotes.toscrape.com': {- 'concurrency': 1,- 'delay': 2,- 'randomize_delay': False+ "quotes.toscrape.com": {+ "concurrency": 1,+ "delay": 2,+ "randomize_delay": False},- 'books.toscrape.com': {- 'delay': 3,- 'randomize_delay': False+ "books.toscrape.com": {+ "delay": 3,+ "randomize_delay": False}}}@@ -33,21 +33,21 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):def start_requests(self):self.times = {None: []}- slots = list(self.custom_settings.get('DOWNLOAD_SLOTS', {}).keys()) + [None]+ slots = list(self.custom_settings.get("DOWNLOAD_SLOTS", {}).keys()) + [None]for slot in slots:url = self.mockserver.url(f"/?downloader_slot={slot}")self.times[slot] = []- yield Request(url, callback=self.parse, meta={'download_slot': slot})+ yield Request(url, callback=self.parse, meta={"download_slot": slot})def parse(self, response):- slot = response.meta.get('download_slot', None)+ slot = response.meta.get("download_slot", None)self.times[slot].append(time.time())url = self.mockserver.url(f"/?downloader_slot={slot}&req=2")- yield Request(url, callback=self.not_parse, meta={'download_slot': slot})+ yield Request(url, callback=self.not_parse, meta={"download_slot": slot})def not_parse(self, response):- slot = response.meta.get('download_slot', None)+ slot = response.meta.get("download_slot", None)self.times[slot].append(time.time())@@ -70,6 +70,9 @@ class CrawlTestCase(TestCase):tolerance = 0.3delays_real = {k: v[1] - v[0] for k, v in times.items()}- error_delta = {k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay) for k, v in slots.items()}+ error_delta = {+ k: 1 - min(delays_real[k], v.delay) / max(delays_real[k], v.delay)+ for k, v in slots.items()+ }self.assertTrue(max(list(error_delta.values())) < tolerance)