Scrapy, a fast high-level web crawling & scraping framework for Python.
Allow disabling the AutoThrottle extension for a given slot (#6246)
Adrián Chavesd7581c6b41e9
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rstindex 5370d77b3..8a13b8976 100644--- a/docs/topics/autothrottle.rst+++ b/docs/topics/autothrottle.rst@@ -47,6 +47,18 @@ effect, but there are some important differences:AutoThrottle doesn't have these issues.+Disabling throttling on a downloader slot+=========================================++It is possible to disable AutoThrottle for a specific download slot at run time+by setting its ``throttle`` attribute to ``False``, e.g. using+:setting:`DOWNLOAD_SLOTS`.++Note, however, that AutoThrottle still determines the starting delay of every+slot by setting the ``download_delay`` attribute on the running spider. You+might want to set a custom value for the ``delay`` attribute of the slot, e.g.+using :setting:`DOWNLOAD_SLOTS`.+Throttling algorithm====================diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 439aedc18..2bd9cf1ed 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -835,7 +835,7 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:.. setting:: DOWNLOAD_SLOTSDOWNLOAD_SLOTS-----------------+--------------Default: ``{}``@@ -844,7 +844,12 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:.. code-block:: pythonDOWNLOAD_SLOTS = {- "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},+ "quotes.toscrape.com": {+ "concurrency": 1,+ "delay": 2,+ "randomize_delay": False,+ "throttle": False,+ },"books.toscrape.com": {"delay": 3, "randomize_delay": False},}@@ -856,6 +861,9 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:- :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``- :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``+ There is no global setting for ``throttle``, whose default value is+ ``None``.+.. setting:: DOWNLOAD_TIMEOUTdiff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 666282856..ecd3e8b56 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -2,7 +2,7 @@ import randomfrom collections import dequefrom datetime import datetimefrom time import time-from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast+from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, castfrom twisted.internet import taskfrom twisted.internet.defer import Deferred@@ -24,10 +24,18 @@ if TYPE_CHECKING:class Slot:"""Downloader slot"""- def __init__(self, concurrency: int, delay: float, randomize_delay: bool):+ def __init__(+ self,+ concurrency: int,+ delay: float,+ randomize_delay: bool,+ *,+ throttle: Optional[bool] = None,+ ):self.concurrency: int = concurrencyself.delay: float = delayself.randomize_delay: bool = randomize_delay+ self.throttle = throttleself.active: Set[Request] = set()self.queue: Deque[Tuple[Request, Deferred]] = deque()@@ -52,13 +60,15 @@ class Slot:return (f"{cls_name}(concurrency={self.concurrency!r}, "f"delay={self.delay:.2f}, "- f"randomize_delay={self.randomize_delay!r})"+ f"randomize_delay={self.randomize_delay!r}, "+ f"throttle={self.throttle!r})")def __str__(self) -> str:return (f"<downloader.Slot concurrency={self.concurrency!r} "f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "+ f"throttle={self.throttle!r} "f"len(active)={len(self.active)} len(queue)={len(self.queue)} "f"len(transferring)={len(self.transferring)} "f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"@@ -127,7 +137,8 @@ class Downloader:slot_settings.get("delay", delay),)randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)- new_slot = Slot(conc, delay, randomize_delay)+ throttle = slot_settings.get("throttle", None)+ new_slot = Slot(conc, delay, randomize_delay, throttle=throttle)self.slots[key] = new_slotreturn key, self.slots[key]diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.pyindex bf4e6bb63..217e61a81 100644--- a/scrapy/extensions/throttle.py+++ b/scrapy/extensions/throttle.py@@ -62,7 +62,7 @@ class AutoThrottle:) -> None:key, slot = self._get_slot(request, spider)latency = request.meta.get("download_latency")- if latency is None or slot is None:+ if latency is None or slot is None or slot.throttle is False:returnolddelay = slot.delaydiff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 9a6e9e4ff..81cff4947 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -7,5 +7,6 @@ class SlotTest(unittest.TestCase):def test_repr(self):slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)self.assertEqual(- repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)"+ repr(slot),+ "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)",)diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 9d4072d19..ea8c5b4f0 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -3,8 +3,10 @@ import timefrom twisted.internet import deferfrom twisted.trial.unittest import TestCase+from scrapy import Request+from scrapy.core.downloader import Downloader, Slotfrom scrapy.crawler import CrawlerRunner-from scrapy.http import Request+from scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServerfrom tests.spiders import MetaSpider@@ -20,6 +22,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):"concurrency": 1,"delay": 2,"randomize_delay": False,+ "throttle": False,},"books.toscrape.com": {"delay": 3, "randomize_delay": False},},@@ -70,3 +73,27 @@ class CrawlTestCase(TestCase):}self.assertTrue(max(list(error_delta.values())) < tolerance)+++def test_params():+ params = {+ "concurrency": 1,+ "delay": 2,+ "randomize_delay": False,+ "throttle": False,+ }+ settings = {+ "DOWNLOAD_SLOTS": {+ "example.com": params,+ },+ }+ crawler = get_crawler(settings_dict=settings)+ downloader = Downloader(crawler)+ downloader._slot_gc_loop.stop() # Prevent an unclean reactor.+ request = Request("https://example.com")+ _, actual = downloader._get_slot(request, spider=None)+ expected = Slot(**params)+ for param in params:+ assert getattr(expected, param) == getattr(+ actual, param+ ), f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}"diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.pyindex dae4ea966..722a05c26 100644--- a/tests/test_extension_throttle.py+++ b/tests/test_extension_throttle.py@@ -157,16 +157,17 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)@pytest.mark.parametrize(- ("meta", "slot"),+ ("meta", "slot", "throttle"),(- ({}, None),- ({"download_latency": 1.0}, None),- ({"download_slot": "foo"}, None),- ({"download_slot": "foo"}, "foo"),- ({"download_latency": 1.0, "download_slot": "foo"}, None),+ ({}, None, None),+ ({"download_latency": 1.0}, None, None),+ ({"download_slot": "foo"}, None, None),+ ({"download_slot": "foo"}, "foo", None),+ ({"download_latency": 1.0, "download_slot": "foo"}, None, None),+ ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False),),)-def test_skipped(meta, slot):+def test_skipped(meta, slot, throttle):crawler = get_crawler()at = build_from_crawler(AutoThrottle, crawler)spider = TestSpider()@@ -177,7 +178,9 @@ def test_skipped(meta, slot):crawler.engine.downloader = Mock()crawler.engine.downloader.slots = {}if slot is not None:- crawler.engine.downloader.slots[slot] = object()+ _slot = Mock()+ _slot.throttle = throttle+ crawler.engine.downloader.slots[slot] = _slotat._adjust_delay = None # Raise exception if called.at._response_downloaded(None, request, spider)