Scrapy, a fast high-level web crawling & scraping framework for Python.
Replace Slot.throttle with Request.meta['dont_throttle']
Adrián Chavesf57fc454beb4
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rstindex 8a13b8976..fbfdd0647 100644--- a/docs/topics/autothrottle.rst+++ b/docs/topics/autothrottle.rst@@ -47,12 +47,17 @@ effect, but there are some important differences:AutoThrottle doesn't have these issues.-Disabling throttling on a downloader slot-=========================================+.. reqmeta:: dont_throttle-It is possible to disable AutoThrottle for a specific download slot at run time-by setting its ``throttle`` attribute to ``False``, e.g. using-:setting:`DOWNLOAD_SLOTS`.+Disabling the throttling of a request+=====================================++To disable AutoThrottle for a specific request, set the ``dont_throttle``+request metadata key to ``True``:++.. code-block:: python++ yield Request("https://example.com", meta={"dont_throttle": True})Note, however, that AutoThrottle still determines the starting delay of everyslot by setting the ``download_delay`` attribute on the running spider. Youdiff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 3c2843bc1..18b5cbdd0 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -675,6 +675,7 @@ Those are:* :reqmeta:`dont_obey_robotstxt`* :reqmeta:`dont_redirect`* :reqmeta:`dont_retry`+* :reqmeta:`dont_throttle`* :reqmeta:`download_fail_on_dataloss`* :reqmeta:`download_latency`* :reqmeta:`download_maxsize`diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 116e8226e..cce4a7b3e 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -845,12 +845,7 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:.. code-block:: pythonDOWNLOAD_SLOTS = {- "quotes.toscrape.com": {- "concurrency": 1,- "delay": 2,- "randomize_delay": False,- "throttle": False,- },+ "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},"books.toscrape.com": {"delay": 3, "randomize_delay": False},}@@ -862,9 +857,6 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:- :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``- :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``- There is no global setting for ``throttle``, whose default value is- ``None``.-.. setting:: DOWNLOAD_TIMEOUTdiff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.pyindex 1cc0422b7..5040741e2 100644--- a/scrapy/core/downloader/__init__.py+++ b/scrapy/core/downloader/__init__.py@@ -36,13 +36,10 @@ class Slot:concurrency: int,delay: float,randomize_delay: bool,- *,- throttle: bool | None = None,):self.concurrency: int = concurrencyself.delay: float = delayself.randomize_delay: bool = randomize_delay- self.throttle = throttleself.active: set[Request] = set()self.queue: deque[tuple[Request, Deferred[Response]]] = deque()@@ -67,15 +64,13 @@ class Slot:return (f"{cls_name}(concurrency={self.concurrency!r}, "f"delay={self.delay:.2f}, "- f"randomize_delay={self.randomize_delay!r}, "- f"throttle={self.throttle!r})"+ f"randomize_delay={self.randomize_delay!r})")def __str__(self) -> str:return (f"<downloader.Slot concurrency={self.concurrency!r} "f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "- f"throttle={self.throttle!r} "f"len(active)={len(self.active)} len(queue)={len(self.queue)} "f"len(transferring)={len(self.transferring)} "f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"@@ -146,8 +141,7 @@ class Downloader:slot_settings.get("delay", delay),)randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)- throttle = slot_settings.get("throttle", None)- new_slot = Slot(conc, delay, randomize_delay, throttle=throttle)+ new_slot = Slot(conc, delay, randomize_delay)self.slots[key] = new_slotreturn key, self.slots[key]diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.pyindex d4b4f0e9d..fbac48b1e 100644--- a/scrapy/extensions/throttle.py+++ b/scrapy/extensions/throttle.py@@ -64,7 +64,11 @@ class AutoThrottle:) -> None:key, slot = self._get_slot(request, spider)latency = request.meta.get("download_latency")- if latency is None or slot is None or slot.throttle is False:+ if (+ latency is None+ or slot is None+ or request.meta.get("dont_throttle", False) is True+ ):returnolddelay = slot.delaydiff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.pyindex 81cff4947..d929a9369 100644--- a/tests/test_core_downloader.py+++ b/tests/test_core_downloader.py@@ -8,5 +8,5 @@ class SlotTest(unittest.TestCase):slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)self.assertEqual(repr(slot),- "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)",+ "Slot(concurrency=8, delay=0.10, randomize_delay=True)",)diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex ea8c5b4f0..55f9ecac9 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -80,7 +80,6 @@ def test_params():"concurrency": 1,"delay": 2,"randomize_delay": False,- "throttle": False,}settings = {"DOWNLOAD_SLOTS": {diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.pyindex 722a05c26..602b48e78 100644--- a/tests/test_extension_throttle.py+++ b/tests/test_extension_throttle.py@@ -157,17 +157,20 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)@pytest.mark.parametrize(- ("meta", "slot", "throttle"),+ ("meta", "slot"),(- ({}, None, None),- ({"download_latency": 1.0}, None, None),- ({"download_slot": "foo"}, None, None),- ({"download_slot": "foo"}, "foo", None),- ({"download_latency": 1.0, "download_slot": "foo"}, None, None),- ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False),+ ({}, None),+ ({"download_latency": 1.0}, None),+ ({"download_slot": "foo"}, None),+ ({"download_slot": "foo"}, "foo"),+ ({"download_latency": 1.0, "download_slot": "foo"}, None),+ (+ {"download_latency": 1.0, "download_slot": "foo", "dont_throttle": True},+ "foo",+ ),),)-def test_skipped(meta, slot, throttle):+def test_skipped(meta, slot):crawler = get_crawler()at = build_from_crawler(AutoThrottle, crawler)spider = TestSpider()@@ -178,9 +181,7 @@ def test_skipped(meta, slot, throttle):crawler.engine.downloader = Mock()crawler.engine.downloader.slots = {}if slot is not None:- _slot = Mock()- _slot.throttle = throttle- crawler.engine.downloader.slots[slot] = _slot+ crawler.engine.downloader.slots[slot] = object()at._adjust_delay = None # Raise exception if called.at._response_downloaded(None, request, spider)