Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Allow disabling the AutoThrottle extension for a given slot (#6246)

Adrián Chavesd7581c6b41e9
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index 5370d77b3..8a13b8976 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -47,6 +47,18 @@ effect, but there are some important differences:
AutoThrottle doesn't have these issues.
+Disabling throttling on a downloader slot
+=========================================
+
+It is possible to disable AutoThrottle for a specific download slot at run time
+by setting its ``throttle`` attribute to ``False``, e.g. using
+:setting:`DOWNLOAD_SLOTS`.
+
+Note, however, that AutoThrottle still determines the starting delay of every
+slot by setting the ``download_delay`` attribute on the running spider. You
+might want to set a custom value for the ``delay`` attribute of the slot, e.g.
+using :setting:`DOWNLOAD_SLOTS`.
+
Throttling algorithm
====================
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 439aedc18..2bd9cf1ed 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -835,7 +835,7 @@ The default HTTPS handler uses HTTP/1.1. To use HTTP/2:
.. setting:: DOWNLOAD_SLOTS
DOWNLOAD_SLOTS
-----------------
+--------------
Default: ``{}``
@@ -844,7 +844,12 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
.. code-block:: python
DOWNLOAD_SLOTS = {
- "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
+ "quotes.toscrape.com": {
+ "concurrency": 1,
+ "delay": 2,
+ "randomize_delay": False,
+ "throttle": False,
+ },
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
}
@@ -856,6 +861,9 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
- :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
- :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
+ There is no global setting for ``throttle``, whose default value is
+ ``None``.
+
.. setting:: DOWNLOAD_TIMEOUT
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 666282856..ecd3e8b56 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -2,7 +2,7 @@ import random
from collections import deque
from datetime import datetime
from time import time
-from typing import TYPE_CHECKING, Any, Deque, Dict, Set, Tuple, cast
+from typing import TYPE_CHECKING, Any, Deque, Dict, Optional, Set, Tuple, cast
from twisted.internet import task
from twisted.internet.defer import Deferred
@@ -24,10 +24,18 @@ if TYPE_CHECKING:
class Slot:
"""Downloader slot"""
- def __init__(self, concurrency: int, delay: float, randomize_delay: bool):
+ def __init__(
+ self,
+ concurrency: int,
+ delay: float,
+ randomize_delay: bool,
+ *,
+ throttle: Optional[bool] = None,
+ ):
self.concurrency: int = concurrency
self.delay: float = delay
self.randomize_delay: bool = randomize_delay
+ self.throttle = throttle
self.active: Set[Request] = set()
self.queue: Deque[Tuple[Request, Deferred]] = deque()
@@ -52,13 +60,15 @@ class Slot:
return (
f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
- f"randomize_delay={self.randomize_delay!r})"
+ f"randomize_delay={self.randomize_delay!r}, "
+ f"throttle={self.throttle!r})"
)
def __str__(self) -> str:
return (
f"<downloader.Slot concurrency={self.concurrency!r} "
f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
+ f"throttle={self.throttle!r} "
f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
f"len(transferring)={len(self.transferring)} "
f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
@@ -127,7 +137,8 @@ class Downloader:
slot_settings.get("delay", delay),
)
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
- new_slot = Slot(conc, delay, randomize_delay)
+ throttle = slot_settings.get("throttle", None)
+ new_slot = Slot(conc, delay, randomize_delay, throttle=throttle)
self.slots[key] = new_slot
return key, self.slots[key]
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index bf4e6bb63..217e61a81 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -62,7 +62,7 @@ class AutoThrottle:
) -> None:
key, slot = self._get_slot(request, spider)
latency = request.meta.get("download_latency")
- if latency is None or slot is None:
+ if latency is None or slot is None or slot.throttle is False:
return
olddelay = slot.delay
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index 9a6e9e4ff..81cff4947 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -7,5 +7,6 @@ class SlotTest(unittest.TestCase):
def test_repr(self):
slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
self.assertEqual(
- repr(slot), "Slot(concurrency=8, delay=0.10, randomize_delay=True)"
+ repr(slot),
+ "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)",
)
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 9d4072d19..ea8c5b4f0 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -3,8 +3,10 @@ import time
from twisted.internet import defer
from twisted.trial.unittest import TestCase
+from scrapy import Request
+from scrapy.core.downloader import Downloader, Slot
from scrapy.crawler import CrawlerRunner
-from scrapy.http import Request
+from scrapy.utils.test import get_crawler
from tests.mockserver import MockServer
from tests.spiders import MetaSpider
@@ -20,6 +22,7 @@ class DownloaderSlotsSettingsTestSpider(MetaSpider):
"concurrency": 1,
"delay": 2,
"randomize_delay": False,
+ "throttle": False,
},
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
},
@@ -70,3 +73,27 @@ class CrawlTestCase(TestCase):
}
self.assertTrue(max(list(error_delta.values())) < tolerance)
+
+
+def test_params():
+ params = {
+ "concurrency": 1,
+ "delay": 2,
+ "randomize_delay": False,
+ "throttle": False,
+ }
+ settings = {
+ "DOWNLOAD_SLOTS": {
+ "example.com": params,
+ },
+ }
+ crawler = get_crawler(settings_dict=settings)
+ downloader = Downloader(crawler)
+ downloader._slot_gc_loop.stop() # Prevent an unclean reactor.
+ request = Request("https://example.com")
+ _, actual = downloader._get_slot(request, spider=None)
+ expected = Slot(**params)
+ for param in params:
+ assert getattr(expected, param) == getattr(
+ actual, param
+ ), f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}"
diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py
index dae4ea966..722a05c26 100644
--- a/tests/test_extension_throttle.py
+++ b/tests/test_extension_throttle.py
@@ -157,16 +157,17 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
@pytest.mark.parametrize(
- ("meta", "slot"),
+ ("meta", "slot", "throttle"),
(
- ({}, None),
- ({"download_latency": 1.0}, None),
- ({"download_slot": "foo"}, None),
- ({"download_slot": "foo"}, "foo"),
- ({"download_latency": 1.0, "download_slot": "foo"}, None),
+ ({}, None, None),
+ ({"download_latency": 1.0}, None, None),
+ ({"download_slot": "foo"}, None, None),
+ ({"download_slot": "foo"}, "foo", None),
+ ({"download_latency": 1.0, "download_slot": "foo"}, None, None),
+ ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False),
),
)
-def test_skipped(meta, slot):
+def test_skipped(meta, slot, throttle):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
spider = TestSpider()
@@ -177,7 +178,9 @@ def test_skipped(meta, slot):
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
if slot is not None:
- crawler.engine.downloader.slots[slot] = object()
+ _slot = Mock()
+ _slot.throttle = throttle
+ crawler.engine.downloader.slots[slot] = _slot
at._adjust_delay = None # Raise exception if called.
at._response_downloaded(None, request, spider)