Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Replace Slot.throttle with Request.meta['dont_throttle']

Adrián Chavesf57fc454beb4
diff --git a/docs/topics/autothrottle.rst b/docs/topics/autothrottle.rst
index 8a13b8976..fbfdd0647 100644
--- a/docs/topics/autothrottle.rst
+++ b/docs/topics/autothrottle.rst
@@ -47,12 +47,17 @@ effect, but there are some important differences:
AutoThrottle doesn't have these issues.
-Disabling throttling on a downloader slot
-=========================================
+.. reqmeta:: dont_throttle
-It is possible to disable AutoThrottle for a specific download slot at run time
-by setting its ``throttle`` attribute to ``False``, e.g. using
-:setting:`DOWNLOAD_SLOTS`.
+Disabling the throttling of a request
+=====================================
+
+To disable AutoThrottle for a specific request, set the ``dont_throttle``
+request metadata key to ``True``:
+
+.. code-block:: python
+
+ yield Request("https://example.com", meta={"dont_throttle": True})
Note, however, that AutoThrottle still determines the starting delay of every
slot by setting the ``download_delay`` attribute on the running spider. You
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 3c2843bc1..18b5cbdd0 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -675,6 +675,7 @@ Those are:
* :reqmeta:`dont_obey_robotstxt`
* :reqmeta:`dont_redirect`
* :reqmeta:`dont_retry`
+* :reqmeta:`dont_throttle`
* :reqmeta:`download_fail_on_dataloss`
* :reqmeta:`download_latency`
* :reqmeta:`download_maxsize`
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 116e8226e..cce4a7b3e 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -845,12 +845,7 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
.. code-block:: python
DOWNLOAD_SLOTS = {
- "quotes.toscrape.com": {
- "concurrency": 1,
- "delay": 2,
- "randomize_delay": False,
- "throttle": False,
- },
+ "quotes.toscrape.com": {"concurrency": 1, "delay": 2, "randomize_delay": False},
"books.toscrape.com": {"delay": 3, "randomize_delay": False},
}
@@ -862,9 +857,6 @@ Allows to define concurrency/delay parameters on per slot (domain) basis:
- :setting:`CONCURRENT_REQUESTS_PER_DOMAIN`: ``concurrency``
- :setting:`RANDOMIZE_DOWNLOAD_DELAY`: ``randomize_delay``
- There is no global setting for ``throttle``, whose default value is
- ``None``.
-
.. setting:: DOWNLOAD_TIMEOUT
diff --git a/scrapy/core/downloader/__init__.py b/scrapy/core/downloader/__init__.py
index 1cc0422b7..5040741e2 100644
--- a/scrapy/core/downloader/__init__.py
+++ b/scrapy/core/downloader/__init__.py
@@ -36,13 +36,10 @@ class Slot:
concurrency: int,
delay: float,
randomize_delay: bool,
- *,
- throttle: bool | None = None,
):
self.concurrency: int = concurrency
self.delay: float = delay
self.randomize_delay: bool = randomize_delay
- self.throttle = throttle
self.active: set[Request] = set()
self.queue: deque[tuple[Request, Deferred[Response]]] = deque()
@@ -67,15 +64,13 @@ class Slot:
return (
f"{cls_name}(concurrency={self.concurrency!r}, "
f"delay={self.delay:.2f}, "
- f"randomize_delay={self.randomize_delay!r}, "
- f"throttle={self.throttle!r})"
+ f"randomize_delay={self.randomize_delay!r})"
)
def __str__(self) -> str:
return (
f"<downloader.Slot concurrency={self.concurrency!r} "
f"delay={self.delay:.2f} randomize_delay={self.randomize_delay!r} "
- f"throttle={self.throttle!r} "
f"len(active)={len(self.active)} len(queue)={len(self.queue)} "
f"len(transferring)={len(self.transferring)} "
f"lastseen={datetime.fromtimestamp(self.lastseen).isoformat()}>"
@@ -146,8 +141,7 @@ class Downloader:
slot_settings.get("delay", delay),
)
randomize_delay = slot_settings.get("randomize_delay", self.randomize_delay)
- throttle = slot_settings.get("throttle", None)
- new_slot = Slot(conc, delay, randomize_delay, throttle=throttle)
+ new_slot = Slot(conc, delay, randomize_delay)
self.slots[key] = new_slot
return key, self.slots[key]
diff --git a/scrapy/extensions/throttle.py b/scrapy/extensions/throttle.py
index d4b4f0e9d..fbac48b1e 100644
--- a/scrapy/extensions/throttle.py
+++ b/scrapy/extensions/throttle.py
@@ -64,7 +64,11 @@ class AutoThrottle:
) -> None:
key, slot = self._get_slot(request, spider)
latency = request.meta.get("download_latency")
- if latency is None or slot is None or slot.throttle is False:
+ if (
+ latency is None
+ or slot is None
+ or request.meta.get("dont_throttle", False) is True
+ ):
return
olddelay = slot.delay
diff --git a/tests/test_core_downloader.py b/tests/test_core_downloader.py
index 81cff4947..d929a9369 100644
--- a/tests/test_core_downloader.py
+++ b/tests/test_core_downloader.py
@@ -8,5 +8,5 @@ class SlotTest(unittest.TestCase):
slot = Slot(concurrency=8, delay=0.1, randomize_delay=True)
self.assertEqual(
repr(slot),
- "Slot(concurrency=8, delay=0.10, randomize_delay=True, throttle=None)",
+ "Slot(concurrency=8, delay=0.10, randomize_delay=True)",
)
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index ea8c5b4f0..55f9ecac9 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -80,7 +80,6 @@ def test_params():
"concurrency": 1,
"delay": 2,
"randomize_delay": False,
- "throttle": False,
}
settings = {
"DOWNLOAD_SLOTS": {
diff --git a/tests/test_extension_throttle.py b/tests/test_extension_throttle.py
index 722a05c26..602b48e78 100644
--- a/tests/test_extension_throttle.py
+++ b/tests/test_extension_throttle.py
@@ -157,17 +157,20 @@ def test_startdelay_definition(min_spider, min_setting, start_setting, expected)
@pytest.mark.parametrize(
- ("meta", "slot", "throttle"),
+ ("meta", "slot"),
(
- ({}, None, None),
- ({"download_latency": 1.0}, None, None),
- ({"download_slot": "foo"}, None, None),
- ({"download_slot": "foo"}, "foo", None),
- ({"download_latency": 1.0, "download_slot": "foo"}, None, None),
- ({"download_latency": 1.0, "download_slot": "foo"}, "foo", False),
+ ({}, None),
+ ({"download_latency": 1.0}, None),
+ ({"download_slot": "foo"}, None),
+ ({"download_slot": "foo"}, "foo"),
+ ({"download_latency": 1.0, "download_slot": "foo"}, None),
+ (
+ {"download_latency": 1.0, "download_slot": "foo", "dont_throttle": True},
+ "foo",
+ ),
),
)
-def test_skipped(meta, slot, throttle):
+def test_skipped(meta, slot):
crawler = get_crawler()
at = build_from_crawler(AutoThrottle, crawler)
spider = TestSpider()
@@ -178,9 +181,7 @@ def test_skipped(meta, slot, throttle):
crawler.engine.downloader = Mock()
crawler.engine.downloader.slots = {}
if slot is not None:
- _slot = Mock()
- _slot.throttle = throttle
- crawler.engine.downloader.slots[slot] = _slot
+ crawler.engine.downloader.slots[slot] = object()
at._adjust_delay = None # Raise exception if called.
at._response_downloaded(None, request, spider)