Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(spiders): Auto throttle based on website speed/protection

Karim shoairca5196242184
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 3972ff8..2d19016 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -13,6 +13,7 @@ from scrapling.spiders.session import SessionManager
from scrapling.spiders.request import Request, Response
from scrapling.spiders.robotstxt import RobotsTxtManager
from scrapling.spiders.result import CrawlStats, ItemList
+from scrapling.spiders.throttle import AutoThrottle, parse_retry_after
from scrapling.spiders.cache import ResponseCacheManager
from scrapling.spiders.checkpoint import CheckpointManager, CheckpointData
from scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator
@@ -60,6 +61,18 @@ class CrawlerEngine:
else:
self._cache_manager = None
+ if self.spider.autothrottle_enabled:
+ self._autothrottle: Optional[AutoThrottle] = AutoThrottle(
+ start_delay=self.spider.autothrottle_start_delay,
+ max_delay=self.spider.autothrottle_max_delay,
+ target_concurrency=(
+ self.spider.autothrottle_target_concurrency or self.spider.concurrent_requests_per_domain or 1.0
+ ),
+ block_backoff=self.spider.autothrottle_block_backoff,
+ )
+ else:
+ self._autothrottle = None
+
self._global_limiter = CapacityLimiter(spider.concurrent_requests)
self._domain_limiters: dict[str, CapacityLimiter] = {}
self._allowed_domains: set[str] = spider.allowed_domains or set()
@@ -177,9 +190,11 @@ class CrawlerEngine:
self.stats.robots_disallowed_count += 1
log.info(f"Request disallowed by robots.txt: {request.url}")
return
- delay = await self._get_domain_delay(request)
+ floor = await self._get_domain_delay(request)
else:
- delay = self.spider.download_delay
+ floor = self.spider.download_delay
+
+ delay = floor
if self._cache_manager and request._fp is not None:
cached = await self._cache_manager.get(request._fp)
@@ -194,6 +209,9 @@ class CrawlerEngine:
return
async with self._rate_limiter(request.domain):
+ if self._autothrottle:
+ delay = self._autothrottle.delay_for(request.domain, floor)
+
if delay:
await anyio.sleep(delay)
@@ -202,7 +220,9 @@ class CrawlerEngine:
if request._session_kwargs.get("proxies"):
self.stats.proxies.append(dict(request._session_kwargs["proxies"]))
try:
+ started_at = anyio.current_time()
response = await self.session_manager.fetch(request)
+ latency = anyio.current_time() - started_at
self.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)
self.stats.increment_response_bytes(request.domain, len(response.body))
self.stats.increment_status(response.status)
@@ -216,7 +236,13 @@ class CrawlerEngine:
self.stats.cache_misses += 1
await self._cache_manager.put(request._fp, response, request._session_kwargs.get("method", "GET"))
- if await self.spider.is_blocked(response):
+ blocked = await self.spider.is_blocked(response)
+ if self._autothrottle:
+ ok = 200 <= response.status < 300 and not blocked
+ retry_after = None if ok else parse_retry_after(response.headers)
+ self._autothrottle.record(request.domain, latency, ok, floor, retry_after)
+
+ if blocked:
self.stats.blocked_requests_count += 1
if request._retry_count < self.spider.max_blocked_retries:
retry_request = request.copy()
@@ -333,6 +359,8 @@ class CrawlerEngine:
self._domain_limiters.clear()
if self._robots_manager:
self._domain_delays.clear()
+ if self._autothrottle:
+ self._autothrottle.reset()
# Check for existing checkpoint
resuming = (await self._restore_from_checkpoint()) if self._checkpoint_system_enabled else False
@@ -342,6 +370,7 @@ class CrawlerEngine:
self.stats.concurrent_requests = self.spider.concurrent_requests
self.stats.concurrent_requests_per_domain = self.spider.concurrent_requests_per_domain
self.stats.download_delay = self.spider.download_delay
+ self.stats.autothrottle_enabled = self.spider.autothrottle_enabled
await self.spider.on_start(resuming=resuming)
await self._prefetch_robots_txt()
@@ -408,6 +437,9 @@ class CrawlerEngine:
if not self.paused and self._checkpoint_system_enabled:
await self._checkpoint_manager.cleanup()
+ if self._autothrottle:
+ self.stats.autothrottle_delays = dict(self._autothrottle.delays)
+
self.stats.log_levels_counter = self.spider._log_counter.get_counts()
self.stats.end_time = anyio.current_time()
log.info(_dump(self.stats.to_dict()))
diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.py
index 3710750..796bbc2 100644
--- a/scrapling/spiders/result.py
+++ b/scrapling/spiders/result.py
@@ -56,7 +56,9 @@ class CrawlStats:
start_time: float = 0.0
end_time: float = 0.0
download_delay: float = 0.0
+ autothrottle_enabled: bool = False
blocked_requests_count: int = 0
+ autothrottle_delays: Dict = field(default_factory=dict)
custom_stats: Dict = field(default_factory=dict)
response_status_count: Dict = field(default_factory=dict)
domains_response_bytes: Dict = field(default_factory=dict)
@@ -91,6 +93,8 @@ class CrawlStats:
"items_dropped": self.items_dropped,
"elapsed_seconds": round(self.elapsed_seconds, 2),
"download_delay": round(self.download_delay, 2),
+ "autothrottle_enabled": self.autothrottle_enabled,
+ "autothrottle_delays": {domain: round(delay, 2) for domain, delay in self.autothrottle_delays.items()},
"concurrent_requests": self.concurrent_requests,
"concurrent_requests_per_domain": self.concurrent_requests_per_domain,
"requests_count": self.requests_count,
diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.py
index edd9d8b..4875077 100644
--- a/scrapling/spiders/spider.py
+++ b/scrapling/spiders/spider.py
@@ -85,6 +85,13 @@ class Spider(ABC):
download_delay: float = 0.0
max_blocked_retries: int = 3
+ # AutoThrottle settings
+ autothrottle_enabled: bool = False
+ autothrottle_start_delay: float = 5.0
+ autothrottle_max_delay: float = 60.0
+ autothrottle_target_concurrency: Optional[float] = None
+ autothrottle_block_backoff: bool = True
+
# Fingerprint adjustments
fp_include_kwargs: bool = False
fp_keep_fragments: bool = False
diff --git a/scrapling/spiders/throttle.py b/scrapling/spiders/throttle.py
new file mode 100644
index 0000000..6b39d69
--- /dev/null
+++ b/scrapling/spiders/throttle.py
@@ -0,0 +1,101 @@
+from email.utils import parsedate_to_datetime
+from datetime import datetime, timezone
+
+from scrapling.core.utils import log
+from scrapling.core._types import Dict, Mapping, Optional
+
+BLOCK_BACKOFF_FACTOR = 2.0
+
+
+def parse_retry_after(headers: Mapping[str, str]) -> Optional[float]:
+ """Return how many seconds a `Retry-After` header asks us to wait, or `None` when it's missing or unreadable.
+
+ :param headers: The response headers to look the value up in.
+ """
+ value = next((headers[key] for key in headers if key.lower() == "retry-after"), "").strip()
+ if not value:
+ return None
+
+ try:
+ return max(float(value), 0.0)
+ except ValueError:
+ pass
+
+ try: # The header can be an HTTP date instead of a number of seconds
+ return max((parsedate_to_datetime(value) - datetime.now(timezone.utc)).total_seconds(), 0.0)
+ except (TypeError, ValueError):
+ log.debug(f"Ignoring an unreadable `Retry-After` header: {value!r}")
+ return None
+
+
+class AutoThrottle:
+ """Adjusts the per-domain delay from the observed response latency, so the spider speeds up on fast
+ servers and backs off on slow or hostile ones."""
+
+ def __init__(
+ self,
+ start_delay: float = 5.0,
+ max_delay: float = 60.0,
+ target_concurrency: float = 1.0,
+ block_backoff: bool = True,
+ ):
+ """
+ :param start_delay: The delay used for the first request to a domain.
+ :param max_delay: The highest delay the throttle is allowed to reach.
+ :param target_concurrency: How many requests the spider aims to have in flight per domain. The engine
+ passes the spider's `concurrent_requests_per_domain` here, or 1 when it's unlimited.
+ :param block_backoff: Double the delay of a domain whenever it blocks us, or wait what its `Retry-After`
+ header asks for.
+ """
+ if target_concurrency <= 0:
+ raise ValueError("`target_concurrency` must be higher than 0")
+ if max_delay < start_delay:
+ raise ValueError("`autothrottle_max_delay` can't be lower than `autothrottle_start_delay`")
+
+ self.start_delay = start_delay
+ self.max_delay = max_delay
+ self.target_concurrency = target_concurrency
+ self.block_backoff = block_backoff
+ self.delays: Dict[str, float] = {}
+
+ def delay_for(self, domain: str, floor: float = 0.0) -> float:
+ """Return the current delay for a domain, starting it at `start_delay` the first time.
+
+ :param domain: The domain the request belongs to.
+ :param floor: The lowest delay allowed, which is the spider's own delay for this domain.
+ """
+ if domain not in self.delays:
+ self.delays[domain] = min(max(floor, self.start_delay), self.max_delay)
+ return self.delays[domain]
+
+ def record(
+ self, domain: str, latency: float, ok: bool, floor: float = 0.0, retry_after: Optional[float] = None
+ ) -> float:
+ """Feed a finished request back into the throttle and return the domain's new delay.
+
+ :param domain: The domain the request belongs to.
+ :param latency: How long the request took in seconds.
+ :param ok: Whether the response was a healthy one, so a non-blocked 2xx.
+ :param floor: The lowest delay allowed, which is the spider's own delay for this domain.
+ :param retry_after: How long the website asked us to wait, when it did.
+ """
+ current_delay = self.delay_for(domain, floor)
+ target_delay = latency / self.target_concurrency
+ new_delay = max((current_delay + target_delay) / 2, target_delay)
+
+ if not ok:
+ penalty = current_delay
+ if self.block_backoff:
+ penalty = retry_after if retry_after is not None else current_delay * BLOCK_BACKOFF_FACTOR
+ new_delay = max(new_delay, penalty, current_delay) # A block can never speed the spider up
+
+ new_delay = min(max(new_delay, floor), self.max_delay)
+ self.delays[domain] = new_delay
+ log.debug(
+ f"AutoThrottle ({domain}): latency={latency:.2f}s, ok={ok}, delay {current_delay:.2f}s -> {new_delay:.2f}s"
+ )
+ return new_delay
+
+ def reset(self) -> None:
+ """Drop every learned delay."""
+ self.delays.clear()
diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.py
index 3b990ed..b29114c 100644
--- a/tests/spiders/test_cache.py
+++ b/tests/spiders/test_cache.py
@@ -224,6 +224,11 @@ class MockSpider:
self.concurrent_requests_per_domain = 0
self.download_delay = 0.0
self.max_blocked_retries = 3
+ self.autothrottle_enabled = False
+ self.autothrottle_start_delay = 5.0
+ self.autothrottle_max_delay = 60.0
+ self.autothrottle_target_concurrency = None
+ self.autothrottle_block_backoff = True
self.allowed_domains: Set[str] = set()
self.fp_include_kwargs = False
self.fp_include_headers = False
diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py
index 10cbfdc..8e4bac2 100644
--- a/tests/spiders/test_engine.py
+++ b/tests/spiders/test_engine.py
@@ -78,6 +78,11 @@ class MockSpider:
concurrent_requests_per_domain: int = 0,
download_delay: float = 0.0,
max_blocked_retries: int = 3,
+ autothrottle_enabled: bool = False,
+ autothrottle_start_delay: float = 5.0,
+ autothrottle_max_delay: float = 60.0,
+ autothrottle_target_concurrency: float | None = None,
+ autothrottle_block_backoff: bool = True,
allowed_domains: Set[str] | None = None,
fp_include_kwargs: bool = False,
fp_include_headers: bool = False,
@@ -92,6 +97,11 @@ class MockSpider:
self.concurrent_requests_per_domain = concurrent_requests_per_domain
self.download_delay = download_delay
self.max_blocked_retries = max_blocked_retries
+ self.autothrottle_enabled = autothrottle_enabled
+ self.autothrottle_start_delay = autothrottle_start_delay
+ self.autothrottle_max_delay = autothrottle_max_delay
+ self.autothrottle_target_concurrency = autothrottle_target_concurrency
+ self.autothrottle_block_backoff = autothrottle_block_backoff
self.allowed_domains = allowed_domains or set()
self.fp_include_kwargs = fp_include_kwargs
self.fp_include_headers = fp_include_headers
diff --git a/tests/spiders/test_force_stop_checkpoint.py b/tests/spiders/test_force_stop_checkpoint.py
index a26e552..41b8d8f 100644
--- a/tests/spiders/test_force_stop_checkpoint.py
+++ b/tests/spiders/test_force_stop_checkpoint.py
@@ -71,6 +71,11 @@ class SlowSpider:
self.concurrent_requests_per_domain = 0
self.download_delay = 0.0
self.max_blocked_retries = 3
+ self.autothrottle_enabled = False
+ self.autothrottle_start_delay = 5.0
+ self.autothrottle_max_delay = 60.0
+ self.autothrottle_target_concurrency = None
+ self.autothrottle_block_backoff = True
self.allowed_domains = set()
self.fp_include_kwargs = False
self.fp_include_headers = False
diff --git a/tests/spiders/test_throttle.py b/tests/spiders/test_throttle.py
new file mode 100644
index 0000000..7705199
--- /dev/null
+++ b/tests/spiders/test_throttle.py
@@ -0,0 +1,392 @@
+"""Tests for the AutoThrottle algorithm and its integration with the crawling engine."""
+
+import anyio
+import pytest
+
+from scrapling.spiders.engine import CrawlerEngine
+from scrapling.spiders.request import Request
+from scrapling.spiders.session import SessionManager
+from scrapling.spiders.throttle import AutoThrottle, parse_retry_after
+from scrapling.engines.toolbelt.custom import Response
+from scrapling.core._types import Any, Dict, Set, AsyncGenerator
+
+
+def _make_response(url: str = "https://example.com", status: int = 200, retry_after: str = "") -> Response:
+ headers = {"content-type": "text/html"}
+ if retry_after:
+ headers["Retry-After"] = retry_after
+
+ return Response(
+ url=url,
+ content=b"<html>hello</html>",
+ status=status,
+ reason="OK",
+ encoding="utf-8",
+ cookies={},
+ headers=headers,
+ request_headers={"user-agent": "test"},
+ method="GET",
+ )
+
+
+class TestAutoThrottleValidation:
+ """Test the constructor guards"""
+
+ def test_invalid_target_concurrency(self):
+ for value in (0, -1.0):
+ with pytest.raises(ValueError, match="target_concurrency"):
+ AutoThrottle(target_concurrency=value)
+
+ def test_max_delay_lower_than_start_delay(self):
+ with pytest.raises(ValueError, match="max_delay"):
+ AutoThrottle(start_delay=10.0, max_delay=5.0)
+
+
+class TestAutoThrottleMath:
+ """Test the delay calculation itself"""
+
+ def test_starts_at_start_delay(self):
+ throttle = AutoThrottle(start_delay=5.0)
+
+ assert throttle.delay_for("example.com") == 5.0
+
+ def test_floor_raises_the_starting_delay(self):
+ throttle = AutoThrottle(start_delay=1.0)
+
+ assert throttle.delay_for("example.com", floor=3.0) == 3.0
+
+ def test_start_delay_is_capped_by_max_delay(self):
+ throttle = AutoThrottle(start_delay=5.0, max_delay=20.0)
+
+ assert throttle.delay_for("example.com", floor=100.0) == 20.0
+
+ def test_converges_toward_latency_over_target_concurrency(self):
+ throttle = AutoThrottle(start_delay=5.0)
+
+ for _ in range(20):
+ delay = throttle.record("example.com", latency=2.0, ok=True)
+
+ assert delay == pytest.approx(2.0, abs=0.01)
+
+ def test_target_concurrency_divides_the_latency(self):
+ throttle = AutoThrottle(start_delay=5.0, target_concurrency=4.0)
+
+ for _ in range(20):
+ delay = throttle.record("example.com", latency=2.0, ok=True)
+
+ assert delay == pytest.approx(0.5, abs=0.01)
+
+ def test_latency_spike_jumps_straight_to_target(self):
+ """A slow response must not be averaged down, it takes effect immediately"""
+ throttle = AutoThrottle(start_delay=1.0)
+
+ assert throttle.record("example.com", latency=10.0, ok=True) == 10.0
+
+ def test_non_ok_response_never_decreases_the_delay(self):
+ throttle = AutoThrottle(start_delay=5.0, block_backoff=False)
+
+ assert throttle.record("example.com", latency=0.1, ok=False) == 5.0
+
+ def test_non_ok_response_can_still_increase_the_delay(self):
+ throttle = AutoThrottle(start_delay=1.0, block_backoff=False)
+
+ assert throttle.record("example.com", latency=8.0, ok=False) == 8.0
+
+ def test_delay_is_clamped_between_floor_and_max(self):
+ throttle = AutoThrottle(start_delay=1.0, max_delay=3.0)
+
+ assert throttle.record("example.com", latency=100.0, ok=True) == 3.0
+ assert throttle.record("other.com", latency=0.001, ok=True, floor=2.0) == 2.0
+
+ def test_domains_are_throttled_independently(self):
+ throttle = AutoThrottle(start_delay=1.0)
+
+ throttle.record("slow.com", latency=9.0, ok=True)
+ throttle.record("fast.com", latency=0.5, ok=True)
+
+ assert throttle.delays["slow.com"] == 9.0
+ assert throttle.delays["fast.com"] < 1.0
+
+ def test_reset_clears_every_domain(self):
+ throttle = AutoThrottle(start_delay=1.0)
+ throttle.record("example.com", latency=9.0, ok=True)
+
+ throttle.reset()
+
+ assert throttle.delays == {}
+ assert throttle.delay_for("example.com") == 1.0
+
+
+class TestBlockBackoff:
+ """Test the doubling that kicks in when a website blocks us"""
+
+ def test_each_block_doubles_the_delay(self):
+ """Blocks are often served fast, so latency alone would never slow the spider down"""
+ throttle = AutoThrottle(start_delay=1.0, max_delay=60.0)
+
+ assert [throttle.record("example.com", latency=0.05, ok=False) for _ in range(5)] == [2.0, 4.0, 8.0, 16.0, 32.0]
+
+ def test_backoff_stops_at_max_delay(self):
+ throttle = AutoThrottle(start_delay=1.0, max_delay=5.0)
+
+ for _ in range(10):
+ delay = throttle.record("example.com", latency=0.05, ok=False)
+
+ assert delay == 5.0
+
+ def test_healthy_responses_bring_the_delay_back_down(self):
+ """Once the website stops blocking, the normal averaging has to recover"""
+ throttle = AutoThrottle(start_delay=1.0, max_delay=60.0)
+ for _ in range(4):
+ throttle.record("example.com", latency=0.05, ok=False)
+ assert throttle.delays["example.com"] == 16.0
+
+ for _ in range(20):
+ delay = throttle.record("example.com", latency=0.5, ok=True)
+
+ assert delay == pytest.approx(0.5, abs=0.01)
+
+ def test_disabled_backoff_only_freezes_the_delay(self):
+ throttle = AutoThrottle(start_delay=1.0, block_backoff=False)
+
+ assert [throttle.record("example.com", latency=0.05, ok=False) for _ in range(3)] == [1.0, 1.0, 1.0]
+
+ def test_retry_after_wins_over_the_doubling(self):
+ throttle = AutoThrottle(start_delay=1.0, max_delay=60.0)
+
+ assert throttle.record("example.com", latency=0.05, ok=False, retry_after=30.0) == 30.0
+
+ def test_retry_after_is_capped_by_max_delay(self):
+ throttle = AutoThrottle(start_delay=1.0, max_delay=60.0)
+
+ assert throttle.record("example.com", latency=0.05, ok=False, retry_after=3600.0) == 60.0
+
+ def test_retry_after_never_speeds_the_spider_up(self):
+ """A tiny or zero Retry-After on a block must not undo the backoff"""
+ throttle = AutoThrottle(start_delay=10.0, max_delay=60.0)
+
+ assert throttle.record("example.com", latency=0.05, ok=False, retry_after=0.0) == 10.0
+
+ def test_retry_after_is_ignored_when_backoff_is_disabled(self):
+ throttle = AutoThrottle(start_delay=1.0, block_backoff=False)
+
+ assert throttle.record("example.com", latency=0.05, ok=False, retry_after=30.0) == 1.0
+
+
+class TestParseRetryAfter:
+ """Test reading the `Retry-After` header"""
+
+ def test_missing_header(self):
+ assert parse_retry_after({}) is None
+ assert parse_retry_after({"Content-Type": "text/html"}) is None
+
+ def test_seconds(self):
+ assert parse_retry_after({"Retry-After": "120"}) == 120.0
+
+ def test_header_name_is_case_insensitive(self):
+ """Engines don't agree on the casing of header names"""
+ assert parse_retry_after({"retry-after": "120"}) == 120.0
+ assert parse_retry_after({"RETRY-AFTER": " 120 "}) == 120.0
+
+ def test_http_date(self):
+ from email.utils import format_datetime
+ from datetime import datetime, timedelta, timezone
+
+ future = datetime.now(timezone.utc) + timedelta(seconds=90)
+
+ assert parse_retry_after({"Retry-After": format_datetime(future)}) == pytest.approx(90.0, abs=5.0)
+
+ def test_past_http_date_is_zero(self):
+ assert parse_retry_after({"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}) == 0.0
+
+ def test_negative_seconds_are_clamped(self):
+ assert parse_retry_after({"Retry-After": "-5"}) == 0.0
+
+ def test_garbage_is_ignored(self):
+ for value in ("soon", "", " ", "12 seconds"):
+ assert parse_retry_after({"Retry-After": value}) is None
+
+
+# ---------------------------------------------------------------------------
+# Engine integration
+# ---------------------------------------------------------------------------
+
+
+class MockSession:
+ def __init__(self, latency: float = 0.05, status: int = 200, retry_after: str = ""):
+ self._is_alive = False
+ self.latency = latency
+ self.status = status
+ self.retry_after = retry_after
+ self.fetch_count = 0
+
+ async def __aenter__(self):
+ self._is_alive = True
+ return self
+
+ async def __aexit__(self, *args):
+ self._is_alive = False
+
+ async def fetch(self, url: str, **kwargs):
+ self.fetch_count += 1
+ await anyio.sleep(self.latency)
+ return _make_response(url=url, status=self.status, retry_after=self.retry_after)
+
+
+class _LogCounterStub:
+ def get_counts(self) -> Dict[str, int]:
+ return {"debug": 0, "info": 0, "warning": 0, "error": 0, "critical": 0}
+
+
+class MockSpider:
+ def __init__(self, enabled: bool = True, start_delay: float = 0.2, blocked: bool = False, urls: int = 6):
+ self.concurrent_requests = 4
+ self.concurrent_requests_per_domain = 1
+ self.download_delay = 0.0
+ self.max_blocked_retries = 0
+ self.autothrottle_enabled = enabled
+ self.autothrottle_start_delay = start_delay
+ self.autothrottle_max_delay = 1.0
+ self.autothrottle_target_concurrency = None
+ self.autothrottle_block_backoff = True
+ self.allowed_domains: Set[str] = set()
+ self.fp_include_kwargs = False
+ self.fp_include_headers = False
+ self.fp_keep_fragments = False
+ self.robots_txt_obey = False
+ self.development_mode = False
+ self.development_cache_dir = None
+ self.start_urls: list[str] = []
+ self.name = "test_throttle_spider"
+ self._log_counter = _LogCounterStub()
+ self._blocked = blocked
+ self._urls = urls
+
+ async def parse(self, response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:
+ yield {"url": str(response)}
+
+ async def on_start(self, resuming: bool = False) -> None:
+ pass
+
+ async def on_close(self) -> None:
+ pass
+
+ async def on_error(self, request: Request, error: Exception) -> None:
+ pass
+
+ async def on_scraped_item(self, item: Dict[str, Any]) -> Dict[str, Any] | None:
+ return item
+
+ async def is_blocked(self, response) -> bool:
+ return self._blocked
+
+ async def retry_blocked_request(self, request: Request, response) -> Request:
+ return request
+
+ async def start_requests(self) -> AsyncGenerator[Request, None]:
+ for index in range(self._urls):
+ yield Request(f"https://example.com/page{index}", sid="default")
+
+
+async def _crawl(spider: MockSpider, session: MockSession):
+ manager = SessionManager()
+ manager.add("default", session)
+ engine = CrawlerEngine(spider, manager)
+ return engine, await engine.crawl()
+
+
+class TestAutoThrottleIntegration:
+ """Test how the engine drives the throttle"""
+
+ @pytest.mark.anyio
+ async def test_explicit_target_concurrency_wins(self):
+ """The order is autothrottle_target_concurrency, then concurrent_requests_per_domain, then 1"""
+ spider = MockSpider()
+ spider.autothrottle_target_concurrency = 2.5
+ spider.concurrent_requests_per_domain = 4
+ engine, _ = await _crawl(spider, MockSession(latency=0.01))
+
+ assert engine._autothrottle.target_concurrency == 2.5
+
+ @pytest.mark.anyio
+ async def test_target_concurrency_falls_back_to_the_per_domain_limit(self):
+ spider = MockSpider()
+ spider.autothrottle_target_concurrency = None
+ spider.concurrent_requests_per_domain = 4
+ engine, _ = await _crawl(spider, MockSession(latency=0.01))
+
+ assert engine._autothrottle.target_concurrency == 4
+
+ @pytest.mark.anyio
+ async def test_target_concurrency_falls_back_to_one(self):
+ spider = MockSpider()
+ spider.autothrottle_target_concurrency = None
+ spider.concurrent_requests_per_domain = 0
+ engine, _ = await _crawl(spider, MockSession(latency=0.01))
+
+ assert engine._autothrottle.target_concurrency == 1.0
+
+ @pytest.mark.anyio
+ async def test_disabled_by_default_leaves_no_state(self):
+ engine, stats = await _crawl(MockSpider(enabled=False), MockSession())
+
+ assert engine._autothrottle is None
+ assert stats.autothrottle_enabled is False
+ assert stats.autothrottle_delays == {}
+
+ @pytest.mark.anyio
+ async def test_fast_responses_speed_the_crawl_up(self):
+ """A server answering in ~0.05s must pull the delay down from the 0.2s start"""
+ spider = MockSpider(start_delay=0.2)
+ engine, stats = await _crawl(spider, MockSession(latency=0.05))
+
+ assert stats.autothrottle_enabled is True
+ assert list(stats.autothrottle_delays) == ["example.com"]
+ assert stats.autothrottle_delays["example.com"] < 0.2
+ assert stats.autothrottle_delays["example.com"] >= 0.05
+ assert stats.autothrottle_delays == engine._autothrottle.delays
+
+ @pytest.mark.anyio
+ async def test_blocked_responses_slow_the_crawl_down(self):
+ """Content-based blocks come back as fast 200s, so only the backoff can react to them"""
+ spider = MockSpider(start_delay=0.05, blocked=True, urls=4)
+ spider.autothrottle_max_delay = 0.4
+ engine, stats = await _crawl(spider, MockSession(latency=0.01))
+
+ assert stats.blocked_requests_count > 0
+ assert stats.autothrottle_delays["example.com"] > 0.05
+
+ @pytest.mark.anyio
+ async def test_error_responses_slow_the_crawl_down(self):
+ spider = MockSpider(start_delay=0.05, urls=4)
+ spider.autothrottle_max_delay = 0.4
+ engine, stats = await _crawl(spider, MockSession(latency=0.01, status=503))
+
+ assert stats.autothrottle_delays["example.com"] > 0.05
+
+ @pytest.mark.anyio
+ async def test_backoff_can_be_turned_off(self):
+ spider = MockSpider(start_delay=0.05, blocked=True, urls=4)
+ spider.autothrottle_block_backoff = False
+ engine, stats = await _crawl(spider, MockSession(latency=0.01))
+
+ assert stats.autothrottle_delays["example.com"] == 0.05
+
+ @pytest.mark.anyio
+ async def test_retry_after_header_is_honored(self):
+ """A rate-limited response carrying `Retry-After` sets the delay directly"""
+ spider = MockSpider(start_delay=0.05, urls=2)
+ spider.autothrottle_max_delay = 5.0
+ engine, stats = await _crawl(spider, MockSession(latency=0.01, status=429, retry_after="3"))
+
+ assert stats.autothrottle_delays["example.com"] == 3.0
+
+ @pytest.mark.anyio
+ async def test_delays_are_included_in_the_stats_dump(self):
+ _, stats = await _crawl(MockSpider(start_delay=0.2), MockSession(latency=0.05))
+ dumped = stats.to_dict()
+
+ assert dumped["autothrottle_enabled"] is True
+ assert dumped["autothrottle_delays"]["example.com"] == pytest.approx(
+ stats.autothrottle_delays["example.com"], abs=0.01
+ )