🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(spiders): Auto throttle based on website speed/protection
Karim shoairca5196242184
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 3972ff8..2d19016 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -13,6 +13,7 @@ from scrapling.spiders.session import SessionManagerfrom scrapling.spiders.request import Request, Responsefrom scrapling.spiders.robotstxt import RobotsTxtManagerfrom scrapling.spiders.result import CrawlStats, ItemList+from scrapling.spiders.throttle import AutoThrottle, parse_retry_afterfrom scrapling.spiders.cache import ResponseCacheManagerfrom scrapling.spiders.checkpoint import CheckpointManager, CheckpointDatafrom scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator@@ -60,6 +61,18 @@ class CrawlerEngine:else:self._cache_manager = None+ if self.spider.autothrottle_enabled:+ self._autothrottle: Optional[AutoThrottle] = AutoThrottle(+ start_delay=self.spider.autothrottle_start_delay,+ max_delay=self.spider.autothrottle_max_delay,+ target_concurrency=(+ self.spider.autothrottle_target_concurrency or self.spider.concurrent_requests_per_domain or 1.0+ ),+ block_backoff=self.spider.autothrottle_block_backoff,+ )+ else:+ self._autothrottle = None+self._global_limiter = CapacityLimiter(spider.concurrent_requests)self._domain_limiters: dict[str, CapacityLimiter] = {}self._allowed_domains: set[str] = spider.allowed_domains or set()@@ -177,9 +190,11 @@ class CrawlerEngine:self.stats.robots_disallowed_count += 1log.info(f"Request disallowed by robots.txt: {request.url}")return- delay = await self._get_domain_delay(request)+ floor = await self._get_domain_delay(request)else:- delay = self.spider.download_delay+ floor = self.spider.download_delay++ delay = floorif self._cache_manager and request._fp is not None:cached = await self._cache_manager.get(request._fp)@@ -194,6 +209,9 @@ class CrawlerEngine:returnasync with self._rate_limiter(request.domain):+ if self._autothrottle:+ delay = self._autothrottle.delay_for(request.domain, floor)+if delay:await anyio.sleep(delay)@@ -202,7 +220,9 @@ class CrawlerEngine:if request._session_kwargs.get("proxies"):self.stats.proxies.append(dict(request._session_kwargs["proxies"]))try:+ started_at = anyio.current_time()response = await self.session_manager.fetch(request)+ latency = anyio.current_time() - started_atself.stats.increment_requests_count(request.sid or self.session_manager.default_session_id)self.stats.increment_response_bytes(request.domain, len(response.body))self.stats.increment_status(response.status)@@ -216,7 +236,13 @@ class CrawlerEngine:self.stats.cache_misses += 1await self._cache_manager.put(request._fp, response, request._session_kwargs.get("method", "GET"))- if await self.spider.is_blocked(response):+ blocked = await self.spider.is_blocked(response)+ if self._autothrottle:+ ok = 200 <= response.status < 300 and not blocked+ retry_after = None if ok else parse_retry_after(response.headers)+ self._autothrottle.record(request.domain, latency, ok, floor, retry_after)++ if blocked:self.stats.blocked_requests_count += 1if request._retry_count < self.spider.max_blocked_retries:retry_request = request.copy()@@ -333,6 +359,8 @@ class CrawlerEngine:self._domain_limiters.clear()if self._robots_manager:self._domain_delays.clear()+ if self._autothrottle:+ self._autothrottle.reset()# Check for existing checkpointresuming = (await self._restore_from_checkpoint()) if self._checkpoint_system_enabled else False@@ -342,6 +370,7 @@ class CrawlerEngine:self.stats.concurrent_requests = self.spider.concurrent_requestsself.stats.concurrent_requests_per_domain = self.spider.concurrent_requests_per_domainself.stats.download_delay = self.spider.download_delay+ self.stats.autothrottle_enabled = self.spider.autothrottle_enabledawait self.spider.on_start(resuming=resuming)await self._prefetch_robots_txt()@@ -408,6 +437,9 @@ class CrawlerEngine:if not self.paused and self._checkpoint_system_enabled:await self._checkpoint_manager.cleanup()+ if self._autothrottle:+ self.stats.autothrottle_delays = dict(self._autothrottle.delays)+self.stats.log_levels_counter = self.spider._log_counter.get_counts()self.stats.end_time = anyio.current_time()log.info(_dump(self.stats.to_dict()))diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.pyindex 3710750..796bbc2 100644--- a/scrapling/spiders/result.py+++ b/scrapling/spiders/result.py@@ -56,7 +56,9 @@ class CrawlStats:start_time: float = 0.0end_time: float = 0.0download_delay: float = 0.0+ autothrottle_enabled: bool = Falseblocked_requests_count: int = 0+ autothrottle_delays: Dict = field(default_factory=dict)custom_stats: Dict = field(default_factory=dict)response_status_count: Dict = field(default_factory=dict)domains_response_bytes: Dict = field(default_factory=dict)@@ -91,6 +93,8 @@ class CrawlStats:"items_dropped": self.items_dropped,"elapsed_seconds": round(self.elapsed_seconds, 2),"download_delay": round(self.download_delay, 2),+ "autothrottle_enabled": self.autothrottle_enabled,+ "autothrottle_delays": {domain: round(delay, 2) for domain, delay in self.autothrottle_delays.items()},"concurrent_requests": self.concurrent_requests,"concurrent_requests_per_domain": self.concurrent_requests_per_domain,"requests_count": self.requests_count,diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.pyindex edd9d8b..4875077 100644--- a/scrapling/spiders/spider.py+++ b/scrapling/spiders/spider.py@@ -85,6 +85,13 @@ class Spider(ABC):download_delay: float = 0.0max_blocked_retries: int = 3+ # AutoThrottle settings+ autothrottle_enabled: bool = False+ autothrottle_start_delay: float = 5.0+ autothrottle_max_delay: float = 60.0+ autothrottle_target_concurrency: Optional[float] = None+ autothrottle_block_backoff: bool = True+# Fingerprint adjustmentsfp_include_kwargs: bool = Falsefp_keep_fragments: bool = Falsediff --git a/scrapling/spiders/throttle.py b/scrapling/spiders/throttle.pynew file mode 100644index 0000000..6b39d69--- /dev/null+++ b/scrapling/spiders/throttle.py@@ -0,0 +1,101 @@+from email.utils import parsedate_to_datetime+from datetime import datetime, timezone++from scrapling.core.utils import log+from scrapling.core._types import Dict, Mapping, Optional++BLOCK_BACKOFF_FACTOR = 2.0+++def parse_retry_after(headers: Mapping[str, str]) -> Optional[float]:+ """Return how many seconds a `Retry-After` header asks us to wait, or `None` when it's missing or unreadable.++ :param headers: The response headers to look the value up in.+ """+ value = next((headers[key] for key in headers if key.lower() == "retry-after"), "").strip()+ if not value:+ return None++ try:+ return max(float(value), 0.0)+ except ValueError:+ pass++ try: # The header can be an HTTP date instead of a number of seconds+ return max((parsedate_to_datetime(value) - datetime.now(timezone.utc)).total_seconds(), 0.0)+ except (TypeError, ValueError):+ log.debug(f"Ignoring an unreadable `Retry-After` header: {value!r}")+ return None+++class AutoThrottle:+ """Adjusts the per-domain delay from the observed response latency, so the spider speeds up on fast+ servers and backs off on slow or hostile ones."""++ def __init__(+ self,+ start_delay: float = 5.0,+ max_delay: float = 60.0,+ target_concurrency: float = 1.0,+ block_backoff: bool = True,+ ):+ """+ :param start_delay: The delay used for the first request to a domain.+ :param max_delay: The highest delay the throttle is allowed to reach.+ :param target_concurrency: How many requests the spider aims to have in flight per domain. The engine+ passes the spider's `concurrent_requests_per_domain` here, or 1 when it's unlimited.+ :param block_backoff: Double the delay of a domain whenever it blocks us, or wait what its `Retry-After`+ header asks for.+ """+ if target_concurrency <= 0:+ raise ValueError("`target_concurrency` must be higher than 0")+ if max_delay < start_delay:+ raise ValueError("`autothrottle_max_delay` can't be lower than `autothrottle_start_delay`")++ self.start_delay = start_delay+ self.max_delay = max_delay+ self.target_concurrency = target_concurrency+ self.block_backoff = block_backoff+ self.delays: Dict[str, float] = {}++ def delay_for(self, domain: str, floor: float = 0.0) -> float:+ """Return the current delay for a domain, starting it at `start_delay` the first time.++ :param domain: The domain the request belongs to.+ :param floor: The lowest delay allowed, which is the spider's own delay for this domain.+ """+ if domain not in self.delays:+ self.delays[domain] = min(max(floor, self.start_delay), self.max_delay)+ return self.delays[domain]++ def record(+ self, domain: str, latency: float, ok: bool, floor: float = 0.0, retry_after: Optional[float] = None+ ) -> float:+ """Feed a finished request back into the throttle and return the domain's new delay.++ :param domain: The domain the request belongs to.+ :param latency: How long the request took in seconds.+ :param ok: Whether the response was a healthy one, so a non-blocked 2xx.+ :param floor: The lowest delay allowed, which is the spider's own delay for this domain.+ :param retry_after: How long the website asked us to wait, when it did.+ """+ current_delay = self.delay_for(domain, floor)+ target_delay = latency / self.target_concurrency+ new_delay = max((current_delay + target_delay) / 2, target_delay)++ if not ok:+ penalty = current_delay+ if self.block_backoff:+ penalty = retry_after if retry_after is not None else current_delay * BLOCK_BACKOFF_FACTOR+ new_delay = max(new_delay, penalty, current_delay) # A block can never speed the spider up++ new_delay = min(max(new_delay, floor), self.max_delay)+ self.delays[domain] = new_delay+ log.debug(+ f"AutoThrottle ({domain}): latency={latency:.2f}s, ok={ok}, delay {current_delay:.2f}s -> {new_delay:.2f}s"+ )+ return new_delay++ def reset(self) -> None:+ """Drop every learned delay."""+ self.delays.clear()diff --git a/tests/spiders/test_cache.py b/tests/spiders/test_cache.pyindex 3b990ed..b29114c 100644--- a/tests/spiders/test_cache.py+++ b/tests/spiders/test_cache.py@@ -224,6 +224,11 @@ class MockSpider:self.concurrent_requests_per_domain = 0self.download_delay = 0.0self.max_blocked_retries = 3+ self.autothrottle_enabled = False+ self.autothrottle_start_delay = 5.0+ self.autothrottle_max_delay = 60.0+ self.autothrottle_target_concurrency = None+ self.autothrottle_block_backoff = Trueself.allowed_domains: Set[str] = set()self.fp_include_kwargs = Falseself.fp_include_headers = Falsediff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.pyindex 10cbfdc..8e4bac2 100644--- a/tests/spiders/test_engine.py+++ b/tests/spiders/test_engine.py@@ -78,6 +78,11 @@ class MockSpider:concurrent_requests_per_domain: int = 0,download_delay: float = 0.0,max_blocked_retries: int = 3,+ autothrottle_enabled: bool = False,+ autothrottle_start_delay: float = 5.0,+ autothrottle_max_delay: float = 60.0,+ autothrottle_target_concurrency: float | None = None,+ autothrottle_block_backoff: bool = True,allowed_domains: Set[str] | None = None,fp_include_kwargs: bool = False,fp_include_headers: bool = False,@@ -92,6 +97,11 @@ class MockSpider:self.concurrent_requests_per_domain = concurrent_requests_per_domainself.download_delay = download_delayself.max_blocked_retries = max_blocked_retries+ self.autothrottle_enabled = autothrottle_enabled+ self.autothrottle_start_delay = autothrottle_start_delay+ self.autothrottle_max_delay = autothrottle_max_delay+ self.autothrottle_target_concurrency = autothrottle_target_concurrency+ self.autothrottle_block_backoff = autothrottle_block_backoffself.allowed_domains = allowed_domains or set()self.fp_include_kwargs = fp_include_kwargsself.fp_include_headers = fp_include_headersdiff --git a/tests/spiders/test_force_stop_checkpoint.py b/tests/spiders/test_force_stop_checkpoint.pyindex a26e552..41b8d8f 100644--- a/tests/spiders/test_force_stop_checkpoint.py+++ b/tests/spiders/test_force_stop_checkpoint.py@@ -71,6 +71,11 @@ class SlowSpider:self.concurrent_requests_per_domain = 0self.download_delay = 0.0self.max_blocked_retries = 3+ self.autothrottle_enabled = False+ self.autothrottle_start_delay = 5.0+ self.autothrottle_max_delay = 60.0+ self.autothrottle_target_concurrency = None+ self.autothrottle_block_backoff = Trueself.allowed_domains = set()self.fp_include_kwargs = Falseself.fp_include_headers = Falsediff --git a/tests/spiders/test_throttle.py b/tests/spiders/test_throttle.pynew file mode 100644index 0000000..7705199--- /dev/null+++ b/tests/spiders/test_throttle.py@@ -0,0 +1,392 @@+"""Tests for the AutoThrottle algorithm and its integration with the crawling engine."""++import anyio+import pytest++from scrapling.spiders.engine import CrawlerEngine+from scrapling.spiders.request import Request+from scrapling.spiders.session import SessionManager+from scrapling.spiders.throttle import AutoThrottle, parse_retry_after+from scrapling.engines.toolbelt.custom import Response+from scrapling.core._types import Any, Dict, Set, AsyncGenerator+++def _make_response(url: str = "https://example.com", status: int = 200, retry_after: str = "") -> Response:+ headers = {"content-type": "text/html"}+ if retry_after:+ headers["Retry-After"] = retry_after++ return Response(+ url=url,+ content=b"<html>hello</html>",+ status=status,+ reason="OK",+ encoding="utf-8",+ cookies={},+ headers=headers,+ request_headers={"user-agent": "test"},+ method="GET",+ )+++class TestAutoThrottleValidation:+ """Test the constructor guards"""++ def test_invalid_target_concurrency(self):+ for value in (0, -1.0):+ with pytest.raises(ValueError, match="target_concurrency"):+ AutoThrottle(target_concurrency=value)++ def test_max_delay_lower_than_start_delay(self):+ with pytest.raises(ValueError, match="max_delay"):+ AutoThrottle(start_delay=10.0, max_delay=5.0)+++class TestAutoThrottleMath:+ """Test the delay calculation itself"""++ def test_starts_at_start_delay(self):+ throttle = AutoThrottle(start_delay=5.0)++ assert throttle.delay_for("example.com") == 5.0++ def test_floor_raises_the_starting_delay(self):+ throttle = AutoThrottle(start_delay=1.0)++ assert throttle.delay_for("example.com", floor=3.0) == 3.0++ def test_start_delay_is_capped_by_max_delay(self):+ throttle = AutoThrottle(start_delay=5.0, max_delay=20.0)++ assert throttle.delay_for("example.com", floor=100.0) == 20.0++ def test_converges_toward_latency_over_target_concurrency(self):+ throttle = AutoThrottle(start_delay=5.0)++ for _ in range(20):+ delay = throttle.record("example.com", latency=2.0, ok=True)++ assert delay == pytest.approx(2.0, abs=0.01)++ def test_target_concurrency_divides_the_latency(self):+ throttle = AutoThrottle(start_delay=5.0, target_concurrency=4.0)++ for _ in range(20):+ delay = throttle.record("example.com", latency=2.0, ok=True)++ assert delay == pytest.approx(0.5, abs=0.01)++ def test_latency_spike_jumps_straight_to_target(self):+ """A slow response must not be averaged down, it takes effect immediately"""+ throttle = AutoThrottle(start_delay=1.0)++ assert throttle.record("example.com", latency=10.0, ok=True) == 10.0++ def test_non_ok_response_never_decreases_the_delay(self):+ throttle = AutoThrottle(start_delay=5.0, block_backoff=False)++ assert throttle.record("example.com", latency=0.1, ok=False) == 5.0++ def test_non_ok_response_can_still_increase_the_delay(self):+ throttle = AutoThrottle(start_delay=1.0, block_backoff=False)++ assert throttle.record("example.com", latency=8.0, ok=False) == 8.0++ def test_delay_is_clamped_between_floor_and_max(self):+ throttle = AutoThrottle(start_delay=1.0, max_delay=3.0)++ assert throttle.record("example.com", latency=100.0, ok=True) == 3.0+ assert throttle.record("other.com", latency=0.001, ok=True, floor=2.0) == 2.0++ def test_domains_are_throttled_independently(self):+ throttle = AutoThrottle(start_delay=1.0)++ throttle.record("slow.com", latency=9.0, ok=True)+ throttle.record("fast.com", latency=0.5, ok=True)++ assert throttle.delays["slow.com"] == 9.0+ assert throttle.delays["fast.com"] < 1.0++ def test_reset_clears_every_domain(self):+ throttle = AutoThrottle(start_delay=1.0)+ throttle.record("example.com", latency=9.0, ok=True)++ throttle.reset()++ assert throttle.delays == {}+ assert throttle.delay_for("example.com") == 1.0+++class TestBlockBackoff:+ """Test the doubling that kicks in when a website blocks us"""++ def test_each_block_doubles_the_delay(self):+ """Blocks are often served fast, so latency alone would never slow the spider down"""+ throttle = AutoThrottle(start_delay=1.0, max_delay=60.0)++ assert [throttle.record("example.com", latency=0.05, ok=False) for _ in range(5)] == [2.0, 4.0, 8.0, 16.0, 32.0]++ def test_backoff_stops_at_max_delay(self):+ throttle = AutoThrottle(start_delay=1.0, max_delay=5.0)++ for _ in range(10):+ delay = throttle.record("example.com", latency=0.05, ok=False)++ assert delay == 5.0++ def test_healthy_responses_bring_the_delay_back_down(self):+ """Once the website stops blocking, the normal averaging has to recover"""+ throttle = AutoThrottle(start_delay=1.0, max_delay=60.0)+ for _ in range(4):+ throttle.record("example.com", latency=0.05, ok=False)+ assert throttle.delays["example.com"] == 16.0++ for _ in range(20):+ delay = throttle.record("example.com", latency=0.5, ok=True)++ assert delay == pytest.approx(0.5, abs=0.01)++ def test_disabled_backoff_only_freezes_the_delay(self):+ throttle = AutoThrottle(start_delay=1.0, block_backoff=False)++ assert [throttle.record("example.com", latency=0.05, ok=False) for _ in range(3)] == [1.0, 1.0, 1.0]++ def test_retry_after_wins_over_the_doubling(self):+ throttle = AutoThrottle(start_delay=1.0, max_delay=60.0)++ assert throttle.record("example.com", latency=0.05, ok=False, retry_after=30.0) == 30.0++ def test_retry_after_is_capped_by_max_delay(self):+ throttle = AutoThrottle(start_delay=1.0, max_delay=60.0)++ assert throttle.record("example.com", latency=0.05, ok=False, retry_after=3600.0) == 60.0++ def test_retry_after_never_speeds_the_spider_up(self):+ """A tiny or zero Retry-After on a block must not undo the backoff"""+ throttle = AutoThrottle(start_delay=10.0, max_delay=60.0)++ assert throttle.record("example.com", latency=0.05, ok=False, retry_after=0.0) == 10.0++ def test_retry_after_is_ignored_when_backoff_is_disabled(self):+ throttle = AutoThrottle(start_delay=1.0, block_backoff=False)++ assert throttle.record("example.com", latency=0.05, ok=False, retry_after=30.0) == 1.0+++class TestParseRetryAfter:+ """Test reading the `Retry-After` header"""++ def test_missing_header(self):+ assert parse_retry_after({}) is None+ assert parse_retry_after({"Content-Type": "text/html"}) is None++ def test_seconds(self):+ assert parse_retry_after({"Retry-After": "120"}) == 120.0++ def test_header_name_is_case_insensitive(self):+ """Engines don't agree on the casing of header names"""+ assert parse_retry_after({"retry-after": "120"}) == 120.0+ assert parse_retry_after({"RETRY-AFTER": " 120 "}) == 120.0++ def test_http_date(self):+ from email.utils import format_datetime+ from datetime import datetime, timedelta, timezone++ future = datetime.now(timezone.utc) + timedelta(seconds=90)++ assert parse_retry_after({"Retry-After": format_datetime(future)}) == pytest.approx(90.0, abs=5.0)++ def test_past_http_date_is_zero(self):+ assert parse_retry_after({"Retry-After": "Wed, 21 Oct 2015 07:28:00 GMT"}) == 0.0++ def test_negative_seconds_are_clamped(self):+ assert parse_retry_after({"Retry-After": "-5"}) == 0.0++ def test_garbage_is_ignored(self):+ for value in ("soon", "", " ", "12 seconds"):+ assert parse_retry_after({"Retry-After": value}) is None+++# ---------------------------------------------------------------------------+# Engine integration+# ---------------------------------------------------------------------------+++class MockSession:+ def __init__(self, latency: float = 0.05, status: int = 200, retry_after: str = ""):+ self._is_alive = False+ self.latency = latency+ self.status = status+ self.retry_after = retry_after+ self.fetch_count = 0++ async def __aenter__(self):+ self._is_alive = True+ return self++ async def __aexit__(self, *args):+ self._is_alive = False++ async def fetch(self, url: str, **kwargs):+ self.fetch_count += 1+ await anyio.sleep(self.latency)+ return _make_response(url=url, status=self.status, retry_after=self.retry_after)+++class _LogCounterStub:+ def get_counts(self) -> Dict[str, int]:+ return {"debug": 0, "info": 0, "warning": 0, "error": 0, "critical": 0}+++class MockSpider:+ def __init__(self, enabled: bool = True, start_delay: float = 0.2, blocked: bool = False, urls: int = 6):+ self.concurrent_requests = 4+ self.concurrent_requests_per_domain = 1+ self.download_delay = 0.0+ self.max_blocked_retries = 0+ self.autothrottle_enabled = enabled+ self.autothrottle_start_delay = start_delay+ self.autothrottle_max_delay = 1.0+ self.autothrottle_target_concurrency = None+ self.autothrottle_block_backoff = True+ self.allowed_domains: Set[str] = set()+ self.fp_include_kwargs = False+ self.fp_include_headers = False+ self.fp_keep_fragments = False+ self.robots_txt_obey = False+ self.development_mode = False+ self.development_cache_dir = None+ self.start_urls: list[str] = []+ self.name = "test_throttle_spider"+ self._log_counter = _LogCounterStub()+ self._blocked = blocked+ self._urls = urls++ async def parse(self, response) -> AsyncGenerator[Dict[str, Any] | Request | None, None]:+ yield {"url": str(response)}++ async def on_start(self, resuming: bool = False) -> None:+ pass++ async def on_close(self) -> None:+ pass++ async def on_error(self, request: Request, error: Exception) -> None:+ pass++ async def on_scraped_item(self, item: Dict[str, Any]) -> Dict[str, Any] | None:+ return item++ async def is_blocked(self, response) -> bool:+ return self._blocked++ async def retry_blocked_request(self, request: Request, response) -> Request:+ return request++ async def start_requests(self) -> AsyncGenerator[Request, None]:+ for index in range(self._urls):+ yield Request(f"https://example.com/page{index}", sid="default")+++async def _crawl(spider: MockSpider, session: MockSession):+ manager = SessionManager()+ manager.add("default", session)+ engine = CrawlerEngine(spider, manager)+ return engine, await engine.crawl()+++class TestAutoThrottleIntegration:+ """Test how the engine drives the throttle"""++ @pytest.mark.anyio+ async def test_explicit_target_concurrency_wins(self):+ """The order is autothrottle_target_concurrency, then concurrent_requests_per_domain, then 1"""+ spider = MockSpider()+ spider.autothrottle_target_concurrency = 2.5+ spider.concurrent_requests_per_domain = 4+ engine, _ = await _crawl(spider, MockSession(latency=0.01))++ assert engine._autothrottle.target_concurrency == 2.5++ @pytest.mark.anyio+ async def test_target_concurrency_falls_back_to_the_per_domain_limit(self):+ spider = MockSpider()+ spider.autothrottle_target_concurrency = None+ spider.concurrent_requests_per_domain = 4+ engine, _ = await _crawl(spider, MockSession(latency=0.01))++ assert engine._autothrottle.target_concurrency == 4++ @pytest.mark.anyio+ async def test_target_concurrency_falls_back_to_one(self):+ spider = MockSpider()+ spider.autothrottle_target_concurrency = None+ spider.concurrent_requests_per_domain = 0+ engine, _ = await _crawl(spider, MockSession(latency=0.01))++ assert engine._autothrottle.target_concurrency == 1.0++ @pytest.mark.anyio+ async def test_disabled_by_default_leaves_no_state(self):+ engine, stats = await _crawl(MockSpider(enabled=False), MockSession())++ assert engine._autothrottle is None+ assert stats.autothrottle_enabled is False+ assert stats.autothrottle_delays == {}++ @pytest.mark.anyio+ async def test_fast_responses_speed_the_crawl_up(self):+ """A server answering in ~0.05s must pull the delay down from the 0.2s start"""+ spider = MockSpider(start_delay=0.2)+ engine, stats = await _crawl(spider, MockSession(latency=0.05))++ assert stats.autothrottle_enabled is True+ assert list(stats.autothrottle_delays) == ["example.com"]+ assert stats.autothrottle_delays["example.com"] < 0.2+ assert stats.autothrottle_delays["example.com"] >= 0.05+ assert stats.autothrottle_delays == engine._autothrottle.delays++ @pytest.mark.anyio+ async def test_blocked_responses_slow_the_crawl_down(self):+ """Content-based blocks come back as fast 200s, so only the backoff can react to them"""+ spider = MockSpider(start_delay=0.05, blocked=True, urls=4)+ spider.autothrottle_max_delay = 0.4+ engine, stats = await _crawl(spider, MockSession(latency=0.01))++ assert stats.blocked_requests_count > 0+ assert stats.autothrottle_delays["example.com"] > 0.05++ @pytest.mark.anyio+ async def test_error_responses_slow_the_crawl_down(self):+ spider = MockSpider(start_delay=0.05, urls=4)+ spider.autothrottle_max_delay = 0.4+ engine, stats = await _crawl(spider, MockSession(latency=0.01, status=503))++ assert stats.autothrottle_delays["example.com"] > 0.05++ @pytest.mark.anyio+ async def test_backoff_can_be_turned_off(self):+ spider = MockSpider(start_delay=0.05, blocked=True, urls=4)+ spider.autothrottle_block_backoff = False+ engine, stats = await _crawl(spider, MockSession(latency=0.01))++ assert stats.autothrottle_delays["example.com"] == 0.05++ @pytest.mark.anyio+ async def test_retry_after_header_is_honored(self):+ """A rate-limited response carrying `Retry-After` sets the delay directly"""+ spider = MockSpider(start_delay=0.05, urls=2)+ spider.autothrottle_max_delay = 5.0+ engine, stats = await _crawl(spider, MockSession(latency=0.01, status=429, retry_after="3"))++ assert stats.autothrottle_delays["example.com"] == 3.0++ @pytest.mark.anyio+ async def test_delays_are_included_in_the_stats_dump(self):+ _, stats = await _crawl(MockSpider(start_delay=0.2), MockSession(latency=0.05))+ dumped = stats.to_dict()++ assert dumped["autothrottle_enabled"] is True+ assert dumped["autothrottle_delays"]["example.com"] == pytest.approx(+ stats.autothrottle_delays["example.com"], abs=0.01+ )