Scrapy, a fast high-level web crawling & scraping framework for Python.
Extend benchmarks (#7887)
Adriane0128c20c3e7
diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.pyindex 7b5ca0cb9..a066ed00a 100644--- a/tests/benchmarks/__init__.py+++ b/tests/benchmarks/__init__.py@@ -1,14 +1,53 @@from __future__ import annotations+import asynciofrom typing import TYPE_CHECKING, Any+from scrapy.http import Responsefrom scrapy.utils.test import get_crawlerif TYPE_CHECKING:- from scrapy import Spider+ from scrapy import Request, Spiderfrom scrapy.crawler import Crawler+class NullDownloadHandler:+ """Download handler that returns an empty response without doing any I/O.++ It lets benchmarks measure the engine, the scheduler and the middlewares+ without also measuring HTTP parsing and socket handling, and reach as many+ hostnames as they need without DNS resolution.++ It yields control to the event loop once per request, so that requests can+ be in progress at the same time and concurrency limits apply. The peak+ number of requests in progress is tracked in the+ ``benchmark/peak_concurrency`` stat.+ """++ lazy = False++ def __init__(self, crawler: Crawler):+ self._crawler = crawler+ self._active = 0++ @classmethod+ def from_crawler(cls, crawler: Crawler) -> NullDownloadHandler:+ return cls(crawler)++ async def download_request(self, request: Request) -> Response:+ self._active += 1+ assert self._crawler.stats+ self._crawler.stats.max_value("benchmark/peak_concurrency", self._active)+ try:+ await asyncio.sleep(0)+ return Response(request.url, request=request)+ finally:+ self._active -= 1++ async def close(self) -> None:+ pass++def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler:"""Run a crawl to completion and return its crawler.diff --git a/tests/benchmarks/test_crawl.py b/tests/benchmarks/test_crawl.pyindex 0fdfe742b..d3bf0fdd6 100644--- a/tests/benchmarks/test_crawl.py+++ b/tests/benchmarks/test_crawl.py@@ -7,13 +7,14 @@ import pytestfrom scrapy import Field, Item, Request, Spiderfrom scrapy.linkextractors import LinkExtractor-from tests.benchmarks import crawl+from tests.benchmarks import NullDownloadHandler, crawlif TYPE_CHECKING:from collections.abc import AsyncIteratorfrom pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]+ from scrapy.crawler import Crawlerfrom scrapy.http import Responsefrom tests.mockserver.http import MockServer@@ -22,6 +23,22 @@ pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeePAGES = 100LINKS_PER_PAGE = 5+# Requests per crawl of the benchmarks that use NullDownloadHandler. The broad+# crawl scenarios split them differently between hostnames and pages per+# hostname.+REQUESTS = 200+BROAD_DEEP_PAGES = 10++# Requests per crawl and delay of the benchmark that measures delayed requests,+# where wall time, unlike in the other benchmarks, is a function of the delay.+DELAYED_REQUESTS = 50+DELAY = 0.005++NULL_SETTINGS: dict[str, Any] = {+ "DOWNLOAD_HANDLERS": {"http": NullDownloadHandler},+ "LOG_ENABLED": False,+}+class _Page(Item):url = Field()@@ -45,11 +62,43 @@ class _FollowSpider(Spider):yield Request(link.url)+class _TreeSpider(Spider):+ """Crawl *pages* pages on each of *domains* hostnames.++ Pages are numbered from 1, and page *n* links to pages *2n* and *2n+1*, so+ that requests also reach the scheduler from callbacks, and not only from+ :meth:`~scrapy.Spider.start`.+ """++ name = "benchmark-tree"+ domains: int = 1+ pages: int = 1++ async def start(self) -> AsyncIterator[Any]:+ for domain in range(self.domains):+ yield Request(f"http://d{domain}.example.com/1")++ def parse(self, response: Response) -> Any:+ page = int(response.url.rpartition("/")[2])+ for child in (page * 2, page * 2 + 1):+ if child <= self.pages:+ yield Request(response.urljoin(f"/{child}"))++class _Pipeline:def process_item(self, item: Any) -> Any:return item+def _crawl_tree(settings: dict[str, Any], *, domains: int, pages: int) -> Crawler:+ crawler = crawl(+ _TreeSpider, {**NULL_SETTINGS, **settings}, domains=domains, pages=pages+ )+ assert crawler.stats+ assert crawler.stats.get_value("downloader/response_count") == domains * pages+ return crawler++def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:"""Per-request overhead of a crawl over HTTP.@@ -67,3 +116,50 @@ def test_overhead_http(benchmark: BenchmarkFixture, mockserver: MockServer) -> Nassert crawler.stats.get_value("item_scraped_count") == PAGES + 1benchmark(run)+++def test_overhead_engine(benchmark: BenchmarkFixture) -> None:+ """Per-request overhead of a crawl of a single hostname without any I/O."""++ def run() -> None:+ crawler = _crawl_tree({}, domains=1, pages=REQUESTS)+ assert crawler.stats+ assert crawler.stats.get_value("benchmark/peak_concurrency") > 1++ benchmark(run)+++ ("domains", "pages"),+ [+ pytest.param(REQUESTS, 1, id="shallow"),+ pytest.param(REQUESTS // BROAD_DEEP_PAGES, BROAD_DEEP_PAGES, id="deep"),+ ],+)+def test_overhead_broad(benchmark: BenchmarkFixture, domains: int, pages: int) -> None:+ """Per-request overhead of a broad crawl.++ The shallow scenario, which reaches a single page of every hostname, pays+ the cost of tracking a hostname for the first time on every request, and+ gets its requests from :meth:`~scrapy.Spider.start`. The deep scenario,+ which reaches the same number of pages spread over fewer hostnames,+ amortizes that cost, and instead keeps several requests per hostname+ waiting in the scheduler.+ """+ benchmark(lambda: _crawl_tree({}, domains=domains, pages=pages))+++def test_overhead_concurrency(benchmark: BenchmarkFixture) -> None:+ """Overhead of a crawl limited to 1 request at a time on a single hostname."""+ settings = {"CONCURRENT_REQUESTS_PER_DOMAIN": 1}+ benchmark(lambda: _crawl_tree(settings, domains=1, pages=REQUESTS))+++def test_overhead_delay(benchmark: BenchmarkFixture) -> None:+ """Overhead of a crawl where every request waits for a download delay.++ The delay is not randomized, so that wall time, and hence the number of+ reactor iterations that the crawl needs, does not change between runs.+ """+ settings = {"DOWNLOAD_DELAY": DELAY, "RANDOMIZE_DOWNLOAD_DELAY": False}+ benchmark(lambda: _crawl_tree(settings, domains=1, pages=DELAYED_REQUESTS))