Scrapy, a fast high-level web crawling & scraping framework for Python.
Set up CodSpeed (#7831)
Adrianaa5ded25398f
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.ymlnew file mode 100644index 000000000..6930c4c1c--- /dev/null+++ b/.github/workflows/codspeed.yml@@ -0,0 +1,45 @@+---+name: codspeed++on:+ push:+ branches:+ - master+ pull_request:+ paths:+ - scrapy/**+ - tests/benchmarks/**+ - .github/workflows/codspeed.yml+ - tox.ini+ workflow_dispatch:++concurrency:+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}+ cancel-in-progress: true++permissions: {}++jobs:+ benchmark:+ runs-on: ubuntu-latest+ permissions:+ contents: read+ id-token: write # OIDC authentication with CodSpeed++ steps:+ - uses: actions/checkout@v6++ - name: Set up Python 3.14+ uses: actions/setup-python@v6+ with:+ python-version: '3.14'+ - name: Install dependencies+ run: |+ pip install --upgrade pip+ pip install --upgrade tox+ tox -n -e benchmark+ - name: Run benchmarks+ uses: CodSpeedHQ/action@v4+ with:+ mode: simulation+ run: tox -e benchmarkdiff --git a/conftest.py b/conftest.pyindex 27c398792..5a535c168 100644--- a/conftest.py+++ b/conftest.py@@ -54,6 +54,9 @@ if not H2_ENABLED:if find_spec("httpx2") is None and find_spec("httpx") is None:collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")+if find_spec("pytest_codspeed") is None:+ collect_ignore.append("tests/benchmarks")+def pytest_addoption(parser, pluginmanager):if pluginmanager.hasplugin("twisted"):diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.pynew file mode 100644index 000000000..7b5ca0cb9--- /dev/null+++ b/tests/benchmarks/__init__.py@@ -0,0 +1,28 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any++from scrapy.utils.test import get_crawler++if TYPE_CHECKING:+ from scrapy import Spider+ from scrapy.crawler import Crawler+++def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler:+ """Run a crawl to completion and return its crawler.++ Unlike the rest of the test suite, benchmarks run without ``pytest-twisted``+ and drive the reactor themselves, since the code being measured must be+ callable synchronously by ``pytest-codspeed``.+ """+ from twisted.internet import reactor++ crawler = get_crawler(spidercls, settings)+ result: list[Any] = []+ crawler.crawl(**kwargs).addBoth(result.append)+ while not result:+ reactor.iterate(0.001)+ if isinstance(result[0], BaseException):+ raise result[0]+ return crawlerdiff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.pynew file mode 100644index 000000000..55356083d--- /dev/null+++ b/tests/benchmarks/conftest.py@@ -0,0 +1,27 @@+from __future__ import annotations++from typing import TYPE_CHECKING++import pytest++from scrapy.utils.reactor import install_reactor++if TYPE_CHECKING:+ from collections.abc import Generator++[email protected](scope="session", autouse=True)+def running_reactor() -> Generator[None]:+ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")++ from twisted.internet import reactor++ # Marks the reactor as running without blocking, so that crawls can be+ # driven with reactor.iterate(), see tests.benchmarks.crawl().+ reactor.startRunning(installSignalHandlers=False)++ yield++ reactor.stop()+ # Lets the shutdown event triggers run, e.g. to join the thread pool.+ reactor.iterate(0)diff --git a/tests/benchmarks/test_benchmark_crawl.py b/tests/benchmarks/test_benchmark_crawl.pynew file mode 100644index 000000000..4ad0e672b--- /dev/null+++ b/tests/benchmarks/test_benchmark_crawl.py@@ -0,0 +1,64 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any+from urllib.parse import urlencode++import pytest++from scrapy import Field, Item, Request, Spider+from scrapy.linkextractors import LinkExtractor+from tests.benchmarks import crawl++if TYPE_CHECKING:+ from collections.abc import AsyncIterator++ from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]++ from scrapy.http import Response+ from tests.mockserver.http import MockServer++pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")++PAGES = 100+LINKS_PER_PAGE = 5+++class _Page(Item):+ url = Field()+ anchors = Field()+++class _FollowSpider(Spider):+ name = "benchmark"+ url: str+ link_extractor = LinkExtractor()++ async def start(self) -> AsyncIterator[Any]:+ yield Request(self.url, dont_filter=True)++ def parse(self, response: Response) -> Any:+ yield _Page(+ url=response.url,+ anchors=response.css("a::text").getall(),+ )+ for link in self.link_extractor.extract_links(response): # type: ignore[arg-type]+ yield Request(link.url)+++class _Pipeline:+ def process_item(self, item: Any) -> Any:+ return item+++def test_benchmark_crawl(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:+ """Crawl of a set of interlinked pages served over HTTP."""+ query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"})+ url = mockserver.url(f"/follow?{query}")+ settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False}++ def run() -> None:+ crawler = crawl(_FollowSpider, settings, url=url)+ assert crawler.stats+ assert crawler.stats.get_value("item_scraped_count") == PAGES + 1++ benchmark(run)diff --git a/tox.ini b/tox.iniindex e10a7cc0c..18e1579c9 100644--- a/tox.ini+++ b/tox.ini@@ -30,6 +30,7 @@ envlist =botocorepypy3pypy3-extra-deps+ benchmarkminversion = 1.7.0[test-requirements]@@ -317,3 +318,20 @@ setenv ={[min]setenv}commands =pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore+++# CPU benchmarks, tracked on CodSpeed.+#+# pytest-twisted is left out on purpose: benchmarked code must be callable+# synchronously, so tests/benchmarks drives the reactor itself.++[testenv:benchmark]+basepython = python3.14+deps =+ pytest >= 8.4.1+ pytest-codspeed+passenv =+ *codspeed*+ *ci*+commands =+ pytest {posargs:tests/benchmarks} --codspeed --codspeed-mode=simulation