Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Set up CodSpeed (#7831)

Adrianaa5ded25398f
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
new file mode 100644
index 000000000..6930c4c1c
--- /dev/null
+++ b/.github/workflows/codspeed.yml
@@ -0,0 +1,45 @@
+---
+name: codspeed
+
+on:
+ push:
+ branches:
+ - master
+ pull_request:
+ paths:
+ - scrapy/**
+ - tests/benchmarks/**
+ - .github/workflows/codspeed.yml
+ - tox.ini
+ workflow_dispatch:
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+ cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+ benchmark:
+ runs-on: ubuntu-latest
+ permissions:
+ contents: read
+ id-token: write # OIDC authentication with CodSpeed
+
+ steps:
+ - uses: actions/checkout@v6
+
+ - name: Set up Python 3.14
+ uses: actions/setup-python@v6
+ with:
+ python-version: '3.14'
+ - name: Install dependencies
+ run: |
+ pip install --upgrade pip
+ pip install --upgrade tox
+ tox -n -e benchmark
+ - name: Run benchmarks
+ uses: CodSpeedHQ/action@v4
+ with:
+ mode: simulation
+ run: tox -e benchmark
diff --git a/conftest.py b/conftest.py
index 27c398792..5a535c168 100644
--- a/conftest.py
+++ b/conftest.py
@@ -54,6 +54,9 @@ if not H2_ENABLED:
if find_spec("httpx2") is None and find_spec("httpx") is None:
collect_ignore.append("scrapy/core/downloader/handlers/_httpx.py")
+if find_spec("pytest_codspeed") is None:
+ collect_ignore.append("tests/benchmarks")
+
def pytest_addoption(parser, pluginmanager):
if pluginmanager.hasplugin("twisted"):
diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py
new file mode 100644
index 000000000..7b5ca0cb9
--- /dev/null
+++ b/tests/benchmarks/__init__.py
@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from scrapy.utils.test import get_crawler
+
+if TYPE_CHECKING:
+ from scrapy import Spider
+ from scrapy.crawler import Crawler
+
+
+def crawl(spidercls: type[Spider], settings: dict[str, Any], **kwargs: Any) -> Crawler:
+ """Run a crawl to completion and return its crawler.
+
+ Unlike the rest of the test suite, benchmarks run without ``pytest-twisted``
+ and drive the reactor themselves, since the code being measured must be
+ callable synchronously by ``pytest-codspeed``.
+ """
+ from twisted.internet import reactor
+
+ crawler = get_crawler(spidercls, settings)
+ result: list[Any] = []
+ crawler.crawl(**kwargs).addBoth(result.append)
+ while not result:
+ reactor.iterate(0.001)
+ if isinstance(result[0], BaseException):
+ raise result[0]
+ return crawler
diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py
new file mode 100644
index 000000000..55356083d
--- /dev/null
+++ b/tests/benchmarks/conftest.py
@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import pytest
+
+from scrapy.utils.reactor import install_reactor
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
+
+
[email protected](scope="session", autouse=True)
+def running_reactor() -> Generator[None]:
+ install_reactor("twisted.internet.asyncioreactor.AsyncioSelectorReactor")
+
+ from twisted.internet import reactor
+
+ # Marks the reactor as running without blocking, so that crawls can be
+ # driven with reactor.iterate(), see tests.benchmarks.crawl().
+ reactor.startRunning(installSignalHandlers=False)
+
+ yield
+
+ reactor.stop()
+ # Lets the shutdown event triggers run, e.g. to join the thread pool.
+ reactor.iterate(0)
diff --git a/tests/benchmarks/test_benchmark_crawl.py b/tests/benchmarks/test_benchmark_crawl.py
new file mode 100644
index 000000000..4ad0e672b
--- /dev/null
+++ b/tests/benchmarks/test_benchmark_crawl.py
@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+from urllib.parse import urlencode
+
+import pytest
+
+from scrapy import Field, Item, Request, Spider
+from scrapy.linkextractors import LinkExtractor
+from tests.benchmarks import crawl
+
+if TYPE_CHECKING:
+ from collections.abc import AsyncIterator
+
+ from pytest_codspeed import BenchmarkFixture # type: ignore[import-not-found]
+
+ from scrapy.http import Response
+ from tests.mockserver.http import MockServer
+
+pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")
+
+PAGES = 100
+LINKS_PER_PAGE = 5
+
+
+class _Page(Item):
+ url = Field()
+ anchors = Field()
+
+
+class _FollowSpider(Spider):
+ name = "benchmark"
+ url: str
+ link_extractor = LinkExtractor()
+
+ async def start(self) -> AsyncIterator[Any]:
+ yield Request(self.url, dont_filter=True)
+
+ def parse(self, response: Response) -> Any:
+ yield _Page(
+ url=response.url,
+ anchors=response.css("a::text").getall(),
+ )
+ for link in self.link_extractor.extract_links(response): # type: ignore[arg-type]
+ yield Request(link.url)
+
+
+class _Pipeline:
+ def process_item(self, item: Any) -> Any:
+ return item
+
+
+def test_benchmark_crawl(benchmark: BenchmarkFixture, mockserver: MockServer) -> None:
+ """Crawl of a set of interlinked pages served over HTTP."""
+ query = urlencode({"total": PAGES, "show": LINKS_PER_PAGE, "order": "desc"})
+ url = mockserver.url(f"/follow?{query}")
+ settings = {"ITEM_PIPELINES": {_Pipeline: 100}, "LOG_ENABLED": False}
+
+ def run() -> None:
+ crawler = crawl(_FollowSpider, settings, url=url)
+ assert crawler.stats
+ assert crawler.stats.get_value("item_scraped_count") == PAGES + 1
+
+ benchmark(run)
diff --git a/tox.ini b/tox.ini
index e10a7cc0c..18e1579c9 100644
--- a/tox.ini
+++ b/tox.ini
@@ -30,6 +30,7 @@ envlist =
botocore
pypy3
pypy3-extra-deps
+ benchmark
minversion = 1.7.0
[test-requirements]
@@ -317,3 +318,20 @@ setenv =
{[min]setenv}
commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore
+
+
+# CPU benchmarks, tracked on CodSpeed.
+#
+# pytest-twisted is left out on purpose: benchmarked code must be callable
+# synchronously, so tests/benchmarks drives the reactor itself.
+
+[testenv:benchmark]
+basepython = python3.14
+deps =
+ pytest >= 8.4.1
+ pytest-codspeed
+passenv =
+ *codspeed*
+ *ci*
+commands =
+ pytest {posargs:tests/benchmarks} --codspeed --codspeed-mode=simulation