Scrapy, a fast high-level web crawling & scraping framework for Python.
Drop PyPy support from docs and tests. (#8358)
Andrey Rakhmatullinc81046ed4760
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex a7f2debf4..7938edef0 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -21,10 +21,9 @@ env:UV_PYTHON_PREFERENCE: only-systemjobs:- # pre-commit and the static checks gate the test jobs except the slowest- # ones, so that a pull request that fails them costs a couple of minutes of- # CI instead of dozens of test jobs. The gated jobs still finish well before- # the ungated ones, so the gate does not lengthen the overall CI time.+ # pre-commit and the static checks gate the test jobs, so that a pull request+ # that fails them costs a couple of minutes of CI instead of dozens of test+ # jobs.pre-commit:if: github.event.action != 'closed'runs-on: ubuntu-latest@@ -99,76 +98,6 @@ jobs:echo "::error::$message"echo "$message" >> "$GITHUB_STEP_SUMMARY"- # PyPy is the slowest test job, so it runs without waiting for the gate to- # keep the overall CI time unchanged.- pypy:- if: github.event.action != 'closed'- name: tests (ubuntu, ${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})- runs-on: ubuntu-latest- timeout-minutes: 60- env:- # A hanging test would otherwise keep the job running until GitHub kills- # it, without telling which test hung.- PYTEST_ADDOPTS: --timeout=120 ${{ matrix.coverage && '-n logical' || '-n logical --no-cov' }}- strategy:- fail-fast: false- matrix:- include:- # pinned due to https://github.com/pypy/pypy/issues/5388- - python-version: pypy3.11-7.3.20- env:- TOXENV: min-pypy3- - python-version: pypy3.11-7.3.20- env:- TOXENV: pypy3-extra-deps- coverage: true-- steps: &test-steps- - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1- with:- persist-credentials: false-- - name: Set up Python ${{ matrix.python-version }}- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0- with:- python-version: ${{ matrix.python-version }}-- - name: Install system libraries- if: runner.os == 'Linux' && (contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min'))- run: |- sudo apt-get update- sudo apt-get install libxml2-dev libxslt-dev-- - name: Set up uv- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0- with:- cache-dependency-glob: |- pyproject.toml- tox.ini-- - name: Install mitmproxy- env:- # mitmproxy needs a newer Python than the oldest matrix entries, so let- # uv download one where no system interpreter is new enough.- UV_PYTHON_PREFERENCE: system- # mitmproxy has no PyPy wheels, so run it on CPython regardless of the- # interpreter under test.- run: uv tool install --python cpython mitmproxy-- - name: Run tests- env: ${{ matrix.env }}- run: uvx --with tox-uv tox-- - name: Upload coverage report- if: ${{ matrix.coverage }}- uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1-- - name: Upload test results- if: ${{ !cancelled() }}- uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1- with:- report_type: test_results-ubuntu:needs: [pre-commit, checks]name: tests (ubuntu, ${{ matrix.python-version }}, ${{ matrix.env.TOXENV }})@@ -246,7 +175,49 @@ jobs:env:TOXENV: no-reactor- steps: *test-steps+ steps: &test-steps+ - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1+ with:+ persist-credentials: false++ - name: Set up Python ${{ matrix.python-version }}+ uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0+ with:+ python-version: ${{ matrix.python-version }}++ - name: Install system libraries+ if: runner.os == 'Linux' && contains(matrix.env.TOXENV, 'min')+ run: |+ sudo apt-get update+ sudo apt-get install libxml2-dev libxslt-dev++ - name: Set up uv+ uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0+ with:+ cache-dependency-glob: |+ pyproject.toml+ tox.ini++ - name: Install mitmproxy+ env:+ # mitmproxy needs a newer Python than the oldest matrix entries, so let+ # uv download one where no system interpreter is new enough.+ UV_PYTHON_PREFERENCE: system+ run: uv tool install --python cpython mitmproxy++ - name: Run tests+ env: ${{ matrix.env }}+ run: uvx --with tox-uv tox++ - name: Upload coverage report+ if: ${{ matrix.coverage }}+ uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1++ - name: Upload test results+ if: ${{ !cancelled() }}+ uses: codecov/codecov-action@303a32d7a59b442fa8d48b6a1cc6825c09c847a5 # v7.1.1+ with:+ report_type: test_resultswindows:needs: [pre-commit, checks]diff --git a/docs/intro/install.rst b/docs/intro/install.rstindex 8e6c60b0d..dfc21cb84 100644--- a/docs/intro/install.rst+++ b/docs/intro/install.rst@@ -9,8 +9,7 @@ Installation guideSupported Python versions=========================-Scrapy requires Python 3.11+, either the CPython implementation (default) or-the PyPy implementation (see :ref:`python:implementations`).+Scrapy requires Python 3.11+... _intro-install-scrapy:@@ -267,29 +266,6 @@ After any of these workarounds, you should be able to install Scrapy::pip install Scrapy-PyPy-------We recommend using the latest PyPy version.-For PyPy3, only Linux installation was tested.--Most Scrapy dependencies now have binary wheels for CPython, but not for PyPy.-This means that these dependencies will be built during installation. On macOS,-you are likely to face an issue with building the Cryptography dependency. The-solution to this problem is described `here-<https://github.com/pyca/cryptography/issues/2692#issuecomment-272773481>`_,-that is to ``brew install openssl`` and then export the flags that this command-recommends (only needed when installing Scrapy). Installing Scrapy on Linux has-no special issues beyond installing the build dependencies. Installing Scrapy-with PyPy on Windows has not been tested.--You can check that Scrapy is installed correctly by running ``scrapy bench``.-If this command gives errors such as-``TypeError: ... got 2 unexpected keyword arguments``, this means-that the ``PyPyDispatcher`` dependency wasn't installed. To fix this issue, run-``pip install 'PyPyDispatcher>=2.1.0'``.--.. _intro-install-troubleshooting:Troubleshootingdiff --git a/pyproject.toml b/pyproject.tomlindex 65888b461..74645df47 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -50,7 +50,6 @@ classifiers = ["Programming Language :: Python :: 3.14","Programming Language :: Python :: 3.15","Programming Language :: Python :: Implementation :: CPython",- "Programming Language :: Python :: Implementation :: PyPy","Topic :: Internet :: WWW/HTTP","Topic :: Software Development :: Libraries :: Application Frameworks","Topic :: Software Development :: Libraries :: Python Modules",@@ -315,8 +314,7 @@ filterwarnings = [# ResourceWarning, re-emitted by pytest.warns() with the outer filters# applied, when GC instead runs while a pytest.warns() block for an# unrelated warning is active, failing that unrelated test. The message- # wording varies across Python implementations and versions (PyPy leaves- # the prefix empty), so match on the object repr.+ # wording varies across Python versions, so match on the object repr."ignore:.*<socket\\.socket fd=:pytest.PytestUnraisableExceptionWarning","ignore:unclosed <socket\\.socket fd=:ResourceWarning","ignore:.*<function (_SelectorTransport|_ProactorBasePipeTransport|_ProactorSocketTransport|BaseEventLoop)\\.__del__:pytest.PytestUnraisableExceptionWarning",@@ -324,10 +322,6 @@ filterwarnings = [# These are triggered by garbage collecting an unclosed AiohttpDownloadHandler."ignore:.*<function (ClientSession|BaseConnector)\\.__del__:pytest.PytestUnraisableExceptionWarning","ignore:Unclosed (client session|connector) <aiohttp\\.:ResourceWarning",- # PyDispatcher iterates its connection registry while a weakref callback- # may mutate it, which raises RuntimeError on PyPy, where several weakrefs- # can die within a single collection.- "ignore:.*<function connect\\.<locals>\\.remove:pytest.PytestUnraisableExceptionWarning",# queuelib's test helpers (reused by tests/test_squeues.py) leave queue# files and SQLite connections open; see the comment above about the two# forms this can surface as. sqlite3's warning message has no path todiff --git a/scrapy/extensions/postprocessing.py b/scrapy/extensions/postprocessing.pyindex 5828ae52e..21e552c62 100644--- a/scrapy/extensions/postprocessing.py+++ b/scrapy/extensions/postprocessing.py@@ -82,9 +82,6 @@ class LZMAPlugin:- `lzma_preset`- `lzma_filters`- .. note::- ``lzma_filters`` cannot be used in pypy version 7.3.1 and older.-See :py:class:`lzma.LZMAFile` for more info about parameters."""diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex 2be0bd434..3cd230408 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -9,7 +9,7 @@ from typing import TYPE_CHECKINGtry:import brotli-except ImportError:+except ImportError: # pragma: no coverimport brotlicffi as brotliif sys.version_info >= (3, 14):diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 216b5cd92..3864814fc 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -307,7 +307,7 @@ def global_object_name(obj: Any) -> str:return f"{obj.__module__}.{obj.__qualname__}"-if hasattr(sys, "pypy_version_info"):+if hasattr(sys, "pypy_version_info"): # pragma: no coverdef garbage_collect() -> None:# Collecting weakreferences can take two collections on PyPy.diff --git a/scrapy/utils/trackref.py b/scrapy/utils/trackref.pyindex 25cc7909a..d569ff81c 100644--- a/scrapy/utils/trackref.py+++ b/scrapy/utils/trackref.py@@ -5,11 +5,6 @@ If you want live objects for a particular class to be tracked, you only have tosubclass from object_ref (instead of object).This library has a minimal performance impact.--.. note:: PyPy uses a tracing garbage collector, so objects may- remain in the ``live_refs`` longer than expected, even after they- go out of scope. If deterministic behavior is required, you may need- to explicitly trigger garbage collection or call ``trackref.live_refs.clear()``."""from __future__ import annotationsdiff --git a/tests/benchmarks/test_httpcompression.py b/tests/benchmarks/test_httpcompression.pyindex dfbdddb69..c349ad41a 100644--- a/tests/benchmarks/test_httpcompression.py+++ b/tests/benchmarks/test_httpcompression.py@@ -5,6 +5,7 @@ from gzip import GzipFilefrom io import BytesIOfrom typing import TYPE_CHECKING, Any+import brotliimport pytestfrom scrapy import Request, Spider@@ -22,11 +23,6 @@ if TYPE_CHECKING:pytest.importorskip("pytest_codspeed", reason="Benchmarks require pytest-codspeed")-try:- import brotli-except ImportError:- import brotlicffi as brotli-if sys.version_info >= (3, 14):from compression import zstdelse:diff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex dc156947a..00f483ada 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -267,10 +267,7 @@ class ClosingSpider(scrapy.Spider):class TestInteractiveShell:- # Starting an interactive shell involves an interpreter start-up, Scrapy- # imports and shell imports, which on PyPy with coverage enabled can take- # well over 10 seconds.- TIMEOUT = 60+ TIMEOUT = 10def test_fetch(self, mockserver: MockServer) -> None:args = (diff --git a/tests/test_crawler_subprocess.py b/tests/test_crawler_subprocess.pyindex 0b564e352..78cfc8465 100644--- a/tests/test_crawler_subprocess.py+++ b/tests/test_crawler_subprocess.py@@ -20,9 +20,8 @@ from tests.utils.decorators import coroutine_testif TYPE_CHECKING:from tests.mockserver.http import MockServer-# Guards against a hung subprocess. Generous, because starting a script is-# slow on PyPy, slower still with coverage measurement on.-SCRIPT_TIMEOUT = 60+# Guards against a hung subprocess.+SCRIPT_TIMEOUT = 10class ScriptRunnerMixin(ABC):diff --git a/tests/test_feedexport_postprocess.py b/tests/test_feedexport_postprocess.pyindex f56b4a5f7..69e903582 100644--- a/tests/test_feedexport_postprocess.py+++ b/tests/test_feedexport_postprocess.py@@ -5,7 +5,6 @@ import gzipimport lzmaimport marshalimport pickle-import sysimport tempfilefrom io import BytesIOfrom typing import IO, TYPE_CHECKING@@ -323,10 +322,6 @@ class TestFeedPostProcessedExports:async def test_lzma_plugin_filters(self, mockserver: MockServer, tmp_path: Path) -> None:- if "PyPy" in sys.version:- # https://foss.heptapod.net/pypy/pypy/-/issues/3527- pytest.skip("lzma filters doesn't work in PyPy")-filters = [{"id": lzma.FILTER_LZMA2}]compressed = lzma.compress(self.expected, filters=filters)filename = tmp_path / "filters"diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.pyindex 472180885..ec6a4896f 100644--- a/tests/test_http_response_text.py+++ b/tests/test_http_response_text.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport codecsimport gc-import platformimport weakreffrom typing import castfrom unittest import mock@@ -272,10 +271,6 @@ class TestTextResponse(TestResponseBase):assert response.selector.css("title::text").getall() == ["Some page"]assert response.selector.re("Some (.*)</title>") == ["page"]- @pytest.mark.skipif(- platform.python_implementation() != "CPython",- reason="Relies on reference counting",- )def test_selector_response_refcounting(self):response = self.response_class("http://www.example.com", body=b"<a>b</a>")selector = response.selectordiff --git a/tests/test_spidermiddleware.py b/tests/test_spidermiddleware.pyindex a2906613f..30ad58561 100644--- a/tests/test_spidermiddleware.py+++ b/tests/test_spidermiddleware.py@@ -1,7 +1,6 @@from __future__ import annotationsimport gc-import platformimport weakreffrom collections.abc import AsyncIterator, Iterablefrom inspect import isasyncgen@@ -107,10 +106,6 @@ class TestProcessSpiderExceptionReRaise(TestSpiderMiddleware):with pytest.raises(ZeroDivisionError):await collect_asyncgen(it)- @pytest.mark.skipif(- platform.python_implementation() != "CPython",- reason="Relies on reference counting",- )@coroutine_testasync def test_response_refcounting(self):class RaiseExceptionProcessSpiderOutputMiddleware:diff --git a/tests/test_squeues.py b/tests/test_squeues.pyindex c25321ff3..8e4772c8c 100644--- a/tests/test_squeues.py+++ b/tests/test_squeues.py@@ -167,7 +167,7 @@ class PickleFifoDiskQueueTest(t.FifoDiskQueueTest, FifoQueueTestMixin):match=r"Can't (get|pickle) local object|Can't pickle .*: it's not found as",) as exc_info:q.push(lambda x: x)- if sys.version_info >= (3, 14) or hasattr(sys, "pypy_version_info"):+ if sys.version_info >= (3, 14):assert isinstance(exc_info.value.__context__, pickle.PicklingError)else:assert isinstance(exc_info.value.__context__, AttributeError)diff --git a/tests/test_utils_datatypes.py b/tests/test_utils_datatypes.pyindex 5dcc59886..81fc47d47 100644--- a/tests/test_utils_datatypes.py+++ b/tests/test_utils_datatypes.py@@ -14,7 +14,6 @@ from scrapy.utils._datatypes import (LocalWeakReferencedCache,SequenceExclude,)-from scrapy.utils.python import garbage_collectclass TestCaseInsensitiveDict:@@ -366,10 +365,6 @@ class TestLocalWeakReferencedCache:assert cache[r2] == 2assert cache[r3] == 3del r2-- # PyPy takes longer to collect dead references- garbage_collect()-assert len(cache) == 1def test_cache_non_weak_referenceable_objects(self):@@ -402,9 +397,6 @@ class TestLocalWeakReferencedCache:for _ in range(maximum // 2):refs.pop()- # PyPy takes longer to collect dead references- garbage_collect()-assert len(cache) == maximum // 2for i, r in enumerate(refs):assert r in cachediff --git a/tests/test_utils_httpobj.py b/tests/test_utils_httpobj.pyindex eb9852784..e3bb8b68c 100644--- a/tests/test_utils_httpobj.py+++ b/tests/test_utils_httpobj.py@@ -8,7 +8,6 @@ import pytestfrom scrapy.http import Request, Responsefrom scrapy.utils import httpobjfrom scrapy.utils.httpobj import urlparse_cached-from scrapy.utils.python import garbage_collectdef test_urlparse_cached() -> None:@@ -54,7 +53,6 @@ def test_urlparse_cached_lifetime(url_first: bool, holders: list[str]) -> None:while objects:assert urlparse_cached(Request(url)) is parsedobjects.pop(0)- garbage_collect()assert urlparse_cached(Request(url)) is not parseddiff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 12606d33b..81961ba18 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -2,7 +2,6 @@ from __future__ import annotationsimport functoolsimport operator-import platformimport sysfrom typing import TYPE_CHECKING, Any, TypeVar@@ -188,7 +187,7 @@ def test_get_func_args():assert get_func_args(str.split, stripself=True) == ["sep", "maxsplit"]assert get_func_args(" ".join, stripself=True) == ["iterable"]- if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy":+ if sys.version_info >= (3, 13):assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"]else:# The interpreter returns a wrong signature@@ -199,8 +198,7 @@ def test_get_func_args():# inspect.signature() fails for theseassert get_func_args(type) == []- if platform.python_implementation() != "PyPy":- assert get_func_args(max) == []+ assert get_func_args(max) == []@pytest.mark.skipif(diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.pyindex 07daa149c..f34656000 100644--- a/tests/test_utils_trackref.py+++ b/tests/test_utils_trackref.py@@ -1,6 +1,5 @@from __future__ import annotations-import sysfrom io import StringIOfrom unittest import mock@@ -66,64 +65,19 @@ Foo 1 oldest: 0s ago\n\n""")-_IS_PYPY = "PyPy" in sys.version--def test_get_oldest():- """- Verify that `get_oldest` returns the oldest live instance of a class.-- The test runs in two passes to expose differences between:- - CPython (reference counting, immediate destruction)- - PyPy (tracing GC, delayed destruction)-- Since `trackref` relies on weak references, delayed GC on PyPy can leave- stale entries in `live_refs`, affecting results unless explicitly cleared.- """-- def _delete_o1() -> None:- """Delete `o1` and ensure it is actually collected on PyPy."""- nonlocal o1 # type: ignore[misc]- del o1-- if _IS_PYPY:- # On PyPy, `del` only removes the local reference. The object may- # still exist until the GC runs, so we force a collection cycle.- garbage_collect()-- def _do_asserts() -> None:- assert trackref.get_oldest("Foo") is o1- assert trackref.get_oldest("Bar") is o2- # Ensure the newer Foo is not incorrectly considered the oldest- assert trackref.get_oldest("Foo") is not o3- assert trackref.get_oldest("XXX") is None-- o1, o2, o3 = Foo(), Bar(), Foo()-- _do_asserts()-- # Remove the oldest Foo instance; o3 should now become the oldest- _delete_o1()- assert trackref.get_oldest("Foo") is o3-- # PyPy-specific behavior where stale references may persist- # unless the registry is explicitly cleared.- if _IS_PYPY:- trackref.live_refs.clear()-+ """Verify that `get_oldest` returns the oldest live instance of a class."""o1, o2, o3 = Foo(), Bar(), Foo()-- _do_asserts()-- _delete_o1()+ assert trackref.get_oldest("Foo") is o1+ assert trackref.get_oldest("Bar") is o2+ assert trackref.get_oldest("XXX") is None+ del o1assert trackref.get_oldest("Foo") is o3def test_get_oldest_all_dead():o1 = Foo()del o1- if _IS_PYPY:- garbage_collect()# Foo is still a key of live_refs, but it no longer tracks any instance.assert trackref.get_oldest("Foo") is Nonediff --git a/tox.ini b/tox.iniindex 56326cb2e..a430f9431 100644--- a/tox.ini+++ b/tox.ini@@ -22,15 +22,12 @@ envlist =min-default-reactormin-no-reactormin-botocore- min-pypy3py{310,311,312,313,314,315}extra-depsdefault-reactorno-reactorno-reactor-extra-depsbotocore- pypy3- pypy3-extra-depsbenchmarkminversion = 1.7.0@@ -145,8 +142,7 @@ deps =aioftp==0.26.1aiohttp==3.13.3backports.zstd==1.3.0- brotli==1.2.0; implementation_name != "pypy"- brotlicffi==1.2.0.0; implementation_name == "pypy"+ brotli==1.2.0charset-normalizer==3.4.0cryptography==41.0.5cssselect==1.2.0@@ -197,7 +193,7 @@ deps =psutilptpython # optional for shell wrapper testsrobotexclusionrulesparser- uvloop; platform_system != "Windows" and implementation_name != "pypy"+ uvloop; platform_system != "Windows"[testenv:min-extra-deps]basepython = {[min]basepython}@@ -216,7 +212,7 @@ deps =psutil==5.9.0ptpython==3.0.23robotexclusionrulesparser==1.6.2- uvloop==0.17.0; platform_system != "Windows" and implementation_name != "pypy"+ uvloop==0.17.0; platform_system != "Windows"setenv ={[min]setenv}SCRAPY_COVERAGE_PATCH=_exit # redundant with "extra-deps"'s subprocess coverage of scrapy/commands/*@@ -310,53 +306,6 @@ setenv ={[min]setenv}SCRAPY_COVERAGE_PATCH=_exit # redundant with "py"'s subprocess coverage of scrapy/commands/*-[testenv:pypy3]-basepython = pypy3-commands =- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=pypy3.junit.xml -o junit_family=legacy --durations=10 scrapy tests}--[testenv:pypy3-extra-deps]-basepython = pypy3-deps =- {[testenv:extra-deps]deps}-setenv =- # Spawning an interpreter is slow on PyPy, and having each one start a- # coverage session on top makes the tests that do it take 2-3 times as- # long. The CPython jobs cover those code paths, so here coverage only- # measures the test process itself, and _exit is a no-op stand-in for the- # subprocess patch, which coverage offers no way to turn off.- SCRAPY_COVERAGE_PATCH=_exit--[testenv:min-pypy3]-basepython = pypy3.11-deps =- PyPyDispatcher==2.1.0- {[test-requirements]deps}- aioftp==0.26.1- aiohttp==3.13.3- pytest==8.4.0- Protego==0.2.0- Twisted==21.7.0- backports.zstd==1.3.0- brotlicffi==1.2.0.0- charset-normalizer==3.4.0- cryptography==44.0.2- cssselect==1.2.0- itemadapter==0.1.0- itemloaders==1.0.5- lxml==5.3.2- parsel==1.8.1- pyOpenSSL==24.3.0- queuelib==1.8.0- service_identity==24.2.0- w3lib==2.1.1- zope.interface==5.1.0-commands =- ; disabling coverage- pytest {posargs:--durations=10 scrapy tests}-setenv =- {[min]setenv}-[testenv:docs-tests]changedir = docsdeps =