Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove remaining PyPy support. (#8367)
Andrey Rakhmatullin1481630d439a
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex f870a7fd2..fbc56d7d5 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -186,7 +186,7 @@ jobs:# mitmproxy needs a newer Python than the oldest matrix entries, so let# uv download one where no system interpreter is new enough.UV_PYTHON_PREFERENCE: system- run: uv tool install --python cpython mitmproxy+ run: uv tool install mitmproxy- name: Run testsenv: ${{ matrix.env }}diff --git a/.github/workflows/tests-vcs-deps.yml b/.github/workflows/tests-vcs-deps.ymlindex a0b6adb21..0a8fc94fe 100644--- a/.github/workflows/tests-vcs-deps.yml+++ b/.github/workflows/tests-vcs-deps.yml@@ -50,7 +50,7 @@ jobs:tox.ini- name: Install mitmproxy- run: uv tool install --python cpython mitmproxy+ run: uv tool install mitmproxy- name: Run testsrun: uvx --with tox-uv toxdiff --git a/docs/news.rst b/docs/news.rstindex fd007243b..b19033f8b 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -9,6 +9,11 @@ Scrapy VERSION (unreleased)Backward-incompatible changes~~~~~~~~~~~~~~~~~~~~~~~~~~~~~+- Scrapy no longer officially supports PyPy. It may still work if you+ manually remove ``PyDispatcher`` and install ``PyPyDispatcher`` but this is+ not guaranteed.+ (TBD)+- Changes to the ``fetch()`` shortcut of the :ref:`Scrapy shell<topics-shell>`:diff --git a/pyproject.toml b/pyproject.tomlindex ec9343f0c..5007475e8 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -7,9 +7,12 @@ name = "Scrapy"dynamic = ["version"]description = "A high-level Web Crawling and Web Scraping framework"dependencies = [+ "PyDispatcher>=2.0.5","Twisted>=21.7.0","aioftp>=0.26.1","aiohttp>=3.13.3",+ 'backports.zstd>=1.3.0; python_version < "3.14"',+ "brotli>=1.2.0","charset-normalizer>=3.4.0","cryptography>=41.0.5","cssselect>=1.2.0",@@ -29,12 +32,6 @@ dependencies = ['typing-extensions>=4.6.0; python_version < "3.13"',"w3lib>=2.1.1","zope.interface>=5.1.0",- # Platform-specific dependencies- 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',- 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',- 'backports.zstd>=1.3.0; python_version < "3.14"',- 'brotli>=1.2.0; implementation_name != "pypy"',- 'brotlicffi>=1.2.0.0; implementation_name == "pypy"',]classifiers = ["Development Status :: 5 - Production/Stable",@@ -83,7 +80,7 @@ s3 = ["aioboto3>=9.0.0", "boto3>=1.35.2"]socks = ["python-socks[asyncio]>=2.6.0"]twisted-http2 = ["Twisted[http2]>=21.7.0"]uvloop = [- "uvloop>=0.17.0; platform_system != 'Windows' and implementation_name != 'pypy'",+ "uvloop>=0.17.0; platform_system != 'Windows'",][project.scripts]@@ -147,7 +144,6 @@ module = ["aiobotocore.*","bpython","brotli",- "brotlicffi","google.*","ipdb","psutil",diff --git a/scrapy/cmdline.py b/scrapy/cmdline.pyindex f26641851..1f18e21c4 100644--- a/scrapy/cmdline.py+++ b/scrapy/cmdline.py@@ -14,7 +14,6 @@ from scrapy.crawler import AsyncCrawlerProcess, CrawlerProcessfrom scrapy.exceptions import UsageErrorfrom scrapy.utils.misc import walk_modules_iterfrom scrapy.utils.project import get_project_settings, inside_project-from scrapy.utils.python import garbage_collectfrom scrapy.utils.reactor import _asyncio_reactor_pathif TYPE_CHECKING:@@ -241,10 +240,4 @@ def _run_command_profiled(if __name__ == "__main__":- try:- execute()- finally:- # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit:- # http://doc.pypy.org/en/latest/cpython_differences.html- # ?highlight=gc.collect#differences-related-to-garbage-collection-strategies- garbage_collect()+ execute()diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.pyindex 3cd230408..47204cce2 100644--- a/scrapy/utils/_compression.py+++ b/scrapy/utils/_compression.py@@ -7,10 +7,7 @@ from gzip import GzipFilefrom io import BytesIOfrom typing import TYPE_CHECKING-try:- import brotli-except ImportError: # pragma: no cover- import brotlicffi as brotli+import brotliif sys.version_info >= (3, 14):from compression import zstddiff --git a/scrapy/utils/_shell.py b/scrapy/utils/_shell.pyindex 27730e1a3..171611c0c 100644--- a/scrapy/utils/_shell.py+++ b/scrapy/utils/_shell.py@@ -2,6 +2,7 @@ from __future__ import annotationsimport asyncioimport code+import sysfrom collections.abc import Callablefrom functools import partial, wrapsfrom typing import TYPE_CHECKING, Any, TypeAlias@@ -66,6 +67,8 @@ def _embed_ptpython_shell(namespace: dict[str, Any] | None = None, banner: str = "") -> EmbedFuncT:"""Start a ptpython shell"""+ if sys.platform == "win32" and not sys.stdout.isatty():+ raise ImportError("ptpython needs a console on Windows")import ptpython.repl # noqa: PLC0415@wraps(_embed_ptpython_shell)diff --git a/scrapy/utils/python.py b/scrapy/utils/python.pyindex 42cc46dcb..66b47088e 100644--- a/scrapy/utils/python.py+++ b/scrapy/utils/python.py@@ -310,17 +310,13 @@ def global_object_name(obj: Any) -> str:return f"{obj.__module__}.{obj.__qualname__}"-if hasattr(sys, "pypy_version_info"): # pragma: no cover-- def garbage_collect() -> None:- # Collecting weakreferences can take two collections on PyPy.- gc.collect()- gc.collect()--else:-- def garbage_collect() -> None:- gc.collect()+def garbage_collect() -> None: # pragma: no cover+ warnings.warn(+ "scrapy.utils.python.garbage_collect() is deprecated, use gc.collect() instead.",+ category=ScrapyDeprecationWarning,+ stacklevel=2,+ )+ gc.collect()class MutableChain(Iterable[_T]): # pragma: no coverdiff --git a/tests/test_utils_shell.py b/tests/test_utils_shell.pyindex bb28e8b44..828ff5fa7 100644--- a/tests/test_utils_shell.py+++ b/tests/test_utils_shell.py@@ -94,17 +94,32 @@ def test_get_shell_embed_func_ipython():assert shell.__name__ == "_embed_ipython_shell"-def test_get_shell_embed_func_ptpython():- pytest.importorskip("ptpython")- shell = get_shell_embed_func(["ptpython"])+ ("platform", "isatty", "expected"),+ [+ ("linux", False, "_embed_ptpython_shell"),+ ("win32", True, "_embed_ptpython_shell"),+ ("win32", False, "_embed_standard_shell"),+ ],+)+def test_get_shell_embed_func_ptpython(+ monkeypatch: pytest.MonkeyPatch, platform: str, isatty: bool, expected: str+) -> None:+ # Import ptpython before faking the platform it is imported on.+ pytest.importorskip("ptpython.repl")+ monkeypatch.setattr(sys, "platform", platform)+ monkeypatch.setattr(sys.stdout, "isatty", lambda: isatty)+ shell = get_shell_embed_func(["ptpython", "python"])assert shell is not None- assert shell.__name__ == "_embed_ptpython_shell"+ assert shell.__name__ == expected-def test_get_shell_embed_func_default():+def test_get_shell_embed_func_default(monkeypatch: pytest.MonkeyPatch) -> None:# with no shells given, the first available shell in preference order# (ptpython, ipython, bpython, python) is returned; the standard shell# is always available, so the default is never None+ # on Windows, ptpython is only available when stdout is a console+ monkeypatch.setattr(sys.stdout, "isatty", lambda: True)shell = get_shell_embed_func()assert shell is not Noneif find_spec("ptpython") is not None:diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.pyindex f34656000..ad5878e7a 100644--- a/tests/test_utils_trackref.py+++ b/tests/test_utils_trackref.py@@ -1,12 +1,12 @@from __future__ import annotations+import gcfrom io import StringIOfrom unittest import mockimport pytestfrom scrapy.utils import trackref-from scrapy.utils.python import garbage_collectclass Foo(trackref.object_ref):@@ -95,6 +95,6 @@ def test_run_time_classes() -> None:base = type("Baz", (trackref.object_ref,), {})base()del base- garbage_collect()+ gc.collect()assert not list(trackref.iter_all("Baz"))assert sum(1 for cls in trackref.live_refs if cls.__name__ == "Baz") == 0diff --git a/tox.ini b/tox.iniindex 4935c4841..dfc5a853d 100644--- a/tox.ini+++ b/tox.ini@@ -261,7 +261,6 @@ commands_pre =git+https://github.com/boto/boto3 \git+https://github.com/bpython/bpython \git+https://github.com/google/brotli \- git+https://github.com/python-hyper/brotlicffi \git+https://github.com/googleapis/python-storage \git+https://github.com/pydantic/httpx2\#subdirectory=src/httpx2 \git+https://github.com/ipython/ipython \