Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove remaining PyPy support. (#8367)

Andrey Rakhmatullin1481630d439a
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f870a7fd2..fbc56d7d5 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -186,7 +186,7 @@ jobs:
# mitmproxy needs a newer Python than the oldest matrix entries, so let
# uv download one where no system interpreter is new enough.
UV_PYTHON_PREFERENCE: system
- run: uv tool install --python cpython mitmproxy
+ run: uv tool install mitmproxy
- name: Run tests
env: ${{ matrix.env }}
diff --git a/.github/workflows/tests-vcs-deps.yml b/.github/workflows/tests-vcs-deps.yml
index a0b6adb21..0a8fc94fe 100644
--- a/.github/workflows/tests-vcs-deps.yml
+++ b/.github/workflows/tests-vcs-deps.yml
@@ -50,7 +50,7 @@ jobs:
tox.ini
- name: Install mitmproxy
- run: uv tool install --python cpython mitmproxy
+ run: uv tool install mitmproxy
- name: Run tests
run: uvx --with tox-uv tox
diff --git a/docs/news.rst b/docs/news.rst
index fd007243b..b19033f8b 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -9,6 +9,11 @@ Scrapy VERSION (unreleased)
Backward-incompatible changes
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- Scrapy no longer officially supports PyPy. It may still work if you
+ manually remove ``PyDispatcher`` and install ``PyPyDispatcher`` but this is
+ not guaranteed.
+ (TBD)
+
- Changes to the ``fetch()`` shortcut of the :ref:`Scrapy shell
<topics-shell>`:
diff --git a/pyproject.toml b/pyproject.toml
index ec9343f0c..5007475e8 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -7,9 +7,12 @@ name = "Scrapy"
dynamic = ["version"]
description = "A high-level Web Crawling and Web Scraping framework"
dependencies = [
+ "PyDispatcher>=2.0.5",
"Twisted>=21.7.0",
"aioftp>=0.26.1",
"aiohttp>=3.13.3",
+ 'backports.zstd>=1.3.0; python_version < "3.14"',
+ "brotli>=1.2.0",
"charset-normalizer>=3.4.0",
"cryptography>=41.0.5",
"cssselect>=1.2.0",
@@ -29,12 +32,6 @@ dependencies = [
'typing-extensions>=4.6.0; python_version < "3.13"',
"w3lib>=2.1.1",
"zope.interface>=5.1.0",
- # Platform-specific dependencies
- 'PyDispatcher>=2.0.5; platform_python_implementation == "CPython"',
- 'PyPyDispatcher>=2.1.0; platform_python_implementation == "PyPy"',
- 'backports.zstd>=1.3.0; python_version < "3.14"',
- 'brotli>=1.2.0; implementation_name != "pypy"',
- 'brotlicffi>=1.2.0.0; implementation_name == "pypy"',
]
classifiers = [
"Development Status :: 5 - Production/Stable",
@@ -83,7 +80,7 @@ s3 = ["aioboto3>=9.0.0", "boto3>=1.35.2"]
socks = ["python-socks[asyncio]>=2.6.0"]
twisted-http2 = ["Twisted[http2]>=21.7.0"]
uvloop = [
- "uvloop>=0.17.0; platform_system != 'Windows' and implementation_name != 'pypy'",
+ "uvloop>=0.17.0; platform_system != 'Windows'",
]
[project.scripts]
@@ -147,7 +144,6 @@ module = [
"aiobotocore.*",
"bpython",
"brotli",
- "brotlicffi",
"google.*",
"ipdb",
"psutil",
diff --git a/scrapy/cmdline.py b/scrapy/cmdline.py
index f26641851..1f18e21c4 100644
--- a/scrapy/cmdline.py
+++ b/scrapy/cmdline.py
@@ -14,7 +14,6 @@ from scrapy.crawler import AsyncCrawlerProcess, CrawlerProcess
from scrapy.exceptions import UsageError
from scrapy.utils.misc import walk_modules_iter
from scrapy.utils.project import get_project_settings, inside_project
-from scrapy.utils.python import garbage_collect
from scrapy.utils.reactor import _asyncio_reactor_path
if TYPE_CHECKING:
@@ -241,10 +240,4 @@ def _run_command_profiled(
if __name__ == "__main__":
- try:
- execute()
- finally:
- # Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect() on exit:
- # http://doc.pypy.org/en/latest/cpython_differences.html
- # ?highlight=gc.collect#differences-related-to-garbage-collection-strategies
- garbage_collect()
+ execute()
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
index 3cd230408..47204cce2 100644
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -7,10 +7,7 @@ from gzip import GzipFile
from io import BytesIO
from typing import TYPE_CHECKING
-try:
- import brotli
-except ImportError: # pragma: no cover
- import brotlicffi as brotli
+import brotli
if sys.version_info >= (3, 14):
from compression import zstd
diff --git a/scrapy/utils/_shell.py b/scrapy/utils/_shell.py
index 27730e1a3..171611c0c 100644
--- a/scrapy/utils/_shell.py
+++ b/scrapy/utils/_shell.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import code
+import sys
from collections.abc import Callable
from functools import partial, wraps
from typing import TYPE_CHECKING, Any, TypeAlias
@@ -66,6 +67,8 @@ def _embed_ptpython_shell(
namespace: dict[str, Any] | None = None, banner: str = ""
) -> EmbedFuncT:
"""Start a ptpython shell"""
+ if sys.platform == "win32" and not sys.stdout.isatty():
+ raise ImportError("ptpython needs a console on Windows")
import ptpython.repl # noqa: PLC0415
@wraps(_embed_ptpython_shell)
diff --git a/scrapy/utils/python.py b/scrapy/utils/python.py
index 42cc46dcb..66b47088e 100644
--- a/scrapy/utils/python.py
+++ b/scrapy/utils/python.py
@@ -310,17 +310,13 @@ def global_object_name(obj: Any) -> str:
return f"{obj.__module__}.{obj.__qualname__}"
-if hasattr(sys, "pypy_version_info"): # pragma: no cover
-
- def garbage_collect() -> None:
- # Collecting weakreferences can take two collections on PyPy.
- gc.collect()
- gc.collect()
-
-else:
-
- def garbage_collect() -> None:
- gc.collect()
+def garbage_collect() -> None: # pragma: no cover
+ warnings.warn(
+ "scrapy.utils.python.garbage_collect() is deprecated, use gc.collect() instead.",
+ category=ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ gc.collect()
class MutableChain(Iterable[_T]): # pragma: no cover
diff --git a/tests/test_utils_shell.py b/tests/test_utils_shell.py
index bb28e8b44..828ff5fa7 100644
--- a/tests/test_utils_shell.py
+++ b/tests/test_utils_shell.py
@@ -94,17 +94,32 @@ def test_get_shell_embed_func_ipython():
assert shell.__name__ == "_embed_ipython_shell"
-def test_get_shell_embed_func_ptpython():
- pytest.importorskip("ptpython")
- shell = get_shell_embed_func(["ptpython"])
+ ("platform", "isatty", "expected"),
+ [
+ ("linux", False, "_embed_ptpython_shell"),
+ ("win32", True, "_embed_ptpython_shell"),
+ ("win32", False, "_embed_standard_shell"),
+ ],
+)
+def test_get_shell_embed_func_ptpython(
+ monkeypatch: pytest.MonkeyPatch, platform: str, isatty: bool, expected: str
+) -> None:
+ # Import ptpython before faking the platform it is imported on.
+ pytest.importorskip("ptpython.repl")
+ monkeypatch.setattr(sys, "platform", platform)
+ monkeypatch.setattr(sys.stdout, "isatty", lambda: isatty)
+ shell = get_shell_embed_func(["ptpython", "python"])
assert shell is not None
- assert shell.__name__ == "_embed_ptpython_shell"
+ assert shell.__name__ == expected
-def test_get_shell_embed_func_default():
+def test_get_shell_embed_func_default(monkeypatch: pytest.MonkeyPatch) -> None:
# with no shells given, the first available shell in preference order
# (ptpython, ipython, bpython, python) is returned; the standard shell
# is always available, so the default is never None
+ # on Windows, ptpython is only available when stdout is a console
+ monkeypatch.setattr(sys.stdout, "isatty", lambda: True)
shell = get_shell_embed_func()
assert shell is not None
if find_spec("ptpython") is not None:
diff --git a/tests/test_utils_trackref.py b/tests/test_utils_trackref.py
index f34656000..ad5878e7a 100644
--- a/tests/test_utils_trackref.py
+++ b/tests/test_utils_trackref.py
@@ -1,12 +1,12 @@
from __future__ import annotations
+import gc
from io import StringIO
from unittest import mock
import pytest
from scrapy.utils import trackref
-from scrapy.utils.python import garbage_collect
class Foo(trackref.object_ref):
@@ -95,6 +95,6 @@ def test_run_time_classes() -> None:
base = type("Baz", (trackref.object_ref,), {})
base()
del base
- garbage_collect()
+ gc.collect()
assert not list(trackref.iter_all("Baz"))
assert sum(1 for cls in trackref.live_refs if cls.__name__ == "Baz") == 0
diff --git a/tox.ini b/tox.ini
index 4935c4841..dfc5a853d 100644
--- a/tox.ini
+++ b/tox.ini
@@ -261,7 +261,6 @@ commands_pre =
git+https://github.com/boto/boto3 \
git+https://github.com/bpython/bpython \
git+https://github.com/google/brotli \
- git+https://github.com/python-hyper/brotlicffi \
git+https://github.com/googleapis/python-storage \
git+https://github.com/pydantic/httpx2\#subdirectory=src/httpx2 \
git+https://github.com/ipython/ipython \