Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Drop Python 3.10 support. (#8261)

Andrey Rakhmatullinc000f5cdff44
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index aa88e3271..a7f2debf4 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -52,10 +52,10 @@ jobs:
- python-version: "3.14"
env:
TOXENV: pylint
- - python-version: "3.10"
+ - python-version: "3.11"
env:
TOXENV: mypy
- - python-version: "3.10"
+ - python-version: "3.11"
env:
TOXENV: mypy-tests
# Keep in sync with .readthedocs.yml.
@@ -182,9 +182,6 @@ jobs:
fail-fast: false
matrix:
include:
- - python-version: "3.10"
- env:
- TOXENV: py
- python-version: "3.11"
env:
TOXENV: py
@@ -208,19 +205,19 @@ jobs:
coverage: true
# min deps
- - python-version: "3.10.19"
+ - python-version: "3.11.17"
env:
TOXENV: min
coverage: true
- - python-version: "3.10.19"
+ - python-version: "3.11.17"
env:
TOXENV: min-default-reactor
coverage: true
- - python-version: "3.10.19"
+ - python-version: "3.11.17"
env:
TOXENV: min-extra-deps
coverage: true
- - python-version: "3.10.19"
+ - python-version: "3.11.17"
env:
TOXENV: min-botocore
coverage: true
@@ -262,7 +259,7 @@ jobs:
fail-fast: false
matrix:
include:
- - python-version: "3.10"
+ - python-version: "3.11"
env:
TOXENV: py
- python-version: "3.14"
@@ -274,10 +271,10 @@ jobs:
TOXENV: no-reactor
# min deps
- - python-version: "3.10.11"
+ - python-version: "3.11.9"
env:
TOXENV: min
- - python-version: "3.10.11"
+ - python-version: "3.11.9"
env:
TOXENV: min-extra-deps
@@ -309,7 +306,7 @@ jobs:
fail-fast: false
matrix:
include:
- - python-version: "3.10"
+ - python-version: "3.11"
env:
TOXENV: py
- python-version: "3.14"
diff --git a/README.rst b/README.rst
index 651294add..712b66dfd 100644
--- a/README.rst
+++ b/README.rst
@@ -32,7 +32,7 @@
:alt: Ask DeepWiki
Scrapy_ is a web scraping framework to extract structured data from websites.
-It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_
+It is cross-platform, and requires Python 3.11+. It is maintained by Zyte_
(formerly Scrapinghub) and `many other contributors`_.
.. _many other contributors: https://github.com/scrapy/scrapy/graphs/contributors
diff --git a/docs/contributing.rst b/docs/contributing.rst
index fd70b771c..18c6e9616 100644
--- a/docs/contributing.rst
+++ b/docs/contributing.rst
@@ -367,15 +367,15 @@ To run a specific test (say ``tests/test_loader.py``) use:
To run the tests on a specific :doc:`tox <tox:index>` environment, use
``-e <name>`` with an environment name from ``tox.ini``. For example, to run
-the tests with Python 3.10 use::
+the tests with Python 3.14 use::
- tox -e py310
+ tox -e py314
You can also specify a comma-separated list of environments, and use :ref:`tox’s
parallel mode <tox:parallel_mode>` to run the tests on multiple environments in
parallel::
- tox -e py39,py310 -p auto
+ tox -e py313,py314 -p auto
To pass command-line options to :doc:`pytest <pytest:index>`, add them after
``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the
@@ -385,9 +385,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::
tox -- scrapy tests -x # stop after first failure
You can also use the `pytest-xdist`_ plugin. For example, to run all tests on
-the Python 3.10 :doc:`tox <tox:index>` environment using all your CPU cores::
+the Python 3.14 :doc:`tox <tox:index>` environment using all your CPU cores::
- tox -e py310 -- scrapy tests -n auto
+ tox -e py314 -- scrapy tests -n auto
To see coverage report install :doc:`coverage <coverage:index>`
(``pip install coverage``) and run:
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index aa6bef579..b7cd9f330 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -236,10 +236,6 @@ Per-request ``bindaddress`` No (not supported by the library)
TLS implementation Standard library ``ssl``
=========================== =======================================
-Other limitations:
-
-- HTTPS proxies for HTTPS destinations are not supported on Python < 3.11.
-
.. _twisted-http2-handler:
H2DownloadHandler
diff --git a/pyproject.toml b/pyproject.toml
index 5e3f603c3..c22bf3400 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,14 +15,14 @@ dependencies = [
"defusedxml>=0.7.1",
"formerly>=0.1.0",
"itemadapter>=0.1.0",
- "itemloaders>=1.0.1",
- "lxml>=4.6.4",
+ "itemloaders>=1.0.5",
+ "lxml>=4.9.4",
"packaging",
"parsel>=1.8.1",
"platformdirs>=2.0.0",
"protego>=0.1.15",
"pyOpenSSL>=24.3.0",
- "queuelib>=1.6.1",
+ "queuelib>=1.8.0",
"service_identity>=24.2.0",
"tldextract",
'typing-extensions>=4.5.0; python_version < "3.13"',
@@ -43,7 +43,6 @@ classifiers = [
"Operating System :: OS Independent",
"Programming Language :: Python",
"Programming Language :: Python :: 3",
- "Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
@@ -58,7 +57,7 @@ classifiers = [
license = "BSD-3-Clause"
license-files = ["LICENSE", "AUTHORS"]
readme = "README.rst"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
authors = [{ name = "Scrapy developers", email = "[email protected]" }]
maintainers = [{ name = "Pablo Hoffman", email = "[email protected]" }]
@@ -74,14 +73,14 @@ bpython = ["bpython>=0.7.1"]
color = ["colorlog>=6.6.0"]
gcs = ["google-cloud-storage>=1.30.0"]
httpx = ["httpx2[http2,socks]>=2.0.0"]
-images = ["Pillow>=8.3.2"]
+images = ["Pillow>=9.2.0"]
ipython = ["ipython>=8.15.0"]
ptpython = ["ptpython>=3.0.23"]
robotparser = ["robotexclusionrulesparser>=1.6.2"]
s3 = ["boto3>=1.20.0"]
twisted-http2 = ["Twisted[http2]>=21.7.0"]
uvloop = [
- "uvloop>=0.16.0; platform_system != 'Windows' and implementation_name != 'pypy'",
+ "uvloop>=0.17.0; platform_system != 'Windows' and implementation_name != 'pypy'",
]
[project.scripts]
@@ -340,6 +339,10 @@ filterwarnings = [
"ignore:Coverage disabled via --no-cov switch!",
]
+[tool.ruff]
+# TODO
+target-version = "py310"
+
[tool.ruff.lint]
extend-select = [
# flake8-builtins
diff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.py
index ff119215a..17279977f 100644
--- a/scrapy/core/scheduler.py
+++ b/scrapy/core/scheduler.py
@@ -14,7 +14,6 @@ from scrapy.utils.conf import _job_dir
from scrapy.utils.misc import build_from_crawler, load_object
if TYPE_CHECKING:
- # requires queuelib >= 1.6.2
from queuelib.queue import BaseQueue
# typing.Self requires Python 3.11
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 832089e30..a501fa711 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -721,8 +721,7 @@ class FeedExporter:
try:
slot.exporter.export_item(exported_item)
except Exception as e:
- if sys.version_info >= (3, 11):
- e.add_note(f"Item: {exported_item!r}")
+ e.add_note(f"Item: {exported_item!r}")
raise
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old one
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index c97f119ca..931d066f5 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -78,7 +78,7 @@ class ImagesPipeline(FilesPipeline):
self._ImageOps = ImageOps
except ImportError:
raise NotConfigured(
- "ImagesPipeline requires installing Pillow 8.3.2 or later"
+ "ImagesPipeline requires the Scrapy[images] extra to be installed"
) from None
super().__init__(store_uri, crawler=crawler)
@@ -235,13 +235,7 @@ class ImagesPipeline(FilesPipeline):
if size:
image = image.copy()
- try:
- # Image.Resampling.LANCZOS was added in Pillow 9.1.0
- # remove this try except block,
- # when updating the minimum requirements for Pillow.
- resampling_filter = self._Image.Resampling.LANCZOS
- except AttributeError:
- resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]
+ resampling_filter = self._Image.Resampling.LANCZOS
image.thumbnail(size, resampling_filter)
elif target_format == image_format:
image.format = target_format
diff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.py
index cbf0807a5..5bfca9260 100644
--- a/scrapy/utils/reactor.py
+++ b/scrapy/utils/reactor.py
@@ -152,7 +152,7 @@ def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop:
else:
try:
with catch_warnings():
- # In Python 3.10.9, 3.11.1, 3.12 and 3.13, a DeprecationWarning
+ # In Python 3.11.1-3.13.x, a DeprecationWarning
# is emitted about the lack of a current event loop, because in
# Python 3.14 and later `get_event_loop` will raise a
# RuntimeError in that event. Because our code is already
diff --git a/tests/test_downloader_handler_aiohttp.py b/tests/test_downloader_handler_aiohttp.py
index 99bb763c2..46b061fa1 100644
--- a/tests/test_downloader_handler_aiohttp.py
+++ b/tests/test_downloader_handler_aiohttp.py
@@ -1,6 +1,5 @@
from __future__ import annotations
-import sys
from typing import TYPE_CHECKING, Any
import pytest
@@ -103,19 +102,11 @@ class TestHttpProxy(AiohttpDownloadHandlerMixin, TestHttpProxyBase):
class TestHttpsProxy(AiohttpDownloadHandlerMixin, TestHttpProxyBase):
is_secure = True
- @property
- def handler_supports_tls_in_tls(self) -> bool:
- return sys.version_info >= (3, 11)
-
@pytest.mark.requires_mitmproxy
class TestMitmProxy(AiohttpDownloadHandlerMixin, TestMitmProxyBase):
handler_supports_socks: bool = False
- @property
- def handler_supports_tls_in_tls(self) -> bool:
- return sys.version_info >= (3, 11)
-
@pytest.mark.requires_internet
class TestRealWebsite(AiohttpDownloadHandlerMixin, TestRealWebsiteBase):
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index de623b0c5..cce00dea8 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -5,7 +5,6 @@ import json
import logging
import marshal
import pickle
-import sys
import tempfile
from logging import getLogger
from pathlib import Path
@@ -537,9 +536,6 @@ class TestFeedExport(TestFeedExportBase):
assert not listener.start_without_finish
assert not listener.finish_without_start
- @pytest.mark.skipif(
- sys.version_info < (3, 11), reason="BaseException.add_note() is 3.11+"
- )
@coroutine_test
async def test_export_item_exception_mentions_item(
self, caplog: pytest.LogCaptureFixture
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index 04c946f57..60f79926f 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -8,7 +8,6 @@ from abc import ABC, abstractmethod
from pathlib import Path
from shutil import rmtree
from tempfile import mkdtemp
-from types import SimpleNamespace
from typing import Any
import attr
@@ -49,7 +48,7 @@ class TestImagesPipeline:
def test_missing_pillow(self, monkeypatch: pytest.MonkeyPatch) -> None:
monkeypatch.setitem(sys.modules, "PIL", None)
- with pytest.raises(NotConfigured, match="requires installing Pillow"):
+ with pytest.raises(NotConfigured, match=r"requires the Scrapy\[images\] extra"):
ImagesPipeline(self.tempdir, crawler=get_crawler())
def test_file_path(self):
@@ -370,26 +369,6 @@ class TestImagesPipeline:
assert converted.mode == "RGB"
assert converted.getcolors() == [(10000, (128, 128, 128))]
- def test_convert_image_legacy_resampling_filter(
- self, monkeypatch: pytest.MonkeyPatch
- ) -> None:
- """Pillow older than 9.1.0 has Image.ANTIALIAS instead of
- Image.Resampling.LANCZOS."""
- # Image.LANCZOS is the only spelling that exists in every supported
- # Pillow version, but Pillow defines it dynamically, hence the ignore.
- monkeypatch.setattr(
- self.pipeline,
- "_Image",
- SimpleNamespace(ANTIALIAS=Image.LANCZOS), # type: ignore[attr-defined]
- )
- im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255))
-
- thumbnail, _ = self.pipeline.convert_image(
- im, size=(10, 25), image_format="JPEG", response_body=buf
- )
-
- assert thumbnail.size == (10, 10)
-
@pytest.mark.parametrize(
"bad_type",
[
diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py
index f07928510..bcb134135 100644
--- a/tests/test_pqueues.py
+++ b/tests/test_pqueues.py
@@ -3,7 +3,6 @@ from typing import Any
from unittest.mock import Mock
import pytest
-import queuelib
from scrapy.core.downloader import Downloader
from scrapy.http.request import Request
@@ -48,24 +47,7 @@ class TestPriorityQueue:
assert dequeued.priority == req1.priority
assert not queue.close()
- def test_no_peek_raises(self):
- if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
- pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined")
- temp_dir = tempfile.mkdtemp()
- queue = build_from_crawler(
- ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir
- )
- queue.push(Request("https://example.org"))
- with pytest.raises(
- NotImplementedError,
- match="The underlying queue class does not implement 'peek'",
- ):
- queue.peek()
- queue.close()
-
def test_peek(self):
- if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
- pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined")
temp_dir = tempfile.mkdtemp()
queue = build_from_crawler(
ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir
@@ -168,19 +150,7 @@ class TestDownloaderAwarePriorityQueue:
assert len(self.queue) == 0
assert self.queue.pop() is None
- def test_no_peek_raises(self):
- if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
- pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined")
- self.queue.push(Request("https://example.org"))
- with pytest.raises(
- NotImplementedError,
- match="The underlying queue class does not implement 'peek'",
- ):
- self.queue.peek()
-
def test_peek(self):
- if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):
- pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined")
assert len(self.queue) == 0
req1 = Request("https://example.org/1")
req2 = Request("https://example.org/2")
diff --git a/tests/test_utils_python.py b/tests/test_utils_python.py
index 3e46dca18..46b9a69ee 100644
--- a/tests/test_utils_python.py
+++ b/tests/test_utils_python.py
@@ -189,16 +189,19 @@ def test_get_func_args():
assert get_func_args(" ".join, stripself=True) == ["iterable"]
if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy":
- # the correct and correctly extracted signature
assert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"]
- elif platform.python_implementation() == "CPython":
- # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature
- # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951)
- assert get_func_args(operator.itemgetter(2), stripself=True) in [
- [],
- ["args", "kwargs"],
+ else:
+ # The interpreter returns a wrong signature
+ assert get_func_args(operator.itemgetter(2), stripself=True) == [
+ "args",
+ "kwargs",
]
+ # inspect.signature() fails for these
+ assert get_func_args(type) == []
+ if platform.python_implementation() != "PyPy":
+ assert get_func_args(max) == []
+
@pytest.mark.skipif(
sys.version_info < (3, 14),
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index aad138513..9135b1171 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -1497,6 +1497,8 @@ class TestHttpWithCrawlerBase(ABC):
class TestHttpProxyBase(ABC):
is_secure = False
+ # whether the handler supports HTTPS proxies with HTTPS destinations
+ handler_supports_tls_in_tls: bool = True
expected_http_proxy_request_body = b"http://example.com"
@property
@@ -1504,11 +1506,6 @@ class TestHttpProxyBase(ABC):
def download_handler_cls(self) -> type[DownloadHandlerProtocol]:
raise NotImplementedError
- # whether the handler supports HTTPS proxies with HTTPS destinations
- @property
- def handler_supports_tls_in_tls(self) -> bool:
- return True
-
@pytest.fixture(scope="session")
def proxy_mockserver(self) -> Generator[ProxyEchoMockServer]:
with ProxyEchoMockServer() as proxy:
@@ -1583,17 +1580,14 @@ PROXY_KINDS = ["http", "https", "socks5"]
class TestMitmProxyBase(ABC):
handler_supports_socks: bool = False
+ # whether the handler supports HTTPS proxies with HTTPS destinations
+ handler_supports_tls_in_tls: bool = True
@property
@abstractmethod
def settings_dict(self) -> dict[str, Any] | None:
raise NotImplementedError
- # whether the handler supports HTTPS proxies with HTTPS destinations
- @property
- def handler_supports_tls_in_tls(self) -> bool:
- return True
-
def _maybe_skip(self, proxy_kind: str, https_dest: bool) -> None:
if proxy_kind == "socks5" and not self.handler_supports_socks:
pytest.skip("SOCKS proxies are not supported")
diff --git a/tox.ini b/tox.ini
index bb8347ba2..2969d14d2 100644
--- a/tox.ini
+++ b/tox.ini
@@ -67,7 +67,7 @@ commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules}
[testenv:mypy]
-basepython = python3.10
+basepython = python3.11
deps =
mypy==2.3.1
typing-extensions==4.16.0
@@ -100,7 +100,7 @@ commands =
mypy {posargs:scrapy tests}
[testenv:mypy-tests]
-basepython = python3.10
+basepython = python3.11
deps =
{[test-requirements]deps}
{[testenv:mypy]deps}
@@ -135,7 +135,7 @@ commands =
twine check dist/*
[min]
-basepython = python3.10
+basepython = python3.11
deps =
# pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0
pytest==8.4.0
@@ -150,11 +150,12 @@ deps =
cssselect==1.2.0
formerly==0.1.0
itemadapter==0.1.0
- lxml==4.6.4
+ itemloaders==1.0.5
+ lxml==4.9.4
parsel==1.8.1
platformdirs==2.0.0
pyOpenSSL==24.3.0
- queuelib==1.6.1
+ queuelib==1.8.0
service_identity==24.2.0
typing-extensions==4.5.0
w3lib==2.1.1
@@ -197,7 +198,7 @@ deps =
basepython = {[min]basepython}
deps =
{[min]deps}
- Pillow==8.3.2
+ Pillow==9.2.0
Twisted[http2]==21.7.0
boto3==1.20.0
bpython==0.7.1
@@ -207,7 +208,7 @@ deps =
ipython==8.15.0
ptpython==3.0.23
robotexclusionrulesparser==1.6.2
- uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
+ uvloop==0.17.0; platform_system != "Windows" and implementation_name != "pypy"
setenv =
{[min]setenv}
SCRAPY_COVERAGE_PATCH=_exit # redundant with "extra-deps"'s subprocess coverage of scrapy/commands/*
@@ -332,10 +333,11 @@ deps =
cryptography==44.0.2
cssselect==1.2.0
itemadapter==0.1.0
+ itemloaders==1.0.5
lxml==5.3.2
parsel==1.8.1
pyOpenSSL==24.3.0
- queuelib==1.6.1
+ queuelib==1.8.0
service_identity==24.2.0
w3lib==2.1.1
zope.interface==5.1.0