Scrapy, a fast high-level web crawling & scraping framework for Python.
Drop Python 3.10 support. (#8261)
Andrey Rakhmatullinc000f5cdff44
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.ymlindex aa88e3271..a7f2debf4 100644--- a/.github/workflows/ci.yml+++ b/.github/workflows/ci.yml@@ -52,10 +52,10 @@ jobs:- python-version: "3.14"env:TOXENV: pylint- - python-version: "3.10"+ - python-version: "3.11"env:TOXENV: mypy- - python-version: "3.10"+ - python-version: "3.11"env:TOXENV: mypy-tests# Keep in sync with .readthedocs.yml.@@ -182,9 +182,6 @@ jobs:fail-fast: falsematrix:include:- - python-version: "3.10"- env:- TOXENV: py- python-version: "3.11"env:TOXENV: py@@ -208,19 +205,19 @@ jobs:coverage: true# min deps- - python-version: "3.10.19"+ - python-version: "3.11.17"env:TOXENV: mincoverage: true- - python-version: "3.10.19"+ - python-version: "3.11.17"env:TOXENV: min-default-reactorcoverage: true- - python-version: "3.10.19"+ - python-version: "3.11.17"env:TOXENV: min-extra-depscoverage: true- - python-version: "3.10.19"+ - python-version: "3.11.17"env:TOXENV: min-botocorecoverage: true@@ -262,7 +259,7 @@ jobs:fail-fast: falsematrix:include:- - python-version: "3.10"+ - python-version: "3.11"env:TOXENV: py- python-version: "3.14"@@ -274,10 +271,10 @@ jobs:TOXENV: no-reactor# min deps- - python-version: "3.10.11"+ - python-version: "3.11.9"env:TOXENV: min- - python-version: "3.10.11"+ - python-version: "3.11.9"env:TOXENV: min-extra-deps@@ -309,7 +306,7 @@ jobs:fail-fast: falsematrix:include:- - python-version: "3.10"+ - python-version: "3.11"env:TOXENV: py- python-version: "3.14"diff --git a/README.rst b/README.rstindex 651294add..712b66dfd 100644--- a/README.rst+++ b/README.rst@@ -32,7 +32,7 @@:alt: Ask DeepWikiScrapy_ is a web scraping framework to extract structured data from websites.-It is cross-platform, and requires Python 3.10+. It is maintained by Zyte_+It is cross-platform, and requires Python 3.11+. It is maintained by Zyte_(formerly Scrapinghub) and `many other contributors`_... _many other contributors: https://github.com/scrapy/scrapy/graphs/contributorsdiff --git a/docs/contributing.rst b/docs/contributing.rstindex fd70b771c..18c6e9616 100644--- a/docs/contributing.rst+++ b/docs/contributing.rst@@ -367,15 +367,15 @@ To run a specific test (say ``tests/test_loader.py``) use:To run the tests on a specific :doc:`tox <tox:index>` environment, use``-e <name>`` with an environment name from ``tox.ini``. For example, to run-the tests with Python 3.10 use::+the tests with Python 3.14 use::- tox -e py310+ tox -e py314You can also specify a comma-separated list of environments, and use :ref:`tox’sparallel mode <tox:parallel_mode>` to run the tests on multiple environments inparallel::- tox -e py39,py310 -p auto+ tox -e py313,py314 -p autoTo pass command-line options to :doc:`pytest <pytest:index>`, add them after``--`` in your call to :doc:`tox <tox:index>`. Using ``--`` overrides the@@ -385,9 +385,9 @@ default positional arguments (``scrapy tests``) after ``--`` as well::tox -- scrapy tests -x # stop after first failureYou can also use the `pytest-xdist`_ plugin. For example, to run all tests on-the Python 3.10 :doc:`tox <tox:index>` environment using all your CPU cores::+the Python 3.14 :doc:`tox <tox:index>` environment using all your CPU cores::- tox -e py310 -- scrapy tests -n auto+ tox -e py314 -- scrapy tests -n autoTo see coverage report install :doc:`coverage <coverage:index>`(``pip install coverage``) and run:diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex aa6bef579..b7cd9f330 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -236,10 +236,6 @@ Per-request ``bindaddress`` No (not supported by the library)TLS implementation Standard library ``ssl``=========================== =======================================-Other limitations:--- HTTPS proxies for HTTPS destinations are not supported on Python < 3.11.-.. _twisted-http2-handler:H2DownloadHandlerdiff --git a/pyproject.toml b/pyproject.tomlindex 5e3f603c3..c22bf3400 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -15,14 +15,14 @@ dependencies = ["defusedxml>=0.7.1","formerly>=0.1.0","itemadapter>=0.1.0",- "itemloaders>=1.0.1",- "lxml>=4.6.4",+ "itemloaders>=1.0.5",+ "lxml>=4.9.4","packaging","parsel>=1.8.1","platformdirs>=2.0.0","protego>=0.1.15","pyOpenSSL>=24.3.0",- "queuelib>=1.6.1",+ "queuelib>=1.8.0","service_identity>=24.2.0","tldextract",'typing-extensions>=4.5.0; python_version < "3.13"',@@ -43,7 +43,6 @@ classifiers = ["Operating System :: OS Independent","Programming Language :: Python","Programming Language :: Python :: 3",- "Programming Language :: Python :: 3.10","Programming Language :: Python :: 3.11","Programming Language :: Python :: 3.12","Programming Language :: Python :: 3.13",@@ -58,7 +57,7 @@ classifiers = [license = "BSD-3-Clause"license-files = ["LICENSE", "AUTHORS"]readme = "README.rst"-requires-python = ">=3.10"+requires-python = ">=3.11"authors = [{ name = "Scrapy developers", email = "[email protected]" }]maintainers = [{ name = "Pablo Hoffman", email = "[email protected]" }]@@ -74,14 +73,14 @@ bpython = ["bpython>=0.7.1"]color = ["colorlog>=6.6.0"]gcs = ["google-cloud-storage>=1.30.0"]httpx = ["httpx2[http2,socks]>=2.0.0"]-images = ["Pillow>=8.3.2"]+images = ["Pillow>=9.2.0"]ipython = ["ipython>=8.15.0"]ptpython = ["ptpython>=3.0.23"]robotparser = ["robotexclusionrulesparser>=1.6.2"]s3 = ["boto3>=1.20.0"]twisted-http2 = ["Twisted[http2]>=21.7.0"]uvloop = [- "uvloop>=0.16.0; platform_system != 'Windows' and implementation_name != 'pypy'",+ "uvloop>=0.17.0; platform_system != 'Windows' and implementation_name != 'pypy'",][project.scripts]@@ -340,6 +339,10 @@ filterwarnings = ["ignore:Coverage disabled via --no-cov switch!",]+[tool.ruff]+# TODO+target-version = "py310"+[tool.ruff.lint]extend-select = [# flake8-builtinsdiff --git a/scrapy/core/scheduler.py b/scrapy/core/scheduler.pyindex ff119215a..17279977f 100644--- a/scrapy/core/scheduler.py+++ b/scrapy/core/scheduler.py@@ -14,7 +14,6 @@ from scrapy.utils.conf import _job_dirfrom scrapy.utils.misc import build_from_crawler, load_objectif TYPE_CHECKING:- # requires queuelib >= 1.6.2from queuelib.queue import BaseQueue# typing.Self requires Python 3.11diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 832089e30..a501fa711 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -721,8 +721,7 @@ class FeedExporter:try:slot.exporter.export_item(exported_item)except Exception as e:- if sys.version_info >= (3, 11):- e.add_note(f"Item: {exported_item!r}")+ e.add_note(f"Item: {exported_item!r}")raiseslot.itemcount += 1# create new slot for each slot with itemcount == FEED_EXPORT_BATCH_ITEM_COUNT and close the old onediff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex c97f119ca..931d066f5 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -78,7 +78,7 @@ class ImagesPipeline(FilesPipeline):self._ImageOps = ImageOpsexcept ImportError:raise NotConfigured(- "ImagesPipeline requires installing Pillow 8.3.2 or later"+ "ImagesPipeline requires the Scrapy[images] extra to be installed") from Nonesuper().__init__(store_uri, crawler=crawler)@@ -235,13 +235,7 @@ class ImagesPipeline(FilesPipeline):if size:image = image.copy()- try:- # Image.Resampling.LANCZOS was added in Pillow 9.1.0- # remove this try except block,- # when updating the minimum requirements for Pillow.- resampling_filter = self._Image.Resampling.LANCZOS- except AttributeError:- resampling_filter = self._Image.ANTIALIAS # type: ignore[attr-defined]+ resampling_filter = self._Image.Resampling.LANCZOSimage.thumbnail(size, resampling_filter)elif target_format == image_format:image.format = target_formatdiff --git a/scrapy/utils/reactor.py b/scrapy/utils/reactor.pyindex cbf0807a5..5bfca9260 100644--- a/scrapy/utils/reactor.py+++ b/scrapy/utils/reactor.py@@ -152,7 +152,7 @@ def set_asyncio_event_loop(event_loop_path: str | None) -> AbstractEventLoop:else:try:with catch_warnings():- # In Python 3.10.9, 3.11.1, 3.12 and 3.13, a DeprecationWarning+ # In Python 3.11.1-3.13.x, a DeprecationWarning# is emitted about the lack of a current event loop, because in# Python 3.14 and later `get_event_loop` will raise a# RuntimeError in that event. Because our code is alreadydiff --git a/tests/test_downloader_handler_aiohttp.py b/tests/test_downloader_handler_aiohttp.pyindex 99bb763c2..46b061fa1 100644--- a/tests/test_downloader_handler_aiohttp.py+++ b/tests/test_downloader_handler_aiohttp.py@@ -1,6 +1,5 @@from __future__ import annotations-import sysfrom typing import TYPE_CHECKING, Anyimport pytest@@ -103,19 +102,11 @@ class TestHttpProxy(AiohttpDownloadHandlerMixin, TestHttpProxyBase):class TestHttpsProxy(AiohttpDownloadHandlerMixin, TestHttpProxyBase):is_secure = True- @property- def handler_supports_tls_in_tls(self) -> bool:- return sys.version_info >= (3, 11)-@pytest.mark.requires_mitmproxyclass TestMitmProxy(AiohttpDownloadHandlerMixin, TestMitmProxyBase):handler_supports_socks: bool = False- @property- def handler_supports_tls_in_tls(self) -> bool:- return sys.version_info >= (3, 11)-@pytest.mark.requires_internetclass TestRealWebsite(AiohttpDownloadHandlerMixin, TestRealWebsiteBase):diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex de623b0c5..cce00dea8 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -5,7 +5,6 @@ import jsonimport loggingimport marshalimport pickle-import sysimport tempfilefrom logging import getLoggerfrom pathlib import Path@@ -537,9 +536,6 @@ class TestFeedExport(TestFeedExportBase):assert not listener.start_without_finishassert not listener.finish_without_start- @pytest.mark.skipif(- sys.version_info < (3, 11), reason="BaseException.add_note() is 3.11+"- )@coroutine_testasync def test_export_item_exception_mentions_item(self, caplog: pytest.LogCaptureFixturediff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex 04c946f57..60f79926f 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -8,7 +8,6 @@ from abc import ABC, abstractmethodfrom pathlib import Pathfrom shutil import rmtreefrom tempfile import mkdtemp-from types import SimpleNamespacefrom typing import Anyimport attr@@ -49,7 +48,7 @@ class TestImagesPipeline:def test_missing_pillow(self, monkeypatch: pytest.MonkeyPatch) -> None:monkeypatch.setitem(sys.modules, "PIL", None)- with pytest.raises(NotConfigured, match="requires installing Pillow"):+ with pytest.raises(NotConfigured, match=r"requires the Scrapy\[images\] extra"):ImagesPipeline(self.tempdir, crawler=get_crawler())def test_file_path(self):@@ -370,26 +369,6 @@ class TestImagesPipeline:assert converted.mode == "RGB"assert converted.getcolors() == [(10000, (128, 128, 128))]- def test_convert_image_legacy_resampling_filter(- self, monkeypatch: pytest.MonkeyPatch- ) -> None:- """Pillow older than 9.1.0 has Image.ANTIALIAS instead of- Image.Resampling.LANCZOS."""- # Image.LANCZOS is the only spelling that exists in every supported- # Pillow version, but Pillow defines it dynamically, hence the ignore.- monkeypatch.setattr(- self.pipeline,- "_Image",- SimpleNamespace(ANTIALIAS=Image.LANCZOS), # type: ignore[attr-defined]- )- im, buf = _create_image("JPEG", "RGB", (100, 100), (0, 127, 255))-- thumbnail, _ = self.pipeline.convert_image(- im, size=(10, 25), image_format="JPEG", response_body=buf- )-- assert thumbnail.size == (10, 10)-@pytest.mark.parametrize("bad_type",[diff --git a/tests/test_pqueues.py b/tests/test_pqueues.pyindex f07928510..bcb134135 100644--- a/tests/test_pqueues.py+++ b/tests/test_pqueues.py@@ -3,7 +3,6 @@ from typing import Anyfrom unittest.mock import Mockimport pytest-import queuelibfrom scrapy.core.downloader import Downloaderfrom scrapy.http.request import Request@@ -48,24 +47,7 @@ class TestPriorityQueue:assert dequeued.priority == req1.priorityassert not queue.close()- def test_no_peek_raises(self):- if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):- pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined")- temp_dir = tempfile.mkdtemp()- queue = build_from_crawler(- ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir- )- queue.push(Request("https://example.org"))- with pytest.raises(- NotImplementedError,- match="The underlying queue class does not implement 'peek'",- ):- queue.peek()- queue.close()-def test_peek(self):- if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):- pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined")temp_dir = tempfile.mkdtemp()queue = build_from_crawler(ScrapyPriorityQueue, self.crawler, FifoMemoryQueue, temp_dir@@ -168,19 +150,7 @@ class TestDownloaderAwarePriorityQueue:assert len(self.queue) == 0assert self.queue.pop() is None- def test_no_peek_raises(self):- if hasattr(queuelib.queue.FifoMemoryQueue, "peek"):- pytest.skip("queuelib.queue.FifoMemoryQueue.peek is defined")- self.queue.push(Request("https://example.org"))- with pytest.raises(- NotImplementedError,- match="The underlying queue class does not implement 'peek'",- ):- self.queue.peek()-def test_peek(self):- if not hasattr(queuelib.queue.FifoMemoryQueue, "peek"):- pytest.skip("queuelib.queue.FifoMemoryQueue.peek is undefined")assert len(self.queue) == 0req1 = Request("https://example.org/1")req2 = Request("https://example.org/2")diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 3e46dca18..46b9a69ee 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -189,16 +189,19 @@ def test_get_func_args():assert get_func_args(" ".join, stripself=True) == ["iterable"]if sys.version_info >= (3, 13) or platform.python_implementation() == "PyPy":- # the correct and correctly extracted signatureassert get_func_args(operator.itemgetter(2), stripself=True) == ["obj"]- elif platform.python_implementation() == "CPython":- # ["args", "kwargs"] is a correct result for the pre-3.13 incorrect function signature- # [] is an incorrect result on even older CPython (https://github.com/python/cpython/issues/86951)- assert get_func_args(operator.itemgetter(2), stripself=True) in [- [],- ["args", "kwargs"],+ else:+ # The interpreter returns a wrong signature+ assert get_func_args(operator.itemgetter(2), stripself=True) == [+ "args",+ "kwargs",]+ # inspect.signature() fails for these+ assert get_func_args(type) == []+ if platform.python_implementation() != "PyPy":+ assert get_func_args(max) == []+@pytest.mark.skipif(sys.version_info < (3, 14),diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex aad138513..9135b1171 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -1497,6 +1497,8 @@ class TestHttpWithCrawlerBase(ABC):class TestHttpProxyBase(ABC):is_secure = False+ # whether the handler supports HTTPS proxies with HTTPS destinations+ handler_supports_tls_in_tls: bool = Trueexpected_http_proxy_request_body = b"http://example.com"@property@@ -1504,11 +1506,6 @@ class TestHttpProxyBase(ABC):def download_handler_cls(self) -> type[DownloadHandlerProtocol]:raise NotImplementedError- # whether the handler supports HTTPS proxies with HTTPS destinations- @property- def handler_supports_tls_in_tls(self) -> bool:- return True-@pytest.fixture(scope="session")def proxy_mockserver(self) -> Generator[ProxyEchoMockServer]:with ProxyEchoMockServer() as proxy:@@ -1583,17 +1580,14 @@ PROXY_KINDS = ["http", "https", "socks5"]class TestMitmProxyBase(ABC):handler_supports_socks: bool = False+ # whether the handler supports HTTPS proxies with HTTPS destinations+ handler_supports_tls_in_tls: bool = True@property@abstractmethoddef settings_dict(self) -> dict[str, Any] | None:raise NotImplementedError- # whether the handler supports HTTPS proxies with HTTPS destinations- @property- def handler_supports_tls_in_tls(self) -> bool:- return True-def _maybe_skip(self, proxy_kind: str, https_dest: bool) -> None:if proxy_kind == "socks5" and not self.handler_supports_socks:pytest.skip("SOCKS proxies are not supported")diff --git a/tox.ini b/tox.iniindex bb8347ba2..2969d14d2 100644--- a/tox.ini+++ b/tox.ini@@ -67,7 +67,7 @@ commands =pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules}[testenv:mypy]-basepython = python3.10+basepython = python3.11deps =mypy==2.3.1typing-extensions==4.16.0@@ -100,7 +100,7 @@ commands =mypy {posargs:scrapy tests}[testenv:mypy-tests]-basepython = python3.10+basepython = python3.11deps ={[test-requirements]deps}{[testenv:mypy]deps}@@ -135,7 +135,7 @@ commands =twine check dist/*[min]-basepython = python3.10+basepython = python3.11deps =# pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0pytest==8.4.0@@ -150,11 +150,12 @@ deps =cssselect==1.2.0formerly==0.1.0itemadapter==0.1.0- lxml==4.6.4+ itemloaders==1.0.5+ lxml==4.9.4parsel==1.8.1platformdirs==2.0.0pyOpenSSL==24.3.0- queuelib==1.6.1+ queuelib==1.8.0service_identity==24.2.0typing-extensions==4.5.0w3lib==2.1.1@@ -197,7 +198,7 @@ deps =basepython = {[min]basepython}deps ={[min]deps}- Pillow==8.3.2+ Pillow==9.2.0Twisted[http2]==21.7.0boto3==1.20.0bpython==0.7.1@@ -207,7 +208,7 @@ deps =ipython==8.15.0ptpython==3.0.23robotexclusionrulesparser==1.6.2- uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"+ uvloop==0.17.0; platform_system != "Windows" and implementation_name != "pypy"setenv ={[min]setenv}SCRAPY_COVERAGE_PATCH=_exit # redundant with "extra-deps"'s subprocess coverage of scrapy/commands/*@@ -332,10 +333,11 @@ deps =cryptography==44.0.2cssselect==1.2.0itemadapter==0.1.0+ itemloaders==1.0.5lxml==5.3.2parsel==1.8.1pyOpenSSL==24.3.0- queuelib==1.6.1+ queuelib==1.8.0service_identity==24.2.0w3lib==2.1.1zope.interface==5.1.0