Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

CI: Promote Python 3.15 to stable, drop redundant test jobs (#8361)

Adrianedef0249ae91
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7938edef0..9639fe458 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -48,7 +48,7 @@ jobs:
fail-fast: false
matrix:
include:
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: pylint
- python-version: "3.11"
@@ -61,10 +61,10 @@ jobs:
- python-version: "3.14"
env:
TOXENV: docs
- - python-version: "3.13"
+ - python-version: "3.14"
env:
TOXENV: docs-tests
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: twinecheck
@@ -114,21 +114,15 @@ jobs:
- python-version: "3.11"
env:
TOXENV: py
- - python-version: "3.12"
- env:
- TOXENV: py
- - python-version: "3.13"
- env:
- TOXENV: py
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: py
coverage: true
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: default-reactor
coverage: true
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: no-reactor
coverage: true
@@ -151,30 +145,19 @@ jobs:
TOXENV: min-botocore
coverage: true
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: extra-deps
coverage: true
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: no-reactor-extra-deps
coverage: true
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: botocore
coverage: true
- # 3.15 pre-release
- - python-version: "3.15.0-rc.1"
- env:
- TOXENV: py
- - python-version: "3.15.0-rc.1"
- env:
- TOXENV: default-reactor
- - python-version: "3.15.0-rc.1"
- env:
- TOXENV: no-reactor
-
steps: &test-steps
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
@@ -233,11 +216,14 @@ jobs:
- python-version: "3.11"
env:
TOXENV: py
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: py
coverage: true
- - python-version: "3.14"
+ - python-version: "3.15"
+ env:
+ TOXENV: default-reactor
+ - python-version: "3.15"
env:
TOXENV: no-reactor
@@ -249,21 +235,10 @@ jobs:
env:
TOXENV: min-extra-deps
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: extra-deps
- # 3.15 pre-release
- - python-version: "3.15.0-rc.1"
- env:
- TOXENV: py
- - python-version: "3.15.0-rc.1"
- env:
- TOXENV: default-reactor
- - python-version: "3.15.0-rc.1"
- env:
- TOXENV: no-reactor
-
steps: *test-steps
macos:
@@ -280,19 +255,11 @@ jobs:
- python-version: "3.11"
env:
TOXENV: py
- - python-version: "3.14"
+ - python-version: "3.15"
env:
TOXENV: py
coverage: true
- - python-version: "3.14"
- env:
- TOXENV: no-reactor
-
- # 3.15 pre-release
- - python-version: "3.15.0-rc.1"
- env:
- TOXENV: py
- - python-version: "3.15.0-rc.1"
+ - python-version: "3.15"
env:
TOXENV: no-reactor
diff --git a/.github/workflows/codspeed-walltime.yml b/.github/workflows/codspeed-walltime.yml
index 5015de840..60aafaac5 100644
--- a/.github/workflows/codspeed-walltime.yml
+++ b/.github/workflows/codspeed-walltime.yml
@@ -44,10 +44,10 @@ jobs:
with:
persist-credentials: false
- - name: Set up Python 3.14
+ - name: Set up Python 3.15
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
- python-version: '3.14'
+ python-version: '3.15'
- name: Set up uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
index f71be8e7a..73e39850a 100644
--- a/.github/workflows/codspeed.yml
+++ b/.github/workflows/codspeed.yml
@@ -38,10 +38,10 @@ jobs:
with:
persist-credentials: false
- - name: Set up Python 3.14
+ - name: Set up Python 3.15
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
- python-version: '3.14'
+ python-version: '3.15'
- name: Set up uv
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
index 11d17720c..f547be33c 100644
--- a/.github/workflows/publish.yml
+++ b/.github/workflows/publish.yml
@@ -22,7 +22,7 @@ jobs:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
- python-version: "3.14"
+ python-version: "3.15"
- run: |
python -m pip install --upgrade build
python -m build
diff --git a/.github/workflows/tests-vcs-deps.yml b/.github/workflows/tests-vcs-deps.yml
index 1a11e235c..a0b6adb21 100644
--- a/.github/workflows/tests-vcs-deps.yml
+++ b/.github/workflows/tests-vcs-deps.yml
@@ -32,7 +32,6 @@ jobs:
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.15"
- allow-prereleases: true
# Dependencies that ship wheels on PyPI are built from source here, so
# their build dependencies are needed: libxml2 and libxslt for lxml,
diff --git a/.github/workflows/upgrade-ci-pins.yml b/.github/workflows/upgrade-ci-pins.yml
index 7cd04f910..c76bff143 100644
--- a/.github/workflows/upgrade-ci-pins.yml
+++ b/.github/workflows/upgrade-ci-pins.yml
@@ -31,7 +31,7 @@ jobs:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
- python-version: "3.14"
+ python-version: "3.15"
- uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
with:
enable-cache: false
diff --git a/docs/intro/tutorial.rst b/docs/intro/tutorial.rst
index 522ea6cff..c0fbfb799 100644
--- a/docs/intro/tutorial.rst
+++ b/docs/intro/tutorial.rst
@@ -439,8 +439,8 @@ We get a list of selectors for the quote HTML elements with:
.. code-block:: pycon
>>> response.css("div.quote")
- [<Selector query="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
- <Selector query="descendant-or-self::div[@class and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
+ [<Selector query="descendant-or-self::div[@class and contains(@class, 'quote') and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
+ <Selector query="descendant-or-self::div[@class and contains(@class, 'quote') and contains(concat(' ', normalize-space(@class), ' '), ' quote ')]" data='<div class="quote" itemscope itemtype...'>,
...]
Each of the selectors returned by the query above allows us to run further
diff --git a/docs/requirements.txt b/docs/requirements.txt
index c2d5b3ef0..c64f1fa56 100644
--- a/docs/requirements.txt
+++ b/docs/requirements.txt
@@ -1,31 +1,42 @@
# This file was autogenerated by uv via the following command:
-# uv pip compile requirements.in -o requirements.txt
+# uv pip compile requirements.in -o requirements.txt --python-version 3.14
+aiohappyeyeballs==2.7.1
+ # via aiohttp
+aiohttp==3.14.5
+ # via scrapy
+aiosignal==1.4.0
+ # via aiohttp
alabaster==1.0.0
# via sphinx
-annotated-types==0.7.0
+annotated-types==0.8.0
# via pydantic
attrs==26.1.0
# via
+ # aiohttp
# service-identity
# twisted
automat==25.4.16
# via twisted
babel==2.18.0
# via sphinx
-certifi==2026.2.25
+brotli==1.2.0
+ # via scrapy
+certifi==2026.7.22
# via requests
-cffi==2.0.0
+cffi==2.1.1
# via cryptography
-charset-normalizer==3.4.6
- # via requests
+charset-normalizer==3.5.2
+ # via
+ # requests
+ # scrapy
constantly==23.10.4
# via twisted
-cryptography==46.0.6
+cryptography==50.0.2
# via
# pyopenssl
# scrapy
# service-identity
-cssselect==1.4.0
+cssselect==1.6.0
# via
# parsel
# scrapy
@@ -37,22 +48,27 @@ docutils==0.22.4
# sphinx-llm-friendly
# sphinx-rtd-theme
# sphinx-scrapy
-filelock==3.25.2
+filelock==4.2.0
# via tldextract
-h2==4.3.0
+frozenlist==1.8.0
+ # via
+ # aiohttp
+ # aiosignal
+h2==4.4.1
# via -r requirements.in
-hpack==4.1.0
+hpack==4.2.0
# via h2
hyperframe==6.1.0
# via h2
hyperlink==21.0.0
# via twisted
-idna==3.11
+idna==3.20
# via
# hyperlink
# requests
# tldextract
-imagesize==2.0.0
+ # yarl
+imagesize==2.0.1
# via sphinx
incremental==24.11.0
# via twisted
@@ -60,7 +76,7 @@ itemadapter==0.13.1
# via
# itemloaders
# scrapy
-itemloaders==1.4.0
+itemloaders==1.5.0
# via scrapy
jinja2==3.1.6
# via sphinx
@@ -68,49 +84,52 @@ jmespath==1.1.0
# via
# itemloaders
# parsel
-lxml==6.0.2
+lxml==6.1.3
# via
# parsel
# scrapy
-markupsafe==3.0.3
+markupsafe==3.0.4
# via jinja2
-packaging==26.0
+multidict==7.1.0
+ # via
+ # aiohttp
+ # yarl
+packaging==26.3
# via
# incremental
- # parsel
# scrapy
# scrapy-spider-metadata
# sphinx
# sphinx-scrapy
-parsel==1.11.0
+parsel==1.12.1
# via
# itemloaders
# scrapy
-protego==0.6.0
+platformdirs==4.13.3
# via scrapy
-pyasn1==0.6.3
+propcache==0.5.4
# via
- # pyasn1-modules
- # service-identity
-pyasn1-modules==0.4.2
- # via service-identity
-pycparser==3.0
+ # aiohttp
+ # yarl
+protego==0.7.0
+ # via scrapy
+pycparser==3.11
# via cffi
-pydantic==2.12.5
+pydantic==2.14.1
# via
# -r requirements.in
# scrapy-spider-metadata
-pydantic-core==2.41.5
+pydantic-core==2.50.1
# via pydantic
pydispatcher==2.0.7
# via scrapy
-pygments==2.19.2
+pygments==2.21.0
# via sphinx
-pyopenssl==26.0.0
+pyopenssl==26.4.0
# via scrapy
-queuelib==1.9.0
+queuelib==1.10.0
# via scrapy
-requests==2.33.0
+requests==2.34.2
# via
# requests-file
# sphinx
@@ -120,13 +139,13 @@ requests-file==3.0.1
# via tldextract
roman-numerals==4.1.0
# via sphinx
-scrapy==2.14.2
+scrapy==2.19.0
# via scrapy-spider-metadata
scrapy-spider-metadata==0.2.0
# via -r requirements.in
-service-identity==24.2.0
+service-identity==26.1.0
# via scrapy
-snowballstemmer==3.0.1
+snowballstemmer==3.1.1
# via sphinx
sphinx==9.1.0
# via
@@ -181,26 +200,27 @@ sphinxcontrib-youtube==1.5.0
# via sphinx-scrapy
tabulate==0.10.0
# via sphinx-llm-friendly
-tldextract==5.3.1
+tldextract==5.4.0
# via scrapy
-twisted==25.5.0
+twisted==26.4.0
# via scrapy
-typing-extensions==4.15.0
+typing-extensions==4.16.0
# via
# pydantic
# pydantic-core
- # pyopenssl
# twisted
# typing-inspection
-typing-inspection==0.4.2
+typing-inspection==0.4.4
# via pydantic
-urllib3==2.6.3
+urllib3==2.8.0
# via requests
-w3lib==2.4.1
+w3lib==2.5.0
# via
# parsel
# scrapy
-zope-interface==8.2
+yarl==1.25.1
+ # via aiohttp
+zope-interface==8.7
# via
# scrapy
# twisted
diff --git a/tox.ini b/tox.ini
index a430f9431..2f8d8a10a 100644
--- a/tox.ini
+++ b/tox.ini
@@ -115,7 +115,7 @@ commands =
[testenv:pylint]
# Some checks are Python-version-dependent, so pin the version used in CI.
-basepython = python3.14
+basepython = python3.15
deps =
{[testenv:extra-deps]deps}
pylint==4.1.1
@@ -358,7 +358,7 @@ commands =
# synchronously, so tests/benchmarks drives the event loop itself.
[testenv:benchmark]
-basepython = python3.14
+basepython = python3.15
deps =
pytest >= 8.4.1
pytest-codspeed