Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

tox.ini: pinned, typing → min, mypy (#7595)

Adrian2d0a898e2dc5
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
index 49ea3277a..ed2388a59 100644
--- a/.github/workflows/checks.yml
+++ b/.github/workflows/checks.yml
@@ -22,10 +22,10 @@ jobs:
TOXENV: pylint
- python-version: "3.10"
env:
- TOXENV: typing
+ TOXENV: mypy
- python-version: "3.10"
env:
- TOXENV: typing-tests
+ TOXENV: mypy-tests
# Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.
- python-version: "3.14"
env:
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 524c79cdb..f51dd9799 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -45,26 +45,26 @@ jobs:
env:
TOXENV: pypy3
- # pinned deps
+ # min deps
- python-version: "3.10.19"
env:
- TOXENV: pinned
+ TOXENV: min
- python-version: "3.10.19"
env:
- TOXENV: default-reactor-pinned
+ TOXENV: min-default-reactor
- python-version: "3.10.19"
env:
- TOXENV: no-reactor-pinned
+ TOXENV: min-no-reactor
# pinned due to https://github.com/pypy/pypy/issues/5388
- python-version: pypy3.11-7.3.20
env:
- TOXENV: pypy3-pinned
+ TOXENV: min-pypy3
- python-version: "3.10.19"
env:
- TOXENV: extra-deps-pinned
+ TOXENV: min-extra-deps
- python-version: "3.10.19"
env:
- TOXENV: botocore-pinned
+ TOXENV: min-botocore
- python-version: "3.14"
env:
@@ -92,7 +92,7 @@ jobs:
python-version: ${{ matrix.python-version }}
- name: Install system libraries
- if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned')
+ if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min')
run: |
sudo apt-get update
sudo apt-get install libxml2-dev libxslt-dev
diff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.yml
index 840c7f68e..f413782bc 100644
--- a/.github/workflows/tests-windows.yml
+++ b/.github/workflows/tests-windows.yml
@@ -41,13 +41,13 @@ jobs:
env:
TOXENV: no-reactor
- # pinned deps
+ # min deps
- python-version: "3.10.11"
env:
- TOXENV: pinned
+ TOXENV: min
- python-version: "3.10.11"
env:
- TOXENV: extra-deps-pinned
+ TOXENV: min-extra-deps
- python-version: "3.14"
env:
diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py
index 4436efd9b..578e84359 100644
--- a/tests/test_dependencies.py
+++ b/tests/test_dependencies.py
@@ -15,14 +15,14 @@ class TestScrapyUtils:
See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011
"""
- if not os.environ.get("_SCRAPY_PINNED", None):
- pytest.skip("Not in a pinned environment")
+ if not os.environ.get("_SCRAPY_MIN", None):
+ pytest.skip("Not in a min environment")
tox_config_file_path = Path(__file__).parent / ".." / "tox.ini"
config_parser = ConfigParser()
config_parser.read(tox_config_file_path)
pattern = r"Twisted==([\d.]+)"
- match = re.search(pattern, config_parser["pinned"]["deps"])
+ match = re.search(pattern, config_parser["min"]["deps"])
pinned_twisted_version_string = match[1]
assert twisted_version.short() == pinned_twisted_version_string
diff --git a/tox.ini b/tox.ini
index e21e1f6cf..50ac7aa86 100644
--- a/tox.ini
+++ b/tox.ini
@@ -6,7 +6,31 @@
[tox]
requires =
sphinx-scrapy[tox] @ git+https://github.com/scrapy/[email protected]
-envlist = pre-commit,pylint,typing,py,docs
+envlist =
+ pre-commit
+ pylint
+ mypy
+ mypy-tests
+ twinecheck
+ docs
+ docs-tests
+ docs-links
+ docs-coverage
+ min
+ min-extra-deps
+ min-default-reactor
+ min-no-reactor
+ min-botocore
+ min-pypy3
+ py{310,311,312,313,314}
+ extra-deps
+ default-reactor
+ no-reactor
+ no-reactor-extra-deps
+ botocore
+ mitmproxy
+ pypy3
+ pypy3-extra-deps
minversion = 1.7.0
[test-requirements]
@@ -41,7 +65,7 @@ download = true
commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules}
-[testenv:typing]
+[testenv:mypy]
basepython = python3.10
deps =
mypy==2.1.0
@@ -71,11 +95,11 @@ deps =
commands =
mypy {posargs:scrapy tests}
-[testenv:typing-tests]
+[testenv:mypy-tests]
basepython = python3.10
deps =
{[test-requirements]deps}
- {[testenv:typing]deps}
+ {[testenv:mypy]deps}
pytest-mypy-testing==0.2.0
commands =
pytest {posargs:tests_typing}
@@ -105,7 +129,7 @@ commands =
python -m build --sdist
twine check dist/*
-[pinned]
+[min]
basepython = python3.10
deps =
# pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0
@@ -125,18 +149,18 @@ deps =
zope.interface==5.1.0
{[test-requirements]deps}
setenv =
- _SCRAPY_PINNED=true
+ _SCRAPY_MIN=true
commands =
- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=pinned.junit.xml -o junit_family=legacy --durations=10 scrapy tests}
+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests}
-[testenv:pinned]
-basepython = {[pinned]basepython}
+[testenv:min]
+basepython = {[min]basepython}
deps =
- {[pinned]deps}
+ {[min]deps}
PyDispatcher==2.0.5
setenv =
- {[pinned]setenv}
-commands = {[pinned]commands}
+ {[min]setenv}
+commands = {[min]commands}
[testenv:extra-deps]
basepython = python3
@@ -155,10 +179,10 @@ deps =
uvloop; platform_system != "Windows" and implementation_name != "pypy"
zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests
-[testenv:extra-deps-pinned]
-basepython = {[pinned]basepython}
+[testenv:min-extra-deps]
+basepython = {[min]basepython}
deps =
- {[pinned]deps}
+ {[min]deps}
Pillow==8.3.2
Twisted[http2]==21.7.0
boto3==1.20.0
@@ -172,19 +196,19 @@ deps =
uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
zstandard==0.16.0; implementation_name != "pypy"
setenv =
- {[pinned]setenv}
-commands = {[pinned]commands}
+ {[min]setenv}
+commands = {[min]commands}
[testenv:default-reactor]
commands =
{[testenv]commands} --reactor=default
-[testenv:default-reactor-pinned]
-basepython = {[pinned]basepython}
-deps = {[testenv:pinned]deps}
-commands = {[pinned]commands} --reactor=default
+[testenv:min-default-reactor]
+basepython = {[min]basepython}
+deps = {[testenv:min]deps}
+commands = {[min]commands} --reactor=default
setenv =
- {[pinned]setenv}
+ {[min]setenv}
[testenv:no-reactor]
deps =
@@ -200,14 +224,14 @@ deps =
commands =
{[testenv]commands} -p no:twisted --reactor=none
-[testenv:no-reactor-pinned]
-basepython = {[pinned]basepython}
+[testenv:min-no-reactor]
+basepython = {[min]basepython}
deps =
- {[testenv:pinned]deps}
+ {[testenv:min]deps}
pytest-asyncio
-commands = {[pinned]commands} -p no:twisted --reactor=none
+commands = {[min]commands} -p no:twisted --reactor=none
setenv =
- {[pinned]setenv}
+ {[min]setenv}
[testenv:pypy3]
basepython = pypy3
@@ -221,7 +245,7 @@ deps =
{[testenv:extra-deps]deps}
commands = {[testenv:pypy3]commands}
-[testenv:pypy3-pinned]
+[testenv:min-pypy3]
basepython = pypy3.11
deps =
PyPyDispatcher==2.1.0
@@ -248,7 +272,7 @@ commands =
; disabling coverage
pytest {posargs:--durations=10 scrapy tests}
setenv =
- {[pinned]setenv}
+ {[min]setenv}
[testenv:docs-tests]
changedir = docs
@@ -283,17 +307,17 @@ deps =
commands =
pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore
-[testenv:botocore-pinned]
-basepython = {[pinned]basepython}
+[testenv:min-botocore]
+basepython = {[min]basepython}
deps =
- {[pinned]deps}
+ {[min]deps}
botocore==1.13.45
# botocore 1.13.45 requires urllib3>=1.20,<1.26; requests 2.33.0 requires urllib3>=1.26,<3
requests<2.33.0
setenv =
- {[pinned]setenv}
+ {[min]setenv}
commands =
- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy} -m requires_botocore
+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore
# Run proxy tests that use mitmproxy in a separate env to avoid installing