Scrapy, a fast high-level web crawling & scraping framework for Python.
tox.ini: pinned, typing → min, mypy (#7595)
Adrian2d0a898e2dc5
diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.ymlindex 49ea3277a..ed2388a59 100644--- a/.github/workflows/checks.yml+++ b/.github/workflows/checks.yml@@ -22,10 +22,10 @@ jobs:TOXENV: pylint- python-version: "3.10"env:- TOXENV: typing+ TOXENV: mypy- python-version: "3.10"env:- TOXENV: typing-tests+ TOXENV: mypy-tests# Keep in sync with pyproject.toml tool.sphinx-scrapy.python-version.- python-version: "3.14"env:diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex 524c79cdb..f51dd9799 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -45,26 +45,26 @@ jobs:env:TOXENV: pypy3- # pinned deps+ # min deps- python-version: "3.10.19"env:- TOXENV: pinned+ TOXENV: min- python-version: "3.10.19"env:- TOXENV: default-reactor-pinned+ TOXENV: min-default-reactor- python-version: "3.10.19"env:- TOXENV: no-reactor-pinned+ TOXENV: min-no-reactor# pinned due to https://github.com/pypy/pypy/issues/5388- python-version: pypy3.11-7.3.20env:- TOXENV: pypy3-pinned+ TOXENV: min-pypy3- python-version: "3.10.19"env:- TOXENV: extra-deps-pinned+ TOXENV: min-extra-deps- python-version: "3.10.19"env:- TOXENV: botocore-pinned+ TOXENV: min-botocore- python-version: "3.14"env:@@ -92,7 +92,7 @@ jobs:python-version: ${{ matrix.python-version }}- name: Install system libraries- if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'pinned')+ if: contains(matrix.python-version, 'pypy') || contains(matrix.env.TOXENV, 'min')run: |sudo apt-get updatesudo apt-get install libxml2-dev libxslt-devdiff --git a/.github/workflows/tests-windows.yml b/.github/workflows/tests-windows.ymlindex 840c7f68e..f413782bc 100644--- a/.github/workflows/tests-windows.yml+++ b/.github/workflows/tests-windows.yml@@ -41,13 +41,13 @@ jobs:env:TOXENV: no-reactor- # pinned deps+ # min deps- python-version: "3.10.11"env:- TOXENV: pinned+ TOXENV: min- python-version: "3.10.11"env:- TOXENV: extra-deps-pinned+ TOXENV: min-extra-deps- python-version: "3.14"env:diff --git a/tests/test_dependencies.py b/tests/test_dependencies.pyindex 4436efd9b..578e84359 100644--- a/tests/test_dependencies.py+++ b/tests/test_dependencies.py@@ -15,14 +15,14 @@ class TestScrapyUtils:See https://github.com/scrapy/scrapy/pull/4814#issuecomment-706230011"""- if not os.environ.get("_SCRAPY_PINNED", None):- pytest.skip("Not in a pinned environment")+ if not os.environ.get("_SCRAPY_MIN", None):+ pytest.skip("Not in a min environment")tox_config_file_path = Path(__file__).parent / ".." / "tox.ini"config_parser = ConfigParser()config_parser.read(tox_config_file_path)pattern = r"Twisted==([\d.]+)"- match = re.search(pattern, config_parser["pinned"]["deps"])+ match = re.search(pattern, config_parser["min"]["deps"])pinned_twisted_version_string = match[1]assert twisted_version.short() == pinned_twisted_version_stringdiff --git a/tox.ini b/tox.iniindex e21e1f6cf..50ac7aa86 100644--- a/tox.ini+++ b/tox.ini@@ -6,7 +6,31 @@[tox]requires =sphinx-scrapy[tox] @ git+https://github.com/scrapy/[email protected]-envlist = pre-commit,pylint,typing,py,docs+envlist =+ pre-commit+ pylint+ mypy+ mypy-tests+ twinecheck+ docs+ docs-tests+ docs-links+ docs-coverage+ min+ min-extra-deps+ min-default-reactor+ min-no-reactor+ min-botocore+ min-pypy3+ py{310,311,312,313,314}+ extra-deps+ default-reactor+ no-reactor+ no-reactor-extra-deps+ botocore+ mitmproxy+ pypy3+ pypy3-extra-depsminversion = 1.7.0[test-requirements]@@ -41,7 +65,7 @@ download = truecommands =pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report= --cov-report=term-missing --cov-report=xml --junitxml=testenv.junit.xml -o junit_family=legacy --durations=10 scrapy tests --doctest-modules}-[testenv:typing]+[testenv:mypy]basepython = python3.10deps =mypy==2.1.0@@ -71,11 +95,11 @@ deps =commands =mypy {posargs:scrapy tests}-[testenv:typing-tests]+[testenv:mypy-tests]basepython = python3.10deps ={[test-requirements]deps}- {[testenv:typing]deps}+ {[testenv:mypy]deps}pytest-mypy-testing==0.2.0commands =pytest {posargs:tests_typing}@@ -105,7 +129,7 @@ commands =python -m build --sdisttwine check dist/*-[pinned]+[min]basepython = python3.10deps =# pytest 8.4.1 adds support for Twisted 25.5.0 but drops support for Twisted < 24.10.0@@ -125,18 +149,18 @@ deps =zope.interface==5.1.0{[test-requirements]deps}setenv =- _SCRAPY_PINNED=true+ _SCRAPY_MIN=truecommands =- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=pinned.junit.xml -o junit_family=legacy --durations=10 scrapy tests}+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= --junitxml=min.junit.xml -o junit_family=legacy --durations=10 scrapy tests}-[testenv:pinned]-basepython = {[pinned]basepython}+[testenv:min]+basepython = {[min]basepython}deps =- {[pinned]deps}+ {[min]deps}PyDispatcher==2.0.5setenv =- {[pinned]setenv}-commands = {[pinned]commands}+ {[min]setenv}+commands = {[min]commands}[testenv:extra-deps]basepython = python3@@ -155,10 +179,10 @@ deps =uvloop; platform_system != "Windows" and implementation_name != "pypy"zstandard; implementation_name != "pypy" # optional for HTTP compress downloader middleware tests-[testenv:extra-deps-pinned]-basepython = {[pinned]basepython}+[testenv:min-extra-deps]+basepython = {[min]basepython}deps =- {[pinned]deps}+ {[min]deps}Pillow==8.3.2Twisted[http2]==21.7.0boto3==1.20.0@@ -172,19 +196,19 @@ deps =uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"zstandard==0.16.0; implementation_name != "pypy"setenv =- {[pinned]setenv}-commands = {[pinned]commands}+ {[min]setenv}+commands = {[min]commands}[testenv:default-reactor]commands ={[testenv]commands} --reactor=default-[testenv:default-reactor-pinned]-basepython = {[pinned]basepython}-deps = {[testenv:pinned]deps}-commands = {[pinned]commands} --reactor=default+[testenv:min-default-reactor]+basepython = {[min]basepython}+deps = {[testenv:min]deps}+commands = {[min]commands} --reactor=defaultsetenv =- {[pinned]setenv}+ {[min]setenv}[testenv:no-reactor]deps =@@ -200,14 +224,14 @@ deps =commands ={[testenv]commands} -p no:twisted --reactor=none-[testenv:no-reactor-pinned]-basepython = {[pinned]basepython}+[testenv:min-no-reactor]+basepython = {[min]basepython}deps =- {[testenv:pinned]deps}+ {[testenv:min]deps}pytest-asyncio-commands = {[pinned]commands} -p no:twisted --reactor=none+commands = {[min]commands} -p no:twisted --reactor=nonesetenv =- {[pinned]setenv}+ {[min]setenv}[testenv:pypy3]basepython = pypy3@@ -221,7 +245,7 @@ deps ={[testenv:extra-deps]deps}commands = {[testenv:pypy3]commands}-[testenv:pypy3-pinned]+[testenv:min-pypy3]basepython = pypy3.11deps =PyPyDispatcher==2.1.0@@ -248,7 +272,7 @@ commands =; disabling coveragepytest {posargs:--durations=10 scrapy tests}setenv =- {[pinned]setenv}+ {[min]setenv}[testenv:docs-tests]changedir = docs@@ -283,17 +307,17 @@ deps =commands =pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore.junit.xml -o junit_family=legacy} -m requires_botocore-[testenv:botocore-pinned]-basepython = {[pinned]basepython}+[testenv:min-botocore]+basepython = {[min]basepython}deps =- {[pinned]deps}+ {[min]deps}botocore==1.13.45# botocore 1.13.45 requires urllib3>=1.20,<1.26; requests 2.33.0 requires urllib3>=1.26,<3requests<2.33.0setenv =- {[pinned]setenv}+ {[min]setenv}commands =- pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=botocore-pinned.junit.xml -o junit_family=legacy} -m requires_botocore+ pytest {posargs:--cov-config=pyproject.toml --cov=scrapy --cov-report=xml --cov-report= tests --junitxml=min-botocore.junit.xml -o junit_family=legacy} -m requires_botocore# Run proxy tests that use mitmproxy in a separate env to avoid installing