Scrapy, a fast high-level web crawling & scraping framework for Python.
Provide separate test environments for botocore and boto3 extras; add extra-deps-pinned and remote extras from pinned
Adrián Chaves590955fac8de
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.ymlindex 8fcf90a18..96b26a1f8 100644--- a/.github/workflows/tests-ubuntu.yml+++ b/.github/workflows/tests-ubuntu.yml@@ -37,10 +37,19 @@ jobs:- python-version: pypy3.7env:TOXENV: pypy3-pinned+ - python-version: 3.7.13+ env:+ TOXENV: extra-deps-pinned+ - python-version: 3.7.13+ env:+ TOXENV: botocore-pinned- python-version: "3.11"env:TOXENV: extra-deps+ - python-version: "3.11"+ env:+ TOXENV: botocoresteps:- uses: actions/checkout@v3diff --git a/tox.ini b/tox.iniindex 5c2f583d9..f94d7f751 100644--- a/tox.ini+++ b/tox.ini@@ -88,11 +88,6 @@ deps =# mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies# above, hence we do not install it in pinned environments at the moment-- # Extras- botocore==1.4.87- google-cloud-storage==1.29.0- Pillow==7.1.0setenv =_SCRAPY_PINNED=trueinstall_command =@@ -119,14 +114,26 @@ setenv =basepython = python3deps ={[testenv]deps}- botocore>=1.4.87+ boto3google-cloud-storage# Twisted[http2] currently forces old mitmproxy because of h2 version# restrictions in their deps, so we need to pin old markupsafe here too.markupsafe < 2.1.0robotexclusionrulesparser- Pillow>=4.0.0- Twisted[http2]>=17.9.0+ Pillow+ Twisted[http2]++[testenv:extra-deps-pinned]+basepython = python3.7+deps =+ {[pinned]deps}+ boto3==1.0.0+ google-cloud-storage==1.29.0+ Pillow==7.1.0+ robotexclusionrulesparser==1.6.2+install_command = {[pinned]install_command}+setenv =+ {[pinned]setenv}[testenv:asyncio]commands =@@ -185,3 +192,24 @@ deps = {[docs]deps}setenv = {[docs]setenv}commands =sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck+++# Run S3 tests with botocore installed but without boto3.++[testenv:botocore]+deps =+ {[testenv]deps}+ botocore>=1.4.87+commands =+ pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}++[testenv:botocore-pinned]+basepython = python3.7+deps =+ {[pinned]deps}+ botocore==1.4.87+install_command = {[pinned]install_command}+setenv =+ {[pinned]setenv}+commands =+ pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}