Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Provide separate test environments for botocore and boto3 extras; add extra-deps-pinned and remote extras from pinned

Adrián Chaves590955fac8de
diff --git a/.github/workflows/tests-ubuntu.yml b/.github/workflows/tests-ubuntu.yml
index 8fcf90a18..96b26a1f8 100644
--- a/.github/workflows/tests-ubuntu.yml
+++ b/.github/workflows/tests-ubuntu.yml
@@ -37,10 +37,19 @@ jobs:
- python-version: pypy3.7
env:
TOXENV: pypy3-pinned
+ - python-version: 3.7.13
+ env:
+ TOXENV: extra-deps-pinned
+ - python-version: 3.7.13
+ env:
+ TOXENV: botocore-pinned
- python-version: "3.11"
env:
TOXENV: extra-deps
+ - python-version: "3.11"
+ env:
+ TOXENV: botocore
steps:
- uses: actions/checkout@v3
diff --git a/tox.ini b/tox.ini
index 5c2f583d9..f94d7f751 100644
--- a/tox.ini
+++ b/tox.ini
@@ -88,11 +88,6 @@ deps =
# mitmproxy 4.0.4+ requires upgrading some of the pinned dependencies
# above, hence we do not install it in pinned environments at the moment
-
- # Extras
- botocore==1.4.87
- google-cloud-storage==1.29.0
- Pillow==7.1.0
setenv =
_SCRAPY_PINNED=true
install_command =
@@ -119,14 +114,26 @@ setenv =
basepython = python3
deps =
{[testenv]deps}
- botocore>=1.4.87
+ boto3
google-cloud-storage
# Twisted[http2] currently forces old mitmproxy because of h2 version
# restrictions in their deps, so we need to pin old markupsafe here too.
markupsafe < 2.1.0
robotexclusionrulesparser
- Pillow>=4.0.0
- Twisted[http2]>=17.9.0
+ Pillow
+ Twisted[http2]
+
+[testenv:extra-deps-pinned]
+basepython = python3.7
+deps =
+ {[pinned]deps}
+ boto3==1.0.0
+ google-cloud-storage==1.29.0
+ Pillow==7.1.0
+ robotexclusionrulesparser==1.6.2
+install_command = {[pinned]install_command}
+setenv =
+ {[pinned]setenv}
[testenv:asyncio]
commands =
@@ -185,3 +192,24 @@ deps = {[docs]deps}
setenv = {[docs]setenv}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
+
+# Run S3 tests with botocore installed but without boto3.
+
+[testenv:botocore]
+deps =
+ {[testenv]deps}
+ botocore>=1.4.87
+commands =
+ pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}
+
+[testenv:botocore-pinned]
+basepython = python3.7
+deps =
+ {[pinned]deps}
+ botocore==1.4.87
+install_command = {[pinned]install_command}
+setenv =
+ {[pinned]setenv}
+commands =
+ pytest --cov=scrapy --cov-report=xml --cov-report= {posargs:tests -k s3}