Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

[MRG+1] Drop py34 support - Update CI envs (#3892)

elacuesta5dbeece8da66
diff --git a/.travis.yml b/.travis.yml
index 138e81c64..0190a7f4d 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,5 +1,5 @@
language: python
-dist: trusty
+dist: xenial
branches:
only:
- master
@@ -7,32 +7,28 @@ branches:
- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/
matrix:
include:
- - python: 2.7
- env: TOXENV=py27
- - python: 2.7
- env: TOXENV=jessie
- - python: 2.7
- env: TOXENV=pypy
- - python: 2.7
- env: TOXENV=pypy3
- - python: 3.4
- env: TOXENV=py34
- - python: 3.5
- env: TOXENV=py35
- - python: 3.6
- env: TOXENV=py36
- - python: 3.7
- env: TOXENV=py37
- dist: xenial
- sudo: true
- - python: 3.6
- env: TOXENV=docs
- - python: 3.7
- env: TOXENV=py37-extra-deps
- dist: xenial
- sudo: true
- - python: 2.7
- env: TOXENV=py27-extra-deps
+ - env: TOXENV=py27
+ python: 2.7
+ - env: TOXENV=py27-pinned
+ python: 2.7
+ - env: TOXENV=py27-extra-deps
+ python: 2.7
+ - env: TOXENV=pypy
+ python: 2.7
+ - env: TOXENV=pypy3
+ python: 3.5
+ - env: TOXENV=py35
+ python: 3.5
+ - env: TOXENV=py35-pinned
+ python: 3.5
+ - env: TOXENV=py36
+ python: 3.6
+ - env: TOXENV=py37
+ python: 3.7
+ - env: TOXENV=py37-extra-deps
+ python: 3.7
+ - env: TOXENV=docs
+ python: 3.6
install:
- |
if [ "$TOXENV" = "pypy" ]; then
diff --git a/README.rst b/README.rst
index c28d217ff..bd82bff06 100644
--- a/README.rst
+++ b/README.rst
@@ -40,7 +40,7 @@ https://scrapy.org
Requirements
============
-* Python 2.7 or Python 3.4+
+* Python 2.7 or Python 3.5+
* Works on Linux, Windows, Mac OSX, BSD
Install
diff --git a/docs/faq.rst b/docs/faq.rst
index 44f2b97bd..9733471bf 100644
--- a/docs/faq.rst
+++ b/docs/faq.rst
@@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars
What Python versions does Scrapy support?
-----------------------------------------
-Scrapy is supported under Python 2.7 and Python 3.4+
+Scrapy is supported under Python 2.7 and Python 3.5+
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
Python 2.6 support was dropped starting at Scrapy 0.20.
Python 3 support was added in Scrapy 1.1.
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index daec7fcb7..2bf98dbdc 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -7,7 +7,7 @@ Installation guide
Installing Scrapy
=================
-Scrapy runs on Python 2.7 and Python 3.4 or above
+Scrapy runs on Python 2.7 and Python 3.5 or above
under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).
If you're using `Anaconda`_ or `Miniconda`_, you can install the package from
diff --git a/requirements-py2.txt b/requirements-py2.txt
index 0771aae3a..9e6944240 100644
--- a/requirements-py2.txt
+++ b/requirements-py2.txt
@@ -1,10 +1,17 @@
-Twisted>=13.1.0
-lxml
-pyOpenSSL
-cssselect>=0.9
-queuelib
-w3lib>=1.17.0
-six>=1.5.2
+parsel>=1.5.0
PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity
+w3lib>=1.17.0
+
+pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+cryptography>=2.0 # Earlier versions would fail to install
+
+# Reference versions taken from
+# https://packages.ubuntu.com/xenial/python/
+# https://packages.ubuntu.com/xenial/zope/
+cssselect>=0.9.1
+lxml>=3.5.0
+service_identity>=16.0.0
+six>=1.10.0
+Twisted>=16.0.0
+zope.interface>=4.1.3
diff --git a/requirements-py3.txt b/requirements-py3.txt
index 478ed0010..cd183a525 100644
--- a/requirements-py3.txt
+++ b/requirements-py3.txt
@@ -1,11 +1,17 @@
+parsel>=1.5.0
+PyDispatcher>=2.0.5
Twisted>=17.9.0
-lxml;python_version!="3.4"
-lxml<=4.3.5;python_version=="3.4"
-pyOpenSSL>=0.13.1
-cssselect>=0.9
-queuelib>=1.1.1
w3lib>=1.17.0
-six>=1.5.2
-PyDispatcher>=2.0.5
-parsel>=1.5
-service_identity
+
+pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"
+queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"
+cryptography>=2.0 # Earlier versions would fail to install
+
+# Reference versions taken from
+# https://packages.ubuntu.com/xenial/python/
+# https://packages.ubuntu.com/xenial/zope/
+cssselect>=0.9.1
+lxml>=3.5.0
+service_identity>=16.0.0
+six>=1.10.0
+zope.interface>=4.1.3
diff --git a/setup.py b/setup.py
index ee0aaabf0..37892cfbf 100644
--- a/setup.py
+++ b/setup.py
@@ -53,7 +53,6 @@ setup(
'Programming Language :: Python :: 2',
'Programming Language :: Python :: 2.7',
'Programming Language :: Python :: 3',
- 'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
@@ -63,20 +62,21 @@ setup(
'Topic :: Software Development :: Libraries :: Application Frameworks',
'Topic :: Software Development :: Libraries :: Python Modules',
],
- python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',
+ python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',
install_requires=[
- 'Twisted>=13.1.0;python_version!="3.4"',
- 'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"',
- 'w3lib>=1.17.0',
- 'queuelib',
- 'lxml;python_version!="3.4"',
- 'lxml<=4.3.5;python_version=="3.4"',
- 'pyOpenSSL',
- 'cssselect>=0.9',
- 'six>=1.5.2',
- 'parsel>=1.5',
+ 'Twisted>=16.0.0;python_version=="2.7"',
+ 'Twisted>=17.9.0;python_version>="3.5"',
+ 'cryptography>=2.0',
+ 'cssselect>=0.9.1',
+ 'lxml>=3.5.0',
+ 'parsel>=1.5.0',
'PyDispatcher>=2.0.5',
- 'service_identity',
+ 'pyOpenSSL>=16.2.0',
+ 'queuelib>=1.4.2',
+ 'service_identity>=16.0.0',
+ 'six>=1.10.0',
+ 'w3lib>=1.17.0',
+ 'zope.interface>=4.1.3',
],
extras_require=extras_require,
)
diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txt
index be809b151..f621eb4eb 100644
--- a/tests/requirements-py2.txt
+++ b/tests/requirements-py2.txt
@@ -1,14 +1,15 @@
# Tests requirements
-mock
+brotlipy
+jmespath
mitmproxy==0.10.1
+mock
netlib==0.10.1
pytest
pytest-cov
pytest-twisted
pytest-xdist
-jmespath
-brotlipy
testfixtures
+
# optional for shell wrapper tests
bpython
ipython<6.0
diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txt
index ed7bf0be0..cb67bc40e 100644
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@ -1,13 +1,14 @@
+# Tests requirements
+jmespath
+leveldb; sys_platform != "win32"
pytest
pytest-cov
pytest-twisted
pytest-xdist
testfixtures
-jmespath
-leveldb; sys_platform != "win32"
-botocore
+
# optional for shell wrapper tests
bpython
-ipython
brotlipy
+ipython
pywin32; sys_platform == "win32"
diff --git a/tox.ini b/tox.ini
index c918731f4..c3502c2ca 100644
--- a/tox.ini
+++ b/tox.ini
@@ -11,10 +11,10 @@ deps =
-ctests/constraints.txt
-rrequirements-py2.txt
# Extras
- botocore
+ botocore>=1.3.23
google-cloud-storage
- Pillow != 3.0.0
leveldb
+ Pillow>=3.4.2
-rtests/requirements-py2.txt
passenv =
S3_TEST_FILE_URI
@@ -25,72 +25,74 @@ passenv =
commands =
py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
-[testenv:trusty]
+[testenv:py27-pinned]
basepython = python2.7
deps =
- pyOpenSSL==0.13
- lxml==3.3.3
- Twisted==13.2.0
- boto==2.20.1
- Pillow==2.3.0
- cssselect==0.9.1
- zope.interface==4.0.5
- -rtests/requirements-py2.txt
-
-[testenv:jessie]
-# https://packages.debian.org/en/jessie/python/
-# https://packages.debian.org/en/jessie/zope/
-basepython = python2.7
-deps =
- cryptography==0.6.1
- pyOpenSSL==0.14
- lxml==3.4.0
- Twisted==14.0.2
- boto==2.34.0
- Pillow==2.6.1
+ -ctests/constraints.txt
+ cryptography==2.0
cssselect==0.9.1
- zope.interface==4.1.1
+ lxml==3.5.0
+ parsel==1.5.0
+ PyDispatcher==2.0.5
+ pyOpenSSL==16.2.0
+ queuelib==1.4.2
+ service_identity==16.0.0
+ six==1.10.0
+ Twisted==16.0.0
+ w3lib==1.17.0
+ zope.interface==4.1.3
-rtests/requirements-py2.txt
-# Not used directly but allows boto GCE plugins to load.
-# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262
- google-compute-engine==2.8.12
-
-[testenv:trunk]
-basepython = python2.7
-commands =
- pip install -U https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib
- pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib
- py.test --cov=scrapy --cov-report= {posargs:scrapy tests}
+ # Extras
+ botocore==1.3.23
+ Pillow==3.4.2
[testenv:pypy]
basepython = pypy
commands =
py.test {posargs:scrapy tests}
-[testenv:py34]
-basepython = python3.4
+[testenv:py35]
+basepython = python3.5
deps =
-ctests/constraints.txt
-rrequirements-py3.txt
- # Extras
- Pillow
-rtests/requirements-py3.txt
+ # Extras
+ botocore>=1.3.23
+ Pillow>=3.4.2
-[testenv:py35]
+[testenv:py35-pinned]
basepython = python3.5
-deps = {[testenv:py34]deps}
+deps =
+ -ctests/constraints.txt
+ cryptography==2.0
+ cssselect==0.9.1
+ lxml==3.5.0
+ parsel==1.5.0
+ PyDispatcher==2.0.5
+ pyOpenSSL==16.2.0
+ queuelib==1.4.2
+ service_identity==16.0.0
+ six==1.10.0
+ Twisted==17.9.0
+ w3lib==1.17.0
+ zope.interface==4.1.3
+ -rtests/requirements-py3.txt
+ # Extras
+ botocore==1.3.23
+ Pillow==3.4.2
[testenv:py36]
basepython = python3.6
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}
[testenv:py37]
basepython = python3.7
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}
[testenv:pypy3]
basepython = pypy3
-deps = {[testenv:py34]deps}
+deps = {[testenv:py35]deps}
commands =
py.test {posargs:scrapy tests}
@@ -119,14 +121,14 @@ commands =
[testenv:py37-extra-deps]
basepython = python3.7
-deps =
- {[testenv:py34]deps}
+deps =
+ {[testenv:py35]deps}
reppy
robotexclusionrulesparser
[testenv:py27-extra-deps]
basepython = python2.7
-deps =
+deps =
{[testenv]deps}
reppy
robotexclusionrulesparser