Scrapy, a fast high-level web crawling & scraping framework for Python.
[MRG+1] Drop py34 support - Update CI envs (#3892)
elacuesta5dbeece8da66
diff --git a/.travis.yml b/.travis.ymlindex 138e81c64..0190a7f4d 100644--- a/.travis.yml+++ b/.travis.yml@@ -1,5 +1,5 @@language: python-dist: trusty+dist: xenialbranches:only:- master@@ -7,32 +7,28 @@ branches:- /^\d\.\d+\.\d+(rc\d+|\.dev\d+)?$/matrix:include:- - python: 2.7- env: TOXENV=py27- - python: 2.7- env: TOXENV=jessie- - python: 2.7- env: TOXENV=pypy- - python: 2.7- env: TOXENV=pypy3- - python: 3.4- env: TOXENV=py34- - python: 3.5- env: TOXENV=py35- - python: 3.6- env: TOXENV=py36- - python: 3.7- env: TOXENV=py37- dist: xenial- sudo: true- - python: 3.6- env: TOXENV=docs- - python: 3.7- env: TOXENV=py37-extra-deps- dist: xenial- sudo: true- - python: 2.7- env: TOXENV=py27-extra-deps+ - env: TOXENV=py27+ python: 2.7+ - env: TOXENV=py27-pinned+ python: 2.7+ - env: TOXENV=py27-extra-deps+ python: 2.7+ - env: TOXENV=pypy+ python: 2.7+ - env: TOXENV=pypy3+ python: 3.5+ - env: TOXENV=py35+ python: 3.5+ - env: TOXENV=py35-pinned+ python: 3.5+ - env: TOXENV=py36+ python: 3.6+ - env: TOXENV=py37+ python: 3.7+ - env: TOXENV=py37-extra-deps+ python: 3.7+ - env: TOXENV=docs+ python: 3.6install:- |if [ "$TOXENV" = "pypy" ]; thendiff --git a/README.rst b/README.rstindex c28d217ff..bd82bff06 100644--- a/README.rst+++ b/README.rst@@ -40,7 +40,7 @@ https://scrapy.orgRequirements============-* Python 2.7 or Python 3.4++* Python 2.7 or Python 3.5+* Works on Linux, Windows, Mac OSX, BSDInstalldiff --git a/docs/faq.rst b/docs/faq.rstindex 44f2b97bd..9733471bf 100644--- a/docs/faq.rst+++ b/docs/faq.rst@@ -69,7 +69,7 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML parsWhat Python versions does Scrapy support?------------------------------------------Scrapy is supported under Python 2.7 and Python 3.4++Scrapy is supported under Python 2.7 and Python 3.5+under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).Python 2.6 support was dropped starting at Scrapy 0.20.Python 3 support was added in Scrapy 1.1.diff --git a/docs/intro/install.rst b/docs/intro/install.rstindex daec7fcb7..2bf98dbdc 100644--- a/docs/intro/install.rst+++ b/docs/intro/install.rst@@ -7,7 +7,7 @@ Installation guideInstalling Scrapy=================-Scrapy runs on Python 2.7 and Python 3.4 or above+Scrapy runs on Python 2.7 and Python 3.5 or aboveunder CPython (default Python implementation) and PyPy (starting with PyPy 5.9).If you're using `Anaconda`_ or `Miniconda`_, you can install the package fromdiff --git a/requirements-py2.txt b/requirements-py2.txtindex 0771aae3a..9e6944240 100644--- a/requirements-py2.txt+++ b/requirements-py2.txt@@ -1,10 +1,17 @@-Twisted>=13.1.0-lxml-pyOpenSSL-cssselect>=0.9-queuelib-w3lib>=1.17.0-six>=1.5.2+parsel>=1.5.0PyDispatcher>=2.0.5-parsel>=1.5-service_identity+w3lib>=1.17.0++pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"+queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"+cryptography>=2.0 # Earlier versions would fail to install++# Reference versions taken from+# https://packages.ubuntu.com/xenial/python/+# https://packages.ubuntu.com/xenial/zope/+cssselect>=0.9.1+lxml>=3.5.0+service_identity>=16.0.0+six>=1.10.0+Twisted>=16.0.0+zope.interface>=4.1.3diff --git a/requirements-py3.txt b/requirements-py3.txtindex 478ed0010..cd183a525 100644--- a/requirements-py3.txt+++ b/requirements-py3.txt@@ -1,11 +1,17 @@+parsel>=1.5.0+PyDispatcher>=2.0.5Twisted>=17.9.0-lxml;python_version!="3.4"-lxml<=4.3.5;python_version=="3.4"-pyOpenSSL>=0.13.1-cssselect>=0.9-queuelib>=1.1.1w3lib>=1.17.0-six>=1.5.2-PyDispatcher>=2.0.5-parsel>=1.5-service_identity++pyOpenSSL>=16.2.0 # Earlier versions fail with "AttributeError: module 'lib' has no attribute 'SSL_ST_INIT'"+queuelib>=1.4.2 # Earlier versions fail with "AttributeError: '...QueueTest' object has no attribute 'qpath'"+cryptography>=2.0 # Earlier versions would fail to install++# Reference versions taken from+# https://packages.ubuntu.com/xenial/python/+# https://packages.ubuntu.com/xenial/zope/+cssselect>=0.9.1+lxml>=3.5.0+service_identity>=16.0.0+six>=1.10.0+zope.interface>=4.1.3diff --git a/setup.py b/setup.pyindex ee0aaabf0..37892cfbf 100644--- a/setup.py+++ b/setup.py@@ -53,7 +53,6 @@ setup('Programming Language :: Python :: 2','Programming Language :: Python :: 2.7','Programming Language :: Python :: 3',- 'Programming Language :: Python :: 3.4','Programming Language :: Python :: 3.5','Programming Language :: Python :: 3.6','Programming Language :: Python :: 3.7',@@ -63,20 +62,21 @@ setup('Topic :: Software Development :: Libraries :: Application Frameworks','Topic :: Software Development :: Libraries :: Python Modules',],- python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*',+ python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*',install_requires=[- 'Twisted>=13.1.0;python_version!="3.4"',- 'Twisted>=13.1.0,<=19.2.0;python_version=="3.4"',- 'w3lib>=1.17.0',- 'queuelib',- 'lxml;python_version!="3.4"',- 'lxml<=4.3.5;python_version=="3.4"',- 'pyOpenSSL',- 'cssselect>=0.9',- 'six>=1.5.2',- 'parsel>=1.5',+ 'Twisted>=16.0.0;python_version=="2.7"',+ 'Twisted>=17.9.0;python_version>="3.5"',+ 'cryptography>=2.0',+ 'cssselect>=0.9.1',+ 'lxml>=3.5.0',+ 'parsel>=1.5.0','PyDispatcher>=2.0.5',- 'service_identity',+ 'pyOpenSSL>=16.2.0',+ 'queuelib>=1.4.2',+ 'service_identity>=16.0.0',+ 'six>=1.10.0',+ 'w3lib>=1.17.0',+ 'zope.interface>=4.1.3',],extras_require=extras_require,)diff --git a/tests/requirements-py2.txt b/tests/requirements-py2.txtindex be809b151..f621eb4eb 100644--- a/tests/requirements-py2.txt+++ b/tests/requirements-py2.txt@@ -1,14 +1,15 @@# Tests requirements-mock+brotlipy+jmespathmitmproxy==0.10.1+mocknetlib==0.10.1pytestpytest-covpytest-twistedpytest-xdist-jmespath-brotlipytestfixtures+# optional for shell wrapper testsbpythonipython<6.0diff --git a/tests/requirements-py3.txt b/tests/requirements-py3.txtindex ed7bf0be0..cb67bc40e 100644--- a/tests/requirements-py3.txt+++ b/tests/requirements-py3.txt@@ -1,13 +1,14 @@+# Tests requirements+jmespath+leveldb; sys_platform != "win32"pytestpytest-covpytest-twistedpytest-xdisttestfixtures-jmespath-leveldb; sys_platform != "win32"-botocore+# optional for shell wrapper testsbpython-ipythonbrotlipy+ipythonpywin32; sys_platform == "win32"diff --git a/tox.ini b/tox.iniindex c918731f4..c3502c2ca 100644--- a/tox.ini+++ b/tox.ini@@ -11,10 +11,10 @@ deps =-ctests/constraints.txt-rrequirements-py2.txt# Extras- botocore+ botocore>=1.3.23google-cloud-storage- Pillow != 3.0.0leveldb+ Pillow>=3.4.2-rtests/requirements-py2.txtpassenv =S3_TEST_FILE_URI@@ -25,72 +25,74 @@ passenv =commands =py.test --cov=scrapy --cov-report= {posargs:scrapy tests}-[testenv:trusty]+[testenv:py27-pinned]basepython = python2.7deps =- pyOpenSSL==0.13- lxml==3.3.3- Twisted==13.2.0- boto==2.20.1- Pillow==2.3.0- cssselect==0.9.1- zope.interface==4.0.5- -rtests/requirements-py2.txt--[testenv:jessie]-# https://packages.debian.org/en/jessie/python/-# https://packages.debian.org/en/jessie/zope/-basepython = python2.7-deps =- cryptography==0.6.1- pyOpenSSL==0.14- lxml==3.4.0- Twisted==14.0.2- boto==2.34.0- Pillow==2.6.1+ -ctests/constraints.txt+ cryptography==2.0cssselect==0.9.1- zope.interface==4.1.1+ lxml==3.5.0+ parsel==1.5.0+ PyDispatcher==2.0.5+ pyOpenSSL==16.2.0+ queuelib==1.4.2+ service_identity==16.0.0+ six==1.10.0+ Twisted==16.0.0+ w3lib==1.17.0+ zope.interface==4.1.3-rtests/requirements-py2.txt-# Not used directly but allows boto GCE plugins to load.-# https://github.com/GoogleCloudPlatform/compute-image-packages/issues/262- google-compute-engine==2.8.12--[testenv:trunk]-basepython = python2.7-commands =- pip install -U https://github.com/scrapy/w3lib/archive/master.zip#egg=w3lib- pip install -U https://github.com/scrapy/queuelib/archive/master.zip#egg=queuelib- py.test --cov=scrapy --cov-report= {posargs:scrapy tests}+ # Extras+ botocore==1.3.23+ Pillow==3.4.2[testenv:pypy]basepython = pypycommands =py.test {posargs:scrapy tests}-[testenv:py34]-basepython = python3.4+[testenv:py35]+basepython = python3.5deps =-ctests/constraints.txt-rrequirements-py3.txt- # Extras- Pillow-rtests/requirements-py3.txt+ # Extras+ botocore>=1.3.23+ Pillow>=3.4.2-[testenv:py35]+[testenv:py35-pinned]basepython = python3.5-deps = {[testenv:py34]deps}+deps =+ -ctests/constraints.txt+ cryptography==2.0+ cssselect==0.9.1+ lxml==3.5.0+ parsel==1.5.0+ PyDispatcher==2.0.5+ pyOpenSSL==16.2.0+ queuelib==1.4.2+ service_identity==16.0.0+ six==1.10.0+ Twisted==17.9.0+ w3lib==1.17.0+ zope.interface==4.1.3+ -rtests/requirements-py3.txt+ # Extras+ botocore==1.3.23+ Pillow==3.4.2[testenv:py36]basepython = python3.6-deps = {[testenv:py34]deps}+deps = {[testenv:py35]deps}[testenv:py37]basepython = python3.7-deps = {[testenv:py34]deps}+deps = {[testenv:py35]deps}[testenv:pypy3]basepython = pypy3-deps = {[testenv:py34]deps}+deps = {[testenv:py35]deps}commands =py.test {posargs:scrapy tests}@@ -119,14 +121,14 @@ commands =[testenv:py37-extra-deps]basepython = python3.7-deps =- {[testenv:py34]deps}+deps =+ {[testenv:py35]deps}reppyrobotexclusionrulesparser[testenv:py27-extra-deps]basepython = python2.7-deps =+deps ={[testenv]deps}reppyrobotexclusionrulesparser