Scrapy, a fast high-level web crawling & scraping framework for Python.
Upgrade PyPy for CI, and test both 3.5 (oldest) and 3.6 (newest) (#4504)
Adrián Chavesd29bec60d795
diff --git a/.travis.yml b/.travis.ymlindex b403ac54c..db720b918 100644--- a/.travis.yml+++ b/.travis.yml@@ -18,19 +18,25 @@ matrix:- env: TOXENV=typingpython: 3.8- - env: TOXENV=pypy3- env: TOXENV=pinnedpython: 3.5.2- - env: TOXENV=asyncio+ - env: TOXENV=asyncio-pinnedpython: 3.5.2 # We use additional code to support 3.5.3 and earlier+ - env: TOXENV=pypy3-pinned PYPY_VERSION=3-v5.9.0+- env: TOXENV=pypython: 3.5- env: TOXENV=asynciopython: 3.5 # We use specific code to support >= 3.5.4, < 3.6+ - env: TOXENV=pypy3 PYPY_VERSION=3.5-v7.0.0+- env: TOXENV=pypython: 3.6+ - env: TOXENV=pypy3 PYPY_VERSION=3.6-v7.3.1+- env: TOXENV=pypython: 3.7+- env: TOXENV=py PYPI_RELEASE_JOB=truepython: 3.8dist: bionic@@ -42,9 +48,9 @@ matrix:dist: bionicinstall:- |- if [ "$TOXENV" = "pypy3" ]; then- export PYPY_VERSION="pypy3.5-5.9-beta-linux_x86_64-portable"- wget "https://bitbucket.org/squeaky/portable-pypy/downloads/${PYPY_VERSION}.tar.bz2"+ if [[ ! -z "$PYPY_VERSION" ]]; then+ export PYPY_VERSION="pypy$PYPY_VERSION-linux64"+ wget "https://bitbucket.org/pypy/pypy/downloads/${PYPY_VERSION}.tar.bz2"tar -jxf ${PYPY_VERSION}.tar.bz2virtualenv --python="$PYPY_VERSION/bin/pypy3" "$HOME/virtualenvs/$PYPY_VERSION"source "$HOME/virtualenvs/$PYPY_VERSION/bin/activate"diff --git a/docs/faq.rst b/docs/faq.rstindex d5ea3cb87..ea2c8216f 100644--- a/docs/faq.rst+++ b/docs/faq.rst@@ -64,20 +64,6 @@ Here's an example spider using BeautifulSoup API, with ``lxml`` as the HTML pars.. _BeautifulSoup's official documentation: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#specifying-the-parser-to-use-.. _faq-python-versions:--What Python versions does Scrapy support?--------------------------------------------Scrapy is supported under Python 3.5.2+-under CPython (default Python implementation) and PyPy (starting with PyPy 5.9).-Python 3 support was added in Scrapy 1.1.-PyPy support was added in Scrapy 1.4, PyPy3 support was added in Scrapy 1.5.-Python 2 support was dropped in Scrapy 2.0.--.. note::- For Python 3 support on Windows, it is recommended to use- Anaconda/Miniconda as :ref:`outlined in the installation guide <intro-install-windows>`.Did Scrapy "steal" X from Django?---------------------------------diff --git a/docs/intro/install.rst b/docs/intro/install.rstindex fb64d443c..6d65ae2ee 100644--- a/docs/intro/install.rst+++ b/docs/intro/install.rst@@ -4,12 +4,18 @@Installation guide==================+.. _faq-python-versions:++Supported Python versions+=========================++Scrapy requires Python 3.5.2+, either the CPython implementation (default) or+the PyPy 5.9+ implementation (see :ref:`python:implementations`).++Installing Scrapy=================-Scrapy runs on Python 3.5.2 or above under CPython (default Python-implementation) and PyPy (starting with PyPy 5.9).-If you're using `Anaconda`_ or `Miniconda`_, you can install the package fromthe `conda-forge`_ channel, which has up-to-date packages for Linux, Windowsand macOS.diff --git a/setup.py b/setup.pyindex f8d9b491b..58090f7a2 100644--- a/setup.py+++ b/setup.py@@ -18,12 +18,39 @@ def has_environment_marker_platform_impl_support():return parse_version(setuptools_version) >= parse_version('18.5')+install_requires = [+ 'Twisted>=17.9.0',+ 'cryptography>=2.0',+ 'cssselect>=0.9.1',+ 'itemloaders>=1.0.1',+ 'lxml>=3.5.0',+ 'parsel>=1.5.0',+ 'PyDispatcher>=2.0.5',+ 'pyOpenSSL>=16.2.0',+ 'queuelib>=1.4.2',+ 'service_identity>=16.0.0',+ 'w3lib>=1.17.0',+ 'zope.interface>=4.1.3',+ 'protego>=0.1.15',+ 'itemadapter>=0.1.0',+]extras_require = {}if has_environment_marker_platform_impl_support():+ extras_require[':platform_python_implementation == "CPython"'] = [+ 'lxml>=3.5.0',+ ]extras_require[':platform_python_implementation == "PyPy"'] = [+ # Earlier lxml versions are affected by+ # https://bitbucket.org/pypy/pypy/issues/2498/cython-on-pypy-3-dict-object-has-no,+ # which was fixed in Cython 0.26, released on 2017-06-19, and used to+ # generate the C headers of lxml release tarballs published since then, the+ # first of which was:+ 'lxml>=4.0.0','PyPyDispatcher>=2.1.0',]+else:+ install_requires.append('lxml>=3.5.0')setup(@@ -67,21 +94,6 @@ setup('Topic :: Software Development :: Libraries :: Python Modules',],python_requires='>=3.5.2',- install_requires=[- 'Twisted>=17.9.0',- 'cryptography>=2.0',- 'cssselect>=0.9.1',- 'itemloaders>=1.0.1',- 'lxml>=3.5.0',- 'parsel>=1.5.0',- 'PyDispatcher>=2.0.5',- 'pyOpenSSL>=16.2.0',- 'queuelib>=1.4.2',- 'service_identity>=16.0.0',- 'w3lib>=1.17.0',- 'zope.interface>=4.1.3',- 'protego>=0.1.15',- 'itemadapter>=0.1.0',- ],+ install_requires=install_requires,extras_require=extras_require,)diff --git a/tests/test_proxy_connect.py b/tests/test_proxy_connect.pyindex fc5658ae7..a56e3c39a 100644--- a/tests/test_proxy_connect.py+++ b/tests/test_proxy_connect.py@@ -60,6 +60,8 @@ def _wrong_credentials(proxy_url):@skipIf(sys.version_info < (3, 5, 4),"requires mitmproxy < 3.0.0, which these tests do not support")+@skipIf("pypy" in sys.executable,+ "mitmproxy does not support PyPy")@skipIf(platform.system() == 'Windows' and sys.version_info < (3, 7),"mitmproxy does not support Windows when running Python < 3.7")class ProxyConnectTestCase(TestCase):diff --git a/tests/test_utils_python.py b/tests/test_utils_python.pyindex 65e6ba876..ebce3c079 100644--- a/tests/test_utils_python.py+++ b/tests/test_utils_python.py@@ -4,6 +4,7 @@ import operatorimport platformimport unittestfrom itertools import count+from sys import version_infofrom warnings import catch_warningsfrom scrapy.utils.python import (@@ -214,9 +215,12 @@ class UtilsPythonTestCase(unittest.TestCase):else:self.assertEqual(get_func_args(str.split, stripself=True), ['sep', 'maxsplit'])- self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])self.assertEqual(get_func_args(operator.itemgetter(2), stripself=True), ['obj'])+ if version_info < (3, 6):+ self.assertEqual(get_func_args(" ".join, stripself=True), ['list'])+ else:+ self.assertEqual(get_func_args(" ".join, stripself=True), ['iterable'])def test_without_none_values(self):self.assertEqual(without_none_values([1, None, 3, 4]), [1, 3, 4])diff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex c1c5945c2..ee64d455c 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -413,7 +413,9 @@ class WebClientCustomCiphersSSLTestCase(WebClientSSLTestCase):self.getURL("payload"), body=s, contextFactory=client_context_factory).addCallback(self.assertEqual, to_bytes(s))- def testPayloadDefaultCiphers(self):+ def testPayloadDisabledCipher(self):s = "0123456789" * 10- d = getPage(self.getURL("payload"), body=s, contextFactory=ScrapyClientContextFactory())+ settings = Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'ECDHE-RSA-AES256-GCM-SHA384'})+ client_context_factory = create_instance(ScrapyClientContextFactory, settings=settings, crawler=None)+ d = getPage(self.getURL("payload"), body=s, contextFactory=client_context_factory)return self.assertFailure(d, OpenSSL.SSL.Error)diff --git a/tox.ini b/tox.iniindex 5d79739bb..4557c63e3 100644--- a/tox.ini+++ b/tox.ini@@ -58,11 +58,6 @@ deps =commands =pylint conftest.py docs extras scrapy setup.py tests-[testenv:pypy3]-basepython = pypy3-commands =- py.test {posargs:--durations=10 docs scrapy tests}-[pinned]deps =-ctests/constraints.txt@@ -85,7 +80,6 @@ deps =Pillow==3.4.2[testenv:pinned]-basepython = python3deps ={[pinned]deps}lxml==3.5.0@@ -104,6 +98,27 @@ deps =reppyrobotexclusionrulesparser+[testenv:asyncio]+commands =+ {[testenv]commands} --reactor=asyncio++[testenv:asyncio-pinned]+commands = {[testenv:asyncio]commands}+deps = {[testenv:pinned]deps}++[testenv:pypy3]+basepython = pypy3+commands =+ py.test {posargs:--durations=10 docs scrapy tests}++[testenv:pypy3-pinned]+basepython = {[testenv:pypy3]basepython}+commands = {[testenv:pypy3]commands}+deps =+ {[pinned]deps}+ lxml==4.0.0+ PyPyDispatcher==2.1.0+[docs]changedir = docsdeps =@@ -135,7 +150,3 @@ deps = {[docs]deps}setenv = {[docs]setenv}commands =sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck--[testenv:asyncio]-commands =- {[testenv]commands} --reactor=asyncio