Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'origin/master' into asyncio-base

Andrey Rakhmatullin20289be810b0
diff --cc .travis.yml
index 98dab01f2,f9d0dc8be..82167e10a
--- a/.travis.yml
+++ b/.travis.yml
@@@ -12,25 -12,20 +12,24 @@@ matrix
- env: TOXENV=flake8
python: 3.8
- env: TOXENV=pypy3
- python: 3.5
- env: TOXENV=py35
python: 3.5
- - env: TOXENV=py35-pinned
+ - env: TOXENV=pinned
python: 3.5
+ - env: TOXENV=py35-asyncio
+ python: 3.5
- env: TOXENV=py36
python: 3.6
- env: TOXENV=py37
python: 3.7
- env: TOXENV=py38
python: 3.8
- - env: TOXENV=py38-extra-deps
+ - env: TOXENV=extra-deps
python: 3.8
+ - env: TOXENV=py38-asyncio
+ python: 3.8
- env: TOXENV=docs
- python: 3.6
+ python: 3.7 # Keep in sync with .readthedocs.yml
install:
- |
if [ "$TOXENV" = "pypy3" ]; then
diff --cc pytest.ini
index 7b62a1bd8,50fc4ac00..c3f3292bb
--- a/pytest.ini
+++ b/pytest.ini
@@@ -114,13 -111,12 +113,13 @@@ flake8-ignore
# scrapy/spiders
scrapy/spiders/__init__.py E501 E402
scrapy/spiders/crawl.py E501
- scrapy/spiders/feed.py E501 E261
+ scrapy/spiders/feed.py E501
scrapy/spiders/sitemap.py E501
# scrapy/utils
+ scrapy/utils/asyncio.py E501
scrapy/utils/benchserver.py E501
- scrapy/utils/conf.py E402 E502 E501
- scrapy/utils/console.py E261 E306 E305
+ scrapy/utils/conf.py E402 E501
+ scrapy/utils/console.py E306 E305
scrapy/utils/datatypes.py E501 E226
scrapy/utils/decorators.py E501
scrapy/utils/defer.py E501 E128
@@@ -225,11 -219,10 +222,11 @@@
tests/test_spider.py E501
tests/test_spidermiddleware.py E501 E226
tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
- tests/test_spidermiddleware_offsite.py E501 E128 E111 W293
- tests/test_spidermiddleware_output_chain.py E501 W293 E226
- tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E261 E124 E501 E241 E121
+ tests/test_spidermiddleware_offsite.py E501 E128 E111
+ tests/test_spidermiddleware_output_chain.py E501 E226
+ tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
tests/test_squeues.py E501 E701 E741
+ tests/test_utils_asyncio.py E501
tests/test_utils_conf.py E501 E303 E128
tests/test_utils_curl.py E501
tests/test_utils_datatypes.py E402 E501 E305
diff --cc tox.ini
index 795c20233,1ec8f52e4..ed0d4c9ab
--- a/tox.ini
+++ b/tox.ini
@@@ -99,24 -96,3 +96,17 @@@ changedir = {[docs]changedir
deps = {[docs]deps}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
- [testenv:py38-extra-deps]
- basepython = python3.8
- deps =
- {[testenv]deps}
- reppy
- robotexclusionrulesparser
-
+[asyncio]
+commands =
+ py.test --cov=scrapy --cov-report= --reactor=asyncio {posargs:scrapy tests}
+
+[testenv:py35-asyncio]
+basepython = python3.5
+deps = {[testenv]deps}
+commands = {[asyncio]commands}
+
+[testenv:py38-asyncio]
+basepython = python3.8
+deps = {[testenv]deps}
+commands = {[asyncio]commands}