Scrapy, a fast high-level web crawling & scraping framework for Python.
Add tests that check asyncio support.
Andrey Rakhmatullincc19ab5439f2
diff --git a/conftest.py b/conftest.pyindex d54ce155c..24e31f130 100644--- a/conftest.py+++ b/conftest.py@@ -27,3 +27,7 @@ def pytest_collection_modifyitems(session, config, items):items[:] = [item for item in items if isinstance(item, Flake8Item)]except ImportError:pass++def reactor_pytest(request):+ request.cls.reactor_pytest = request.config.getoption("--reactor")diff --git a/tests/test_commands.py b/tests/test_commands.pyindex 536379170..8aa7ee109 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -9,6 +9,7 @@ from tempfile import mkdtempfrom contextlib import contextmanagerfrom threading import Timer+from pytest import markfrom twisted.trial import unittestimport scrapy@@ -178,6 +179,7 @@ class MiscCommandsTest(CommandTest):self.assertEqual(0, self.call('list'))[email protected]('reactor_pytest')class RunSpiderCommandTest(CommandTest):debug_log_spider = """@@ -295,6 +297,11 @@ class BadSpider(scrapy.Spider):self.assertIn("start_requests", log)self.assertIn("badspider.py", log)+ def test_asyncio_supported(self):+ if self.reactor_pytest == 'asyncio':+ log = self.get_log(self.debug_log_spider)+ self.assertIn("DEBUG: Asyncio support enabled", log)+class BenchCommandTest(CommandTest):diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 8eb2389e2..151acb459 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -1,14 +1,16 @@import loggingimport warnings+from pytest import raises, mark+from testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.trial import unittest-from pytest import raisesimport scrapyfrom scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcessfrom scrapy.settings import Settings, default_settingsfrom scrapy.spiderloader import SpiderLoader+from scrapy.utils.asyncio import is_asyncio_supportedfrom scrapy.utils.log import configure_logging, get_scrapy_root_handlerfrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.misc import load_object@@ -203,6 +205,15 @@ class NoRequestsSpider(scrapy.Spider):return []+class AsyncioSpider(scrapy.Spider):+ name = 'asyncio'++ def start_requests(self):+ self.logger.info('Asyncio support: %s', is_asyncio_supported())+ return []++[email protected]('reactor_pytest')class CrawlerRunnerHasSpider(unittest.TestCase):@defer.inlineCallbacks@@ -245,3 +256,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase):yield runner.crawl(NoRequestsSpider)self.assertEqual(runner.bootstrap_failed, True)++ @defer.inlineCallbacks+ def test_asyncio_supported(self):+ runner = CrawlerRunner()+ with LogCapture() as log:+ yield runner.crawl(AsyncioSpider)+ log.check_present(('asyncio', 'INFO', 'Asyncio support: %s' % (self.reactor_pytest == 'asyncio')))diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.pynew file mode 100644index 000000000..e34d3002a--- /dev/null+++ b/tests/test_utils_asyncio.py@@ -0,0 +1,17 @@+from unittest import TestCase++from pytest import mark++from scrapy.utils.asyncio import is_asyncio_supported, install_asyncio_reactor++[email protected]('reactor_pytest')+class AsyncioTest(TestCase):++ def test_is_asyncio_supported(self):+ # the result should depend only on the pytest --reactor argument+ self.assertEquals(is_asyncio_supported(), self.reactor_pytest == 'asyncio')++ def test_install_asyncio_reactor(self):+ # this should do nothing+ install_asyncio_reactor()diff --git a/tox.ini b/tox.iniindex fd75d18e2..844956e5f 100644--- a/tox.ini+++ b/tox.ini@@ -106,3 +106,9 @@ deps ={[testenv]deps}reppyrobotexclusionrulesparser++[testenv:py38-no-asyncio]+basepython = python3.8+deps = {[testenv]deps}+commands =+ py.test --cov=scrapy --cov-report= --reactor=default {posargs:scrapy tests}