Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add tests that check asyncio support.

Andrey Rakhmatullincc19ab5439f2
diff --git a/conftest.py b/conftest.py
index d54ce155c..24e31f130 100644
--- a/conftest.py
+++ b/conftest.py
@@ -27,3 +27,7 @@ def pytest_collection_modifyitems(session, config, items):
items[:] = [item for item in items if isinstance(item, Flake8Item)]
except ImportError:
pass
+
+def reactor_pytest(request):
+ request.cls.reactor_pytest = request.config.getoption("--reactor")
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 536379170..8aa7ee109 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -9,6 +9,7 @@ from tempfile import mkdtemp
from contextlib import contextmanager
from threading import Timer
+from pytest import mark
from twisted.trial import unittest
import scrapy
@@ -178,6 +179,7 @@ class MiscCommandsTest(CommandTest):
self.assertEqual(0, self.call('list'))
[email protected]('reactor_pytest')
class RunSpiderCommandTest(CommandTest):
debug_log_spider = """
@@ -295,6 +297,11 @@ class BadSpider(scrapy.Spider):
self.assertIn("start_requests", log)
self.assertIn("badspider.py", log)
+ def test_asyncio_supported(self):
+ if self.reactor_pytest == 'asyncio':
+ log = self.get_log(self.debug_log_spider)
+ self.assertIn("DEBUG: Asyncio support enabled", log)
+
class BenchCommandTest(CommandTest):
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8eb2389e2..151acb459 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,14 +1,16 @@
import logging
import warnings
+from pytest import raises, mark
+from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial import unittest
-from pytest import raises
import scrapy
from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
from scrapy.settings import Settings, default_settings
from scrapy.spiderloader import SpiderLoader
+from scrapy.utils.asyncio import is_asyncio_supported
from scrapy.utils.log import configure_logging, get_scrapy_root_handler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.misc import load_object
@@ -203,6 +205,15 @@ class NoRequestsSpider(scrapy.Spider):
return []
+class AsyncioSpider(scrapy.Spider):
+ name = 'asyncio'
+
+ def start_requests(self):
+ self.logger.info('Asyncio support: %s', is_asyncio_supported())
+ return []
+
+
[email protected]('reactor_pytest')
class CrawlerRunnerHasSpider(unittest.TestCase):
@defer.inlineCallbacks
@@ -245,3 +256,10 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
yield runner.crawl(NoRequestsSpider)
self.assertEqual(runner.bootstrap_failed, True)
+
+ @defer.inlineCallbacks
+ def test_asyncio_supported(self):
+ runner = CrawlerRunner()
+ with LogCapture() as log:
+ yield runner.crawl(AsyncioSpider)
+ log.check_present(('asyncio', 'INFO', 'Asyncio support: %s' % (self.reactor_pytest == 'asyncio')))
diff --git a/tests/test_utils_asyncio.py b/tests/test_utils_asyncio.py
new file mode 100644
index 000000000..e34d3002a
--- /dev/null
+++ b/tests/test_utils_asyncio.py
@@ -0,0 +1,17 @@
+from unittest import TestCase
+
+from pytest import mark
+
+from scrapy.utils.asyncio import is_asyncio_supported, install_asyncio_reactor
+
+
[email protected]('reactor_pytest')
+class AsyncioTest(TestCase):
+
+ def test_is_asyncio_supported(self):
+ # the result should depend only on the pytest --reactor argument
+ self.assertEquals(is_asyncio_supported(), self.reactor_pytest == 'asyncio')
+
+ def test_install_asyncio_reactor(self):
+ # this should do nothing
+ install_asyncio_reactor()
diff --git a/tox.ini b/tox.ini
index fd75d18e2..844956e5f 100644
--- a/tox.ini
+++ b/tox.ini
@@ -106,3 +106,9 @@ deps =
{[testenv]deps}
reppy
robotexclusionrulesparser
+
+[testenv:py38-no-asyncio]
+basepython = python3.8
+deps = {[testenv]deps}
+commands =
+ py.test --cov=scrapy --cov-report= --reactor=default {posargs:scrapy tests}