Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'origin/master' into asyncio-base
Andrey Rakhmatullin352ddbb33695
diff --cc tests/mockserver.pyindex d09fbc171,fe28176d4..ae249e55a--- a/tests/mockserver.py+++ b/tests/mockserver.py@@@ -3,10 -3,9 +3,10 @@@ import oimport randomimport sysfrom subprocess import Popen, PIPE+ from urllib.parse import urlencodefrom OpenSSL import SSL- from six.moves.urllib.parse import urlencode+from twisted.web.server import Site, NOT_DONE_YETfrom twisted.web.resource import Resourcefrom twisted.web.static import Filediff --cc tests/requirements-py3.txtindex 2ac434f41,e9bf310b9..d97c4b8ee--- a/tests/requirements-py3.txt+++ b/tests/requirements-py3.txt@@@ -1,10 -1,10 +1,10 @@@# Tests requirementsjmespathmitmproxy; python_version >= '3.6'- mitmproxy==3.0.4; python_version < '3.6'+ mitmproxy<4.0.0; python_version < '3.6'pytestpytest-cov-pytest-twisted+pytest-twisted >= 1.11pytest-xdistsybiltestfixturesdiff --cc tests/test_crawler.pyindex 3ac45ca1d,e37a2ff0e..9410b0e7a--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@@ -1,10 -1,12 +1,13 @@@import logging+ import os+ import subprocess+ import sysimport warnings+from pytest import raises, mark+from testfixtures import LogCapturefrom twisted.internet import deferfrom twisted.trial import unittest-from pytest import raisesimport scrapyfrom scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess@@@ -257,27 -250,18 +261,43 @@@ class CrawlerRunnerHasSpider(unittest.Tself.assertEqual(runner.bootstrap_failed, True)+ @defer.inlineCallbacks+ def test_crawler_process_asyncio_supported_true(self):+ with LogCapture(level=logging.DEBUG) as log:+ runner = CrawlerProcess(settings={'ASYNCIO_SUPPORT': True})+ yield runner.crawl(NoRequestsSpider)+ if self.reactor_pytest == 'asyncio':+ self.assertIn("Asyncio support enabled", str(log))+ else:+ self.assertNotIn("Asyncio support enabled", str(log))+ self.assertIn("ASYNCIO_SUPPORT is on but the Twisted asyncio reactor is not installed", str(log))++ @defer.inlineCallbacks+ def test_crawler_process_asyncio_supported_false(self):+ runner = CrawlerProcess(settings={'ASYNCIO_SUPPORT': False})+ with LogCapture(level=logging.DEBUG) as log:+ yield runner.crawl(NoRequestsSpider)+ self.assertNotIn("Asyncio support enabled", str(log))++ @defer.inlineCallbacks+ def test_crawler_runner_asyncio_supported(self):+ runner = CrawlerRunner()+ with LogCapture() as log:+ yield runner.crawl(AsyncioSpider)+ log.check_present(('asyncio', 'INFO', 'Asyncio support: %s' % (self.reactor_pytest == 'asyncio')))++++ class CrawlerProcessSubprocess(unittest.TestCase):+ script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')++ def run_script(self, script_name):+ script_path = os.path.join(self.script_dir, script_name)+ args = (sys.executable, script_path)+ p = subprocess.Popen(args, env=get_testenv(),+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)+ stdout, stderr = p.communicate()+ return stderr.decode('utf-8')++ def test_simple(self):+ log = self.run_script('simple.py')+ self.assertIn('Spider closed (finished)', log)