Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'origin/master' into asyncio-base

Andrey Rakhmatullin352ddbb33695
diff --cc tests/mockserver.py
index d09fbc171,fe28176d4..ae249e55a
--- a/tests/mockserver.py
+++ b/tests/mockserver.py
@@@ -3,10 -3,9 +3,10 @@@ import o
import random
import sys
from subprocess import Popen, PIPE
+ from urllib.parse import urlencode
from OpenSSL import SSL
- from six.moves.urllib.parse import urlencode
+
from twisted.web.server import Site, NOT_DONE_YET
from twisted.web.resource import Resource
from twisted.web.static import File
diff --cc tests/requirements-py3.txt
index 2ac434f41,e9bf310b9..d97c4b8ee
--- a/tests/requirements-py3.txt
+++ b/tests/requirements-py3.txt
@@@ -1,10 -1,10 +1,10 @@@
# Tests requirements
jmespath
mitmproxy; python_version >= '3.6'
- mitmproxy==3.0.4; python_version < '3.6'
+ mitmproxy<4.0.0; python_version < '3.6'
pytest
pytest-cov
-pytest-twisted
+pytest-twisted >= 1.11
pytest-xdist
sybil
testfixtures
diff --cc tests/test_crawler.py
index 3ac45ca1d,e37a2ff0e..9410b0e7a
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@@ -1,10 -1,12 +1,13 @@@
import logging
+ import os
+ import subprocess
+ import sys
import warnings
+from pytest import raises, mark
+from testfixtures import LogCapture
from twisted.internet import defer
from twisted.trial import unittest
-from pytest import raises
import scrapy
from scrapy.crawler import Crawler, CrawlerRunner, CrawlerProcess
@@@ -257,27 -250,18 +261,43 @@@ class CrawlerRunnerHasSpider(unittest.T
self.assertEqual(runner.bootstrap_failed, True)
+ @defer.inlineCallbacks
+ def test_crawler_process_asyncio_supported_true(self):
+ with LogCapture(level=logging.DEBUG) as log:
+ runner = CrawlerProcess(settings={'ASYNCIO_SUPPORT': True})
+ yield runner.crawl(NoRequestsSpider)
+ if self.reactor_pytest == 'asyncio':
+ self.assertIn("Asyncio support enabled", str(log))
+ else:
+ self.assertNotIn("Asyncio support enabled", str(log))
+ self.assertIn("ASYNCIO_SUPPORT is on but the Twisted asyncio reactor is not installed", str(log))
+
+ @defer.inlineCallbacks
+ def test_crawler_process_asyncio_supported_false(self):
+ runner = CrawlerProcess(settings={'ASYNCIO_SUPPORT': False})
+ with LogCapture(level=logging.DEBUG) as log:
+ yield runner.crawl(NoRequestsSpider)
+ self.assertNotIn("Asyncio support enabled", str(log))
+
+ @defer.inlineCallbacks
+ def test_crawler_runner_asyncio_supported(self):
+ runner = CrawlerRunner()
+ with LogCapture() as log:
+ yield runner.crawl(AsyncioSpider)
+ log.check_present(('asyncio', 'INFO', 'Asyncio support: %s' % (self.reactor_pytest == 'asyncio')))
++
+
+ class CrawlerProcessSubprocess(unittest.TestCase):
+ script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
+ def run_script(self, script_name):
+ script_path = os.path.join(self.script_dir, script_name)
+ args = (sys.executable, script_path)
+ p = subprocess.Popen(args, env=get_testenv(),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ return stderr.decode('utf-8')
+
+ def test_simple(self):
+ log = self.run_script('simple.py')
+ self.assertIn('Spider closed (finished)', log)