Scrapy, a fast high-level web crawling & scraping framework for Python.
Add a test for a CrawlerProcess script. (#4218)
Andrey Rahmatullin02cdc53fb82e
diff --git a/conftest.py b/conftest.pyindex d54ce155c..d37c22436 100644--- a/conftest.py+++ b/conftest.py@@ -1,12 +1,19 @@+from pathlib import Path+import pytest+def _py_files(folder):+ return (str(p) for p in Path(folder).rglob('*.py'))++collect_ignore = [# not a test, but looks like a test"scrapy/utils/testsite.py",+ # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess+ *_py_files("tests/CrawlerProcess")]-for line in open('tests/ignores.txt'):file_path = line.strip()if file_path and file_path[0] != '#':diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.pynew file mode 100644index 000000000..5f6f1ae30--- /dev/null+++ b/tests/CrawlerProcess/simple.py@@ -0,0 +1,15 @@+import scrapy+from scrapy.crawler import CrawlerProcess+++class NoRequestsSpider(scrapy.Spider):+ name = 'no_request'++ def start_requests(self):+ return []+++process = CrawlerProcess(settings={})++process.crawl(NoRequestsSpider)+process.start()diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex 8eb2389e2..e37a2ff0e 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -1,4 +1,7 @@import logging+import os+import subprocess+import sysimport warningsfrom twisted.internet import defer@@ -14,6 +17,7 @@ from scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.misc import load_objectfrom scrapy.extensions.throttle import AutoThrottlefrom scrapy.extensions import telnet+from scrapy.utils.test import get_testenvclass BaseCrawlerTest(unittest.TestCase):@@ -245,3 +249,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase):yield runner.crawl(NoRequestsSpider)self.assertEqual(runner.bootstrap_failed, True)+++class CrawlerProcessSubprocess(unittest.TestCase):+ script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')++ def run_script(self, script_name):+ script_path = os.path.join(self.script_dir, script_name)+ args = (sys.executable, script_path)+ p = subprocess.Popen(args, env=get_testenv(),+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)+ stdout, stderr = p.communicate()+ return stderr.decode('utf-8')++ def test_simple(self):+ log = self.run_script('simple.py')+ self.assertIn('Spider closed (finished)', log)