Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add a test for a CrawlerProcess script. (#4218)

Andrey Rahmatullin02cdc53fb82e
diff --git a/conftest.py b/conftest.py
index d54ce155c..d37c22436 100644
--- a/conftest.py
+++ b/conftest.py
@@ -1,12 +1,19 @@
+from pathlib import Path
+
import pytest
+def _py_files(folder):
+ return (str(p) for p in Path(folder).rglob('*.py'))
+
+
collect_ignore = [
# not a test, but looks like a test
"scrapy/utils/testsite.py",
+ # contains scripts to be run by tests/test_crawler.py::CrawlerProcessSubprocess
+ *_py_files("tests/CrawlerProcess")
]
-
for line in open('tests/ignores.txt'):
file_path = line.strip()
if file_path and file_path[0] != '#':
diff --git a/tests/CrawlerProcess/simple.py b/tests/CrawlerProcess/simple.py
new file mode 100644
index 000000000..5f6f1ae30
--- /dev/null
+++ b/tests/CrawlerProcess/simple.py
@@ -0,0 +1,15 @@
+import scrapy
+from scrapy.crawler import CrawlerProcess
+
+
+class NoRequestsSpider(scrapy.Spider):
+ name = 'no_request'
+
+ def start_requests(self):
+ return []
+
+
+process = CrawlerProcess(settings={})
+
+process.crawl(NoRequestsSpider)
+process.start()
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 8eb2389e2..e37a2ff0e 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -1,4 +1,7 @@
import logging
+import os
+import subprocess
+import sys
import warnings
from twisted.internet import defer
@@ -14,6 +17,7 @@ from scrapy.utils.spider import DefaultSpider
from scrapy.utils.misc import load_object
from scrapy.extensions.throttle import AutoThrottle
from scrapy.extensions import telnet
+from scrapy.utils.test import get_testenv
class BaseCrawlerTest(unittest.TestCase):
@@ -245,3 +249,19 @@ class CrawlerRunnerHasSpider(unittest.TestCase):
yield runner.crawl(NoRequestsSpider)
self.assertEqual(runner.bootstrap_failed, True)
+
+
+class CrawlerProcessSubprocess(unittest.TestCase):
+ script_dir = os.path.join(os.path.abspath(os.path.dirname(__file__)), 'CrawlerProcess')
+
+ def run_script(self, script_name):
+ script_path = os.path.join(self.script_dir, script_name)
+ args = (sys.executable, script_path)
+ p = subprocess.Popen(args, env=get_testenv(),
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ stdout, stderr = p.communicate()
+ return stderr.decode('utf-8')
+
+ def test_simple(self):
+ log = self.run_script('simple.py')
+ self.assertIn('Spider closed (finished)', log)