Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge pull request #6112 from wRAR/test-shutdown-forced

Adrián Chaves1045856a50d3
diff --git a/tests/CrawlerProcess/sleeping.py b/tests/CrawlerProcess/sleeping.py
index 420d9d328..45479ea4f 100644
--- a/tests/CrawlerProcess/sleeping.py
+++ b/tests/CrawlerProcess/sleeping.py
@@ -14,7 +14,7 @@ class SleepingSpider(scrapy.Spider):
from twisted.internet import reactor
d = Deferred()
- reactor.callLater(3, d.callback, None)
+ reactor.callLater(int(self.sleep), d.callback, None)
await maybe_deferred_to_future(d)
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 7d87eb62c..7918d94b2 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -1,3 +1,4 @@
+import os
import sys
from io import BytesIO
from pathlib import Path
@@ -147,8 +148,10 @@ class InteractiveShellTest(unittest.TestCase):
"scrapy.cmdline",
"shell",
)
+ env = os.environ.copy()
+ env["SCRAPY_PYTHON_SHELL"] = "python"
logfile = BytesIO()
- p = PopenSpawn(args, timeout=5)
+ p = PopenSpawn(args, env=env, timeout=5)
p.logfile_read = logfile
p.expect_exact("Available Scrapy objects")
with MockServer() as mockserver:
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index 60b92377d..0a7f9bac8 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -525,7 +525,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
def test_shutdown_graceful(self):
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
- args = self.get_script_args("sleeping.py")
+ args = self.get_script_args("sleeping.py", "-a", "sleep=3")
p = PopenSpawn(args, timeout=5)
p.expect_exact("Spider opened")
p.expect_exact("Crawled (200)")
@@ -534,14 +534,21 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
p.expect_exact("Spider closed (shutdown)")
p.wait()
+ @defer.inlineCallbacks
def test_shutdown_forced(self):
+ from twisted.internet import reactor
+
sig = signal.SIGINT if sys.platform != "win32" else signal.SIGBREAK
- args = self.get_script_args("sleeping.py")
+ args = self.get_script_args("sleeping.py", "-a", "sleep=10")
p = PopenSpawn(args, timeout=5)
p.expect_exact("Spider opened")
p.expect_exact("Crawled (200)")
p.kill(sig)
p.expect_exact("shutting down gracefully")
+ # sending the second signal too fast often causes problems
+ d = defer.Deferred()
+ reactor.callLater(0.1, d.callback, None)
+ yield d
p.kill(sig)
p.expect_exact("forcing unclean shutdown")
p.wait()