Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Narrow down TestEngine::test_short_timeout() expectations. (#6911)

Andrey Rakhmatullinc9cdf0af3cac
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index b3a5e1906..1928f6d64 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -135,6 +135,9 @@ class ExecutionEngine:
await maybe_deferred_to_future(
self.signals.send_catch_log_deferred(signal=signals.engine_started)
)
+ if _start_request_processing and self.spider is None:
+ # require an opened spider when not run in scrapy shell
+ return
self.running = True
self._closewait = Deferred()
if _start_request_processing:
@@ -541,5 +544,6 @@ class ExecutionEngine:
dfd.addErrback(log_failure("Error while unassigning spider"))
dfd.addBoth(lambda _: self._spider_closed_callback(spider))
+ dfd.addErrback(log_failure("Error running spider_closed_callback"))
return dfd
diff --git a/tests/test_engine.py b/tests/test_engine.py
index a3c5243d5..aa9c1b23d 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -509,7 +509,9 @@ class TestEngine(TestEngineBase):
finally:
timer.cancel()
- assert b"Traceback" not in stderr, stderr
+ stderr_str = stderr.decode("utf-8")
+ assert "AttributeError" not in stderr_str, stderr_str
+ assert "AssertionError" not in stderr_str, stderr_str
def test_request_scheduled_signal(caplog):
diff --git a/tests/test_engine_loop.py b/tests/test_engine_loop.py
index 8211667fb..fd70de3d8 100644
--- a/tests/test_engine_loop.py
+++ b/tests/test_engine_loop.py
@@ -112,7 +112,8 @@ class MainTestCase(TestCase):
with LogCapture(level=ERROR) as log:
await maybe_deferred_to_future(crawler.crawl())
- assert not log.records
+ assert len(log.records) == 1
+ assert log.records[0].msg == "Error running spider_closed_callback"
finish_reason = crawler.stats.get_value("finish_reason")
assert finish_reason == "shutdown", f"{finish_reason=}"
expected_urls = []