Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve cleanup in TestEngineCloseSpider. (#7012)

Andrey Rakhmatullin15655ca83454
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 2558b8dc6..4bfccdb18 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -26,7 +26,6 @@ from scrapy.exceptions import CloseSpider, IgnoreRequest
from scrapy.http import Request, Response
from scrapy.item import Field, Item
from scrapy.linkextractors import LinkExtractor
-from scrapy.signals import request_scheduled
from scrapy.spiders import Spider
from scrapy.utils.defer import (
_schedule_coro,
@@ -621,7 +620,7 @@ def test_request_scheduled_signal(caplog):
engine._start = start()
engine._slot = _Slot(False, Mock(), scheduler)
- crawler.signals.connect(signal_handler, request_scheduled)
+ crawler.signals.connect(signal_handler, signals.request_scheduled)
keep_request = Request("https://keep.example")
engine._schedule_request(keep_request)
drop_request = Request("https://drop.example")
@@ -630,7 +629,7 @@ def test_request_scheduled_signal(caplog):
assert scheduler.enqueued == [keep_request], (
f"{scheduler.enqueued!r} != [{keep_request!r}]"
)
- crawler.signals.disconnect(signal_handler, request_scheduled)
+ crawler.signals.disconnect(signal_handler, signals.request_scheduled)
class TestEngineCloseSpider:
@@ -646,9 +645,13 @@ class TestEngineCloseSpider:
async def test_no_slot(self, crawler: Crawler) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
await engine.open_spider_async()
+ slot = engine._slot
engine._slot = None
with pytest.raises(RuntimeError, match="Engine slot not assigned"):
await engine.close_spider_async()
+ # close it correctly
+ engine._slot = slot
+ await engine.close_spider_async()
@deferred_f_from_coro_f
async def test_no_spider(self, crawler: Crawler) -> None:
@@ -704,9 +707,16 @@ class TestEngineCloseSpider:
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
await engine.open_spider_async()
+ signal_manager = engine.signals
del engine.signals
await engine.close_spider_async()
assert "Error while sending spider_close signal" in caplog.text
+ # send the spider_closed signal to close various components
+ await signal_manager.send_catch_log_async(
+ signal=signals.spider_closed,
+ spider=engine.spider,
+ reason="cancelled",
+ )
@deferred_f_from_coro_f
async def test_exception_stats(