Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import logging
from typing import TYPE_CHECKING, Any
from unittest.mock import Mock
import pytest
from twisted.internet.defer import Deferred
from scrapy import Request, Spider, signals
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
from scrapy.core.engine import EngineState, ExecutionEngine
from scrapy.core.scheduler import BaseScheduler
from scrapy.exceptions import CloseSpider, ScrapyDeprecationWarning
from scrapy.utils.asyncio import sleep
from scrapy.utils.defer import (
_schedule_coro,
deferred_from_coro,
maybe_deferred_to_future,
)
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Callable
from scrapy.crawler import Crawler
from scrapy.http import Response
from scrapy.utils._stopmode import _StopMode
class SignalRecorder:
"""Record lifecycle signals and the engine state at the time each signal
fired."""
SIGNALS = ("engine_started", "engine_stopped", "spider_opened", "spider_closed")
def __init__(self, crawler: Crawler) -> None:
self.crawler = crawler
self.calls: list[tuple[str, EngineState]] = []
self.close_reasons: list[str] = []
self.close_errors: list[bool] = []
# Keep strong references to the handlers: signal connections are weak.
self._handlers = [self._make_handler(name) for name in self.SIGNALS]
for name, handler in zip(self.SIGNALS, self._handlers, strict=True):
crawler.signals.connect(handler, getattr(signals, name))
def _make_handler(self, name: str) -> Callable[..., None]:
def handler(**kwargs: Any) -> None:
assert self.crawler.engine is not None
self.calls.append((name, self.crawler.engine.state))
if name == "spider_closed":
self.close_reasons.append(kwargs["reason"])
self.close_errors.append(kwargs["error"])
return handler
@property
def names(self) -> list[str]:
return [call[0] for call in self.calls]
@property
def states(self) -> list[EngineState]:
return [call[1] for call in self.calls]
class BlockingScheduler(BaseScheduler):
"""A scheduler whose close() blocks until unblocked, to keep the engine
deterministically in the SPIDER_CLOSING state."""
def __init__(self) -> None:
self.entered_close: Deferred[None] = Deferred()
self.unblock_close: Deferred[None] = Deferred()
def has_pending_requests(self) -> bool:
return False
def enqueue_request(self, request: Any) -> bool:
return True
def next_request(self) -> Any:
return None
async def close(self, reason: str) -> None:
self.entered_close.callback(None)
await maybe_deferred_to_future(self.unblock_close)
class ClosingPipeline:
def open_spider(self) -> None:
raise CloseSpider("early", error=True)
def make_engine(crawler: Crawler) -> ExecutionEngine:
"""Create the engine of *crawler* as Crawler.crawl_async() (and scrapy
shell) would, without opening or starting anything."""
crawler.spider = crawler._create_spider()
engine = crawler.engine = ExecutionEngine(crawler)
return engine
async def start_engine(engine: ExecutionEngine) -> Deferred[None]:
"""Start *engine* in the background and complete once it has been started.
Return the Deferred of the ``start_async()`` call, which completes when
the engine is stopped.
"""
started: Deferred[None] = Deferred()
def handler(**kwargs: Any) -> None:
started.callback(None)
engine.crawler.signals.connect(handler, signals.engine_started)
dfd = deferred_from_coro(engine.start_async())
await maybe_deferred_to_future(started)
engine.crawler.signals.disconnect(handler, signals.engine_started)
return dfd
def assert_state(engine: ExecutionEngine, state: EngineState) -> None:
assert engine.state is state, engine.state
def assert_no_invalid_transition(caplog: pytest.LogCaptureFixture) -> None:
assert not [
r for r in caplog.records if "Invalid engine state transition" in r.message
]
NORMAL_SIGNAL_ORDER = [
"spider_opened",
"engine_started",
"spider_closed",
"engine_stopped",
]
NEVER_STARTED_SIGNAL_ORDER = ["spider_opened", "spider_closed"]
class TestNormalLifecycle:
@coroutine_test
async def test_state_progression(self, caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
with caplog.at_level(logging.WARNING, logger="scrapy.core.engine"):
await crawler.crawl_async()
assert_no_invalid_transition(caplog)
engine = crawler.engine
assert engine is not None
assert_state(engine, EngineState.STOPPED)
assert not engine.running
assert not crawler.crawling
assert recorder.names == NORMAL_SIGNAL_ORDER
assert recorder.states == [
EngineState.SPIDER_OPENING,
EngineState.STARTING,
EngineState.SPIDER_CLOSING,
EngineState.STOPPING,
]
assert crawler.stats
assert crawler.stats.get_value("finish_reason") == "finished"
@coroutine_test
async def test_stop_idempotent(self) -> None:
crawler = get_crawler(DefaultSpider)
await crawler.crawl_async()
engine = crawler.engine
assert engine is not None
assert_state(engine, EngineState.STOPPED)
# None of these raises or hangs on a stopped engine.
await engine.stop_async()
await engine.close_async()
await engine.close_spider_async()
assert_state(engine, EngineState.STOPPED)
class TestFetchOnlyLifecycle:
"""The lifecycle used by scrapy shell: the spider is opened, requests are
downloaded, and the engine is never started."""
@coroutine_test
async def test_open_download_close(self, caplog: pytest.LogCaptureFixture) -> None:
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
assert_state(engine, EngineState.CREATED)
with caplog.at_level(logging.WARNING, logger="scrapy.core.engine"):
await engine.open_spider_async(close_if_idle=False)
assert_state(engine, EngineState.SPIDER_OPEN)
assert not engine.running
response = await engine.download_async(Request("data:,"))
assert response.status == 200
# Downloads schedule the slot's next call, which must not start
# request processing or close the idle spider.
assert_state(engine, EngineState.SPIDER_OPEN)
await engine.close_async()
assert_state(engine, EngineState.STOPPED)
assert_no_invalid_transition(caplog)
# The engine was never started, so no engine signals are sent.
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["shutdown"]
class TestEarlyClose:
"""Closes and stops requested before or while the engine starts."""
@coroutine_test
async def test_close_from_engine_started_handler(self) -> None:
"""A close triggered by an engine_started handler must leave the engine
cleanly stopped instead of half-initialized."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
async def stop(**kwargs: Any) -> None:
await crawler.stop_async()
crawler.signals.connect(stop, signals.engine_started)
await crawler.crawl_async()
engine = crawler.engine
assert engine is not None
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NORMAL_SIGNAL_ORDER
assert recorder.states[2] is EngineState.SPIDER_CLOSING
assert recorder.close_reasons == ["shutdown"]
@coroutine_test
async def test_close_during_engine_started_handler(self) -> None:
"""If the spider is closed while an engine_started handler is still
running, engine_stopped is sent only after that handler finishes."""
crawler = get_crawler(DefaultSpider)
events: list[str] = []
async def engine_started(**kwargs: Any) -> None:
assert crawler.engine is not None
await crawler.engine.close_spider_async(reason="early")
assert_state(crawler.engine, EngineState.STOPPING)
events.append("engine_started handler finished")
def engine_stopped(**kwargs: Any) -> None:
events.append("engine_stopped")
crawler.signals.connect(engine_started, signals.engine_started)
crawler.signals.connect(engine_stopped, signals.engine_stopped)
await crawler.crawl_async()
assert crawler.engine is not None
assert_state(crawler.engine, EngineState.STOPPED)
assert events == ["engine_started handler finished", "engine_stopped"]
@coroutine_test
async def test_close_spider_before_start(self) -> None:
"""A spider close that completes before start_async() (e.g. triggered
by the CloseSpider or MemoryUsage extensions) stops the never-started
engine right away."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
await engine.open_spider_async()
await engine.close_spider_async(reason="early")
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["early"]
with pytest.raises(RuntimeError, match="in the STOPPED state"):
await engine.start_async()
@pytest.mark.parametrize("use_deferred", [False, True])
@coroutine_test
async def test_crawl_with_close_before_start(self, use_deferred: bool) -> None:
"""Crawler.crawl_async() and Crawler.crawl() do not start an engine
whose spider was closed while opening, and complete without
hanging."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
async def close(**kwargs: Any) -> None:
assert crawler.engine is not None
await crawler.engine.close_spider_async(reason="early")
crawler.signals.connect(close, signals.spider_opened)
if use_deferred:
await maybe_deferred_to_future(crawler.crawl())
else:
await crawler.crawl_async()
engine = crawler.engine
assert engine is not None
assert_state(engine, EngineState.STOPPED)
assert not crawler.crawling
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["early"]
@coroutine_test
async def test_close_in_progress_at_start(self) -> None:
"""A spider close that is still in progress when start_async() resumes
after the engine_started signal (e.g. one scheduled by the CloseSpider
extension while an engine_started handler was running) must not make
start_async(), and therefore Crawler.crawl_async(), complete before
the engine is stopped."""
crawler = get_crawler(
DefaultSpider, settings_dict={"SCHEDULER": BlockingScheduler}
)
recorder = SignalRecorder(crawler)
close_in_progress: Deferred[None] = Deferred()
crawl_done_at_engine_stopped: list[bool] = []
schedulers: list[BlockingScheduler] = []
async def engine_started(**kwargs: Any) -> None:
engine = crawler.engine
assert engine is not None
scheduler = engine.scheduler
assert isinstance(scheduler, BlockingScheduler)
schedulers.append(scheduler)
_schedule_coro(engine.close_spider_async(reason="early"))
# Return once the close is under way, blocked in the scheduler.
await maybe_deferred_to_future(scheduler.entered_close)
close_in_progress.callback(None)
def engine_stopped(**kwargs: Any) -> None:
crawl_done_at_engine_stopped.append(crawl_dfd.called)
crawler.signals.connect(engine_started, signals.engine_started)
crawler.signals.connect(engine_stopped, signals.engine_stopped)
crawl_dfd = deferred_from_coro(crawler.crawl_async())
await maybe_deferred_to_future(close_in_progress)
engine = crawler.engine
assert engine is not None
assert_state(engine, EngineState.SPIDER_CLOSING)
assert not crawl_dfd.called
schedulers[0].unblock_close.callback(None)
await maybe_deferred_to_future(crawl_dfd)
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NORMAL_SIGNAL_ORDER
assert recorder.close_reasons == ["early"]
# The crawl completed only after the engine was stopped.
assert crawl_done_at_engine_stopped == [False]
@pytest.mark.parametrize("mode", ["graceful", "fast"])
@coroutine_test
async def test_crawler_stop_before_start(self, mode: _StopMode) -> None:
"""Crawler.stop_async() before the engine is started (e.g. from a
spider_opened handler, or on Ctrl-C during a slow spider open) closes
the spider as soon as it is open, and the crawl neither runs nor
hangs."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
async def stop(**kwargs: Any) -> None:
await crawler.stop_async(mode=mode)
crawler.signals.connect(stop, signals.spider_opened)
await crawler.crawl_async()
engine = crawler.engine
assert engine is not None
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["shutdown"]
@pytest.mark.parametrize("mode", ["graceful", "fast"])
@coroutine_test
async def test_crawler_stop_with_open_spider(self, mode: _StopMode) -> None:
"""Crawler.stop_async() with an open spider and a never-started
engine closes the spider and stops the engine."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
crawler.crawling = True # as crawl_async() would set
await engine.open_spider_async()
await crawler.stop_async(mode=mode)
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["shutdown"]
class TestCloseDuringOpen:
@coroutine_test
async def test_close_spider_during_open(self) -> None:
"""A close requested while the spider is opening is deferred until
the open finishes, keeping the spider_opened/spider_closed order."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
state_after_close_call: list[EngineState] = []
async def close(**kwargs: Any) -> None:
await engine.close_spider_async(reason="early")
# The first reason wins, as for any other double close.
await engine.close_spider_async(reason="later")
state_after_close_call.append(engine.state)
crawler.signals.connect(close, signals.spider_opened)
await engine.open_spider_async()
# The close was deferred, not performed inline in the handler.
assert state_after_close_call == [EngineState.SPIDER_OPENING]
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["early"]
@coroutine_test
async def test_close_spider_during_open_with_error(self) -> None:
"""A close requested while the spider is opening keeps its error
flag."""
crawler = get_crawler(DefaultSpider)
engine = make_engine(crawler)
errors: list[bool] = []
async def close(**kwargs: Any) -> None:
await engine.close_spider_async(reason="early", error=True)
def closed(error: bool, **kwargs: Any) -> None:
errors.append(error)
crawler.signals.connect(close, signals.spider_opened)
crawler.signals.connect(closed, signals.spider_closed)
await engine.open_spider_async()
assert_state(engine, EngineState.STOPPED)
assert errors == [True]
@coroutine_test
async def test_close_spider_exception_from_component(self) -> None:
"""CloseSpider raised by a component while the spider is opening
closes the spider once it is open, instead of being raised."""
crawler = get_crawler(
DefaultSpider, settings_dict={"ITEM_PIPELINES": {ClosingPipeline: 1}}
)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
await engine.open_spider_async()
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["early"]
assert recorder.close_errors == [True]
@coroutine_test
async def test_close_spider_exception_from_spider_opened(self) -> None:
"""CloseSpider raised by a spider_opened handler closes the spider
once it is open, instead of being raised."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
def close(**kwargs: Any) -> None:
raise CloseSpider("early", error=True)
crawler.signals.connect(close, signals.spider_opened)
await engine.open_spider_async()
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["early"]
assert recorder.close_errors == [True]
@coroutine_test
async def test_close_spider_call_wins_over_exception(self) -> None:
"""The first close requested while the spider is opening wins, and a
CloseSpider raised by a spider_opened handler only counts as requested
once all spider_opened handlers have finished."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
def raise_close_spider(**kwargs: Any) -> None:
raise CloseSpider("exception")
async def call_close_spider(**kwargs: Any) -> None:
await engine.close_spider_async(reason="call")
crawler.signals.connect(raise_close_spider, signals.spider_opened)
crawler.signals.connect(call_close_spider, signals.spider_opened)
await engine.open_spider_async()
assert_state(engine, EngineState.STOPPED)
assert recorder.close_reasons == ["call"]
@coroutine_test
async def test_close_spider_during_failed_open(self) -> None:
"""A close requested while the spider is opening is performed even if
opening the spider then fails."""
opening: Deferred[None] = Deferred()
fail_open: Deferred[None] = Deferred()
class FailingPipeline:
async def open_spider(self) -> None:
opening.callback(None)
await maybe_deferred_to_future(fail_open)
raise ValueError("open failed")
crawler = get_crawler(
DefaultSpider, settings_dict={"ITEM_PIPELINES": {FailingPipeline: 1}}
)
recorder = SignalRecorder(crawler)
crawl_dfd = deferred_from_coro(crawler.crawl_async())
await maybe_deferred_to_future(opening)
engine = crawler.engine
assert engine is not None
await engine.close_spider_async(reason="early", error=True)
fail_open.callback(None)
with pytest.raises(ValueError, match="open failed"):
await maybe_deferred_to_future(crawl_dfd)
assert_state(engine, EngineState.STOPPED)
assert recorder.close_reasons == ["early"]
assert recorder.close_errors == [True]
@coroutine_test
async def test_stop_during_open(self) -> None:
"""stop_async() while the spider is opening is performed once the
spider is open."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
async def stop(**kwargs: Any) -> None:
await engine.stop_async()
crawler.signals.connect(stop, signals.spider_opened)
await engine.open_spider_async()
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["shutdown"]
@coroutine_test
async def test_close_engine_during_open(self) -> None:
"""close_async() while the spider is opening defers the spider close
and uses its own reason for it."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
async def close(**kwargs: Any) -> None:
await engine.close_async(reason="early")
crawler.signals.connect(close, signals.spider_opened)
await engine.open_spider_async()
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["early"]
class TestDoubleClose:
@coroutine_test
async def test_close_after_close(self) -> None:
"""close_spider_async() after the spider has been closed returns
instead of raising RuntimeError."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
await engine.open_spider_async()
await engine.close_spider_async(reason="first")
assert_state(engine, EngineState.STOPPED)
await engine.close_spider_async(reason="second")
assert recorder.close_reasons == ["first"]
@coroutine_test
async def test_close_while_closing(self) -> None:
"""A concurrent close_spider_async() returns instead of raising or
starting a second close sequence."""
crawler = get_crawler(
DefaultSpider, settings_dict={"SCHEDULER": BlockingScheduler}
)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
await engine.open_spider_async()
assert engine._slot is not None
scheduler = engine._slot.scheduler
assert isinstance(scheduler, BlockingScheduler)
first = deferred_from_coro(engine.close_spider_async(reason="first"))
# Wait until the first close is underway, blocked on scheduler close.
await maybe_deferred_to_future(scheduler.entered_close)
assert_state(engine, EngineState.SPIDER_CLOSING)
await engine.close_spider_async(reason="second")
assert_state(engine, EngineState.SPIDER_CLOSING) # still the first close
scheduler.unblock_close.callback(None)
await maybe_deferred_to_future(first)
assert_state(engine, EngineState.STOPPED)
assert recorder.close_reasons == ["first"]
@coroutine_test
async def test_stop_while_closing(self) -> None:
"""A stop requested while the spider is closing does not wait for the
close, and the close finishes the stop, so that engine_stopped is
still sent after spider_closed."""
crawler = get_crawler(
DefaultSpider, settings_dict={"SCHEDULER": BlockingScheduler}
)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
await engine.open_spider_async(close_if_idle=False)
start_dfd = await start_engine(engine)
assert engine._slot is not None
scheduler = engine._slot.scheduler
assert isinstance(scheduler, BlockingScheduler)
close_dfd = deferred_from_coro(engine.close_spider_async(reason="first"))
await maybe_deferred_to_future(scheduler.entered_close)
assert_state(engine, EngineState.SPIDER_CLOSING)
await engine.stop_async() # returns without waiting for the close
assert_state(engine, EngineState.SPIDER_CLOSING)
assert "engine_stopped" not in recorder.names
assert not start_dfd.called
scheduler.unblock_close.callback(None)
await maybe_deferred_to_future(close_dfd)
await maybe_deferred_to_future(start_dfd)
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NORMAL_SIGNAL_ORDER
assert recorder.close_reasons == ["first"]
@coroutine_test
async def test_stop_from_spider_closed_handler(self) -> None:
"""Stopping the crawler from a spider_closed handler, i.e. from code
that the close sequence awaits, must not deadlock."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
async def stop(**kwargs: Any) -> None:
await crawler.stop_async()
crawler.signals.connect(stop, signals.spider_closed)
await crawler.crawl_async()
engine = crawler.engine
assert engine is not None
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NORMAL_SIGNAL_ORDER
@coroutine_test
async def test_stop_from_engine_stopped_handler(self) -> None:
"""Stopping the crawler from an engine_stopped handler must not
deadlock either."""
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
async def stop(**kwargs: Any) -> None:
await crawler.stop_async()
crawler.signals.connect(stop, signals.engine_stopped)
await crawler.crawl_async()
engine = crawler.engine
assert engine is not None
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NORMAL_SIGNAL_ORDER
class TestRequestsDuringClose:
"""Requests sent while the spider is closing are processed as long as the
close waits for them, and refused afterwards."""
@coroutine_test
async def test_from_callback(self) -> None:
"""Callbacks of the responses in progress when a graceful stop starts
can still download requests and schedule them."""
reached: Deferred[None] = Deferred()
proceed: Deferred[None] = Deferred()
bodies: list[bytes] = []
class CallbackSpider(Spider):
name = "callback"
async def start(self) -> AsyncIterator[Request]:
yield Request("data:,a")
async def parse(self, response: Response) -> None:
reached.callback(None)
await maybe_deferred_to_future(proceed)
assert self.crawler.engine is not None
response = await self.crawler.engine.download_async(Request("data:,b"))
bodies.append(response.body)
self.crawler.engine.crawl(Request("data:,c"))
crawler = get_crawler(CallbackSpider)
crawl_dfd = deferred_from_coro(crawler.crawl_async())
await maybe_deferred_to_future(reached)
stop_dfd = deferred_from_coro(crawler.stop_async())
await sleep(0)
assert crawler.engine is not None
assert_state(crawler.engine, EngineState.SPIDER_CLOSING)
proceed.callback(None)
await maybe_deferred_to_future(stop_dfd)
await maybe_deferred_to_future(crawl_dfd)
assert bodies == [b"b"]
assert crawler.stats.get_value("scheduler/enqueued") == 2
assert crawler.stats.get_value("finish_reason") == "shutdown"
@coroutine_test
async def test_replacement_request(self, caplog: pytest.LogCaptureFixture) -> None:
"""When a downloader middleware replaces a request downloaded with
download_async() after the close has stopped waiting for the requests
in progress, the replacement is not downloaded."""
reached: Deferred[None] = Deferred()
proceed: Deferred[None] = Deferred()
errors: list[Exception] = []
class ReplacingMiddleware:
async def process_request(self, request: Request) -> None:
if request.url == "data:,1":
reached.callback(None)
await maybe_deferred_to_future(proceed)
def process_response(
self, request: Request, response: Response
) -> Request | Response:
if request.url == "data:,1":
return Request("data:,2")
return response
async def download() -> None:
assert crawler.engine is not None
try:
await crawler.engine.download_async(Request("data:,1"))
except Exception as e:
errors.append(e)
def engine_started() -> None:
_schedule_coro(download())
crawler = get_crawler(
DefaultSpider,
{"DOWNLOADER_MIDDLEWARES": {ReplacingMiddleware: 0}},
)
crawler.signals.connect(engine_started, signals.engine_started)
crawl_dfd = deferred_from_coro(crawler.crawl_async())
await maybe_deferred_to_future(reached)
stop_dfd = deferred_from_coro(crawler.stop_async())
await sleep(0)
assert crawler.engine is not None
assert_state(crawler.engine, EngineState.SPIDER_CLOSING)
proceed.callback(None)
await maybe_deferred_to_future(stop_dfd)
await maybe_deferred_to_future(crawl_dfd)
assert len(errors) == 1
assert isinstance(errors[0], RuntimeError)
assert "Cannot download <GET data:,2>: the spider is closing" in str(errors[0])
assert not [r for r in caplog.records if r.levelno >= logging.ERROR]
assert crawler.stats.get_value("downloader/request_count") == 1
assert crawler.stats.get_value("finish_reason") == "shutdown"
@pytest.mark.parametrize("source", ["pipeline", "spider_closed"])
@coroutine_test
async def test_download_after_drain(
self, source: str, caplog: pytest.LogCaptureFixture
) -> None:
"""Pipelines' close_spider() and spider_closed handlers run after the
downloads have finished and cannot start new ones."""
errors: list[Exception] = []
async def download() -> None:
assert crawler.engine is not None
try:
await crawler.engine.download_async(Request("data:,"))
except Exception as e:
errors.append(e)
class DownloadingPipeline:
async def close_spider(self) -> None:
await download()
if source == "pipeline":
crawler = get_crawler(
DefaultSpider, {"ITEM_PIPELINES": {DownloadingPipeline: 0}}
)
else:
crawler = get_crawler(DefaultSpider)
crawler.signals.connect(download, signals.spider_closed)
await crawler.crawl_async()
assert len(errors) == 1
assert isinstance(errors[0], RuntimeError)
assert "spider is closing" in str(errors[0])
assert not [r for r in caplog.records if r.levelno >= logging.ERROR]
assert crawler.stats.get_value("finish_reason") == "finished"
@pytest.mark.parametrize("source", ["pipeline", "spider_closed"])
@coroutine_test
async def test_crawl_after_drain(self, source: str) -> None:
"""Pipelines' close_spider() and spider_closed handlers run after the
downloads have finished and cannot schedule new ones."""
errors: list[Exception] = []
def crawl() -> None:
assert crawler.engine is not None
try:
crawler.engine.crawl(Request("data:,"))
except Exception as e:
errors.append(e)
class CrawlingPipeline:
def close_spider(self) -> None:
crawl()
if source == "pipeline":
crawler = get_crawler(
DefaultSpider, {"ITEM_PIPELINES": {CrawlingPipeline: 0}}
)
else:
crawler = get_crawler(DefaultSpider)
crawler.signals.connect(crawl, signals.spider_closed)
await crawler.crawl_async()
assert len(errors) == 1
assert isinstance(errors[0], RuntimeError)
assert "spider is closing" in str(errors[0])
assert crawler.stats.get_value("scheduler/enqueued") is None
class TestDownloadHandlersClose:
"""The download handlers are closed with the downloader, also when the
engine is never started."""
@pytest.fixture
def closes(self) -> list[None]:
return []
@pytest.fixture
def settings(self, closes: list[None]) -> dict[str, Any]:
class Handler(DataURIDownloadHandler):
async def close(self) -> None:
closes.append(None)
return {"DOWNLOAD_HANDLERS": {"data": Handler}}
@coroutine_test
async def test_crawl(self, settings: dict[str, Any], closes: list[None]) -> None:
crawler = get_crawler(DefaultSpider, settings)
await crawler.crawl_async()
assert len(closes) == 1
@coroutine_test
async def test_crawl_custom_downloader(
self, settings: dict[str, Any], closes: list[None]
) -> None:
"""The handlers of a custom downloader that does not subclass the
default one are closed when the engine stops."""
class CustomDownloader:
def __init__(self, crawler: Crawler) -> None:
self.active: set[Request] = set()
self.handlers = DownloadHandlers(crawler)
def fetch(self, request: Request) -> Deferred[Response]:
raise NotImplementedError
def needs_backout(self) -> bool:
return False
def close(self) -> None:
pass
crawler = get_crawler(
DefaultSpider, {**settings, "DOWNLOADER": CustomDownloader}
)
await crawler.crawl_async()
assert len(closes) == 1
@coroutine_test
async def test_close_while_opening(
self, settings: dict[str, Any], closes: list[None]
) -> None:
crawler = get_crawler(DefaultSpider, settings)
async def spider_opened(**kwargs: Any) -> None:
assert crawler.engine is not None
await crawler.engine.download_async(Request("data:,"))
raise CloseSpider("early")
crawler.signals.connect(spider_opened, signals.spider_opened)
await crawler.crawl_async()
assert crawler.engine is not None
assert_state(crawler.engine, EngineState.STOPPED)
assert len(closes) == 1
@coroutine_test
async def test_close_created(
self, settings: dict[str, Any], closes: list[None]
) -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider, settings))
await engine.close_async()
assert len(closes) == 1
class TestInvalidTransitions:
"""Calls that the state does not allow raise instead of transitioning, and
a transition that the transition table does not allow is logged."""
@coroutine_test
async def test_transition_not_in_table_is_logged(
self, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
with caplog.at_level(logging.WARNING, logger="scrapy.core.engine"):
engine._transition_to(EngineState.RUNNING)
assert "Invalid engine state transition: CREATED → RUNNING" in caplog.text
assert_state(engine, EngineState.RUNNING)
await engine.downloader._close_async()
@coroutine_test
async def test_stop_not_opened(self) -> None:
crawler = get_crawler(DefaultSpider)
engine = ExecutionEngine(crawler)
with pytest.raises(RuntimeError, match="Spider not opened"):
await engine.stop_async()
await engine.close_async()
@coroutine_test
async def test_double_open(self) -> None:
crawler = get_crawler(DefaultSpider)
engine = make_engine(crawler)
await engine.open_spider_async()
with pytest.raises(RuntimeError, match="No free spider slot"):
await engine.open_spider_async()
await engine.close_spider_async()
@coroutine_test
async def test_open_after_stop(self) -> None:
crawler = get_crawler(DefaultSpider)
engine = make_engine(crawler)
await engine.close_async()
assert_state(engine, EngineState.STOPPED)
with pytest.raises(RuntimeError, match="engine has already been stopped"):
await engine.open_spider_async()
@coroutine_test
async def test_start_after_stop(self) -> None:
crawler = get_crawler(DefaultSpider)
engine = make_engine(crawler)
await engine.open_spider_async()
await engine.close_async()
with pytest.raises(RuntimeError, match="in the STOPPED state"):
await engine.start_async()
@pytest.mark.parametrize(
("signal", "message"),
[
(
signals.spider_opened,
"Cannot start the engine in the SPIDER_OPENING state",
),
(
signals.spider_closed,
"Cannot start the engine in the SPIDER_CLOSING state",
),
],
)
@coroutine_test
async def test_start_while_opening_or_closing(
self, signal: object, message: str
) -> None:
crawler = get_crawler(DefaultSpider)
engine = make_engine(crawler)
errors: list[str] = []
async def start(**kwargs: Any) -> None:
try:
await engine.start_async()
except RuntimeError as exc:
errors.append(str(exc))
crawler.signals.connect(start, signal)
await engine.open_spider_async()
await engine.close_spider_async()
assert errors == [message]
@coroutine_test
async def test_close_spider_never_opened(self) -> None:
crawler = get_crawler(DefaultSpider)
engine = ExecutionEngine(crawler)
with pytest.raises(RuntimeError, match="Spider not opened"):
await engine.close_spider_async()
await engine.close_async()
class TestCloseAsync:
"""close_async() must work, and clean everything up, in every state."""
@coroutine_test
async def test_created(self) -> None:
crawler = get_crawler(DefaultSpider)
engine = ExecutionEngine(crawler)
engine.downloader.close = Mock(wraps=engine.downloader.close) # type: ignore[method-assign]
await engine.close_async()
assert_state(engine, EngineState.STOPPED)
engine.downloader.close.assert_called()
@coroutine_test
async def test_created_custom_downloader(self) -> None:
"""A custom downloader that does not subclass the default one is closed
too."""
class CustomDownloader:
def __init__(self, crawler: Crawler) -> None:
self.close = Mock()
def fetch(self, request: Request) -> Deferred[Response]:
raise NotImplementedError
crawler = get_crawler(DefaultSpider, {"DOWNLOADER": CustomDownloader})
engine = ExecutionEngine(crawler)
await engine.close_async()
assert_state(engine, EngineState.STOPPED)
assert isinstance(engine.downloader, CustomDownloader)
engine.downloader.close.assert_called()
@coroutine_test
async def test_open(self) -> None:
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
await engine.open_spider_async()
await engine.close_async()
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NEVER_STARTED_SIGNAL_ORDER
assert recorder.close_reasons == ["shutdown"]
@coroutine_test
async def test_closing(self) -> None:
"""close_async() while the spider is closing leaves the cleanup to
that close instead of waiting for it."""
crawler = get_crawler(
DefaultSpider, settings_dict={"SCHEDULER": BlockingScheduler}
)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
await engine.open_spider_async()
assert engine._slot is not None
scheduler = engine._slot.scheduler
assert isinstance(scheduler, BlockingScheduler)
close_dfd = deferred_from_coro(engine.close_spider_async(reason="first"))
await maybe_deferred_to_future(scheduler.entered_close)
await engine.close_async()
assert_state(engine, EngineState.SPIDER_CLOSING)
scheduler.unblock_close.callback(None)
await maybe_deferred_to_future(close_dfd)
assert_state(engine, EngineState.STOPPED)
assert recorder.close_reasons == ["first"]
@coroutine_test
async def test_running(self) -> None:
crawler = get_crawler(DefaultSpider)
recorder = SignalRecorder(crawler)
engine = make_engine(crawler)
await engine.open_spider_async(close_if_idle=False)
start_dfd = await start_engine(engine)
await engine.close_async(reason="early")
assert_state(engine, EngineState.STOPPED)
assert recorder.names == NORMAL_SIGNAL_ORDER
assert recorder.close_reasons == ["early"]
await maybe_deferred_to_future(start_dfd)
class TestDeprecated:
@coroutine_test
async def test_running_setter(self) -> None:
crawler = get_crawler(DefaultSpider)
engine = ExecutionEngine(crawler)
with pytest.warns(
ScrapyDeprecationWarning,
match="Setting ExecutionEngine.running is deprecated",
):
engine.running = True
assert engine.running is False # setting it has no effect
await engine.close_async()
@coroutine_test
async def test_spider_closed_callback(self) -> None:
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
callback = Mock()
with pytest.warns(ScrapyDeprecationWarning, match="spider_closed_callback"):
engine = crawler.engine = ExecutionEngine(crawler, callback)
await engine.open_spider_async()
await engine.close_spider_async()
callback.assert_called_once_with(crawler.spider)