Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import asyncio
import logging
import subprocess
import sys
from typing import TYPE_CHECKING, Any
from unittest.mock import AsyncMock, Mock, patch
import pytest
from twisted.internet.defer import succeed
from twisted.python.failure import Failure
from scrapy import signals
from scrapy.core.downloader import Downloader
from scrapy.core.downloader.handlers.base import BaseDownloadHandler
from scrapy.core.engine import EngineState, ExecutionEngine, _Slot
from scrapy.core.scheduler import BaseScheduler
from scrapy.exceptions import (
CloseSpider,
DontCloseSpider,
DownloadCancelledError,
IgnoreRequest,
)
from scrapy.http import Request, Response
from scrapy.spiders import Spider
from scrapy.utils.asyncio import sleep
from scrapy.utils.defer import (
_schedule_coro,
deferred_from_coro,
maybe_deferred_to_future,
)
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.spiders import SingleRequestSpider
from tests.utils.bases.engine import TestEngineBase
from tests.utils.decorators import coroutine_test, inline_callbacks_test
from tests.utils.engine import (
AttrsItemsSpider,
CrawlerRun,
DataClassItemsSpider,
DictItemsSpider,
MySpider,
)
if TYPE_CHECKING:
from collections.abc import AsyncIterator, Generator
from twisted.internet.defer import Deferred
from scrapy.crawler import Crawler
from tests.mockserver.http import MockServer
class DupeFilterSpider(MySpider):
async def start(self):
for url in self.start_urls:
yield Request(url) # no dont_filter=True
class ItemZeroDivisionErrorSpider(MySpider):
custom_settings = {
"ITEM_PIPELINES": {
"tests.pipelines.ProcessWithZeroDivisionErrorPipeline": 300,
}
}
class ChangeCloseReasonSpider(MySpider):
@classmethod
def from_crawler(cls, crawler, *args, **kwargs):
spider = cls(*args, **kwargs)
spider._set_crawler(crawler)
crawler.signals.connect(spider.spider_idle, signals.spider_idle)
return spider
def spider_idle(self):
raise CloseSpider(reason="custom_reason")
class ErrorCloseReasonSpider(ChangeCloseReasonSpider):
def spider_idle(self):
raise CloseSpider(reason="custom_reason", error=True)
class TestEngine(TestEngineBase):
@pytest.mark.parametrize(
"spider", [MySpider, DictItemsSpider, AttrsItemsSpider, DataClassItemsSpider]
)
@coroutine_test
async def test_crawler(
self, spider: type[MySpider], mockserver: MockServer
) -> None:
run = CrawlerRun(spider)
await run.run(mockserver)
self._assert_visited_urls(run)
self._assert_scheduled_requests(run, count=9)
self._assert_downloaded_responses(run, count=9)
self._assert_scraped_items(run)
self._assert_signals_caught(run)
self._assert_headers_received(run)
self._assert_bytes_received(run)
@coroutine_test
async def test_crawler_dupefilter(self, mockserver: MockServer) -> None:
run = CrawlerRun(DupeFilterSpider)
await run.run(mockserver)
self._assert_scheduled_requests(run, count=8)
self._assert_dropped_requests(run)
@coroutine_test
async def test_crawler_itemerror(self, mockserver: MockServer) -> None:
run = CrawlerRun(ItemZeroDivisionErrorSpider)
await run.run(mockserver)
self._assert_items_error(run)
@coroutine_test
async def test_crawler_change_close_reason_on_idle(
self, mockserver: MockServer
) -> None:
run = CrawlerRun(ChangeCloseReasonSpider)
await run.run(mockserver)
assert {
"spider": run.crawler.spider,
"reason": "custom_reason",
"error": False,
} == run.signals_caught[signals.spider_closed]
@coroutine_test
async def test_crawler_change_close_reason_and_error_on_idle(
self, mockserver: MockServer
) -> None:
run = CrawlerRun(ErrorCloseReasonSpider)
await run.run(mockserver)
assert {
"spider": run.crawler.spider,
"reason": "custom_reason",
"error": True,
} == run.signals_caught[signals.spider_closed]
assert run.crawler.stats.get_value("finish_reason_error") is True
@coroutine_test
async def test_close_downloader(self):
e = ExecutionEngine(get_crawler(MySpider))
await e.close_async()
def test_close_without_downloader(self):
class CustomException(Exception):
pass
class BadDownloader:
def __init__(self, crawler):
raise CustomException
with pytest.raises(CustomException):
ExecutionEngine(get_crawler(MySpider, {"DOWNLOADER": BadDownloader}))
@inline_callbacks_test
def test_start_already_running_exception(self):
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
e = ExecutionEngine(crawler)
crawler.engine = e
yield deferred_from_coro(e.open_spider_async())
_schedule_coro(e.start_async())
with pytest.raises(RuntimeError, match=r"in the (STARTING|RUNNING) state"):
yield deferred_from_coro(e.start_async())
yield deferred_from_coro(e.stop_async())
@coroutine_test
async def test_stop_async_force_mode_not_supported(self) -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
with pytest.raises(ValueError, match="force stop mode is not supported"):
await engine.stop_async(mode="force")
@coroutine_test
async def test_stop_async_not_running_raises(self) -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
with pytest.raises(RuntimeError, match="Spider not opened"):
await engine.stop_async()
@coroutine_test
async def test_stop_async_reentrant_fast_drops_downloads(self) -> None:
"""A fast stop requested while the spider is already closing drops
the in-flight downloads, and returns without waiting for the close."""
engine = ExecutionEngine(get_crawler(DefaultSpider))
engine._state = EngineState.SPIDER_CLOSING
with patch.object(
engine, "_fast_stop_downloader", new_callable=AsyncMock
) as fast_stop:
await engine.stop_async(mode="fast")
fast_stop.assert_awaited_once()
assert engine._stop_mode == "fast"
@coroutine_test
async def test_stop_async_reentrant_graceful_is_noop(self) -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
engine._state = EngineState.SPIDER_CLOSING
with patch.object(
engine, "_fast_stop_downloader", new_callable=AsyncMock
) as fast_stop:
await engine.stop_async(mode="graceful")
fast_stop.assert_not_called()
@coroutine_test
async def test_handle_downloader_output_ignores_fast_cancelled_failures(
self,
) -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
engine.spider = Mock()
engine._stop_mode = "fast"
enqueue_scrape = Mock()
engine.scraper.enqueue_scrape = enqueue_scrape # type: ignore[method-assign]
result = Failure(DownloadCancelledError("dropped during fast stop"))
await maybe_deferred_to_future(
engine._handle_downloader_output(result, Request("https://example.com"))
)
enqueue_scrape.assert_not_called()
@pytest.mark.only_asyncio
@coroutine_test
async def test_start_already_running_exception_asyncio(self):
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
e = ExecutionEngine(crawler)
crawler.engine = e
await e.open_spider_async()
with pytest.raises(RuntimeError, match=r"in the (STARTING|RUNNING) state"):
await asyncio.gather(e.start_async(), e.start_async())
await e.stop_async()
@coroutine_test
async def test_start_request_processing_exception(
self, caplog: pytest.LogCaptureFixture
) -> None:
class BadRequestFingerprinter:
def fingerprint(self, request):
raise ValueError # to make Scheduler.enqueue_request() fail
class SimpleSpider(Spider):
name = "simple"
async def start(self):
yield Request("data:,")
crawler = get_crawler(
SimpleSpider, {"REQUEST_FINGERPRINTER_CLASS": BadRequestFingerprinter}
)
with caplog.at_level(logging.DEBUG):
await crawler.crawl_async()
assert "Error while processing requests from start()" in caplog.text
assert "Spider closed (shutdown)" in caplog.text
def test_short_timeout(self):
args = (
sys.executable,
"-m",
"scrapy.cmdline",
"fetch",
"-s",
"CLOSESPIDER_TIMEOUT=0.001",
"-s",
"LOG_LEVEL=DEBUG",
"http://toscrape.com",
)
p = subprocess.Popen(
args,
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
)
try:
_, stderr = p.communicate(timeout=15)
except subprocess.TimeoutExpired:
p.kill()
p.communicate()
pytest.fail("Command took too much time to complete")
stderr_str = stderr.decode()
assert "AttributeError" not in stderr_str, stderr_str
assert "AssertionError" not in stderr_str, stderr_str
@coroutine_test
async def test_request_scheduled_signal():
class TestScheduler(BaseScheduler):
def __init__(self) -> None:
self.enqueued: list[Request] = []
def enqueue_request(self, request: Request) -> bool:
self.enqueued.append(request)
return True
def signal_handler(request: Request, spider: Spider) -> None:
if "drop" in request.url:
raise IgnoreRequest
crawler = get_crawler(MySpider)
engine = ExecutionEngine(crawler)
scheduler = build_from_crawler(TestScheduler, crawler)
async def start() -> AsyncIterator[Any]:
return
yield
engine._start = start()
engine._slot = _Slot(False, Mock(), scheduler)
crawler.signals.connect(signal_handler, signals.request_scheduled)
keep_request = Request("https://keep.example")
engine._schedule_request(keep_request)
drop_request = Request("https://drop.example")
engine._schedule_request(drop_request)
assert scheduler.enqueued == [keep_request], (
f"{scheduler.enqueued!r} != [{keep_request!r}]"
)
crawler.signals.disconnect(signal_handler, signals.request_scheduled)
class ClosingPipeline:
def open_spider(self):
raise CloseSpider("pipeline_reason")
class TestCloseSpiderOnStartup:
@coroutine_test
async def test_pipeline(self, caplog: pytest.LogCaptureFixture) -> None:
closed: list[str] = []
def spider_closed(reason: str) -> None:
closed.append(reason)
crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
crawler.signals.connect(spider_closed, signals.spider_closed)
with caplog.at_level(logging.INFO):
await crawler.crawl_async()
assert crawler.stats.get_value("finish_reason") == "pipeline_reason"
assert closed == ["pipeline_reason"]
assert "Traceback" not in caplog.text
@coroutine_test
async def test_spider_opened(self) -> None:
def spider_opened(spider: Spider) -> None:
raise CloseSpider("signal_reason")
crawler = get_crawler(DefaultSpider)
crawler.signals.connect(spider_opened, signals.spider_opened)
await crawler.crawl_async()
assert crawler.stats.get_value("finish_reason") == "signal_reason"
@coroutine_test
async def test_startup_wins_over_spider_opened(self) -> None:
def spider_opened(spider: Spider) -> None:
raise CloseSpider("signal_reason")
crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
crawler.signals.connect(spider_opened, signals.spider_opened)
await crawler.crawl_async()
assert crawler.stats.get_value("finish_reason") == "pipeline_reason"
@inline_callbacks_test
def test_deferred_crawl(self) -> Generator[Deferred[Any], Any, None]:
crawler = get_crawler(DefaultSpider, {"ITEM_PIPELINES": {ClosingPipeline: 1}})
yield crawler.crawl()
assert crawler.stats.get_value("finish_reason") == "pipeline_reason"
class TestMisuse:
"""The engine raises on operations that its state does not allow."""
@coroutine_test
async def test_stop_not_running(self) -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
try:
with pytest.raises(RuntimeError, match="Spider not opened"):
await engine.stop_async()
finally:
await engine.close_async()
def test_invalid_scheduler_class(self) -> None:
class NotAScheduler:
pass
with pytest.raises(
TypeError, match="does not fully implement the scheduler interface"
):
ExecutionEngine(get_crawler(DefaultSpider, {"SCHEDULER": NotAScheduler}))
@coroutine_test
async def test_spider_is_idle_without_slot(self) -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
try:
with pytest.raises(RuntimeError, match="Engine slot not assigned"):
engine.spider_is_idle()
finally:
await engine.close_async()
@coroutine_test
async def test_crawl_without_spider(self) -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
try:
with pytest.raises(RuntimeError, match="No open spider to crawl"):
engine.crawl(Request("data:,"))
finally:
await engine.close_async()
@coroutine_test
async def test_open_spider_twice(self) -> None:
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
engine = crawler.engine = ExecutionEngine(crawler)
await engine.open_spider_async()
try:
with pytest.raises(RuntimeError, match="No free spider slot"):
await engine.open_spider_async()
finally:
await engine.close_async()
@coroutine_test
async def test_progress_before_open() -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
try:
assert engine.requests_in_progress == frozenset()
assert engine.items_in_progress == 0
assert engine.response_active_size == 0
finally:
await engine.close_async()
class BrokenScheduler(BaseScheduler):
"""A scheduler that cannot be built."""
@classmethod
def from_crawler(cls, crawler: Any) -> BrokenScheduler:
raise ValueError("broken scheduler")
def has_pending_requests(self) -> bool:
return False
def enqueue_request(self, request: Request) -> bool:
return True
def next_request(self) -> Request | None:
return None
@coroutine_test
async def test_scheduler_creation_error() -> None:
"""An error while building the scheduler is reported directly, instead of
other errors such as "Engine slot not assigned"."""
crawler = get_crawler(DefaultSpider, {"SCHEDULER": BrokenScheduler})
with pytest.raises(ValueError, match="broken scheduler"):
await crawler.crawl_async()
assert crawler.engine is not None
assert crawler.engine.spider is None
@coroutine_test
async def test_start_without_spider() -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
try:
with pytest.raises(RuntimeError, match="in the CREATED state"):
await engine.start_async()
finally:
await engine.close_async()
@coroutine_test
async def test_stop_while_item_in_pipelines() -> None:
"""A stop that starts while an item is going through the item pipelines
completes once that item is done."""
class StoppingPipeline:
async def process_item(self, item: Any) -> Any:
_schedule_coro(crawler.stop_async())
await sleep(0.1)
return item
class ItemSpider(Spider):
name = "item"
async def start(self) -> AsyncIterator[Any]:
yield {"a": "b"}
crawler = get_crawler(ItemSpider, {"ITEM_PIPELINES": {StoppingPipeline: 1}})
await crawler.crawl_async()
assert crawler.stats.get_value("item_scraped_count") == 1
assert crawler.stats.get_value("finish_reason") == "shutdown"
@coroutine_test
async def test_pause_unpause() -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))
try:
engine.pause()
assert engine.paused
engine.unpause()
assert not engine.paused
finally:
await engine.close_async()
class TestSpiderIdle:
@coroutine_test
async def test_dont_close_spider(self) -> None:
"""A ``DontCloseSpider`` handler keeps the spider open for one more loop."""
idle_calls = 0
def spider_idle() -> None:
nonlocal idle_calls
idle_calls += 1
if idle_calls == 1:
# Schedule a request so that the engine loops again soon
# instead of waiting for the slot heartbeat.
crawler.engine.crawl(Request("data:,"))
raise DontCloseSpider
crawler = get_crawler(DefaultSpider)
crawler.signals.connect(spider_idle, signals.spider_idle)
await crawler.crawl_async()
assert idle_calls == 2
assert crawler.stats.get_value("finish_reason") == "finished"
@coroutine_test
async def test_not_idle_anymore(self) -> None:
"""A handler that schedules a request keeps the spider open."""
urls: list[str] = []
def spider_idle() -> None:
if not urls:
urls.append("data:,")
crawler.engine.crawl(Request(urls[0]))
crawler = get_crawler(DefaultSpider)
crawler.signals.connect(spider_idle, signals.spider_idle)
await crawler.crawl_async()
assert crawler.stats.get_value("downloader/request_count") == 1
@coroutine_test
async def test_request_not_in_downloader(self) -> None:
"""The spider is not idle while a request taken from the scheduler is
not in the downloader yet."""
class SlowFetchDownloader(Downloader):
def fetch(self, request: Request) -> Deferred[Response | Request]: # type: ignore[override]
return deferred_from_coro(self._slow_fetch(request))
async def _slow_fetch(self, request: Request) -> Response | Request:
await sleep(0.1)
return await maybe_deferred_to_future(super().fetch(request))
events: list[str] = []
def response_received() -> None:
events.append("response_received")
def spider_idle() -> None:
events.append("spider_idle")
crawler = get_crawler(SingleRequestSpider, {"DOWNLOADER": SlowFetchDownloader})
crawler.signals.connect(response_received, signals.response_received)
crawler.signals.connect(spider_idle, signals.spider_idle)
await crawler.crawl_async(seed="data:,")
assert events == ["response_received", "spider_idle"]
@coroutine_test
async def test_request_only_in_downloader(self) -> None:
"""The spider is not idle while the downloader has a request that was
not sent through the engine."""
class SlowDownloadHandler(BaseDownloadHandler):
async def download_request(self, request: Request) -> Response:
await sleep(0.1)
return Response(request.url)
events: list[str] = []
class DirectFetchSpider(Spider):
name = "direct_fetch"
async def start(self) -> AsyncIterator[Any]:
yield Request("data:,")
def parse(self, response: Response) -> None:
assert self.crawler.engine is not None
d = self.crawler.engine.downloader.fetch(Request("slow://example.com"))
d.addCallback(self.fetched)
def fetched(self, result: Response | Request) -> None:
events.append("fetched")
# Schedule a request so that the engine loops again soon
# instead of waiting for the slot heartbeat.
assert self.crawler.engine is not None
self.crawler.engine.crawl(Request("data:,b", callback=self.noop))
def noop(self, response: Response) -> None:
pass
def spider_idle() -> None:
events.append("spider_idle")
crawler = get_crawler(
DirectFetchSpider, {"DOWNLOAD_HANDLERS": {"slow": SlowDownloadHandler}}
)
crawler.signals.connect(spider_idle, signals.spider_idle)
await crawler.crawl_async()
assert events == ["fetched", "spider_idle"]
@coroutine_test
async def test_download_wrong_type(caplog: pytest.LogCaptureFixture) -> None:
class WrongTypeDownloader(Downloader):
def fetch(self, request: Request) -> str: # type: ignore[override]
return "not a response"
class WrongTypeSpider(Spider):
name = "wrong_type"
async def start(self):
yield Request("data:,")
crawler = get_crawler(WrongTypeSpider, {"DOWNLOADER": WrongTypeDownloader})
await crawler.crawl_async()
assert "expected Response or Request, got <class 'str'>" in caplog.text
@coroutine_test
async def test_download_async_custom_downloader() -> None:
class CustomDownloader:
def __init__(self, crawler: Crawler) -> None:
self.active: set[Request] = set()
def fetch(self, request: Request) -> Deferred[Response]:
return succeed(Response(request.url))
def needs_backout(self) -> bool:
return False
def close(self) -> None:
pass
responses: list[Response] = []
class DownloadAsyncSpider(Spider):
name = "download_async"
async def start(self):
assert self.crawler.engine is not None
responses.append(
await self.crawler.engine.download_async(Request("data:,"))
)
return
yield
crawler = get_crawler(DownloadAsyncSpider, {"DOWNLOADER": CustomDownloader})
await crawler.crawl_async()
assert [response.url for response in responses] == ["data:,"]
@coroutine_test
async def test_enqueue_scrape_error(
caplog: pytest.LogCaptureFixture, monkeypatch: pytest.MonkeyPatch
) -> None:
def enqueue_scrape(self, result, request):
raise ValueError("enqueue error")
monkeypatch.setattr("scrapy.core.engine.Scraper.enqueue_scrape", enqueue_scrape)
class SimpleSpider(Spider):
name = "simple"
async def start(self):
yield Request("data:,")
crawler = get_crawler(SimpleSpider)
await crawler.crawl_async()
assert "Error while enqueuing scrape" in caplog.text