Scrapy, a fast high-level web crawling & scraping framework for Python.
from __future__ import annotations
import asyncio
from typing import TYPE_CHECKING, Any
import pytest
from twisted.internet.defer import Deferred, fail, succeed
from scrapy import Request, Spider, signals
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.pipelines import ItemPipelineManager
from scrapy.utils.asyncio import call_later
from scrapy.utils.conf import build_component_list
from scrapy.utils.defer import deferred_to_future, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler, get_from_asyncio_queue
from tests.utils.decorators import coroutine_test
if TYPE_CHECKING:
from scrapy.crawler import Crawler
from tests.mockserver.http import MockServer
class SimplePipeline:
def process_item(self, item):
item["pipeline_passed"] = True
return item
class DeprecatedSpiderArgPipeline:
def open_spider(self, spider):
pass
def close_spider(self, spider):
pass
def process_item(self, item, spider):
item["pipeline_passed"] = True
return item
class DeferredPipeline:
def cb(self, item):
item["pipeline_passed"] = True
return item
def open_spider(self):
return succeed(None)
def close_spider(self):
return succeed(None)
def process_item(self, item):
d: Deferred[Any] = Deferred()
d.addCallback(self.cb)
d.callback(item)
return d
class AsyncDefPipeline:
async def process_item(self, item):
d: Deferred[Any] = Deferred()
call_later(0, d.callback, None)
await maybe_deferred_to_future(d)
item["pipeline_passed"] = True
return item
class AsyncDefAsyncioPipeline:
async def process_item(self, item):
d: Deferred[Any] = Deferred()
loop = asyncio.get_running_loop()
loop.call_later(0, d.callback, None)
await deferred_to_future(d)
await asyncio.sleep(0.2)
item["pipeline_passed"] = await get_from_asyncio_queue(True)
return item
class AsyncDefNotAsyncioPipeline:
async def process_item(self, item):
d1: Deferred[Any] = Deferred()
from twisted.internet import reactor
reactor.callLater(0, d1.callback, None)
await d1
d2: Deferred[Any] = Deferred()
reactor.callLater(0, d2.callback, None)
await maybe_deferred_to_future(d2)
item["pipeline_passed"] = True
return item
class ProcessItemExceptionPipeline:
def process_item(self, item):
raise ValueError("process_item error")
class ProcessItemExceptionDeferredPipeline:
def process_item(self, item):
return fail(ValueError("process_item error"))
class ProcessItemExceptionAsyncPipeline:
async def process_item(self, item):
raise ValueError("process_item error")
class OpenSpiderExceptionPipeline:
def open_spider(self):
raise ValueError("open_spider error")
class OpenSpiderExceptionDeferredPipeline:
def open_spider(self):
return fail(ValueError("open_spider error"))
class OpenSpiderExceptionAsyncPipeline:
async def open_spider(self):
raise ValueError("open_spider error")
class ItemSpider(Spider):
name = "itemspider"
mockserver: MockServer
async def start(self):
yield Request(self.mockserver.url("/status?n=200"))
def parse(self, response):
return {"field": 42}
class TestPipeline:
def _on_item_scraped(self, item: dict[str, Any]) -> None:
assert isinstance(item, dict)
assert item.get("pipeline_passed")
self.items.append(item)
def _create_crawler(self, pipeline_class: type) -> Crawler:
settings = {
"ITEM_PIPELINES": {pipeline_class: 1},
}
crawler = get_crawler(ItemSpider, settings)
crawler.signals.connect(self._on_item_scraped, signals.item_scraped)
self.items: list[Any] = []
return crawler
@pytest.mark.parametrize(
"pipeline_class",
[
SimplePipeline,
AsyncDefPipeline,
pytest.param(AsyncDefAsyncioPipeline, marks=pytest.mark.only_asyncio),
pytest.param(
AsyncDefNotAsyncioPipeline, marks=pytest.mark.only_not_asyncio
),
],
)
@coroutine_test
async def test_pipeline(self, mockserver: MockServer, pipeline_class: type) -> None:
crawler = self._create_crawler(pipeline_class)
await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 1
@coroutine_test
async def test_pipeline_deferred(self, mockserver: MockServer) -> None:
crawler = self._create_crawler(DeferredPipeline)
with (
pytest.warns(
ScrapyDeprecationWarning,
match="DeferredPipeline.open_spider returned a Deferred",
),
pytest.warns(
ScrapyDeprecationWarning,
match="DeferredPipeline.close_spider returned a Deferred",
),
pytest.warns(
ScrapyDeprecationWarning,
match="DeferredPipeline.process_item returned a Deferred",
),
):
await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 1
@coroutine_test
async def test_deprecated_spider_arg(self, mockserver: MockServer) -> None:
crawler = self._create_crawler(DeprecatedSpiderArgPipeline)
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument",
),
):
await crawler.crawl_async(mockserver=mockserver)
assert len(self.items) == 1
@pytest.mark.parametrize(
"pipeline_class",
[
ProcessItemExceptionPipeline,
pytest.param(
ProcessItemExceptionDeferredPipeline,
marks=pytest.mark.filterwarnings(
"ignore::scrapy.exceptions.ScrapyDeprecationWarning"
),
),
ProcessItemExceptionAsyncPipeline,
],
)
@coroutine_test
async def test_process_item_exception(
self,
caplog: pytest.LogCaptureFixture,
mockserver: MockServer,
pipeline_class: type,
) -> None:
crawler = self._create_crawler(pipeline_class)
await crawler.crawl_async(mockserver=mockserver)
assert "Error processing {'field': 42}" in caplog.text
assert "process_item error" in caplog.text
@pytest.mark.parametrize(
"pipeline_class",
[
OpenSpiderExceptionPipeline,
pytest.param(
OpenSpiderExceptionDeferredPipeline,
marks=pytest.mark.filterwarnings(
"ignore::scrapy.exceptions.ScrapyDeprecationWarning"
),
),
OpenSpiderExceptionAsyncPipeline,
],
)
@coroutine_test
async def test_open_spider_exception(
self, mockserver: MockServer, pipeline_class: type
) -> None:
crawler = self._create_crawler(pipeline_class)
with pytest.raises(ValueError, match="open_spider error"):
await crawler.crawl_async(mockserver=mockserver)
class TestCustomPipelineManager:
@coroutine_test
async def test_deprecated_process_item_spider_arg(self) -> None:
class CustomPipelineManager(ItemPipelineManager):
def process_item(self, item: Any, spider: Spider) -> Deferred[Any]: # pylint: disable=useless-parent-delegation
return super().process_item(item, spider)
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
itemproc = build_from_crawler(CustomPipelineManager, crawler)
with pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager.process_item\(\) is deprecated, use process_item_async\(\)",
):
itemproc.process_item({}, crawler.spider)
@coroutine_test
async def test_integration_recommended(self, mockserver: MockServer) -> None:
class CustomPipelineManager(ItemPipelineManager):
async def process_item_async(self, item):
return await super().process_item_async(item)
items = []
def _on_item_scraped(item: dict[str, Any]) -> None:
assert isinstance(item, dict)
assert item.get("pipeline_passed")
items.append(item)
crawler = get_crawler(
ItemSpider,
{
"ITEM_PROCESSOR": CustomPipelineManager,
"ITEM_PIPELINES": {SimplePipeline: 1},
},
)
crawler.spider = crawler._create_spider()
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
await crawler.crawl_async(mockserver=mockserver)
assert len(items) == 1
@coroutine_test
async def test_integration_no_async_subclass(self, mockserver: MockServer) -> None:
class CustomPipelineManager(ItemPipelineManager):
def open_spider(self, spider):
with pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\)",
):
return super().open_spider(spider)
def close_spider(self, spider):
with pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\)",
):
return super().close_spider(spider)
def process_item(self, item, spider):
with pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager.process_item\(\) is deprecated, use process_item_async\(\)",
):
return super().process_item(item, spider)
items = []
def _on_item_scraped(item: dict[str, Any]) -> None:
assert isinstance(item, dict)
assert item.get("pipeline_passed")
items.append(item)
crawler = get_crawler(
ItemSpider,
{
"ITEM_PROCESSOR": CustomPipelineManager,
"ITEM_PIPELINES": {SimplePipeline: 1},
},
)
crawler.spider = crawler._create_spider()
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager overrides open_spider\(\) but doesn't override open_spider_async\(\)",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager overrides close_spider\(\) but doesn't override close_spider_async\(\)",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager overrides process_item\(\) but doesn't override process_item_async\(\)",
),
):
await crawler.crawl_async(mockserver=mockserver)
assert len(items) == 1
@coroutine_test
async def test_integration_no_async_not_subclass(
self, mockserver: MockServer
) -> None:
class CustomPipelineManager:
def __init__(self, crawler):
self.pipelines = [
p()
for p in build_component_list(
crawler.settings.getwithbase("ITEM_PIPELINES")
)
]
@classmethod
def from_crawler(cls, crawler):
return cls(crawler)
def open_spider(self, spider):
return succeed(None)
def close_spider(self, spider):
return succeed(None)
def process_item(self, item, spider):
for pipeline in self.pipelines:
item = pipeline.process_item(item)
return succeed(item)
items = []
def _on_item_scraped(item: dict[str, Any]) -> None:
assert isinstance(item, dict)
assert item.get("pipeline_passed")
items.append(item)
crawler = get_crawler(
ItemSpider,
{
"ITEM_PROCESSOR": CustomPipelineManager,
"ITEM_PIPELINES": {SimplePipeline: 1},
},
)
crawler.spider = crawler._create_spider()
crawler.signals.connect(_on_item_scraped, signals.item_scraped)
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager doesn't define a open_spider_async\(\) method",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager doesn't define a close_spider_async\(\) method",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"CustomPipelineManager doesn't define a process_item_async\(\) method",
),
):
await crawler.crawl_async(mockserver=mockserver)
assert len(items) == 1
class TestMiddlewareManagerSpider:
"""Tests for the deprecated spider arg handling in MiddlewareManager.
Here because MiddlewareManager doesn't have methods that could take a spider arg."""
@pytest.fixture
def crawler(self) -> Crawler:
return get_crawler(Spider)
@coroutine_test
async def test_deprecated_spider_arg_no_crawler_spider(
self, crawler: Crawler
) -> None:
"""Crawler is provided, but doesn't have a spider, the methods raise an exception.
The instance passed to a deprecated method is ignored."""
mwman = ItemPipelineManager(crawler=crawler)
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument",
),
):
mwman._add_middleware(DeprecatedSpiderArgPipeline())
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead",
),
pytest.raises(
ValueError,
match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",
),
):
await maybe_deferred_to_future(mwman.open_spider(DefaultSpider()))
with pytest.raises(
ValueError,
match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",
):
await mwman.open_spider_async()
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead",
),
pytest.raises(
ValueError,
match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",
),
):
await maybe_deferred_to_future(mwman.close_spider(DefaultSpider()))
with pytest.raises(
ValueError,
match=r"ItemPipelineManager needs to access self\.crawler\.spider but it is None",
):
await mwman.close_spider_async()
@coroutine_test
async def test_deprecated_spider_arg_with_crawler(self, crawler: Crawler) -> None:
"""Crawler is provided and has a spider, works. The instance passed to a deprecated method
is ignored, even if mismatched."""
mwman = ItemPipelineManager(crawler=crawler)
crawler.spider = crawler._create_spider("foo")
with pytest.warns(
ScrapyDeprecationWarning,
match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead",
):
await maybe_deferred_to_future(mwman.open_spider(DefaultSpider()))
with pytest.warns(
ScrapyDeprecationWarning,
match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead",
):
await maybe_deferred_to_future(mwman.close_spider(DefaultSpider()))
@coroutine_test
async def test_deprecated_spider_arg_without_crawler(self) -> None:
"""The first instance passed to a deprecated method is used. Mismatched ones raise an error."""
with pytest.warns(
ScrapyDeprecationWarning,
match="was called without the crawler argument",
):
mwman = ItemPipelineManager()
spider = DefaultSpider()
with pytest.warns(
ScrapyDeprecationWarning,
match=r"ItemPipelineManager.open_spider\(\) is deprecated, use open_spider_async\(\) instead",
):
await maybe_deferred_to_future(mwman.open_spider(spider))
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead",
),
pytest.raises(
RuntimeError, match="Different instances of Spider were passed"
),
):
await maybe_deferred_to_future(mwman.close_spider(DefaultSpider()))
with pytest.warns(
ScrapyDeprecationWarning,
match=r"ItemPipelineManager.close_spider\(\) is deprecated, use close_spider_async\(\) instead",
):
await maybe_deferred_to_future(mwman.close_spider(spider))
@coroutine_test
async def test_no_spider_arg_without_crawler(self) -> None:
"""If no crawler and no spider arg, raise an error."""
with pytest.warns(
ScrapyDeprecationWarning,
match="was called without the crawler argument",
):
mwman = ItemPipelineManager()
with (
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.open_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.close_spider\(\) requires a spider argument",
),
pytest.warns(
ScrapyDeprecationWarning,
match=r"DeprecatedSpiderArgPipeline.process_item\(\) requires a spider argument",
),
):
mwman._add_middleware(DeprecatedSpiderArgPipeline())
with (
pytest.raises(
ValueError,
match="has no known Spider instance",
),
):
await mwman.open_spider_async()