Scrapy, a fast high-level web crawling & scraping framework for Python.
Document the ITEM_PROCESSOR setting (#7983)
Adrian65b37286cc56
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex 27ef3f7ef..010c9d1c6 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1529,6 +1529,20 @@ Default: ``{}``A dict containing the pipelines enabled by default in Scrapy. You should nevermodify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.+.. setting:: ITEM_PROCESSOR++ITEM_PROCESSOR+--------------++Default: ``"scrapy.pipelines.ItemPipelineManager"``++The :ref:`component <topics-components>` that builds the :ref:`item pipeline+<topics-item-pipeline>` from :setting:`ITEM_PIPELINES` and runs scraped items+through it. It must implement :class:`~scrapy.pipelines.ItemProcessorProtocol`.++.. autoclass:: scrapy.pipelines.ItemProcessorProtocol+ :members:+.. setting:: JOBDIRdiff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.pyindex 383c461e6..e6483d939 100644--- a/scrapy/pipelines/__init__.py+++ b/scrapy/pipelines/__init__.py@@ -8,7 +8,7 @@ from __future__ import annotationsimport asyncioimport warnings-from typing import TYPE_CHECKING, Any, cast+from typing import TYPE_CHECKING, Any, Protocol, castfrom twisted.internet.defer import Deferred, DeferredList, FirstError@@ -28,6 +28,23 @@ if TYPE_CHECKING:from scrapy.settings import Settings+class ItemProcessorProtocol(Protocol):+ """Protocol for item processor implementations.++ See :setting:`ITEM_PROCESSOR`.+ """++ async def open_spider_async(self) -> None:+ """Get the item processor ready to process items."""++ async def process_item_async(self, item: Any) -> Any:+ """Return the processed *item*, or raise+ :exc:`~scrapy.exceptions.DropItem` to drop it."""++ async def close_spider_async(self) -> None:+ """Release any resource that the item processor is using."""++class ItemPipelineManager(MiddlewareManager):component_name = "item pipeline"