Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Document the ITEM_PROCESSOR setting (#7983)

Adrian65b37286cc56
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index 27ef3f7ef..010c9d1c6 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1529,6 +1529,20 @@ Default: ``{}``
A dict containing the pipelines enabled by default in Scrapy. You should never
modify this setting in your project, modify :setting:`ITEM_PIPELINES` instead.
+.. setting:: ITEM_PROCESSOR
+
+ITEM_PROCESSOR
+--------------
+
+Default: ``"scrapy.pipelines.ItemPipelineManager"``
+
+The :ref:`component <topics-components>` that builds the :ref:`item pipeline
+<topics-item-pipeline>` from :setting:`ITEM_PIPELINES` and runs scraped items
+through it. It must implement :class:`~scrapy.pipelines.ItemProcessorProtocol`.
+
+.. autoclass:: scrapy.pipelines.ItemProcessorProtocol
+ :members:
+
.. setting:: JOBDIR
diff --git a/scrapy/pipelines/__init__.py b/scrapy/pipelines/__init__.py
index 383c461e6..e6483d939 100644
--- a/scrapy/pipelines/__init__.py
+++ b/scrapy/pipelines/__init__.py
@@ -8,7 +8,7 @@ from __future__ import annotations
import asyncio
import warnings
-from typing import TYPE_CHECKING, Any, cast
+from typing import TYPE_CHECKING, Any, Protocol, cast
from twisted.internet.defer import Deferred, DeferredList, FirstError
@@ -28,6 +28,23 @@ if TYPE_CHECKING:
from scrapy.settings import Settings
+class ItemProcessorProtocol(Protocol):
+ """Protocol for item processor implementations.
+
+ See :setting:`ITEM_PROCESSOR`.
+ """
+
+ async def open_spider_async(self) -> None:
+ """Get the item processor ready to process items."""
+
+ async def process_item_async(self, item: Any) -> Any:
+ """Return the processed *item*, or raise
+ :exc:`~scrapy.exceptions.DropItem` to drop it."""
+
+ async def close_spider_async(self) -> None:
+ """Release any resource that the item processor is using."""
+
+
class ItemPipelineManager(MiddlewareManager):
component_name = "item pipeline"