Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Recursively apply nested items' own serializers in item exporters (#8057)

Adrian4635b1dc56ee
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index 07417bd6c..b87a99fbe 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -247,10 +247,15 @@ BaseItemExporter
method (in your custom Item Exporters) if you want to control how a
particular field or value will be serialized/exported.
+ .. versionchanged:: VERSION
+ Nested items, and lists of them, are now serialized recursively
+ using their own declared serializers, instead of unchanged.
+
By default, this method looks for a serializer :ref:`declared in the item
field <topics-exporters-serializers>` and returns the result of applying
- that serializer to the value. If no serializer is found, it returns the
- value unchanged.
+ that serializer to the value. If no serializer is found and the value is
+ a nested item, or a list of them, each of them is serialized the same
+ way; otherwise, the value is returned unchanged.
:param field: the field being serialized. If the source :ref:`item object
<item-types>` does not define field metadata, *field* is an empty
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 2667cf4be..5d4ceb93f 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -18,7 +18,6 @@ from xml.sax.xmlreader import AttributesImpl
from itemadapter import ItemAdapter, is_item
-from scrapy.item import Field, Item
from scrapy.utils.python import is_listlike, to_unicode
from scrapy.utils.serialize import ScrapyJSONEncoder
@@ -26,6 +25,8 @@ if TYPE_CHECKING:
from collections.abc import Coroutine
from json import JSONEncoder
+ from scrapy.item import Field
+
logger = logging.getLogger(__name__)
__all__ = [
@@ -74,9 +75,29 @@ class BaseItemExporter(ABC):
def serialize_field(
self, field: Mapping[str, Any] | Field, name: str, value: Any
) -> Any:
- serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x)
+ serializer: Callable[[Any], Any] = field.get(
+ "serializer", self._serialize_value
+ )
return serializer(value)
+ def _serialize_value(self, value: Any) -> Any:
+ if is_item(value):
+ adapter = ItemAdapter(value)
+ field_names = (
+ adapter.field_names()
+ if self.export_empty_fields
+ else self._get_populated_field_names(adapter)
+ )
+ return {
+ name: self.serialize_field(
+ adapter.get_field_meta(name), name, adapter[name]
+ )
+ for name in field_names
+ }
+ if is_listlike(value):
+ return [self._serialize_value(v) for v in value]
+ return value
+
def start_exporting(self) -> Coroutine[Any, Any, None] | None: # noqa: B027
"""Signal the beginning of the exporting process. Some exporters may
use this to generate some required header (for example, the
@@ -314,6 +335,7 @@ class CsvItemExporter(BaseItemExporter):
return serializer(value)
def _join_if_needed(self, value: Any) -> Any:
+ value = self._serialize_value(value)
if isinstance(value, (list, tuple)):
try:
return self._join_multivalued.join(value)
@@ -411,11 +433,9 @@ class PprintItemExporter(BaseItemExporter):
class PythonItemExporter(BaseItemExporter):
- """This is a base class for item exporters that extends
- :class:`BaseItemExporter` with support for nested items.
-
- It serializes items to built-in Python types, so that any serialization
- library (e.g. :mod:`json` or msgpack_) can be used on top of it.
+ """It serializes items to built-in Python types, so that any
+ serialization library (e.g. :mod:`json` or msgpack_) can be used on top
+ of it.
.. _msgpack: https://pypi.org/project/msgpack/
"""
@@ -425,28 +445,10 @@ class PythonItemExporter(BaseItemExporter):
if not self.encoding:
self.encoding = "utf-8"
- def serialize_field(
- self, field: Mapping[str, Any] | Field, name: str, value: Any
- ) -> Any:
- serializer: Callable[[Any], Any] = field.get(
- "serializer", self._serialize_value
- )
- return serializer(value)
-
def _serialize_value(self, value: Any) -> Any:
- if isinstance(value, Item):
- return self.export_item(value)
if isinstance(value, (str, bytes)):
return to_unicode(value, encoding=self.encoding)
- if is_item(value):
- return dict(self._serialize_item(value))
- if is_listlike(value):
- return [self._serialize_value(v) for v in value]
- return value
-
- def _serialize_item(self, item: Any) -> Iterable[tuple[str | bytes, Any]]:
- for key, value in ItemAdapter(item).items():
- yield key, self._serialize_value(value)
+ return super()._serialize_value(value)
def export_item(self, item: Any) -> dict[str | bytes, Any]: # type: ignore[override]
result: dict[str | bytes, Any] = dict(self.get_serialized_fields(item))
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 573d23752..f6f143771 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -145,6 +145,36 @@ class TestBaseItemExporter(ABC):
)
assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "24"
+ def test_nested_item_field_custom_serializer(self):
+ nested = self.custom_field_item_class(name="Nested\xa3", age="22")
+ a = ItemAdapter(self.item_class(name="Top", age=nested))
+ ie = self._get_exporter()
+ assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == {
+ "name": "Nested\xa3",
+ "age": "24",
+ }
+
+ def test_nested_item_list_field_custom_serializer(self):
+ nested = [
+ self.custom_field_item_class(name="Nested\xa3", age="22"),
+ self.custom_field_item_class(name="Other", age="1"),
+ ]
+ a = ItemAdapter(self.item_class(name="Top", age=nested))
+ ie = self._get_exporter()
+ assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == [
+ {"name": "Nested\xa3", "age": "24"},
+ {"name": "Other", "age": "3"},
+ ]
+
+ def test_nested_item_field_ignores_fields_to_export(self):
+ nested = self.custom_field_item_class(name="Nested\xa3", age="22")
+ a = ItemAdapter(self.item_class(name="Top", age=nested))
+ ie = self._get_exporter(fields_to_export=["age"])
+ assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == {
+ "name": "Nested\xa3",
+ "age": "24",
+ }
+
class TestPythonItemExporter(TestBaseItemExporter):
def _get_exporter(self, **kwargs: Any) -> PythonItemExporter: