Scrapy, a fast high-level web crawling & scraping framework for Python.
Recursively apply nested items' own serializers in item exporters (#8057)
Adrian4635b1dc56ee
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rstindex 07417bd6c..b87a99fbe 100644--- a/docs/topics/exporters.rst+++ b/docs/topics/exporters.rst@@ -247,10 +247,15 @@ BaseItemExportermethod (in your custom Item Exporters) if you want to control how aparticular field or value will be serialized/exported.+ .. versionchanged:: VERSION+ Nested items, and lists of them, are now serialized recursively+ using their own declared serializers, instead of unchanged.+By default, this method looks for a serializer :ref:`declared in the itemfield <topics-exporters-serializers>` and returns the result of applying- that serializer to the value. If no serializer is found, it returns the- value unchanged.+ that serializer to the value. If no serializer is found and the value is+ a nested item, or a list of them, each of them is serialized the same+ way; otherwise, the value is returned unchanged.:param field: the field being serialized. If the source :ref:`item object<item-types>` does not define field metadata, *field* is an emptydiff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 2667cf4be..5d4ceb93f 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -18,7 +18,6 @@ from xml.sax.xmlreader import AttributesImplfrom itemadapter import ItemAdapter, is_item-from scrapy.item import Field, Itemfrom scrapy.utils.python import is_listlike, to_unicodefrom scrapy.utils.serialize import ScrapyJSONEncoder@@ -26,6 +25,8 @@ if TYPE_CHECKING:from collections.abc import Coroutinefrom json import JSONEncoder+ from scrapy.item import Field+logger = logging.getLogger(__name__)__all__ = [@@ -74,9 +75,29 @@ class BaseItemExporter(ABC):def serialize_field(self, field: Mapping[str, Any] | Field, name: str, value: Any) -> Any:- serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x)+ serializer: Callable[[Any], Any] = field.get(+ "serializer", self._serialize_value+ )return serializer(value)+ def _serialize_value(self, value: Any) -> Any:+ if is_item(value):+ adapter = ItemAdapter(value)+ field_names = (+ adapter.field_names()+ if self.export_empty_fields+ else self._get_populated_field_names(adapter)+ )+ return {+ name: self.serialize_field(+ adapter.get_field_meta(name), name, adapter[name]+ )+ for name in field_names+ }+ if is_listlike(value):+ return [self._serialize_value(v) for v in value]+ return value+def start_exporting(self) -> Coroutine[Any, Any, None] | None: # noqa: B027"""Signal the beginning of the exporting process. Some exporters mayuse this to generate some required header (for example, the@@ -314,6 +335,7 @@ class CsvItemExporter(BaseItemExporter):return serializer(value)def _join_if_needed(self, value: Any) -> Any:+ value = self._serialize_value(value)if isinstance(value, (list, tuple)):try:return self._join_multivalued.join(value)@@ -411,11 +433,9 @@ class PprintItemExporter(BaseItemExporter):class PythonItemExporter(BaseItemExporter):- """This is a base class for item exporters that extends- :class:`BaseItemExporter` with support for nested items.-- It serializes items to built-in Python types, so that any serialization- library (e.g. :mod:`json` or msgpack_) can be used on top of it.+ """It serializes items to built-in Python types, so that any+ serialization library (e.g. :mod:`json` or msgpack_) can be used on top+ of it... _msgpack: https://pypi.org/project/msgpack/"""@@ -425,28 +445,10 @@ class PythonItemExporter(BaseItemExporter):if not self.encoding:self.encoding = "utf-8"- def serialize_field(- self, field: Mapping[str, Any] | Field, name: str, value: Any- ) -> Any:- serializer: Callable[[Any], Any] = field.get(- "serializer", self._serialize_value- )- return serializer(value)-def _serialize_value(self, value: Any) -> Any:- if isinstance(value, Item):- return self.export_item(value)if isinstance(value, (str, bytes)):return to_unicode(value, encoding=self.encoding)- if is_item(value):- return dict(self._serialize_item(value))- if is_listlike(value):- return [self._serialize_value(v) for v in value]- return value-- def _serialize_item(self, item: Any) -> Iterable[tuple[str | bytes, Any]]:- for key, value in ItemAdapter(item).items():- yield key, self._serialize_value(value)+ return super()._serialize_value(value)def export_item(self, item: Any) -> dict[str | bytes, Any]: # type: ignore[override]result: dict[str | bytes, Any] = dict(self.get_serialized_fields(item))diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 573d23752..f6f143771 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -145,6 +145,36 @@ class TestBaseItemExporter(ABC):)assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "24"+ def test_nested_item_field_custom_serializer(self):+ nested = self.custom_field_item_class(name="Nested\xa3", age="22")+ a = ItemAdapter(self.item_class(name="Top", age=nested))+ ie = self._get_exporter()+ assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == {+ "name": "Nested\xa3",+ "age": "24",+ }++ def test_nested_item_list_field_custom_serializer(self):+ nested = [+ self.custom_field_item_class(name="Nested\xa3", age="22"),+ self.custom_field_item_class(name="Other", age="1"),+ ]+ a = ItemAdapter(self.item_class(name="Top", age=nested))+ ie = self._get_exporter()+ assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == [+ {"name": "Nested\xa3", "age": "24"},+ {"name": "Other", "age": "3"},+ ]++ def test_nested_item_field_ignores_fields_to_export(self):+ nested = self.custom_field_item_class(name="Nested\xa3", age="22")+ a = ItemAdapter(self.item_class(name="Top", age=nested))+ ie = self._get_exporter(fields_to_export=["age"])+ assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == {+ "name": "Nested\xa3",+ "age": "24",+ }+class TestPythonItemExporter(TestBaseItemExporter):def _get_exporter(self, **kwargs: Any) -> PythonItemExporter: