Scrapy, a fast high-level web crawling & scraping framework for Python.
Document how to write a custom item exporter (#7931)
Adrian786ab1049431
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rstindex c43b7e20f..56b995e18 100644--- a/docs/topics/exporters.rst+++ b/docs/topics/exporters.rst@@ -136,6 +136,70 @@ Example:return f"$ {str(value)}"return super().serialize_field(field, name, value)+.. _custom-exporters:++Writing your own item exporter+==============================++To write an item exporter, subclass :class:`BaseItemExporter` and implement+:meth:`~BaseItemExporter.export_item`, where+:meth:`~BaseItemExporter.get_serialized_fields` gives you the ``(name, value)``+pairs to export.++To make your exporter available to the :ref:`feed exports+<topics-feed-exports>`, list it in the :setting:`FEED_EXPORTERS` setting. Feed+exports :ref:`build <from-crawler>` it with the output file as the first+positional argument, and with the ``fields``, ``encoding`` and ``indent``+:ref:`feed options <feed-options>` and every key of ``item_export_kwargs`` as+keyword arguments, so your ``__init__`` method must forward unknown keyword+arguments to :class:`BaseItemExporter`.++The file object belongs to whoever opened it, i.e. to the feed storage in the+case of feed exports, which also closes it. If you need a text file, for+example to use :func:`csv.writer` or another Python API that does not accept a+binary file, wrap it with :class:`io.TextIOWrapper` and call+:meth:`~io.TextIOBase.detach` on the wrapper in+:meth:`~BaseItemExporter.finish_exporting`; otherwise the wrapper closes the+underlying file when it is garbage-collected.++For example, the following item exporter writes items as blocks of+``name: value`` lines:++.. code-block:: python++ from io import TextIOWrapper++ from scrapy.exporters import BaseItemExporter+++ class TextItemExporter(BaseItemExporter):+ def __init__(self, file, item_separator="\n", **kwargs):+ super().__init__(**kwargs)+ self.item_separator = item_separator+ self.stream = TextIOWrapper(+ file, encoding=self.encoding or "utf-8", write_through=True+ )++ def export_item(self, item):+ for name, value in self.get_serialized_fields(item):+ print(f"{name}: {value}", file=self.stream)+ self.stream.write(self.item_separator)++ def finish_exporting(self):+ self.stream.detach()++To use it as the ``txt`` feed format:++.. code-block:: python++ FEED_EXPORTERS = {"txt": "myproject.exporters.TextItemExporter"}+ FEEDS = {+ "items.txt": {+ "format": "txt",+ "item_export_kwargs": {"item_separator": "---\n"},+ },+ }+.. _topics-exporters-reference:Built-in Item Exporters reference@@ -168,6 +232,8 @@ BaseItemExporterExports the given item. This method must be implemented in subclasses.+ .. automethod:: BaseItemExporter.get_serialized_fields+.. method:: serialize_field(field, name, value)Return the serialized value for the given field. You can override thisdiff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex ea600d1a8..7f8aaf059 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -85,11 +85,16 @@ class BaseItemExporter(ABC):declared = (name for name in adapter.field_names() if name in populated)return dict.fromkeys([*declared, *adapter.keys()])- def _get_serialized_fields(+ def get_serialized_fields(self, item: Any, default_value: Any = None, include_empty: bool | None = None) -> Iterable[tuple[str, Any]]:- """Return the fields to export as an iterable of tuples- (name, serialized_value)+ """Return the fields of *item* to export, as an iterable of+ ``(name, serialized_value)`` tuples, taking :attr:`fields_to_export`+ into account and applying :meth:`serialize_field` to every value.++ Fields missing from *item* are exported with *default_value*.++ *include_empty* overrides :attr:`export_empty_fields`."""item = ItemAdapter(item)@@ -136,7 +141,7 @@ class JsonLinesItemExporter(BaseItemExporter):self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs)def export_item(self, item: Any) -> None:- itemdict = dict(self._get_serialized_fields(item))+ itemdict = dict(self.get_serialized_fields(item))data = self.encoder.encode(itemdict) + "\n"self.file.write(to_bytes(data, self.encoding))@@ -176,7 +181,7 @@ class JsonItemExporter(BaseItemExporter):self.file.write(b"]")def export_item(self, item: Any) -> None:- itemdict = dict(self._get_serialized_fields(item))+ itemdict = dict(self.get_serialized_fields(item))data = to_bytes(self.encoder.encode(itemdict), self.encoding)self._add_comma_after_first()self.file.write(data)@@ -216,7 +221,7 @@ class XmlItemExporter(BaseItemExporter):self._beautify_indent(depth=1)self.xg.startElement(self.item_element, AttributesImpl({}))self._beautify_newline()- for name, value in self._get_serialized_fields(item, default_value=""):+ for name, value in self.get_serialized_fields(item, default_value=""):self._export_xml_field(name, value, depth=2)self._beautify_indent(depth=1)self.xg.endElement(self.item_element)@@ -310,7 +315,7 @@ class CsvItemExporter(BaseItemExporter):f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",)self._data_loss_warned = True- fields = self._get_serialized_fields(item, default_value="", include_empty=True)+ fields = self.get_serialized_fields(item, default_value="", include_empty=True)values = list(self._build_row(x for _, x in fields))self.csv_writer.writerow(values)@@ -347,7 +352,7 @@ class PickleItemExporter(BaseItemExporter):self.protocol: int = protocoldef export_item(self, item: Any) -> None:- d = dict(self._get_serialized_fields(item))+ d = dict(self.get_serialized_fields(item))pickle.dump(d, self.file, self.protocol)@@ -365,7 +370,7 @@ class MarshalItemExporter(BaseItemExporter):self.file: BytesIO = filedef export_item(self, item: Any) -> None:- marshal.dump(dict(self._get_serialized_fields(item)), self.file)+ marshal.dump(dict(self.get_serialized_fields(item)), self.file)class PprintItemExporter(BaseItemExporter):@@ -374,7 +379,7 @@ class PprintItemExporter(BaseItemExporter):self.file: BytesIO = filedef export_item(self, item: Any) -> None:- itemdict = dict(self._get_serialized_fields(item))+ itemdict = dict(self.get_serialized_fields(item))self.file.write(to_bytes(pprint.pformat(itemdict) + "\n"))@@ -417,5 +422,5 @@ class PythonItemExporter(BaseItemExporter):yield key, self._serialize_value(value)def export_item(self, item: Any) -> dict[str | bytes, Any]: # type: ignore[override]- result: dict[str | bytes, Any] = dict(self._get_serialized_fields(item))+ result: dict[str | bytes, Any] = dict(self.get_serialized_fields(item))return resultdiff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex b857728ba..99afb8beb 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -108,26 +108,26 @@ class TestBaseItemExporter(ABC):def test_fields_to_export(self):ie = self._get_exporter(fields_to_export=["name"])- assert list(ie._get_serialized_fields(self.i)) == [("name", "John\xa3")]+ assert list(ie.get_serialized_fields(self.i)) == [("name", "John\xa3")]ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1")- _, name = next(iter(ie._get_serialized_fields(self.i)))+ _, name = next(iter(ie.get_serialized_fields(self.i)))assert isinstance(name, str)assert name == "John\xa3"ie = self._get_exporter(fields_to_export={"name": "名稱"})- assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")]+ assert list(ie.get_serialized_fields(self.i)) == [("名稱", "John\xa3")]def test_field_order(self):item = self.item_class(age="22", name="John\xa3")ie = self._get_exporter()- assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"]+ assert [name for name, _ in ie.get_serialized_fields(item)] == ["name", "age"]def test_field_order_dict_item(self):ie = self._get_exporter()- assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"]+ assert [name for name, _ in ie.get_serialized_fields({"age": "22"})] == ["age"]assert [- name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"})+ name for name, _ in ie.get_serialized_fields({"age": "22", "name": "John"})] == ["age", "name"]def test_field_custom_serializer(self):