Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Document how to write a custom item exporter (#7931)

Adrian786ab1049431
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index c43b7e20f..56b995e18 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -136,6 +136,70 @@ Example:
return f"$ {str(value)}"
return super().serialize_field(field, name, value)
+.. _custom-exporters:
+
+Writing your own item exporter
+==============================
+
+To write an item exporter, subclass :class:`BaseItemExporter` and implement
+:meth:`~BaseItemExporter.export_item`, where
+:meth:`~BaseItemExporter.get_serialized_fields` gives you the ``(name, value)``
+pairs to export.
+
+To make your exporter available to the :ref:`feed exports
+<topics-feed-exports>`, list it in the :setting:`FEED_EXPORTERS` setting. Feed
+exports :ref:`build <from-crawler>` it with the output file as the first
+positional argument, and with the ``fields``, ``encoding`` and ``indent``
+:ref:`feed options <feed-options>` and every key of ``item_export_kwargs`` as
+keyword arguments, so your ``__init__`` method must forward unknown keyword
+arguments to :class:`BaseItemExporter`.
+
+The file object belongs to whoever opened it, i.e. to the feed storage in the
+case of feed exports, which also closes it. If you need a text file, for
+example to use :func:`csv.writer` or another Python API that does not accept a
+binary file, wrap it with :class:`io.TextIOWrapper` and call
+:meth:`~io.TextIOBase.detach` on the wrapper in
+:meth:`~BaseItemExporter.finish_exporting`; otherwise the wrapper closes the
+underlying file when it is garbage-collected.
+
+For example, the following item exporter writes items as blocks of
+``name: value`` lines:
+
+.. code-block:: python
+
+ from io import TextIOWrapper
+
+ from scrapy.exporters import BaseItemExporter
+
+
+ class TextItemExporter(BaseItemExporter):
+ def __init__(self, file, item_separator="\n", **kwargs):
+ super().__init__(**kwargs)
+ self.item_separator = item_separator
+ self.stream = TextIOWrapper(
+ file, encoding=self.encoding or "utf-8", write_through=True
+ )
+
+ def export_item(self, item):
+ for name, value in self.get_serialized_fields(item):
+ print(f"{name}: {value}", file=self.stream)
+ self.stream.write(self.item_separator)
+
+ def finish_exporting(self):
+ self.stream.detach()
+
+To use it as the ``txt`` feed format:
+
+.. code-block:: python
+
+ FEED_EXPORTERS = {"txt": "myproject.exporters.TextItemExporter"}
+ FEEDS = {
+ "items.txt": {
+ "format": "txt",
+ "item_export_kwargs": {"item_separator": "---\n"},
+ },
+ }
+
.. _topics-exporters-reference:
Built-in Item Exporters reference
@@ -168,6 +232,8 @@ BaseItemExporter
Exports the given item. This method must be implemented in subclasses.
+ .. automethod:: BaseItemExporter.get_serialized_fields
+
.. method:: serialize_field(field, name, value)
Return the serialized value for the given field. You can override this
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index ea600d1a8..7f8aaf059 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -85,11 +85,16 @@ class BaseItemExporter(ABC):
declared = (name for name in adapter.field_names() if name in populated)
return dict.fromkeys([*declared, *adapter.keys()])
- def _get_serialized_fields(
+ def get_serialized_fields(
self, item: Any, default_value: Any = None, include_empty: bool | None = None
) -> Iterable[tuple[str, Any]]:
- """Return the fields to export as an iterable of tuples
- (name, serialized_value)
+ """Return the fields of *item* to export, as an iterable of
+ ``(name, serialized_value)`` tuples, taking :attr:`fields_to_export`
+ into account and applying :meth:`serialize_field` to every value.
+
+ Fields missing from *item* are exported with *default_value*.
+
+ *include_empty* overrides :attr:`export_empty_fields`.
"""
item = ItemAdapter(item)
@@ -136,7 +141,7 @@ class JsonLinesItemExporter(BaseItemExporter):
self.encoder: JSONEncoder = ScrapyJSONEncoder(**self._kwargs)
def export_item(self, item: Any) -> None:
- itemdict = dict(self._get_serialized_fields(item))
+ itemdict = dict(self.get_serialized_fields(item))
data = self.encoder.encode(itemdict) + "\n"
self.file.write(to_bytes(data, self.encoding))
@@ -176,7 +181,7 @@ class JsonItemExporter(BaseItemExporter):
self.file.write(b"]")
def export_item(self, item: Any) -> None:
- itemdict = dict(self._get_serialized_fields(item))
+ itemdict = dict(self.get_serialized_fields(item))
data = to_bytes(self.encoder.encode(itemdict), self.encoding)
self._add_comma_after_first()
self.file.write(data)
@@ -216,7 +221,7 @@ class XmlItemExporter(BaseItemExporter):
self._beautify_indent(depth=1)
self.xg.startElement(self.item_element, AttributesImpl({}))
self._beautify_newline()
- for name, value in self._get_serialized_fields(item, default_value=""):
+ for name, value in self.get_serialized_fields(item, default_value=""):
self._export_xml_field(name, value, depth=2)
self._beautify_indent(depth=1)
self.xg.endElement(self.item_element)
@@ -310,7 +315,7 @@ class CsvItemExporter(BaseItemExporter):
f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",
)
self._data_loss_warned = True
- fields = self._get_serialized_fields(item, default_value="", include_empty=True)
+ fields = self.get_serialized_fields(item, default_value="", include_empty=True)
values = list(self._build_row(x for _, x in fields))
self.csv_writer.writerow(values)
@@ -347,7 +352,7 @@ class PickleItemExporter(BaseItemExporter):
self.protocol: int = protocol
def export_item(self, item: Any) -> None:
- d = dict(self._get_serialized_fields(item))
+ d = dict(self.get_serialized_fields(item))
pickle.dump(d, self.file, self.protocol)
@@ -365,7 +370,7 @@ class MarshalItemExporter(BaseItemExporter):
self.file: BytesIO = file
def export_item(self, item: Any) -> None:
- marshal.dump(dict(self._get_serialized_fields(item)), self.file)
+ marshal.dump(dict(self.get_serialized_fields(item)), self.file)
class PprintItemExporter(BaseItemExporter):
@@ -374,7 +379,7 @@ class PprintItemExporter(BaseItemExporter):
self.file: BytesIO = file
def export_item(self, item: Any) -> None:
- itemdict = dict(self._get_serialized_fields(item))
+ itemdict = dict(self.get_serialized_fields(item))
self.file.write(to_bytes(pprint.pformat(itemdict) + "\n"))
@@ -417,5 +422,5 @@ class PythonItemExporter(BaseItemExporter):
yield key, self._serialize_value(value)
def export_item(self, item: Any) -> dict[str | bytes, Any]: # type: ignore[override]
- result: dict[str | bytes, Any] = dict(self._get_serialized_fields(item))
+ result: dict[str | bytes, Any] = dict(self.get_serialized_fields(item))
return result
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index b857728ba..99afb8beb 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -108,26 +108,26 @@ class TestBaseItemExporter(ABC):
def test_fields_to_export(self):
ie = self._get_exporter(fields_to_export=["name"])
- assert list(ie._get_serialized_fields(self.i)) == [("name", "John\xa3")]
+ assert list(ie.get_serialized_fields(self.i)) == [("name", "John\xa3")]
ie = self._get_exporter(fields_to_export=["name"], encoding="latin-1")
- _, name = next(iter(ie._get_serialized_fields(self.i)))
+ _, name = next(iter(ie.get_serialized_fields(self.i)))
assert isinstance(name, str)
assert name == "John\xa3"
ie = self._get_exporter(fields_to_export={"name": "名稱"})
- assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")]
+ assert list(ie.get_serialized_fields(self.i)) == [("名稱", "John\xa3")]
def test_field_order(self):
item = self.item_class(age="22", name="John\xa3")
ie = self._get_exporter()
- assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"]
+ assert [name for name, _ in ie.get_serialized_fields(item)] == ["name", "age"]
def test_field_order_dict_item(self):
ie = self._get_exporter()
- assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"]
+ assert [name for name, _ in ie.get_serialized_fields({"age": "22"})] == ["age"]
assert [
- name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"})
+ name for name, _ in ie.get_serialized_fields({"age": "22", "name": "John"})
] == ["age", "name"]
def test_field_custom_serializer(self):