Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Export item fields in declaration order (#7824)

Adrian98696efa809d
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index ecd154122..c43b7e20f 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -211,6 +211,16 @@ BaseItemExporter
- ``None`` (all fields [2]_, default)
+ Fields are exported in declaration order, i.e. the order in which
+ they are defined in the :ref:`item class <item-types>`. For
+ :class:`dict` items, which have no declared fields, the key order of
+ each item is used instead.
+
+ .. versionchanged:: VERSION
+ Fields of non-\ :class:`dict` items used to be exported in the
+ order in which they had been populated, except in
+ :class:`CsvItemExporter`, which has always used declaration order.
+
- A list of fields:
.. code-block:: python
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 5a11df833..ea600d1a8 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -74,6 +74,17 @@ class BaseItemExporter(ABC):
def finish_exporting(self) -> None: # noqa: B027
pass
+ @staticmethod
+ def _get_populated_field_names(adapter: ItemAdapter) -> Iterable[str]:
+ """Return the populated field names of *adapter*, in declaration order.
+
+ Populated fields that are not declared, which some item types allow,
+ come last, in item order.
+ """
+ populated = set(adapter.keys())
+ declared = (name for name in adapter.field_names() if name in populated)
+ return dict.fromkeys([*declared, *adapter.keys()])
+
def _get_serialized_fields(
self, item: Any, default_value: Any = None, include_empty: bool | None = None
) -> Iterable[tuple[str, Any]]:
@@ -86,7 +97,11 @@ class BaseItemExporter(ABC):
include_empty = self.export_empty_fields
if self.fields_to_export is None:
- field_iter = item.field_names() if include_empty else item.keys()
+ field_iter = (
+ item.field_names()
+ if include_empty
+ else self._get_populated_field_names(item)
+ )
elif isinstance(self.fields_to_export, Mapping):
if include_empty:
field_iter = self.fields_to_export.items()
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 3b997767e..b857728ba 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -118,6 +118,18 @@ class TestBaseItemExporter(ABC):
ie = self._get_exporter(fields_to_export={"name": "名稱"})
assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")]
+ def test_field_order(self):
+ item = self.item_class(age="22", name="John\xa3")
+ ie = self._get_exporter()
+ assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"]
+
+ def test_field_order_dict_item(self):
+ ie = self._get_exporter()
+ assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"]
+ assert [
+ name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"})
+ ] == ["age", "name"]
+
def test_field_custom_serializer(self):
i = self.custom_field_item_class(name="John\xa3", age="22")
a = ItemAdapter(i)
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 92c414bef..1cca287af 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -675,14 +675,14 @@ class TestFeedExport(TestFeedExportBase):
formats = {
"csv": b"foo,egg,baz\r\nbar1,spam1,\r\n",
- "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]',
+ "json": b'[\n{"foo": "bar2", "hello": "world2"}\n]',
"jsonlines": (
- b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n'
+ b'{"foo": "bar1", "egg": "spam1"}\n{"foo": "bar2", "hello": "world2"}\n'
),
"xml": (
b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
- b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>"
- b"world2</hello><foo>bar2</foo></item>\n<item><hello>world3"
+ b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><foo>"
+ b"bar2</foo><hello>world2</hello></item>\n<item><hello>world3"
b"</hello><egg>spam3</egg></item>\n</items>"
),
}
@@ -740,8 +740,8 @@ class TestFeedExport(TestFeedExportBase):
"json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]',
"xml": (
b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
- b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>"
- b"world2</hello><foo>bar2</foo></item>\n</items>"
+ b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><foo>"
+ b"bar2</foo><hello>world2</hello></item>\n</items>"
),
"jsonlines": b'{"foo": "bar1", "egg": "spam1"}\n',
}