Scrapy, a fast high-level web crawling & scraping framework for Python.
Export item fields in declaration order (#7824)
Adrian98696efa809d
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rstindex ecd154122..c43b7e20f 100644--- a/docs/topics/exporters.rst+++ b/docs/topics/exporters.rst@@ -211,6 +211,16 @@ BaseItemExporter- ``None`` (all fields [2]_, default)+ Fields are exported in declaration order, i.e. the order in which+ they are defined in the :ref:`item class <item-types>`. For+ :class:`dict` items, which have no declared fields, the key order of+ each item is used instead.++ .. versionchanged:: VERSION+ Fields of non-\ :class:`dict` items used to be exported in the+ order in which they had been populated, except in+ :class:`CsvItemExporter`, which has always used declaration order.+- A list of fields:.. code-block:: pythondiff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 5a11df833..ea600d1a8 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -74,6 +74,17 @@ class BaseItemExporter(ABC):def finish_exporting(self) -> None: # noqa: B027pass+ @staticmethod+ def _get_populated_field_names(adapter: ItemAdapter) -> Iterable[str]:+ """Return the populated field names of *adapter*, in declaration order.++ Populated fields that are not declared, which some item types allow,+ come last, in item order.+ """+ populated = set(adapter.keys())+ declared = (name for name in adapter.field_names() if name in populated)+ return dict.fromkeys([*declared, *adapter.keys()])+def _get_serialized_fields(self, item: Any, default_value: Any = None, include_empty: bool | None = None) -> Iterable[tuple[str, Any]]:@@ -86,7 +97,11 @@ class BaseItemExporter(ABC):include_empty = self.export_empty_fieldsif self.fields_to_export is None:- field_iter = item.field_names() if include_empty else item.keys()+ field_iter = (+ item.field_names()+ if include_empty+ else self._get_populated_field_names(item)+ )elif isinstance(self.fields_to_export, Mapping):if include_empty:field_iter = self.fields_to_export.items()diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 3b997767e..b857728ba 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -118,6 +118,18 @@ class TestBaseItemExporter(ABC):ie = self._get_exporter(fields_to_export={"name": "名稱"})assert list(ie._get_serialized_fields(self.i)) == [("名稱", "John\xa3")]+ def test_field_order(self):+ item = self.item_class(age="22", name="John\xa3")+ ie = self._get_exporter()+ assert [name for name, _ in ie._get_serialized_fields(item)] == ["name", "age"]++ def test_field_order_dict_item(self):+ ie = self._get_exporter()+ assert [name for name, _ in ie._get_serialized_fields({"age": "22"})] == ["age"]+ assert [+ name for name, _ in ie._get_serialized_fields({"age": "22", "name": "John"})+ ] == ["age", "name"]+def test_field_custom_serializer(self):i = self.custom_field_item_class(name="John\xa3", age="22")a = ItemAdapter(i)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 92c414bef..1cca287af 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -675,14 +675,14 @@ class TestFeedExport(TestFeedExportBase):formats = {"csv": b"foo,egg,baz\r\nbar1,spam1,\r\n",- "json": b'[\n{"hello": "world2", "foo": "bar2"}\n]',+ "json": b'[\n{"foo": "bar2", "hello": "world2"}\n]',"jsonlines": (- b'{"foo": "bar1", "egg": "spam1"}\n{"hello": "world2", "foo": "bar2"}\n'+ b'{"foo": "bar1", "egg": "spam1"}\n{"foo": "bar2", "hello": "world2"}\n'),"xml": (b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'- b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>"- b"world2</hello><foo>bar2</foo></item>\n<item><hello>world3"+ b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><foo>"+ b"bar2</foo><hello>world2</hello></item>\n<item><hello>world3"b"</hello><egg>spam3</egg></item>\n</items>"),}@@ -740,8 +740,8 @@ class TestFeedExport(TestFeedExportBase):"json": b'[\n{"foo": "bar1", "egg": "spam1"}\n]',"xml": (b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'- b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>"- b"world2</hello><foo>bar2</foo></item>\n</items>"+ b"<foo>bar1</foo><egg>spam1</egg></item>\n<item><foo>"+ b"bar2</foo><hello>world2</hello></item>\n</items>"),"jsonlines": b'{"foo": "bar1", "egg": "spam1"}\n',}