Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix silent data loss in CsvItemExporter (closes #4002) (#7651)
Shaquon Kelley9d9950df6947
diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex c3540d724..5a11df833 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -5,6 +5,7 @@ Item Exporters are used to export/serialize items into different formats.from __future__ import annotationsimport csv+import loggingimport marshalimport pickleimport pprint@@ -24,6 +25,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoderif TYPE_CHECKING:from json import JSONEncoder+logger = logging.getLogger(__name__)+__all__ = ["BaseItemExporter","CsvItemExporter",@@ -254,6 +257,8 @@ class CsvItemExporter(BaseItemExporter):self.csv_writer = csv.writer(self.stream, **self._kwargs)self._headers_not_written = Trueself._join_multivalued = join_multivalued+ self._autodetected_fields = False+ self._data_loss_warned = Falsedef serialize_field(self, field: Mapping[str, Any] | Field, name: str, value: Any@@ -274,6 +279,22 @@ class CsvItemExporter(BaseItemExporter):self._headers_not_written = Falseself._write_headers_and_set_fields_to_export(item)+ if (+ self._autodetected_fields+ and self.fields_to_export is not None+ and not self._data_loss_warned+ ):+ item_fields = ItemAdapter(item).field_names()+ dropped_fields = set(item_fields) - set(self.fields_to_export)++ if dropped_fields:+ dropped_fields_display = sorted(dropped_fields)+ logger.warning(+ f"CSVExporter dropped fields {dropped_fields_display}. "+ f"To avoid this, fully configure your FEED_EXPORT_FIELDS setting. "+ f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",+ )+ self._data_loss_warned = Truefields = self._get_serialized_fields(item, default_value="", include_empty=True)values = list(self._build_row(x for _, x in fields))self.csv_writer.writerow(values)@@ -293,6 +314,7 @@ class CsvItemExporter(BaseItemExporter):if not self.fields_to_export:# use declared field names, or keys if the item is a dictself.fields_to_export = ItemAdapter(item).field_names()+ self._autodetected_fields = Truefields: Iterable[str]if isinstance(self.fields_to_export, Mapping):fields = self.fields_to_export.values()diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 877e0708b..3b997767e 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -384,6 +384,20 @@ class TestCsvItemExporter(TestBaseItemExporter):errors="xmlcharrefreplace",)+ def test_csv_dropped_fields_warning(self, caplog):+ out = BytesIO()+ exporter = CsvItemExporter(out)+ exporter.start_exporting()++ exporter.export_item({"name": "Apple"})++ with caplog.at_level("WARNING", logger="scrapy.exporters"):+ exporter.export_item({"name": "Banana", "price": 2.00})++ assert len(caplog.records) == 1+ assert "CSVExporter dropped fields" in caplog.text+ assert "price" in caplog.text+class TestCsvItemExporterDataclass(TestCsvItemExporter):item_class = MyDataClass