Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix silent data loss in CsvItemExporter (closes #4002) (#7651)

Shaquon Kelley9d9950df6947
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index c3540d724..5a11df833 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -5,6 +5,7 @@ Item Exporters are used to export/serialize items into different formats.
from __future__ import annotations
import csv
+import logging
import marshal
import pickle
import pprint
@@ -24,6 +25,8 @@ from scrapy.utils.serialize import ScrapyJSONEncoder
if TYPE_CHECKING:
from json import JSONEncoder
+logger = logging.getLogger(__name__)
+
__all__ = [
"BaseItemExporter",
"CsvItemExporter",
@@ -254,6 +257,8 @@ class CsvItemExporter(BaseItemExporter):
self.csv_writer = csv.writer(self.stream, **self._kwargs)
self._headers_not_written = True
self._join_multivalued = join_multivalued
+ self._autodetected_fields = False
+ self._data_loss_warned = False
def serialize_field(
self, field: Mapping[str, Any] | Field, name: str, value: Any
@@ -274,6 +279,22 @@ class CsvItemExporter(BaseItemExporter):
self._headers_not_written = False
self._write_headers_and_set_fields_to_export(item)
+ if (
+ self._autodetected_fields
+ and self.fields_to_export is not None
+ and not self._data_loss_warned
+ ):
+ item_fields = ItemAdapter(item).field_names()
+ dropped_fields = set(item_fields) - set(self.fields_to_export)
+
+ if dropped_fields:
+ dropped_fields_display = sorted(dropped_fields)
+ logger.warning(
+ f"CSVExporter dropped fields {dropped_fields_display}. "
+ f"To avoid this, fully configure your FEED_EXPORT_FIELDS setting. "
+ f"See: https://docs.scrapy.org/en/latest/topics/feed-exports.html#feed-export-fields",
+ )
+ self._data_loss_warned = True
fields = self._get_serialized_fields(item, default_value="", include_empty=True)
values = list(self._build_row(x for _, x in fields))
self.csv_writer.writerow(values)
@@ -293,6 +314,7 @@ class CsvItemExporter(BaseItemExporter):
if not self.fields_to_export:
# use declared field names, or keys if the item is a dict
self.fields_to_export = ItemAdapter(item).field_names()
+ self._autodetected_fields = True
fields: Iterable[str]
if isinstance(self.fields_to_export, Mapping):
fields = self.fields_to_export.values()
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 877e0708b..3b997767e 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -384,6 +384,20 @@ class TestCsvItemExporter(TestBaseItemExporter):
errors="xmlcharrefreplace",
)
+ def test_csv_dropped_fields_warning(self, caplog):
+ out = BytesIO()
+ exporter = CsvItemExporter(out)
+ exporter.start_exporting()
+
+ exporter.export_item({"name": "Apple"})
+
+ with caplog.at_level("WARNING", logger="scrapy.exporters"):
+ exporter.export_item({"name": "Banana", "price": 2.00})
+
+ assert len(caplog.records) == 1
+ assert "CSVExporter dropped fields" in caplog.text
+ assert "price" in caplog.text
+
class TestCsvItemExporterDataclass(TestCsvItemExporter):
item_class = MyDataClass