Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add errors parameter for CsvItemExporter with tests

maranqza2d6fa5adc17
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 95518b3ac..8cd2077b6 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -195,7 +195,7 @@ class XmlItemExporter(BaseItemExporter):
class CsvItemExporter(BaseItemExporter):
- def __init__(self, file, include_headers_line=True, join_multivalued=',', **kwargs):
+ def __init__(self, file, include_headers_line=True, join_multivalued=',', errors=None, **kwargs):
super().__init__(dont_fail=True, **kwargs)
if not self.encoding:
self.encoding = 'utf-8'
@@ -205,7 +205,8 @@ class CsvItemExporter(BaseItemExporter):
line_buffering=False,
write_through=True,
encoding=self.encoding,
- newline='' # Windows needs this https://github.com/scrapy/scrapy/issues/3034
+ newline='', # Windows needs this https://github.com/scrapy/scrapy/issues/3034
+ errors=errors,
)
self.csv_writer = csv.writer(self.stream, **self._kwargs)
self._headers_not_written = True
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 6c25a0064..ebc477e74 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -355,6 +355,23 @@ class CsvItemExporterTest(BaseItemExporterTest):
expected='22,False,3.14,2015-01-01 01:01:01\r\n'
)
+ def test_errors_default(self):
+ with self.assertRaises(UnicodeEncodeError):
+ self.assertExportResult(
+ item=dict(text=u'W\u0275\u200Brd'),
+ expected=None,
+ encoding='windows-1251',
+ )
+
+ def test_errors_xmlcharrefreplace(self):
+ self.assertExportResult(
+ item=dict(text=u'W\u0275\u200Brd'),
+ include_headers_line=False,
+ expected='Wɵ​rd\r\n',
+ encoding='windows-1251',
+ errors='xmlcharrefreplace',
+ )
+
class CsvItemExporterDataclassTest(CsvItemExporterTest):
item_class = TestDataClass