Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'upstream/master' into allow-customizing-export-column-names
Adrián Chaves5138f9a96540
diff --cc scrapy/exporters.pyindex 5e36c3a96,95518b3ac..f404549fc--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@@ -256,17 -243,9 +256,13 @@@ class CsvItemExporter(BaseItemExporter)def _write_headers_and_set_fields_to_export(self, item):if self.include_headers_line:if not self.fields_to_export:- if isinstance(item, dict):- # for dicts try using fields of the first item- self.fields_to_export = list(item.keys())- else:- # use fields declared in Item- self.fields_to_export = list(item.fields.keys())+ # use declared field names, or keys if the item is a dict+ self.fields_to_export = ItemAdapter(item).field_names()- row = list(self._build_row(self.fields_to_export))+ if isinstance(self.fields_to_export, Mapping):+ fields = self.fields_to_export.values()+ else:+ fields = self.fields_to_export+ row = list(self._build_row(fields))self.csv_writer.writerow(row)diff --cc scrapy/utils/conf.pyindex 6a6d38a5c,a83076c47..8f9cb43c6--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@@ -113,8 -115,9 +115,9 @@@ def get_sources(use_closest=True)def feed_complete_default_values_from_settings(feed, settings):out = feed.copy()+ out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])- out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)+ out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))out.setdefault("uri_params", settings["FEED_URI_PARAMS"])if settings["FEED_EXPORT_INDENT"] is None:diff --cc tests/test_exporters.pyindex 83148159f,6c25a0064..171045596--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@@ -80,29 -108,19 +109,27 @@@ class BaseItemExporterTest(unittest.Tesie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')_, name = list(ie._get_serialized_fields(self.i))[0]assert isinstance(name, str)- self.assertEqual(name, u'John\xa3')+ self.assertEqual(name, 'John\xa3')+ ie = self._get_exporter(+ fields_to_export=OrderedDict([('name', u'名稱')])+ )+ self.assertEqual(+ list(ie._get_serialized_fields(self.i)),+ [(u'名稱', u'John\xa3')]+ )+def test_field_custom_serializer(self):- def custom_serializer(value):- return str(int(value) + 2)-- class CustomFieldItem(Item):- name = Field()- age = Field(serializer=custom_serializer)+ i = self.custom_field_item_class(name='John\xa3', age='22')+ a = ItemAdapter(i)+ ie = self._get_exporter()+ self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3')+ self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')- i = CustomFieldItem(name=u'John\xa3', age=u'22')- ie = self._get_exporter()- self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3')- self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')+ class BaseItemExporterDataclassTest(BaseItemExporterTest):+ item_class = TestDataClass+ custom_field_item_class = CustomFieldDataclassclass PythonItemExporterTest(BaseItemExporterTest):@@@ -228,9 -261,13 +270,14 @@@ class MarshalItemExporterTest(BaseItemEself.assertEqual(marshal.load(fp), item)+ class MarshalItemExporterDataclassTest(MarshalItemExporterTest):+ item_class = TestDataClass+ custom_field_item_class = CustomFieldDataclass++class CsvItemExporterTest(BaseItemExporterTest):def _get_exporter(self, **kwargs):+ self.output = tempfile.TemporaryFile()return CsvItemExporter(self.output, **kwargs)def assertCsvEqual(self, first, second, msg=None):@@@ -242,8 -279,7 +289,8 @@@return self.assertEqual(split_csv(first), split_csv(second), msg=msg)def _check_output(self):- self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')+ self.output.seek(0)- self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n')++ self.assertCsvEqual(to_unicode(self.output.read()), 'age,name\r\n22,John\xa3\r\n')def assertExportResult(self, item, expected, **kwargs):fp = BytesIO()diff --cc tests/test_feedexport.pyindex f5d1b0843,b57349848..69a5730fa--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@@ -4,10 -4,10 +4,11 @@@ import oimport randomimport shutilimport string+import sysimport tempfileimport warnings- from collections import OrderedDict+ from abc import ABC, abstractmethod-from collections import defaultdict++from collections import defaultdict, OrderedDictfrom io import BytesIOfrom logging import getLoggerfrom pathlib import Path