Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'upstream/master' into allow-customizing-export-column-names

Adrián Chaves5138f9a96540
diff --cc scrapy/exporters.py
index 5e36c3a96,95518b3ac..f404549fc
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@@ -256,17 -243,9 +256,13 @@@ class CsvItemExporter(BaseItemExporter)
def _write_headers_and_set_fields_to_export(self, item):
if self.include_headers_line:
if not self.fields_to_export:
- if isinstance(item, dict):
- # for dicts try using fields of the first item
- self.fields_to_export = list(item.keys())
- else:
- # use fields declared in Item
- self.fields_to_export = list(item.fields.keys())
+ # use declared field names, or keys if the item is a dict
+ self.fields_to_export = ItemAdapter(item).field_names()
- row = list(self._build_row(self.fields_to_export))
+ if isinstance(self.fields_to_export, Mapping):
+ fields = self.fields_to_export.values()
+ else:
+ fields = self.fields_to_export
+ row = list(self._build_row(fields))
self.csv_writer.writerow(row)
diff --cc scrapy/utils/conf.py
index 6a6d38a5c,a83076c47..8f9cb43c6
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@@ -113,8 -115,9 +115,9 @@@ def get_sources(use_closest=True)
def feed_complete_default_values_from_settings(feed, settings):
out = feed.copy()
+ out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
- out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+ out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
if settings["FEED_EXPORT_INDENT"] is None:
diff --cc tests/test_exporters.py
index 83148159f,6c25a0064..171045596
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@@ -80,29 -108,19 +109,27 @@@ class BaseItemExporterTest(unittest.Tes
ie = self._get_exporter(fields_to_export=['name'], encoding='latin-1')
_, name = list(ie._get_serialized_fields(self.i))[0]
assert isinstance(name, str)
- self.assertEqual(name, u'John\xa3')
+ self.assertEqual(name, 'John\xa3')
+ ie = self._get_exporter(
+ fields_to_export=OrderedDict([('name', u'名稱')])
+ )
+ self.assertEqual(
+ list(ie._get_serialized_fields(self.i)),
+ [(u'名稱', u'John\xa3')]
+ )
+
def test_field_custom_serializer(self):
- def custom_serializer(value):
- return str(int(value) + 2)
-
- class CustomFieldItem(Item):
- name = Field()
- age = Field(serializer=custom_serializer)
+ i = self.custom_field_item_class(name='John\xa3', age='22')
+ a = ItemAdapter(i)
+ ie = self._get_exporter()
+ self.assertEqual(ie.serialize_field(a.get_field_meta('name'), 'name', a['name']), 'John\xa3')
+ self.assertEqual(ie.serialize_field(a.get_field_meta('age'), 'age', a['age']), '24')
- i = CustomFieldItem(name=u'John\xa3', age=u'22')
- ie = self._get_exporter()
- self.assertEqual(ie.serialize_field(i.fields['name'], 'name', i['name']), u'John\xa3')
- self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
+ class BaseItemExporterDataclassTest(BaseItemExporterTest):
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
class PythonItemExporterTest(BaseItemExporterTest):
@@@ -228,9 -261,13 +270,14 @@@ class MarshalItemExporterTest(BaseItemE
self.assertEqual(marshal.load(fp), item)
+ class MarshalItemExporterDataclassTest(MarshalItemExporterTest):
+ item_class = TestDataClass
+ custom_field_item_class = CustomFieldDataclass
+
+
class CsvItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
+ self.output = tempfile.TemporaryFile()
return CsvItemExporter(self.output, **kwargs)
def assertCsvEqual(self, first, second, msg=None):
@@@ -242,8 -279,7 +289,8 @@@
return self.assertEqual(split_csv(first), split_csv(second), msg=msg)
def _check_output(self):
- self.assertCsvEqual(to_unicode(self.output.getvalue()), 'age,name\r\n22,John\xa3\r\n')
+ self.output.seek(0)
- self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n')
++ self.assertCsvEqual(to_unicode(self.output.read()), 'age,name\r\n22,John\xa3\r\n')
def assertExportResult(self, item, expected, **kwargs):
fp = BytesIO()
diff --cc tests/test_feedexport.py
index f5d1b0843,b57349848..69a5730fa
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@@ -4,10 -4,10 +4,11 @@@ import o
import random
import shutil
import string
+import sys
import tempfile
import warnings
- from collections import OrderedDict
+ from abc import ABC, abstractmethod
-from collections import defaultdict
++from collections import defaultdict, OrderedDict
from io import BytesIO
from logging import getLogger
from pathlib import Path