Scrapy, a fast high-level web crawling & scraping framework for Python.
Allow customizing export column names
Adrián Chaves484bd0d22a11
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rstindex f5048d2da..42a93e459 100644--- a/docs/topics/exporters.rst+++ b/docs/topics/exporters.rst@@ -190,14 +190,33 @@ BaseItemExporter.. attribute:: fields_to_export- A list with the name of the fields that will be exported, or None if you- want to export all fields. Defaults to None.+ Fields to export, their order [1]_ and their output names.- Some exporters (like :class:`CsvItemExporter`) respect the order of the- fields defined in this attribute.+ Possible values are:- Some exporters may require fields_to_export list in order to export the- data properly when spiders return dicts (not :class:`~Item` instances).+ - ``None`` (all fields [2]_, default)++ - A list of fields::++ ['field1', 'field2']++ - A dict [3]_ where keys are fields and values are output names::++ {'field1': 'Field 1', 'field2': 'Field 2'}++ .. [1] Not all exporters respect the specified field order.+ .. [2] If you yield items as dicts (not :class:`Item` instances),+ exporters that need to know the fields to export beforehand, like+ :class:`CsvItemExporter`, only export the fields found in the+ first item.+ .. [3] Dicts preserve insertion order since `Python 3.7`_+ (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version+ of Python, use an OrderedDict_ to enforce a specific field order.++ .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html+ .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation+ .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html+ .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict.. attribute:: export_empty_fields@@ -286,8 +305,8 @@ CsvItemExporterExports Items in CSV format to the given file-like object. If the:attr:`fields_to_export` attribute is set, it will be used to define the- CSV columns and their order. The :attr:`export_empty_fields` attribute has- no effect on this exporter.+ CSV columns, their order and their column names. The+ :attr:`export_empty_fields` attribute has no effect on this exporter.:param file: the file-like object to use for exporting the data. Its ``write`` method shouldaccept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex cf70b8aca..968cb8884 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -56,7 +56,7 @@ CSV* :setting:`FEED_FORMAT`: ``csv``* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`- * To specify columns to export and their order use+ * To specify columns to export, their order and their column names, use:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use thisoption, but it is important for CSV because unlike many other exportformats CSV uses a fixed header.@@ -259,18 +259,9 @@ FEED_EXPORT_FIELDSDefault: ``None``-A list of fields to export, optional.-Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.--Use FEED_EXPORT_FIELDS option to define fields to export and their order.--When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields-defined in dicts or :class:`~.Item` subclasses a spider is yielding.--If an exporter requires a fixed set of fields (this is the case for-:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS-is empty or None, then Scrapy tries to infer field names from the-exported data - currently it uses field names from the first item.+Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their+order and their output names. See :attr:`BaseItemExporter.fields_to_export+<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information... setting:: FEED_EXPORT_INDENTdiff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 695c74fec..c05acaca5 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -2,6 +2,7 @@Item Exporters are used to export/serialize items into different formats."""+from collections import Mappingimport csvimport ioimport sys@@ -64,6 +65,14 @@ class BaseItemExporter(object):field_iter = six.iterkeys(item.fields)else:field_iter = six.iterkeys(item)+ elif isinstance(self.fields_to_export, Mapping):+ if include_empty:+ field_iter = self.fields_to_export.items()+ else:+ field_iter = (+ (x, y) for x, y in self.fields_to_export.items()+ if x in item+ )else:if include_empty:field_iter = self.fields_to_export@@ -71,13 +80,22 @@ class BaseItemExporter(object):field_iter = (x for x in self.fields_to_export if x in item)for field_name in field_iter:- if field_name in item:- field = {} if isinstance(item, dict) else item.fields[field_name]- value = self.serialize_field(field, field_name, item[field_name])+ if isinstance(field_name, six.string_types):+ item_field, output_field = field_name, field_name+ else:+ item_field, output_field = field_name+ if item_field in item:+ if isinstance(item, dict):+ field = {}+ else:+ field = item.fields[item_field]+ value = self.serialize_field(+ field, output_field, item[item_field]+ )else:value = default_value- yield field_name, value+ yield output_field, valueclass JsonLinesItemExporter(BaseItemExporter):@@ -259,7 +277,11 @@ class CsvItemExporter(BaseItemExporter):else:# use fields declared in Itemself.fields_to_export = list(item.fields.keys())- row = list(self._build_row(self.fields_to_export))+ if isinstance(self.fields_to_export, Mapping):+ fields = self.fields_to_export.values()+ else:+ fields = self.fields_to_export+ row = list(self._build_row(fields))self.csv_writer.writerow(row)diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex b2f7267a2..1ed476d83 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -201,7 +201,8 @@ class FeedExporter(object):raise NotConfiguredself.store_empty = settings.getbool('FEED_STORE_EMPTY')self._exporting = False- self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None+ self.export_fields = settings.getdictorlist('FEED_EXPORT_FIELDS')+ self.export_fields = self.export_fields or Noneself.indent = Noneif settings.get('FEED_EXPORT_INDENT') is not None:self.indent = settings.getint('FEED_EXPORT_INDENT')diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.pyindex 14c93bef2..69b324e84 100644--- a/scrapy/settings/__init__.py+++ b/scrapy/settings/__init__.py@@ -1,7 +1,7 @@import siximport jsonimport copy-from collections import MutableMapping+from collections import MutableMapping, OrderedDictfrom importlib import import_modulefrom pprint import pformat@@ -198,6 +198,39 @@ class BaseSettings(MutableMapping):value = json.loads(value)return dict(value)+ def getdictorlist(self, name, default=None):+ """Get a setting value as either an ``OrderedDict`` or a list.++ If the setting is already a dict or a list, a copy of it will be+ returned.++ If it is a string it will be evaluated as JSON, or as a comma-separated+ list of strings as a fallback.++ For example, settings populated through environment variables will+ return:++ - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to+ ``'{"key1": "value1", "key2": "value2"}'``++ - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``++ :param name: the setting name+ :type name: string++ :param default: the value to return if no setting is found+ :type default: any+ """+ value = self.get(name, default)+ if value is None:+ return {}+ if isinstance(value, six.string_types):+ try:+ return json.loads(value, object_pairs_hook=OrderedDict)+ except ValueError:+ return value.split(',')+ return copy.deepcopy(value)+def getwithbase(self, name):"""Get a composition of a dictionary-like setting and its `_BASE`counterpart.diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex cd72c661a..1b3dc14a1 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -1,4 +1,7 @@+# -*- coding:utf-8 -*-+from __future__ import absolute_import+from collections import OrderedDictimport reimport jsonimport marshal@@ -83,6 +86,14 @@ class BaseItemExporterTest(unittest.TestCase):assert isinstance(name, six.text_type)self.assertEqual(name, u'John\xa3')+ ie = self._get_exporter(+ fields_to_export=OrderedDict([('name', u'名稱')])+ )+ self.assertEqual(+ list(ie._get_serialized_fields(self.i)),+ [(u'名稱', u'John\xa3')]+ )+def test_field_custom_serializer(self):def custom_serializer(value):return str(int(value) + 2)@@ -214,6 +225,7 @@ class MarshalItemExporterTest(BaseItemExporterTest):class CsvItemExporterTest(BaseItemExporterTest):def _get_exporter(self, **kwargs):+ self.output = tempfile.TemporaryFile()return CsvItemExporter(self.output, **kwargs)def assertCsvEqual(self, first, second, msg=None):@@ -224,7 +236,8 @@ class CsvItemExporterTest(BaseItemExporterTest):return self.assertEqual(csvsplit(first), csvsplit(second), msg)def _check_output(self):- self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')+ self.output.seek(0)+ self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n')def assertExportResult(self, item, expected, **kwargs):fp = BytesIO()diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex eef0384cf..4f72c0ff4 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1,4 +1,5 @@from __future__ import absolute_import+from collections import OrderedDictimport osimport csvimport json@@ -578,6 +579,62 @@ class FeedExportTest(unittest.TestCase):yield self.assertExported(items, header, rows,settings=settings, ordered=True)+ # fields may be defined as a comma-separated list+ header = ["foo", "baz", "hello"]+ settings = {'FEED_EXPORT_FIELDS': ",".join(header)}+ rows = [+ {'foo': 'bar1', 'baz': '', 'hello': ''},+ {'foo': 'bar2', 'baz': '', 'hello': 'world2'},+ {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},+ {'foo': '', 'baz': '', 'hello': 'world4'},+ ]+ yield self.assertExported(items, header, rows,+ settings=settings, ordered=True)++ # fields may also be defined as a JSON array+ header = ["foo", "baz", "hello"]+ settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}+ rows = [+ {'foo': 'bar1', 'baz': '', 'hello': ''},+ {'foo': 'bar2', 'baz': '', 'hello': 'world2'},+ {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},+ {'foo': '', 'baz': '', 'hello': 'world4'},+ ]+ yield self.assertExported(items, header, rows,+ settings=settings, ordered=True)++ # custom output field names can be specified+ header = OrderedDict((+ ("foo", "Foo"),+ ("baz", "Baz"),+ ("hello", "Hello"),+ ))+ settings = {'FEED_EXPORT_FIELDS': header}+ rows = [+ {'Foo': 'bar1', 'Baz': '', 'Hello': ''},+ {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'},+ {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},+ {'Foo': '', 'Baz': '', 'Hello': 'world4'},+ ]+ yield self.assertExported(items, list(header.values()), rows,+ settings=settings, ordered=True)++ # custom output field names can be specified as a JSON object+ header = OrderedDict((+ ("foo", "Foo"),+ ("baz", "Baz"),+ ("hello", "Hello"),+ ))+ settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}+ rows = [+ {'Foo': 'bar1', 'Baz': '', 'Hello': ''},+ {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'},+ {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},+ {'Foo': '', 'Baz': '', 'Hello': 'world4'},+ ]+ yield self.assertExported(items, list(header.values()), rows,+ settings=settings, ordered=True)+@defer.inlineCallbacksdef test_export_dicts(self):# When dicts are used, only keys from the first row are used as