Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Allow customizing export column names

Adrián Chaves484bd0d22a11
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rst
index f5048d2da..42a93e459 100644
--- a/docs/topics/exporters.rst
+++ b/docs/topics/exporters.rst
@@ -190,14 +190,33 @@ BaseItemExporter
.. attribute:: fields_to_export
- A list with the name of the fields that will be exported, or None if you
- want to export all fields. Defaults to None.
+ Fields to export, their order [1]_ and their output names.
- Some exporters (like :class:`CsvItemExporter`) respect the order of the
- fields defined in this attribute.
+ Possible values are:
- Some exporters may require fields_to_export list in order to export the
- data properly when spiders return dicts (not :class:`~Item` instances).
+ - ``None`` (all fields [2]_, default)
+
+ - A list of fields::
+
+ ['field1', 'field2']
+
+ - A dict [3]_ where keys are fields and values are output names::
+
+ {'field1': 'Field 1', 'field2': 'Field 2'}
+
+ .. [1] Not all exporters respect the specified field order.
+ .. [2] If you yield items as dicts (not :class:`Item` instances),
+ exporters that need to know the fields to export beforehand, like
+ :class:`CsvItemExporter`, only export the fields found in the
+ first item.
+ .. [3] Dicts preserve insertion order since `Python 3.7`_
+ (`CPython 3.6`_, `PyPy 2.5`_). If you are using an older version
+ of Python, use an OrderedDict_ to enforce a specific field order.
+
+ .. _Python 3.7: https://docs.python.org/whatsnew/3.7.html
+ .. _CPython 3.6: https://docs.python.org/whatsnew/3.6.html#new-dict-implementation
+ .. _PyPy 2.5: https://morepypy.blogspot.com/2015/02/pypy-250-released.html
+ .. _OrderedDict: https://docs.python.org/library/collections.html#collections.OrderedDict
.. attribute:: export_empty_fields
@@ -286,8 +305,8 @@ CsvItemExporter
Exports Items in CSV format to the given file-like object. If the
:attr:`fields_to_export` attribute is set, it will be used to define the
- CSV columns and their order. The :attr:`export_empty_fields` attribute has
- no effect on this exporter.
+ CSV columns, their order and their column names. The
+ :attr:`export_empty_fields` attribute has no effect on this exporter.
:param file: the file-like object to use for exporting the data. Its ``write`` method should
accept ``bytes`` (a disk file opened in binary mode, a ``io.BytesIO`` object, etc)
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index cf70b8aca..968cb8884 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -56,7 +56,7 @@ CSV
* :setting:`FEED_FORMAT`: ``csv``
* Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- * To specify columns to export and their order use
+ * To specify columns to export, their order and their column names, use
:setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
option, but it is important for CSV because unlike many other export
formats CSV uses a fixed header.
@@ -259,18 +259,9 @@ FEED_EXPORT_FIELDS
Default: ``None``
-A list of fields to export, optional.
-Example: ``FEED_EXPORT_FIELDS = ["foo", "bar", "baz"]``.
-
-Use FEED_EXPORT_FIELDS option to define fields to export and their order.
-
-When FEED_EXPORT_FIELDS is empty or None (default), Scrapy uses fields
-defined in dicts or :class:`~.Item` subclasses a spider is yielding.
-
-If an exporter requires a fixed set of fields (this is the case for
-:ref:`CSV <topics-feed-format-csv>` export format) and FEED_EXPORT_FIELDS
-is empty or None, then Scrapy tries to infer field names from the
-exported data - currently it uses field names from the first item.
+Use the ``FEED_EXPORT_FIELDS`` setting to define the fields to export, their
+order and their output names. See :attr:`BaseItemExporter.fields_to_export
+<scrapy.exporters.BaseItemExporter.fields_to_export>` for more information.
.. setting:: FEED_EXPORT_INDENT
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 695c74fec..c05acaca5 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -2,6 +2,7 @@
Item Exporters are used to export/serialize items into different formats.
"""
+from collections import Mapping
import csv
import io
import sys
@@ -64,6 +65,14 @@ class BaseItemExporter(object):
field_iter = six.iterkeys(item.fields)
else:
field_iter = six.iterkeys(item)
+ elif isinstance(self.fields_to_export, Mapping):
+ if include_empty:
+ field_iter = self.fields_to_export.items()
+ else:
+ field_iter = (
+ (x, y) for x, y in self.fields_to_export.items()
+ if x in item
+ )
else:
if include_empty:
field_iter = self.fields_to_export
@@ -71,13 +80,22 @@ class BaseItemExporter(object):
field_iter = (x for x in self.fields_to_export if x in item)
for field_name in field_iter:
- if field_name in item:
- field = {} if isinstance(item, dict) else item.fields[field_name]
- value = self.serialize_field(field, field_name, item[field_name])
+ if isinstance(field_name, six.string_types):
+ item_field, output_field = field_name, field_name
+ else:
+ item_field, output_field = field_name
+ if item_field in item:
+ if isinstance(item, dict):
+ field = {}
+ else:
+ field = item.fields[item_field]
+ value = self.serialize_field(
+ field, output_field, item[item_field]
+ )
else:
value = default_value
- yield field_name, value
+ yield output_field, value
class JsonLinesItemExporter(BaseItemExporter):
@@ -259,7 +277,11 @@ class CsvItemExporter(BaseItemExporter):
else:
# use fields declared in Item
self.fields_to_export = list(item.fields.keys())
- row = list(self._build_row(self.fields_to_export))
+ if isinstance(self.fields_to_export, Mapping):
+ fields = self.fields_to_export.values()
+ else:
+ fields = self.fields_to_export
+ row = list(self._build_row(fields))
self.csv_writer.writerow(row)
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index b2f7267a2..1ed476d83 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -201,7 +201,8 @@ class FeedExporter(object):
raise NotConfigured
self.store_empty = settings.getbool('FEED_STORE_EMPTY')
self._exporting = False
- self.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None
+ self.export_fields = settings.getdictorlist('FEED_EXPORT_FIELDS')
+ self.export_fields = self.export_fields or None
self.indent = None
if settings.get('FEED_EXPORT_INDENT') is not None:
self.indent = settings.getint('FEED_EXPORT_INDENT')
diff --git a/scrapy/settings/__init__.py b/scrapy/settings/__init__.py
index 14c93bef2..69b324e84 100644
--- a/scrapy/settings/__init__.py
+++ b/scrapy/settings/__init__.py
@@ -1,7 +1,7 @@
import six
import json
import copy
-from collections import MutableMapping
+from collections import MutableMapping, OrderedDict
from importlib import import_module
from pprint import pformat
@@ -198,6 +198,39 @@ class BaseSettings(MutableMapping):
value = json.loads(value)
return dict(value)
+ def getdictorlist(self, name, default=None):
+ """Get a setting value as either an ``OrderedDict`` or a list.
+
+ If the setting is already a dict or a list, a copy of it will be
+ returned.
+
+ If it is a string it will be evaluated as JSON, or as a comma-separated
+ list of strings as a fallback.
+
+ For example, settings populated through environment variables will
+ return:
+
+ - ``OrdetedDict([('key1', 'value1'), ('key2', 'value2')])`` if set to
+ ``'{"key1": "value1", "key2": "value2"}'``
+
+ - ``['one', 'two']`` if set to ``'["one", "two"]'`` or ``'one,two'``
+
+ :param name: the setting name
+ :type name: string
+
+ :param default: the value to return if no setting is found
+ :type default: any
+ """
+ value = self.get(name, default)
+ if value is None:
+ return {}
+ if isinstance(value, six.string_types):
+ try:
+ return json.loads(value, object_pairs_hook=OrderedDict)
+ except ValueError:
+ return value.split(',')
+ return copy.deepcopy(value)
+
def getwithbase(self, name):
"""Get a composition of a dictionary-like setting and its `_BASE`
counterpart.
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index cd72c661a..1b3dc14a1 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -1,4 +1,7 @@
+# -*- coding:utf-8 -*-
+
from __future__ import absolute_import
+from collections import OrderedDict
import re
import json
import marshal
@@ -83,6 +86,14 @@ class BaseItemExporterTest(unittest.TestCase):
assert isinstance(name, six.text_type)
self.assertEqual(name, u'John\xa3')
+ ie = self._get_exporter(
+ fields_to_export=OrderedDict([('name', u'名稱')])
+ )
+ self.assertEqual(
+ list(ie._get_serialized_fields(self.i)),
+ [(u'名稱', u'John\xa3')]
+ )
+
def test_field_custom_serializer(self):
def custom_serializer(value):
return str(int(value) + 2)
@@ -214,6 +225,7 @@ class MarshalItemExporterTest(BaseItemExporterTest):
class CsvItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
+ self.output = tempfile.TemporaryFile()
return CsvItemExporter(self.output, **kwargs)
def assertCsvEqual(self, first, second, msg=None):
@@ -224,7 +236,8 @@ class CsvItemExporterTest(BaseItemExporterTest):
return self.assertEqual(csvsplit(first), csvsplit(second), msg)
def _check_output(self):
- self.assertCsvEqual(to_unicode(self.output.getvalue()), u'age,name\r\n22,John\xa3\r\n')
+ self.output.seek(0)
+ self.assertCsvEqual(to_unicode(self.output.read()), u'age,name\r\n22,John\xa3\r\n')
def assertExportResult(self, item, expected, **kwargs):
fp = BytesIO()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index eef0384cf..4f72c0ff4 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1,4 +1,5 @@
from __future__ import absolute_import
+from collections import OrderedDict
import os
import csv
import json
@@ -578,6 +579,62 @@ class FeedExportTest(unittest.TestCase):
yield self.assertExported(items, header, rows,
settings=settings, ordered=True)
+ # fields may be defined as a comma-separated list
+ header = ["foo", "baz", "hello"]
+ settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
+ rows = [
+ {'foo': 'bar1', 'baz': '', 'hello': ''},
+ {'foo': 'bar2', 'baz': '', 'hello': 'world2'},
+ {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
+ {'foo': '', 'baz': '', 'hello': 'world4'},
+ ]
+ yield self.assertExported(items, header, rows,
+ settings=settings, ordered=True)
+
+ # fields may also be defined as a JSON array
+ header = ["foo", "baz", "hello"]
+ settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+ rows = [
+ {'foo': 'bar1', 'baz': '', 'hello': ''},
+ {'foo': 'bar2', 'baz': '', 'hello': 'world2'},
+ {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
+ {'foo': '', 'baz': '', 'hello': 'world4'},
+ ]
+ yield self.assertExported(items, header, rows,
+ settings=settings, ordered=True)
+
+ # custom output field names can be specified
+ header = OrderedDict((
+ ("foo", "Foo"),
+ ("baz", "Baz"),
+ ("hello", "Hello"),
+ ))
+ settings = {'FEED_EXPORT_FIELDS': header}
+ rows = [
+ {'Foo': 'bar1', 'Baz': '', 'Hello': ''},
+ {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'},
+ {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
+ {'Foo': '', 'Baz': '', 'Hello': 'world4'},
+ ]
+ yield self.assertExported(items, list(header.values()), rows,
+ settings=settings, ordered=True)
+
+ # custom output field names can be specified as a JSON object
+ header = OrderedDict((
+ ("foo", "Foo"),
+ ("baz", "Baz"),
+ ("hello", "Hello"),
+ ))
+ settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+ rows = [
+ {'Foo': 'bar1', 'Baz': '', 'Hello': ''},
+ {'Foo': 'bar2', 'Baz': '', 'Hello': 'world2'},
+ {'Foo': 'bar3', 'Baz': 'quux3', 'Hello': ''},
+ {'Foo': '', 'Baz': '', 'Hello': 'world4'},
+ ]
+ yield self.assertExported(items, list(header.values()), rows,
+ settings=settings, ordered=True)
+
@defer.inlineCallbacks
def test_export_dicts(self):
# When dicts are used, only keys from the first row are used as