Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'upstream/master' into allow-customizing-export-column-names
Adrián Chavese8503217fba4
diff --cc docs/topics/feed-exports.rstindex 6831dcb99,9a13eb82f..52b47347e--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@@ -52,12 -54,14 +54,14 @@@ JSON lineCSV---- * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``- * Exporter used: :class:`~scrapy.exporters.CsvItemExporter`- * To specify columns to export, their order and their column names, use- :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this- option, but it is important for CSV because unlike many other export- formats CSV uses a fixed header.+ - Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``++ - Exporter used: :class:`~scrapy.exporters.CsvItemExporter`+-- To specify columns to export and their order use++- To specify columns to export, their order and their column names, use+ :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this+ option, but it is important for CSV because unlike many other export+ formats CSV uses a fixed header... _topics-feed-format-xml:diff --cc scrapy/utils/conf.pyindex 453d8b487,24a6187b9..00cc53725--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@@ -118,10 -118,10 +118,10 @@@ def feed_complete_default_values_from_sout = feed.copy()out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])- out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)+ out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))out.setdefault("uri_params", settings["FEED_URI_PARAMS"])- out.setdefault("item_export_kwargs", dict())+ out.setdefault("item_export_kwargs", {})if settings["FEED_EXPORT_INDENT"] is None:out.setdefault("indent", None)else:diff --cc tests/test_exporters.pyindex d15b2ad27,b263b3475..6ba7428f6--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@@ -4,9 -4,9 +4,10 @@@ import marshaimport pickleimport tempfileimport unittest+from collections import OrderedDictfrom io import BytesIOfrom datetime import datetime+ from warnings import catch_warnings, filterwarningsimport lxml.etreefrom itemadapter import ItemAdapterdiff --cc tests/test_feedexport.pyindex 25b1ccf63,f0acf1941..83aabbdc7--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@@ -914,76 -959,131 +961,184 @@@ class FeedExportTest(FeedExportTestBaseyield self.assertExportedCsv(items, header, rows_csv, ordered=False)yield self.assertExportedJsonLines(items, rows_jl)- # edge case: FEED_EXPORT_FIELDS==[] means the same as default None+ @defer.inlineCallbacks+ def test_export_items_empty_field_list(self):+ # FEED_EXPORT_FIELDS==[] means the same as default None+ items = [{'foo': 'bar'}]+ header = ["foo"]+ rows = [{'foo': 'bar'}]settings = {'FEED_EXPORT_FIELDS': []}- yield self.assertExportedCsv(items, header, rows_csv, ordered=False)- yield self.assertExportedJsonLines(items, rows_jl, settings)+ yield self.assertExportedCsv(items, header, rows, ordered=False)+ yield self.assertExportedJsonLines(items, rows, settings)- # it is possible to override fields using FEED_EXPORT_FIELDS- header = ["foo", "baz", "hello"]+ @defer.inlineCallbacks+ def test_export_items_field_list(self):+ items = [{'foo': 'bar'}]+ header = ["foo", "baz"]+ rows = [{'foo': 'bar', 'baz': ''}]settings = {'FEED_EXPORT_FIELDS': header}- rows = [- {'foo': 'bar1', 'baz': '', 'hello': ''},- {'foo': 'bar2', 'baz': '', 'hello': 'world2'},- {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},- {'foo': '', 'baz': '', 'hello': 'world4'},- ]- yield self.assertExported(items, header, rows,- settings=settings, ordered=True)+ yield self.assertExported(items, header, rows, settings=settings)++ @defer.inlineCallbacks+ def test_export_items_comma_separated_field_list(self):+ items = [{'foo': 'bar'}]+ header = ["foo", "baz"]+ rows = [{'foo': 'bar', 'baz': ''}]+ settings = {'FEED_EXPORT_FIELDS': ",".join(header)}+ yield self.assertExported(items, header, rows, settings=settings)++ @defer.inlineCallbacks+ def test_export_items_json_field_list(self):+ items = [{'foo': 'bar'}]+ header = ["foo", "baz"]+ rows = [{'foo': 'bar', 'baz': ''}]+ settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}+ yield self.assertExported(items, header, rows, settings=settings)++ @defer.inlineCallbacks+ def test_export_items_field_names(self):+ items = [{'foo': 'bar'}]+ header = OrderedDict((+ ("foo", "Foo"),+ ))+ rows = [{'Foo': 'bar'}]+ settings = {'FEED_EXPORT_FIELDS': header}+ yield self.assertExported(items, list(header.values()), rows,+ settings=settings)++ @pytest.mark.skipif(sys.version_info < (3, 7),+ reason='Only official in Python 3.7+')+ @defer.inlineCallbacks+ def test_export_items_dict_field_names(self):+ items = [{'foo': 'bar'}]+ header = {+ 'baz': 'Baz',+ 'foo': 'Foo',+ }+ rows = [{'Baz': '', 'Foo': 'bar'}]+ settings = {'FEED_EXPORT_FIELDS': header}+ yield self.assertExported(items, ['Baz', 'Foo'], rows,+ settings=settings)++ @defer.inlineCallbacks+ def test_export_items_json_field_names(self):+ items = [{'foo': 'bar'}]+ header = OrderedDict((+ ("foo", "Foo"),+ ))+ rows = [{'Foo': 'bar'}]+ settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}+ yield self.assertExported(items, list(header.values()), rows,+ settings=settings)+ @defer.inlineCallbacks+ def test_export_based_on_item_classes(self):+ items = [+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),+ self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),+ {'hello': 'world3', 'egg': 'spam3'},+ ]++ formats = {+ 'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n',+ 'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]',+ 'jsonlines': (+ b'{"foo": "bar1", "egg": "spam1"}\n'+ b'{"hello": "world2", "foo": "bar2"}\n'+ ),+ 'xml': (+ b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'+ b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'+ b'world2</hello><foo>bar2</foo></item>\n<item><hello>world3'+ b'</hello><egg>spam3</egg></item>\n</items>'+ ),+ }++ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {+ 'format': 'csv',+ 'item_classes': [self.MyItem],+ },+ self._random_temp_filename(): {+ 'format': 'json',+ 'item_classes': [self.MyItem2],+ },+ self._random_temp_filename(): {+ 'format': 'jsonlines',+ 'item_classes': [self.MyItem, self.MyItem2],+ },+ self._random_temp_filename(): {+ 'format': 'xml',+ },+ },+ }++ data = yield self.exported_data(items, settings)+ for fmt, expected in formats.items():+ self.assertEqual(expected, data[fmt])++ @defer.inlineCallbacks+ def test_export_based_on_custom_filters(self):+ items = [+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),+ self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),+ {'hello': 'world3', 'egg': 'spam3'},+ ]++ MyItem = self.MyItem++ class CustomFilter1:+ def __init__(self, feed_options):+ pass++ def accepts(self, item):+ return isinstance(item, MyItem)++ class CustomFilter2(scrapy.extensions.feedexport.ItemFilter):+ def accepts(self, item):+ if 'foo' not in item.fields:+ return False+ return True++ class CustomFilter3(scrapy.extensions.feedexport.ItemFilter):+ def accepts(self, item):+ if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1":+ return True+ return False++ formats = {+ 'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]',+ 'xml': (+ b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'+ b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'+ b'world2</hello><foo>bar2</foo></item>\n</items>'+ ),+ 'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n',+ }++ settings = {+ 'FEEDS': {+ self._random_temp_filename(): {+ 'format': 'json',+ 'item_filter': CustomFilter1,+ },+ self._random_temp_filename(): {+ 'format': 'xml',+ 'item_filter': CustomFilter2,+ },+ self._random_temp_filename(): {+ 'format': 'jsonlines',+ 'item_classes': [self.MyItem, self.MyItem2],+ 'item_filter': CustomFilter3,+ },+ },+ }++ data = yield self.exported_data(items, settings)+ for fmt, expected in formats.items():+ self.assertEqual(expected, data[fmt])+@defer.inlineCallbacksdef test_export_dicts(self):# When dicts are used, only keys from the first row are used as