Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'upstream/master' into allow-customizing-export-column-names

Adrián Chavese8503217fba4
diff --cc docs/topics/feed-exports.rst
index 6831dcb99,9a13eb82f..52b47347e
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@@ -52,12 -54,14 +54,14 @@@ JSON line
CSV
---
- * Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
- * Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
- * To specify columns to export, their order and their column names, use
- :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
- option, but it is important for CSV because unlike many other export
- formats CSV uses a fixed header.
+ - Value for the ``format`` key in the :setting:`FEEDS` setting: ``csv``
+
+ - Exporter used: :class:`~scrapy.exporters.CsvItemExporter`
+
-- To specify columns to export and their order use
++- To specify columns to export, their order and their column names, use
+ :setting:`FEED_EXPORT_FIELDS`. Other feed exporters can also use this
+ option, but it is important for CSV because unlike many other export
+ formats CSV uses a fixed header.
.. _topics-feed-format-xml:
diff --cc scrapy/utils/conf.py
index 453d8b487,24a6187b9..00cc53725
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@@ -118,10 -118,10 +118,10 @@@ def feed_complete_default_values_from_s
out = feed.copy()
out.setdefault("batch_item_count", settings.getint('FEED_EXPORT_BATCH_ITEM_COUNT'))
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
- out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+ out.setdefault("fields", settings.getdictorlist("FEED_EXPORT_FIELDS") or None)
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
- out.setdefault("item_export_kwargs", dict())
+ out.setdefault("item_export_kwargs", {})
if settings["FEED_EXPORT_INDENT"] is None:
out.setdefault("indent", None)
else:
diff --cc tests/test_exporters.py
index d15b2ad27,b263b3475..6ba7428f6
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@@ -4,9 -4,9 +4,10 @@@ import marsha
import pickle
import tempfile
import unittest
+from collections import OrderedDict
from io import BytesIO
from datetime import datetime
+ from warnings import catch_warnings, filterwarnings
import lxml.etree
from itemadapter import ItemAdapter
diff --cc tests/test_feedexport.py
index 25b1ccf63,f0acf1941..83aabbdc7
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@@ -914,76 -959,131 +961,184 @@@ class FeedExportTest(FeedExportTestBase
yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
yield self.assertExportedJsonLines(items, rows_jl)
- # edge case: FEED_EXPORT_FIELDS==[] means the same as default None
+ @defer.inlineCallbacks
+ def test_export_items_empty_field_list(self):
+ # FEED_EXPORT_FIELDS==[] means the same as default None
+ items = [{'foo': 'bar'}]
+ header = ["foo"]
+ rows = [{'foo': 'bar'}]
settings = {'FEED_EXPORT_FIELDS': []}
- yield self.assertExportedCsv(items, header, rows_csv, ordered=False)
- yield self.assertExportedJsonLines(items, rows_jl, settings)
+ yield self.assertExportedCsv(items, header, rows, ordered=False)
+ yield self.assertExportedJsonLines(items, rows, settings)
- # it is possible to override fields using FEED_EXPORT_FIELDS
- header = ["foo", "baz", "hello"]
+ @defer.inlineCallbacks
+ def test_export_items_field_list(self):
+ items = [{'foo': 'bar'}]
+ header = ["foo", "baz"]
+ rows = [{'foo': 'bar', 'baz': ''}]
settings = {'FEED_EXPORT_FIELDS': header}
- rows = [
- {'foo': 'bar1', 'baz': '', 'hello': ''},
- {'foo': 'bar2', 'baz': '', 'hello': 'world2'},
- {'foo': 'bar3', 'baz': 'quux3', 'hello': ''},
- {'foo': '', 'baz': '', 'hello': 'world4'},
- ]
- yield self.assertExported(items, header, rows,
- settings=settings, ordered=True)
+ yield self.assertExported(items, header, rows, settings=settings)
+
+ @defer.inlineCallbacks
+ def test_export_items_comma_separated_field_list(self):
+ items = [{'foo': 'bar'}]
+ header = ["foo", "baz"]
+ rows = [{'foo': 'bar', 'baz': ''}]
+ settings = {'FEED_EXPORT_FIELDS': ",".join(header)}
+ yield self.assertExported(items, header, rows, settings=settings)
+
+ @defer.inlineCallbacks
+ def test_export_items_json_field_list(self):
+ items = [{'foo': 'bar'}]
+ header = ["foo", "baz"]
+ rows = [{'foo': 'bar', 'baz': ''}]
+ settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+ yield self.assertExported(items, header, rows, settings=settings)
+
+ @defer.inlineCallbacks
+ def test_export_items_field_names(self):
+ items = [{'foo': 'bar'}]
+ header = OrderedDict((
+ ("foo", "Foo"),
+ ))
+ rows = [{'Foo': 'bar'}]
+ settings = {'FEED_EXPORT_FIELDS': header}
+ yield self.assertExported(items, list(header.values()), rows,
+ settings=settings)
+
+ @pytest.mark.skipif(sys.version_info < (3, 7),
+ reason='Only official in Python 3.7+')
+ @defer.inlineCallbacks
+ def test_export_items_dict_field_names(self):
+ items = [{'foo': 'bar'}]
+ header = {
+ 'baz': 'Baz',
+ 'foo': 'Foo',
+ }
+ rows = [{'Baz': '', 'Foo': 'bar'}]
+ settings = {'FEED_EXPORT_FIELDS': header}
+ yield self.assertExported(items, ['Baz', 'Foo'], rows,
+ settings=settings)
+
+ @defer.inlineCallbacks
+ def test_export_items_json_field_names(self):
+ items = [{'foo': 'bar'}]
+ header = OrderedDict((
+ ("foo", "Foo"),
+ ))
+ rows = [{'Foo': 'bar'}]
+ settings = {'FEED_EXPORT_FIELDS': json.dumps(header)}
+ yield self.assertExported(items, list(header.values()), rows,
+ settings=settings)
+ @defer.inlineCallbacks
+ def test_export_based_on_item_classes(self):
+ items = [
+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+ self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
+ {'hello': 'world3', 'egg': 'spam3'},
+ ]
+
+ formats = {
+ 'csv': b'baz,egg,foo\r\n,spam1,bar1\r\n',
+ 'json': b'[\n{"hello": "world2", "foo": "bar2"}\n]',
+ 'jsonlines': (
+ b'{"foo": "bar1", "egg": "spam1"}\n'
+ b'{"hello": "world2", "foo": "bar2"}\n'
+ ),
+ 'xml': (
+ b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
+ b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'
+ b'world2</hello><foo>bar2</foo></item>\n<item><hello>world3'
+ b'</hello><egg>spam3</egg></item>\n</items>'
+ ),
+ }
+
+ settings = {
+ 'FEEDS': {
+ self._random_temp_filename(): {
+ 'format': 'csv',
+ 'item_classes': [self.MyItem],
+ },
+ self._random_temp_filename(): {
+ 'format': 'json',
+ 'item_classes': [self.MyItem2],
+ },
+ self._random_temp_filename(): {
+ 'format': 'jsonlines',
+ 'item_classes': [self.MyItem, self.MyItem2],
+ },
+ self._random_temp_filename(): {
+ 'format': 'xml',
+ },
+ },
+ }
+
+ data = yield self.exported_data(items, settings)
+ for fmt, expected in formats.items():
+ self.assertEqual(expected, data[fmt])
+
+ @defer.inlineCallbacks
+ def test_export_based_on_custom_filters(self):
+ items = [
+ self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),
+ self.MyItem2({'hello': 'world2', 'foo': 'bar2'}),
+ {'hello': 'world3', 'egg': 'spam3'},
+ ]
+
+ MyItem = self.MyItem
+
+ class CustomFilter1:
+ def __init__(self, feed_options):
+ pass
+
+ def accepts(self, item):
+ return isinstance(item, MyItem)
+
+ class CustomFilter2(scrapy.extensions.feedexport.ItemFilter):
+ def accepts(self, item):
+ if 'foo' not in item.fields:
+ return False
+ return True
+
+ class CustomFilter3(scrapy.extensions.feedexport.ItemFilter):
+ def accepts(self, item):
+ if isinstance(item, tuple(self.item_classes)) and item['foo'] == "bar1":
+ return True
+ return False
+
+ formats = {
+ 'json': b'[\n{"foo": "bar1", "egg": "spam1"}\n]',
+ 'xml': (
+ b'<?xml version="1.0" encoding="utf-8"?>\n<items>\n<item>'
+ b'<foo>bar1</foo><egg>spam1</egg></item>\n<item><hello>'
+ b'world2</hello><foo>bar2</foo></item>\n</items>'
+ ),
+ 'jsonlines': b'{"foo": "bar1", "egg": "spam1"}\n',
+ }
+
+ settings = {
+ 'FEEDS': {
+ self._random_temp_filename(): {
+ 'format': 'json',
+ 'item_filter': CustomFilter1,
+ },
+ self._random_temp_filename(): {
+ 'format': 'xml',
+ 'item_filter': CustomFilter2,
+ },
+ self._random_temp_filename(): {
+ 'format': 'jsonlines',
+ 'item_classes': [self.MyItem, self.MyItem2],
+ 'item_filter': CustomFilter3,
+ },
+ },
+ }
+
+ data = yield self.exported_data(items, settings)
+ for fmt, expected in formats.items():
+ self.assertEqual(expected, data[fmt])
+
@defer.inlineCallbacks
def test_export_dicts(self):
# When dicts are used, only keys from the first row are used as