Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

kwargs for Item exporters classes test docs

maranqza8e895e68418
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 9fb2189e8..e69a64195 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -303,6 +303,9 @@ For instance::
'store_empty': False,
'fields': None,
'indent': 4,
+ 'item_export_kwargs': {
+ 'export_empty_fields': True,
+ },
},
'/home/user/documents/items.xml': {
'format': 'xml',
@@ -332,6 +335,8 @@ as a fallback value if that key is not provided for a specific feed definition:
- ``indent``: falls back to :setting:`FEED_EXPORT_INDENT`.
+- ``item_export_kwargs``: dict with kwargs for :ref:`Item exporters <topics-exporters>` classes.
+
- ``overwrite``: whether to overwrite the file if it already exists
(``True``) or append to its content (``False``).
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 980825499..f32d48fa5 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -360,6 +360,7 @@ class FeedExporter:
fields_to_export=feed_options['fields'],
encoding=feed_options['encoding'],
indent=feed_options['indent'],
+ **feed_options['item_export_kwargs'],
)
slot = _FeedSlot(
file=file,
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 4e7a9967e..b904c4a03 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -121,6 +121,7 @@ def feed_complete_default_values_from_settings(feed, settings):
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
+ out.setdefault("item_export_kwargs", dict())
if settings["FEED_EXPORT_INDENT"] is None:
out.setdefault("indent", None)
else:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 840e0f87b..e88e4f5ce 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1252,6 +1252,43 @@ class FeedExportTest(FeedExportTestBase):
for fmt in ['json', 'xml', 'csv']:
self.assertIn(f'Error storing {fmt} feed (2 items)', str(log))
+ @defer.inlineCallbacks
+ def test_extend_kwargs(self):
+ items = [{'foo': 'FOO', 'bar': 'BAR'}]
+
+ expected_with_title_csv = 'foo,bar\r\nFOO,BAR\r\n'.encode('utf-8')
+ expected_without_title_csv = 'FOO,BAR\r\n'.encode('utf-8')
+ test_cases = [
+ # with title
+ {
+ 'options': {
+ 'format': 'csv',
+ 'item_export_kwargs': dict(include_headers_line=True),
+ },
+ 'expected': expected_with_title_csv,
+ },
+ # without title
+ {
+ 'options': {
+ 'format': 'csv',
+ 'item_export_kwargs': dict(include_headers_line=False),
+ },
+ 'expected': expected_without_title_csv,
+ },
+ ]
+
+ for row in test_cases:
+ feed_options = row['options']
+ settings = {
+ 'FEEDS': {
+ self._random_temp_filename(): feed_options,
+ },
+ 'FEED_EXPORT_INDENT': None,
+ }
+
+ data = yield self.exported_data(items, settings)
+ self.assertEqual(row['expected'], data[feed_options['format']])
+
class BatchDeliveriesTest(FeedExportTestBase):
__test__ = True