Scrapy, a fast high-level web crawling & scraping framework for Python.
Feed exports: consistent and backwards compatible behaviour on indent
Eugenio Lacuestac7bb2fa8ce26
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rstindex ad559fb35..2ad77c905 100644--- a/docs/topics/exporters.rst+++ b/docs/topics/exporters.rst@@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items::BaseItemExporter-----------------.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=None)+.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=0)This is the (abstract) base class for all Item Exporters. It providessupport for common features used by all (concrete) Item Exporters, such as@@ -218,12 +218,12 @@ BaseItemExporter.. attribute:: indent- Amount of spaces used to indent the output on each level. Defaults to ``None``,- which disables indentation. This argument behaves like ``indent`` in python's- JSON module (both for JSON and XML exporters): "If ``indent`` is a non-negative- integer, then array elements and object members will be pretty-printed with that- indent level. An indent level of 0, or negative, will only insert newlines.- ``None`` (the default) selects the most compact representation"+ Amount of spaces used to indent the output on each level. Defaults to ``0``.++ * ``indent=None`` selects the most compact representation,+ all items in the same line with no indentation+ * ``indent<=0`` each item on it's own line, no indentation+ * ``indent>0`` each item on it's own line, indentated with the provided numeric value.. highlight:: nonediff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex afaa972e5..e57a4e776 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -272,12 +272,12 @@ exported data - currently it uses field names from the first item.FEED_EXPORT_INDENT-------------------Default: ``None``+Default: ``0``Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT``is a non-negative integer, then array elements and object members will be pretty-printed-with that indent level. An indent level of 0, or negative, will only insert newlines.-``None`` (the default) selects the most compact representation+with that indent level. An indent level of ``0``, or negative, will put each item on a new line.+``None`` selects the most compact representationCurrently used by :class:`~scrapy.exporters.JsonItemExporter`and :class:`~scrapy.exporters.XmlItemExporter`diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 1dfa2af85..e2d42b6ab 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -99,8 +99,12 @@ class JsonItemExporter(BaseItemExporter):def __init__(self, file, **kwargs):self._configure(kwargs, dont_fail=True)self.file = file+ # there is a small difference between the behaviour or JsonItemExporter.indent+ # and ScrapyJSONEncoder.indent. ScrapyJSONEncoder.indent=None is needed to prevent+ # the addition of newlines everywhere+ json_indent = self.indent if self.indent is not None and self.indent > 0 else None+ kwargs.setdefault('indent', json_indent)kwargs.setdefault('ensure_ascii', not self.encoding)- kwargs.setdefault('indent', self.indent)self.encoder = ScrapyJSONEncoder(**kwargs)self.first_item = True@@ -137,8 +141,8 @@ class XmlItemExporter(BaseItemExporter):self.encoding = 'utf-8'self.xg = XMLGenerator(file, encoding=self.encoding)- def _beautify_newline(self):- if self.indent is not None:+ def _beautify_newline(self, new_item=False):+ if self.indent is not None and (self.indent > 0 or new_item):self._xg_characters('\n')def _beautify_indent(self, depth=1):@@ -148,7 +152,7 @@ class XmlItemExporter(BaseItemExporter):def start_exporting(self):self.xg.startDocument()self.xg.startElement(self.root_element, {})- self._beautify_newline()+ self._beautify_newline(new_item=True)def export_item(self, item):self._beautify_indent(depth=1)@@ -158,7 +162,7 @@ class XmlItemExporter(BaseItemExporter):self._export_xml_field(name, value, depth=2)self._beautify_indent(depth=1)self.xg.endElement(self.item_element)- self._beautify_newline()+ self._beautify_newline(new_item=True)def finish_exporting(self):self.xg.endElement(self.root_element)diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex fc265e2ba..bbc02cfdb 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -161,7 +161,7 @@ FEED_EXPORTERS_BASE = {'marshal': 'scrapy.exporters.MarshalItemExporter','pickle': 'scrapy.exporters.PickleItemExporter',}-FEED_EXPORT_INDENT = None+FEED_EXPORT_INDENT = 0FILES_STORE_S3_ACL = 'private'diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 2b82bba0c..c66c470a8 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -326,7 +326,7 @@ class FeedExportTest(unittest.TestCase):)for fmt, expctd in formats:- settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True}+ settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True, 'FEED_EXPORT_INDENT': None}data = yield self.exported_no_data(settings)self.assertEqual(data, expctd)@@ -451,9 +451,12 @@ class FeedExportTest(unittest.TestCase):@defer.inlineCallbacksdef test_export_indentation(self):- items = [dict({'foo': ['bar']}), dict({'key': 'value'})]+ items = [+ {'foo': ['bar']},+ {'key': 'value'},+ ]- output = [+ test_cases = [# JSON{'format': 'json',@@ -465,14 +468,8 @@ class FeedExportTest(unittest.TestCase):'indent': -1,'expected': b"""[-{-"foo": [-"bar"-]-},-{-"key": "value"-}+{"foo": ["bar"]},+{"key": "value"}]""",},@@ -481,14 +478,8 @@ class FeedExportTest(unittest.TestCase):'indent': 0,'expected': b"""[-{-"foo": [-"bar"-]-},-{-"key": "value"-}+{"foo": ["bar"]},+{"key": "value"}]""",},@@ -542,7 +533,9 @@ class FeedExportTest(unittest.TestCase):{'format': 'xml','indent': None,- 'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>',+ 'expected': b"""+<?xml version="1.0" encoding="utf-8"?>+<items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>""",},{'format': 'xml',@@ -550,14 +543,8 @@ class FeedExportTest(unittest.TestCase):'expected': b"""<?xml version="1.0" encoding="utf-8"?><items>-<item>-<foo>-<value>bar</value>-</foo>-</item>-<item>-<key>value</key>-</item>+<item><foo><value>bar</value></foo></item>+<item><key>value</key></item></items>""",},{@@ -566,14 +553,8 @@ class FeedExportTest(unittest.TestCase):'expected': b"""<?xml version="1.0" encoding="utf-8"?><items>-<item>-<foo>-<value>bar</value>-</foo>-</item>-<item>-<key>value</key>-</item>+<item><foo><value>bar</value></foo></item>+<item><key>value</key></item></items>""",},{@@ -626,7 +607,8 @@ class FeedExportTest(unittest.TestCase):},]- for row in output:+ for row in test_cases:settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']}data = yield self.exported_data(items, settings)+ print(row['format'], row['indent'])self.assertEqual(row['expected'].strip(), data)