Scrapy, a fast high-level web crawling & scraping framework for Python.
Feed exports: enforce difference between None and 0 on indent
Eugenio Lacuesta766b2c84539d
diff --git a/docs/topics/exporters.rst b/docs/topics/exporters.rstindex 4114eda58..ad559fb35 100644--- a/docs/topics/exporters.rst+++ b/docs/topics/exporters.rst@@ -140,7 +140,7 @@ output examples, which assume you're exporting these two items::BaseItemExporter-----------------.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent_width=None)+.. class:: BaseItemExporter(fields_to_export=None, export_empty_fields=False, encoding='utf-8', indent=None)This is the (abstract) base class for all Item Exporters. It providessupport for common features used by all (concrete) Item Exporters, such as@@ -149,7 +149,7 @@ BaseItemExporterThese features can be configured through the constructor arguments whichpopulate their respective instance attributes: :attr:`fields_to_export`,- :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent_width`.+ :attr:`export_empty_fields`, :attr:`encoding`, :attr:`indent`... method:: export_item(item)@@ -216,10 +216,14 @@ BaseItemExporterencoding). Other value types are passed unchanged to the specificserialization library.- .. attribute:: indent_width+ .. attribute:: indent- Amount of spaces used to indent the output on each level.- Defaults to ``None``, which disables indentation.+ Amount of spaces used to indent the output on each level. Defaults to ``None``,+ which disables indentation. This argument behaves like ``indent`` in python's+ JSON module (both for JSON and XML exporters): "If ``indent`` is a non-negative+ integer, then array elements and object members will be pretty-printed with that+ indent level. An indent level of 0, or negative, will only insert newlines.+ ``None`` (the default) selects the most compact representation".. highlight:: nonediff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex ce3b5fd75..afaa972e5 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -209,7 +209,7 @@ These are the settings used for configuring the feed exports:* :setting:`FEED_STORE_EMPTY`* :setting:`FEED_EXPORT_ENCODING`* :setting:`FEED_EXPORT_FIELDS`- * :setting:`FEED_EXPORT_INDENT_WIDTH`+ * :setting:`FEED_EXPORT_INDENT`.. currentmodule:: scrapy.extensions.feedexport@@ -267,15 +267,17 @@ If an exporter requires a fixed set of fields (this is the case foris empty or None, then Scrapy tries to infer field names from theexported data - currently it uses field names from the first item.-.. setting:: FEED_EXPORT_INDENT_WIDTH+.. setting:: FEED_EXPORT_INDENT-FEED_EXPORT_INDENT_WIDTH-------------------------+FEED_EXPORT_INDENT+------------------Default: ``None``-Amount of spaces to indent on each level.-Set to `None` to disable indentation.+Amount of spaces used to indent the output on each level. If ``FEED_EXPORT_INDENT``+is a non-negative integer, then array elements and object members will be pretty-printed+with that indent level. An indent level of 0, or negative, will only insert newlines.+``None`` (the default) selects the most compact representationCurrently used by :class:`~scrapy.exporters.JsonItemExporter`and :class:`~scrapy.exporters.XmlItemExporter`diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 69e6c15e0..1dfa2af85 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -36,7 +36,7 @@ class BaseItemExporter(object):self.encoding = options.pop('encoding', None)self.fields_to_export = options.pop('fields_to_export', None)self.export_empty_fields = options.pop('export_empty_fields', False)- self.indent_width = options.pop('indent_width', None)+ self.indent = options.pop('indent', None)if not dont_fail and options:raise TypeError("Unexpected options: %s" % ', '.join(options.keys()))@@ -100,20 +100,28 @@ class JsonItemExporter(BaseItemExporter):self._configure(kwargs, dont_fail=True)self.file = filekwargs.setdefault('ensure_ascii', not self.encoding)- self.encoder = ScrapyJSONEncoder(indent=self.indent_width, **kwargs)+ kwargs.setdefault('indent', self.indent)+ self.encoder = ScrapyJSONEncoder(**kwargs)self.first_item = True+ def _beautify_newline(self):+ if self.indent is not None:+ self.file.write(b'\n')+def start_exporting(self):- self.file.write(b"[\n")+ self.file.write(b"[")+ self._beautify_newline()def finish_exporting(self):- self.file.write(b"\n]")+ self._beautify_newline()+ self.file.write(b"]")def export_item(self, item):if self.first_item:self.first_item = Falseelse:- self.file.write(b',\n')+ self.file.write(b',')+ self._beautify_newline()itemdict = dict(self._get_serialized_fields(item))data = self.encoder.encode(itemdict)self.file.write(to_bytes(data, self.encoding))@@ -130,12 +138,12 @@ class XmlItemExporter(BaseItemExporter):self.xg = XMLGenerator(file, encoding=self.encoding)def _beautify_newline(self):- if self.indent_width:+ if self.indent is not None:self._xg_characters('\n')def _beautify_indent(self, depth=1):- if self.indent_width:- self._xg_characters(' ' * self.indent_width * depth)+ if self.indent:+ self._xg_characters(' ' * self.indent * depth)def start_exporting(self):self.xg.startDocument()diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 26024e5e9..5f133fbde 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -172,7 +172,9 @@ class FeedExporter(object):self.store_empty = settings.getbool('FEED_STORE_EMPTY')self._exporting = Falseself.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or None- self.indent_width = settings.getint('FEED_EXPORT_INDENT_WIDTH') or None+ self.indent = None+ if settings.get('FEED_EXPORT_INDENT') is not None:+ self.indent = settings.getint('FEED_EXPORT_INDENT')uripar = settings['FEED_URI_PARAMS']self._uripar = load_object(uripar) if uripar else lambda x, y: None@@ -189,7 +191,7 @@ class FeedExporter(object):storage = self._get_storage(uri)file = storage.open(spider)exporter = self._get_exporter(file, fields_to_export=self.export_fields,- encoding=self.export_encoding, indent_width=self.indent_width)+ encoding=self.export_encoding, indent=self.indent)if self.store_empty:exporter.start_exporting()self._exporting = Truediff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex cca0d3889..fc265e2ba 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -161,7 +161,7 @@ FEED_EXPORTERS_BASE = {'marshal': 'scrapy.exporters.MarshalItemExporter','pickle': 'scrapy.exporters.PickleItemExporter',}-FEED_EXPORT_INDENT_WIDTH = None+FEED_EXPORT_INDENT = NoneFILES_STORE_S3_ACL = 'private'diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex bf002bec7..2b82bba0c 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -319,7 +319,7 @@ class FeedExportTest(unittest.TestCase):@defer.inlineCallbacksdef test_export_no_items_store_empty(self):formats = (- ('json', b'[\n\n]'),+ ('json', b'[]'),('jsonlines', b''),('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),('csv', b''),@@ -425,25 +425,25 @@ class FeedExportTest(unittest.TestCase):header = ['foo']formats = {- 'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'),+ 'json': u'[{"foo": "Test\\u00d6"}]'.encode('utf-8'),'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),}for format, expected in formats.items():- settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT_WIDTH': None}+ settings = {'FEED_FORMAT': format, 'FEED_EXPORT_INDENT': None}data = yield self.exported_data(items, settings)self.assertEqual(expected, data)formats = {- 'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'),+ 'json': u'[{"foo": "Test\xd6"}]'.encode('latin-1'),'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),}- settings = {'FEED_EXPORT_INDENT_WIDTH': None, 'FEED_EXPORT_ENCODING': 'latin-1'}+ settings = {'FEED_EXPORT_INDENT': None, 'FEED_EXPORT_ENCODING': 'latin-1'}for format, expected in formats.items():settings['FEED_FORMAT'] = formatdata = yield self.exported_data(items, settings)@@ -451,48 +451,89 @@ class FeedExportTest(unittest.TestCase):@defer.inlineCallbacksdef test_export_indentation(self):- items = [dict({'foo': ['bar']})]+ items = [dict({'foo': ['bar']}), dict({'key': 'value'})]output = [# JSON{'format': 'json',- 'indent_width': None,- 'expected': b'[\n{"foo": ["bar"]}\n]',+ 'indent': None,+ 'expected': b'[{"foo": ["bar"]},{"key": "value"}]',},{'format': 'json',- 'indent_width': 2,+ 'indent': -1,+ 'expected': b"""+[+{+"foo": [+"bar"+]+},+{+"key": "value"+}+]+""",+ },+ {+ 'format': 'json',+ 'indent': 0,+ 'expected': b"""+[+{+"foo": [+"bar"+]+},+{+"key": "value"+}+]+""",+ },+ {+ 'format': 'json',+ 'indent': 2,'expected': b"""[{"foo": ["bar"]+},+{+ "key": "value"}]""",},{'format': 'json',- 'indent_width': 4,+ 'indent': 4,'expected': b"""[{"foo": ["bar"]+},+{+ "key": "value"}]""",},{'format': 'json',- 'indent_width': 5,+ 'indent': 5,'expected': b"""[{"foo": ["bar"]+},+{+ "key": "value"}]""",},@@ -500,12 +541,44 @@ class FeedExportTest(unittest.TestCase):# XML{'format': 'xml',- 'indent_width': None,- 'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item></items>',+ 'indent': None,+ 'expected': b'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo><value>bar</value></foo></item><item><key>value</key></item></items>',},{'format': 'xml',- 'indent_width': 2,+ 'indent': -1,+ 'expected': b"""+<?xml version="1.0" encoding="utf-8"?>+<items>+<item>+<foo>+<value>bar</value>+</foo>+</item>+<item>+<key>value</key>+</item>+</items>""",+ },+ {+ 'format': 'xml',+ 'indent': 0,+ 'expected': b"""+<?xml version="1.0" encoding="utf-8"?>+<items>+<item>+<foo>+<value>bar</value>+</foo>+</item>+<item>+<key>value</key>+</item>+</items>""",+ },+ {+ 'format': 'xml',+ 'indent': 2,'expected': b"""<?xml version="1.0" encoding="utf-8"?><items>@@ -514,11 +587,14 @@ class FeedExportTest(unittest.TestCase):<value>bar</value></foo></item>+ <item>+ <key>value</key>+ </item></items>""",},{'format': 'xml',- 'indent_width': 4,+ 'indent': 4,'expected': b"""<?xml version="1.0" encoding="utf-8"?><items>@@ -527,11 +603,14 @@ class FeedExportTest(unittest.TestCase):<value>bar</value></foo></item>+ <item>+ <key>value</key>+ </item></items>""",},{'format': 'xml',- 'indent_width': 5,+ 'indent': 5,'expected': b"""<?xml version="1.0" encoding="utf-8"?><items>@@ -540,11 +619,14 @@ class FeedExportTest(unittest.TestCase):<value>bar</value></foo></item>+ <item>+ <key>value</key>+ </item></items>""",},]for row in output:- settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT_WIDTH': row['indent_width']}+ settings = {'FEED_FORMAT': row['format'], 'FEED_EXPORT_INDENT': row['indent']}data = yield self.exported_data(items, settings)self.assertEqual(row['expected'].strip(), data)