Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

PY3: ported json(lines), xml exporters

Elias Dornelesc76190d491fc
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 6f679480d..4138f6192 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -11,7 +11,10 @@ from six.moves import cPickle as pickle
from xml.sax.saxutils import XMLGenerator
from scrapy.utils.serialize import ScrapyJSONEncoder
+from scrapy.utils.python import to_bytes, to_unicode
from scrapy.item import BaseItem
+import warnings
+
__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter',
'CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter',
@@ -83,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter):
def export_item(self, item):
itemdict = dict(self._get_serialized_fields(item))
- self.file.write(self.encoder.encode(itemdict) + '\n')
+ self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n'))
class JsonItemExporter(BaseItemExporter):
@@ -95,18 +98,18 @@ class JsonItemExporter(BaseItemExporter):
self.first_item = True
def start_exporting(self):
- self.file.write("[")
+ self.file.write(b"[")
def finish_exporting(self):
- self.file.write("]")
+ self.file.write(b"]")
def export_item(self, item):
if self.first_item:
self.first_item = False
else:
- self.file.write(',\n')
+ self.file.write(b',\n')
itemdict = dict(self._get_serialized_fields(item))
- self.file.write(self.encoder.encode(itemdict))
+ self.file.write(to_bytes(self.encoder.encode(itemdict)))
class XmlItemExporter(BaseItemExporter):
@@ -136,8 +139,9 @@ class XmlItemExporter(BaseItemExporter):
if hasattr(serialized_value, 'items'):
for subname, value in serialized_value.items():
self._export_xml_field(subname, value)
- elif hasattr(serialized_value, '__iter__'):
- for value in serialized_value:
+ elif (hasattr(serialized_value, '__iter__')
+ and not isinstance(serialized_value, six.string_types)):
+ for value in serialized_value:
self._export_xml_field('value', value)
else:
self._xg_characters(serialized_value)
@@ -150,7 +154,7 @@ class XmlItemExporter(BaseItemExporter):
# and Python 3.x will require unicode, so ">= 2.7.4" should be fine.
if sys.version_info[:3] >= (2, 7, 4):
def _xg_characters(self, serialized_value):
- if not isinstance(serialized_value, unicode):
+ if not isinstance(serialized_value, six.text_type):
serialized_value = serialized_value.decode(self.encoding)
return self.xg.characters(serialized_value)
else:
@@ -177,7 +181,7 @@ class CsvItemExporter(BaseItemExporter):
value = self._join_multivalued.join(value)
except TypeError: # list in value may not contain strings
pass
- return value.encode(self.encoding) if isinstance(value, unicode) else value
+ return value.encode(self.encoding) if isinstance(value, six.text_type) else value
def export_item(self, item):
if self._headers_not_written:
@@ -231,7 +235,7 @@ class PprintItemExporter(BaseItemExporter):
def export_item(self, item):
itemdict = dict(self._get_serialized_fields(item))
- self.file.write(pprint.pformat(itemdict) + '\n')
+ self.file.write(to_bytes(pprint.pformat(itemdict) + '\n'))
class PythonItemExporter(BaseItemExporter):
@@ -240,6 +244,13 @@ class PythonItemExporter(BaseItemExporter):
json, msgpack, binc, etc) can be used on top of it. Its main goal is to
seamless support what BaseItemExporter does plus nested items.
"""
+ def _configure(self, options, dont_fail=False):
+ self.binary = options.pop('binary', True)
+ super(PythonItemExporter, self)._configure(options, dont_fail)
+ if self.binary:
+ warnings.warn(
+ "PythonItemExporter will drop support for binary export in the future",
+ PendingDeprecationWarning)
def serialize_field(self, field, name, value):
serializer = field.get('serializer', self._serialize_value)
@@ -250,9 +261,13 @@ class PythonItemExporter(BaseItemExporter):
return self.export_item(value)
if isinstance(value, dict):
return dict(self._serialize_dict(value))
- if hasattr(value, '__iter__'):
+ if hasattr(value, '__iter__') \
+ and not isinstance(value, six.string_types):
return [self._serialize_value(v) for v in value]
- return value.encode(self.encoding) if isinstance(value, unicode) else value
+ if self.binary:
+ return to_bytes(value, encoding=self.encoding)
+ else:
+ return to_unicode(value, encoding=self.encoding)
def _serialize_dict(self, value):
for key, val in six.iteritems(value):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index c84fb978a..05374e617 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -3,6 +3,7 @@ import re
import json
import unittest
from io import BytesIO
+import six
from six.moves import cPickle as pickle
import lxml.etree
@@ -80,7 +81,7 @@ class BaseItemExporterTest(unittest.TestCase):
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
-class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):
+class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest):
"""Class introduced just to keep old behavior of BaseItemExporterTest for the
test cases that inherit from it while we make changes to exporters one by
one -- a needed refactoring trick because the test cases are quite coupled.
@@ -127,9 +128,9 @@ class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):
self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')
-class PythonItemExporterTest(MidRefactoringBaseItemExporterTest):
+class PythonItemExporterTest(BaseItemExporterTest):
def _get_exporter(self, **kwargs):
- return PythonItemExporter(**kwargs)
+ return PythonItemExporter(binary=False, **kwargs)
def test_nested_item(self):
i1 = TestItem(name=u'Joseph', age='22')
@@ -194,7 +195,8 @@ class PickleItemExporterTest(BaseItemExporterTest):
self.assertEqual(pickle.load(f), i2)
-class CsvItemExporterTest(MidRefactoringBaseItemExporterTest):
[email protected](six.PY2, "TODO")
+class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):
def _get_exporter(self, **kwargs):
return CsvItemExporter(self.output, **kwargs)
@@ -294,13 +296,13 @@ class XmlItemExporterTest(BaseItemExporterTest):
self.assertXmlEquivalent(fp.getvalue(), expected_value)
def _check_output(self):
- expected_value = '<?xml version="1.0" encoding="utf-8"?>\n<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'
+ expected_value = u'<?xml version="1.0" ?>\n<items><item><age>22</age><name>John\xa3</name></item></items>'
self.assertXmlEquivalent(self.output.getvalue(), expected_value)
def test_multivalued_fields(self):
self.assertExportResult(
TestItem(name=[u'John\xa3', u'Doe']),
- '<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'
+ u'<?xml version="1.0" ?>\n<items><item><name><value>John\xa3</value><value>Doe</value></name></item></items>'
)
def test_nested_item(self):
@@ -309,19 +311,19 @@ class XmlItemExporterTest(BaseItemExporterTest):
i3 = TestItem(name=u'buz', age=i2)
self.assertExportResult(i3,
- '<?xml version="1.0" encoding="utf-8"?>\n'
- '<items>'
- '<item>'
- '<age>'
- '<age>'
- '<age>22</age>'
- '<name>foo\xc2\xa3hoo</name>'
- '</age>'
- '<name>bar</name>'
- '</age>'
- '<name>buz</name>'
- '</item>'
- '</items>'
+ u'<?xml version="1.0" ?>\n'
+ u'<items>'
+ u'<item>'
+ u'<age>'
+ u'<age>'
+ u'<age>22</age>'
+ u'<name>foo\xa3hoo</name>'
+ u'</age>'
+ u'<name>bar</name>'
+ u'</age>'
+ u'<name>buz</name>'
+ u'</item>'
+ u'</items>'
)
def test_nested_list_item(self):
@@ -330,16 +332,16 @@ class XmlItemExporterTest(BaseItemExporterTest):
i3 = TestItem(name=u'buz', age=[i1, i2])
self.assertExportResult(i3,
- '<?xml version="1.0" encoding="utf-8"?>\n'
- '<items>'
- '<item>'
- '<age>'
- '<value><name>foo</name></value>'
- '<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
- '</age>'
- '<name>buz</name>'
- '</item>'
- '</items>'
+ u'<?xml version="1.0" ?>\n'
+ u'<items>'
+ u'<item>'
+ u'<age>'
+ u'<value><name>foo</name></value>'
+ u'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'
+ u'</age>'
+ u'<name>buz</name>'
+ u'</item>'
+ u'</items>'
)
@@ -351,7 +353,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
return JsonLinesItemExporter(self.output, **kwargs)
def _check_output(self):
- exported = json.loads(self.output.getvalue().strip())
+ exported = json.loads(to_unicode(self.output.getvalue().strip()))
self.assertEqual(exported, dict(self.i))
def test_nested_item(self):
@@ -361,7 +363,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
- exported = json.loads(self.output.getvalue())
+ exported = json.loads(to_unicode(self.output.getvalue()))
self.assertEqual(exported, self._expected_nested)
def test_extra_keywords(self):
@@ -379,7 +381,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
return JsonItemExporter(self.output, **kwargs)
def _check_output(self):
- exported = json.loads(self.output.getvalue().strip())
+ exported = json.loads(to_unicode(self.output.getvalue().strip()))
self.assertEqual(exported, [dict(self.i)])
def assertTwoItemsExported(self, item):
@@ -387,7 +389,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.export_item(item)
self.ie.export_item(item)
self.ie.finish_exporting()
- exported = json.loads(self.output.getvalue())
+ exported = json.loads(to_unicode(self.output.getvalue()))
self.assertEqual(exported, [dict(item), dict(item)])
def test_two_items(self):
@@ -403,7 +405,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
- exported = json.loads(self.output.getvalue())
+ exported = json.loads(to_unicode(self.output.getvalue()))
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}
self.assertEqual(exported, [expected])
@@ -414,7 +416,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.ie.start_exporting()
self.ie.export_item(i3)
self.ie.finish_exporting()
- exported = json.loads(self.output.getvalue())
+ exported = json.loads(to_unicode(self.output.getvalue()))
expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}
self.assertEqual(exported, [expected])