Scrapy, a fast high-level web crawling & scraping framework for Python.
PY3: ported json(lines), xml exporters
Elias Dornelesc76190d491fc
diff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 6f679480d..4138f6192 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -11,7 +11,10 @@ from six.moves import cPickle as picklefrom xml.sax.saxutils import XMLGeneratorfrom scrapy.utils.serialize import ScrapyJSONEncoder+from scrapy.utils.python import to_bytes, to_unicodefrom scrapy.item import BaseItem+import warnings+__all__ = ['BaseItemExporter', 'PprintItemExporter', 'PickleItemExporter','CsvItemExporter', 'XmlItemExporter', 'JsonLinesItemExporter',@@ -83,7 +86,7 @@ class JsonLinesItemExporter(BaseItemExporter):def export_item(self, item):itemdict = dict(self._get_serialized_fields(item))- self.file.write(self.encoder.encode(itemdict) + '\n')+ self.file.write(to_bytes(self.encoder.encode(itemdict) + '\n'))class JsonItemExporter(BaseItemExporter):@@ -95,18 +98,18 @@ class JsonItemExporter(BaseItemExporter):self.first_item = Truedef start_exporting(self):- self.file.write("[")+ self.file.write(b"[")def finish_exporting(self):- self.file.write("]")+ self.file.write(b"]")def export_item(self, item):if self.first_item:self.first_item = Falseelse:- self.file.write(',\n')+ self.file.write(b',\n')itemdict = dict(self._get_serialized_fields(item))- self.file.write(self.encoder.encode(itemdict))+ self.file.write(to_bytes(self.encoder.encode(itemdict)))class XmlItemExporter(BaseItemExporter):@@ -136,8 +139,9 @@ class XmlItemExporter(BaseItemExporter):if hasattr(serialized_value, 'items'):for subname, value in serialized_value.items():self._export_xml_field(subname, value)- elif hasattr(serialized_value, '__iter__'):- for value in serialized_value:+ elif (hasattr(serialized_value, '__iter__')+ and not isinstance(serialized_value, six.string_types)):+ for value in serialized_value:self._export_xml_field('value', value)else:self._xg_characters(serialized_value)@@ -150,7 +154,7 @@ class XmlItemExporter(BaseItemExporter):# and Python 3.x will require unicode, so ">= 2.7.4" should be fine.if sys.version_info[:3] >= (2, 7, 4):def _xg_characters(self, serialized_value):- if not isinstance(serialized_value, unicode):+ if not isinstance(serialized_value, six.text_type):serialized_value = serialized_value.decode(self.encoding)return self.xg.characters(serialized_value)else:@@ -177,7 +181,7 @@ class CsvItemExporter(BaseItemExporter):value = self._join_multivalued.join(value)except TypeError: # list in value may not contain stringspass- return value.encode(self.encoding) if isinstance(value, unicode) else value+ return value.encode(self.encoding) if isinstance(value, six.text_type) else valuedef export_item(self, item):if self._headers_not_written:@@ -231,7 +235,7 @@ class PprintItemExporter(BaseItemExporter):def export_item(self, item):itemdict = dict(self._get_serialized_fields(item))- self.file.write(pprint.pformat(itemdict) + '\n')+ self.file.write(to_bytes(pprint.pformat(itemdict) + '\n'))class PythonItemExporter(BaseItemExporter):@@ -240,6 +244,13 @@ class PythonItemExporter(BaseItemExporter):json, msgpack, binc, etc) can be used on top of it. Its main goal is toseamless support what BaseItemExporter does plus nested items."""+ def _configure(self, options, dont_fail=False):+ self.binary = options.pop('binary', True)+ super(PythonItemExporter, self)._configure(options, dont_fail)+ if self.binary:+ warnings.warn(+ "PythonItemExporter will drop support for binary export in the future",+ PendingDeprecationWarning)def serialize_field(self, field, name, value):serializer = field.get('serializer', self._serialize_value)@@ -250,9 +261,13 @@ class PythonItemExporter(BaseItemExporter):return self.export_item(value)if isinstance(value, dict):return dict(self._serialize_dict(value))- if hasattr(value, '__iter__'):+ if hasattr(value, '__iter__') \+ and not isinstance(value, six.string_types):return [self._serialize_value(v) for v in value]- return value.encode(self.encoding) if isinstance(value, unicode) else value+ if self.binary:+ return to_bytes(value, encoding=self.encoding)+ else:+ return to_unicode(value, encoding=self.encoding)def _serialize_dict(self, value):for key, val in six.iteritems(value):diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex c84fb978a..05374e617 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -3,6 +3,7 @@ import reimport jsonimport unittestfrom io import BytesIO+import sixfrom six.moves import cPickle as pickleimport lxml.etree@@ -80,7 +81,7 @@ class BaseItemExporterTest(unittest.TestCase):self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')-class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):+class IntermediateRefactoringBaseItemExporterTest(BaseItemExporterTest):"""Class introduced just to keep old behavior of BaseItemExporterTest for thetest cases that inherit from it while we make changes to exporters one byone -- a needed refactoring trick because the test cases are quite coupled.@@ -127,9 +128,9 @@ class MidRefactoringBaseItemExporterTest(BaseItemExporterTest):self.assertEqual(ie.serialize_field(i.fields['age'], 'age', i['age']), '24')-class PythonItemExporterTest(MidRefactoringBaseItemExporterTest):+class PythonItemExporterTest(BaseItemExporterTest):def _get_exporter(self, **kwargs):- return PythonItemExporter(**kwargs)+ return PythonItemExporter(binary=False, **kwargs)def test_nested_item(self):i1 = TestItem(name=u'Joseph', age='22')@@ -194,7 +195,8 @@ class PickleItemExporterTest(BaseItemExporterTest):self.assertEqual(pickle.load(f), i2)-class CsvItemExporterTest(MidRefactoringBaseItemExporterTest):[email protected](six.PY2, "TODO")+class CsvItemExporterTest(IntermediateRefactoringBaseItemExporterTest):def _get_exporter(self, **kwargs):return CsvItemExporter(self.output, **kwargs)@@ -294,13 +296,13 @@ class XmlItemExporterTest(BaseItemExporterTest):self.assertXmlEquivalent(fp.getvalue(), expected_value)def _check_output(self):- expected_value = '<?xml version="1.0" encoding="utf-8"?>\n<items><item><age>22</age><name>John\xc2\xa3</name></item></items>'+ expected_value = u'<?xml version="1.0" ?>\n<items><item><age>22</age><name>John\xa3</name></item></items>'self.assertXmlEquivalent(self.output.getvalue(), expected_value)def test_multivalued_fields(self):self.assertExportResult(TestItem(name=[u'John\xa3', u'Doe']),- '<?xml version="1.0" encoding="utf-8"?>\n<items><item><name><value>John\xc2\xa3</value><value>Doe</value></name></item></items>'+ u'<?xml version="1.0" ?>\n<items><item><name><value>John\xa3</value><value>Doe</value></name></item></items>')def test_nested_item(self):@@ -309,19 +311,19 @@ class XmlItemExporterTest(BaseItemExporterTest):i3 = TestItem(name=u'buz', age=i2)self.assertExportResult(i3,- '<?xml version="1.0" encoding="utf-8"?>\n'- '<items>'- '<item>'- '<age>'- '<age>'- '<age>22</age>'- '<name>foo\xc2\xa3hoo</name>'- '</age>'- '<name>bar</name>'- '</age>'- '<name>buz</name>'- '</item>'- '</items>'+ u'<?xml version="1.0" ?>\n'+ u'<items>'+ u'<item>'+ u'<age>'+ u'<age>'+ u'<age>22</age>'+ u'<name>foo\xa3hoo</name>'+ u'</age>'+ u'<name>bar</name>'+ u'</age>'+ u'<name>buz</name>'+ u'</item>'+ u'</items>')def test_nested_list_item(self):@@ -330,16 +332,16 @@ class XmlItemExporterTest(BaseItemExporterTest):i3 = TestItem(name=u'buz', age=[i1, i2])self.assertExportResult(i3,- '<?xml version="1.0" encoding="utf-8"?>\n'- '<items>'- '<item>'- '<age>'- '<value><name>foo</name></value>'- '<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'- '</age>'- '<name>buz</name>'- '</item>'- '</items>'+ u'<?xml version="1.0" ?>\n'+ u'<items>'+ u'<item>'+ u'<age>'+ u'<value><name>foo</name></value>'+ u'<value><name>bar</name><v2><egg><value>spam</value></egg></v2></value>'+ u'</age>'+ u'<name>buz</name>'+ u'</item>'+ u'</items>')@@ -351,7 +353,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):return JsonLinesItemExporter(self.output, **kwargs)def _check_output(self):- exported = json.loads(self.output.getvalue().strip())+ exported = json.loads(to_unicode(self.output.getvalue().strip()))self.assertEqual(exported, dict(self.i))def test_nested_item(self):@@ -361,7 +363,7 @@ class JsonLinesItemExporterTest(BaseItemExporterTest):self.ie.start_exporting()self.ie.export_item(i3)self.ie.finish_exporting()- exported = json.loads(self.output.getvalue())+ exported = json.loads(to_unicode(self.output.getvalue()))self.assertEqual(exported, self._expected_nested)def test_extra_keywords(self):@@ -379,7 +381,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):return JsonItemExporter(self.output, **kwargs)def _check_output(self):- exported = json.loads(self.output.getvalue().strip())+ exported = json.loads(to_unicode(self.output.getvalue().strip()))self.assertEqual(exported, [dict(self.i)])def assertTwoItemsExported(self, item):@@ -387,7 +389,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):self.ie.export_item(item)self.ie.export_item(item)self.ie.finish_exporting()- exported = json.loads(self.output.getvalue())+ exported = json.loads(to_unicode(self.output.getvalue()))self.assertEqual(exported, [dict(item), dict(item)])def test_two_items(self):@@ -403,7 +405,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):self.ie.start_exporting()self.ie.export_item(i3)self.ie.finish_exporting()- exported = json.loads(self.output.getvalue())+ exported = json.loads(to_unicode(self.output.getvalue()))expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': dict(i1)}}self.assertEqual(exported, [expected])@@ -414,7 +416,7 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):self.ie.start_exporting()self.ie.export_item(i3)self.ie.finish_exporting()- exported = json.loads(self.output.getvalue())+ exported = json.loads(to_unicode(self.output.getvalue()))expected = {'name': u'Jesus', 'age': {'name': 'Maria', 'age': i1}}self.assertEqual(exported, [expected])