Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix JsonItemExporter puts lone comma in the output if encoder fails (#5952)

Adnan Awan5d91ea12d642
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 4538c9ee1..8254ea63e 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -133,6 +133,13 @@ class JsonItemExporter(BaseItemExporter):
if self.indent is not None:
self.file.write(b"\n")
+ def _add_comma_after_first(self):
+ if self.first_item:
+ self.first_item = False
+ else:
+ self.file.write(b",")
+ self._beautify_newline()
+
def start_exporting(self):
self.file.write(b"[")
self._beautify_newline()
@@ -142,14 +149,10 @@ class JsonItemExporter(BaseItemExporter):
self.file.write(b"]")
def export_item(self, item):
- if self.first_item:
- self.first_item = False
- else:
- self.file.write(b",")
- self._beautify_newline()
itemdict = dict(self._get_serialized_fields(item))
- data = self.encoder.encode(itemdict)
- self.file.write(to_bytes(data, self.encoding))
+ data = to_bytes(self.encoder.encode(itemdict), self.encoding)
+ self._add_comma_after_first()
+ self.file.write(data)
class XmlItemExporter(BaseItemExporter):
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 63bebcf7a..cb24ddd8e 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -599,6 +599,20 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
def test_two_dict_items(self):
self.assertTwoItemsExported(ItemAdapter(self.i).asdict())
+ def test_two_items_with_failure_between(self):
+ i1 = TestItem(name="Joseph\xa3", age="22")
+ i2 = TestItem(
+ name="Maria", age=1j
+ ) # Invalid datetimes didn't consistently fail between Python versions
+ i3 = TestItem(name="Jesus", age="44")
+ self.ie.start_exporting()
+ self.ie.export_item(i1)
+ self.assertRaises(TypeError, self.ie.export_item, i2)
+ self.ie.export_item(i3)
+ self.ie.finish_exporting()
+ exported = json.loads(to_unicode(self.output.getvalue()))
+ self.assertEqual(exported, [dict(i1), dict(i3)])
+
def test_nested_item(self):
i1 = self.item_class(name="Joseph\xa3", age="22")
i2 = self.item_class(name="Maria", age=i1)
@@ -637,6 +651,24 @@ class JsonItemExporterTest(JsonLinesItemExporterTest):
self.assertEqual(exported, [item])
+class JsonItemExporterToBytesTest(BaseItemExporterTest):
+ def _get_exporter(self, **kwargs):
+ kwargs["encoding"] = "latin"
+ return JsonItemExporter(self.output, **kwargs)
+
+ def test_two_items_with_failure_between(self):
+ i1 = TestItem(name="Joseph", age="22")
+ i2 = TestItem(name="\u263a", age="11")
+ i3 = TestItem(name="Jesus", age="44")
+ self.ie.start_exporting()
+ self.ie.export_item(i1)
+ self.assertRaises(UnicodeEncodeError, self.ie.export_item, i2)
+ self.ie.export_item(i3)
+ self.ie.finish_exporting()
+ exported = json.loads(to_unicode(self.output.getvalue(), encoding="latin"))
+ self.assertEqual(exported, [dict(i1), dict(i3)])
+
+
class JsonItemExporterDataclassTest(JsonItemExporterTest):
item_class = TestDataClass
custom_field_item_class = CustomFieldDataclass