Scrapy, a fast high-level web crawling & scraping framework for Python.
Feed exporter: start exporting only on first item
Paul Tremberth03ab0772491c
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex c3fc66de5..85d328528 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -170,6 +170,7 @@ class FeedExporter(object):if not self._exporter_supported(self.format):raise NotConfiguredself.store_empty = settings.getbool('FEED_STORE_EMPTY')+ self._exporting = Falseself.export_fields = settings.getlist('FEED_EXPORT_FIELDS') or Noneuripar = settings['FEED_URI_PARAMS']self._uripar = load_object(uripar) if uripar else lambda x, y: None@@ -188,14 +189,18 @@ class FeedExporter(object):file = storage.open(spider)exporter = self._get_exporter(file, fields_to_export=self.export_fields,encoding=self.export_encoding)- exporter.start_exporting()+ if self.store_empty:+ exporter.start_exporting()+ self._exporting = Trueself.slot = SpiderSlot(file, exporter, storage, uri)def close_spider(self, spider):slot = self.slotif not slot.itemcount and not self.store_empty:return- slot.exporter.finish_exporting()+ if self._exporting:+ slot.exporter.finish_exporting()+ self._exporting = Falselogfmt = "%s %%(format)s feed (%%(itemcount)d items) in: %%(uri)s"log_args = {'format': self.format,'itemcount': slot.itemcount,@@ -210,6 +215,9 @@ class FeedExporter(object):def item_scraped(self, item, spider):slot = self.slot+ if not self._exporting:+ slot.exporter.start_exporting()+ self._exporting = Trueslot.exporter.export_item(item)slot.itemcount += 1return itemdiff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 353b21927..e93d2bafb 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -197,6 +197,21 @@ class FeedExportTest(unittest.TestCase):data = yield self.run_and_export(TestSpider, settings)defer.returnValue(data)+ @defer.inlineCallbacks+ def exported_no_data(self, settings):+ """+ Return exported data which a spider yielding no ``items`` would return.+ """+ class TestSpider(scrapy.Spider):+ name = 'testspider'+ start_urls = ['http://localhost:8998/']++ def parse(self, response):+ pass++ data = yield self.run_and_export(TestSpider, settings)+ defer.returnValue(data)+@defer.inlineCallbacksdef assertExportedCsv(self, items, header, rows, settings=None, ordered=True):settings = settings or {}@@ -283,6 +298,32 @@ class FeedExportTest(unittest.TestCase):header = self.MyItem.fields.keys()yield self.assertExported(items, header, rows, ordered=False)+ @defer.inlineCallbacks+ def test_export_no_items_not_store_empty(self):+ formats = ('json',+ 'jsonlines',+ 'xml',+ 'csv',)++ for fmt in formats:+ settings = {'FEED_FORMAT': fmt}+ data = yield self.exported_no_data(settings)+ self.assertEqual(data, b'')++ @defer.inlineCallbacks+ def test_export_no_items_store_empty(self):+ formats = (+ ('json', b'[\n\n]'),+ ('jsonlines', b''),+ ('xml', b'<?xml version="1.0" encoding="utf-8"?>\n<items></items>'),+ ('csv', b''),+ )++ for fmt, expctd in formats:+ settings = {'FEED_FORMAT': fmt, 'FEED_STORE_EMPTY': True}+ data = yield self.exported_no_data(settings)+ self.assertEqual(data, expctd)+@defer.inlineCallbacksdef test_export_multiple_item_classes(self):@@ -376,26 +417,26 @@ class FeedExportTest(unittest.TestCase):def test_export_encoding(self):items = [dict({'foo': u'Test\xd6'})]header = ['foo']-+formats = {'json': u'[\n{"foo": "Test\\u00d6"}\n]'.encode('utf-8'),'jsonlines': u'{"foo": "Test\\u00d6"}\n'.encode('utf-8'),'xml': u'<?xml version="1.0" encoding="utf-8"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('utf-8'),'csv': u'foo\r\nTest\xd6\r\n'.encode('utf-8'),}-+for format in formats:settings = {'FEED_FORMAT': format}data = yield self.exported_data(items, settings)self.assertEqual(formats[format], data)-+formats = {'json': u'[\n{"foo": "Test\xd6"}\n]'.encode('latin-1'),'jsonlines': u'{"foo": "Test\xd6"}\n'.encode('latin-1'),'xml': u'<?xml version="1.0" encoding="latin-1"?>\n<items><item><foo>Test\xd6</foo></item></items>'.encode('latin-1'),'csv': u'foo\r\nTest\xd6\r\n'.encode('latin-1'),}-+for format in formats:settings = {'FEED_FORMAT': format, 'FEED_EXPORT_ENCODING': 'latin-1'}data = yield self.exported_data(items, settings)