Scrapy, a fast high-level web crawling & scraping framework for Python.
Add batch_item_count support in FEEDS setting
BroodingKangaroo88a52198b90f
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex e06116acd..2312c994e 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_infofrom scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.python import without_none_values-logger = logging.getLogger(__name__)@@ -337,7 +336,9 @@ class FeedExporter:slot.exporter.export_item(item)slot.itemcount += 1# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one- if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count:+ if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \+ and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count',+ self.storage_batch_item_count):uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)self._close_slot(slot, spider)slots.append(self._start_new_batch(diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 578cd396b..3bc0c083c 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1327,8 +1327,9 @@ class BatchDeliveriesTest(FeedExportTestBase):'<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>').encode('latin-1')],- 'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),- 'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],+ 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),+ 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],+ 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],}settings = {@@ -1348,8 +1349,15 @@ class BatchDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv','indent': None,- 'fields': ['bar', 'foo'],+ 'fields': ['foo', 'bar'],+ 'encoding': 'utf-8',+ },+ os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {+ 'format': 'jsonlines',+ 'indent': None,+ 'fields': ['foo', 'bar'],'encoding': 'utf-8',+ 'batch_item_count': 0,},},'FEED_STORAGE_BATCH_ITEM_COUNT': 1,@@ -1359,6 +1367,31 @@ class BatchDeliveriesTest(FeedExportTestBase):for expected_batch, got_batch in zip(expected, data[fmt]):self.assertEqual(expected_batch, got_batch)+ @defer.inlineCallbacks+ def test_batch_item_count_feeds_setting(self):+ items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]++ formats = {+ 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'),+ '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],+ }++ settings = {+ 'FEEDS': {+ os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {+ 'format': 'jsonlines',+ 'indent': None,+ 'fields': ['foo', 'bar'],+ 'encoding': 'utf-8',+ 'batch_item_count': 1,+ },+ },+ }+ data = yield self.exported_data(items, settings)+ for fmt, expected in formats.items():+ for expected_batch, got_batch in zip(expected, data[fmt]):+ self.assertEqual(expected_batch, got_batch)+@defer.inlineCallbacksdef test_batch_path_differ(self):"""