Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add batch_item_count support in FEEDS setting

BroodingKangaroo88a52198b90f
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index e06116acd..2312c994e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -25,7 +25,6 @@ from scrapy.utils.log import failure_to_exc_info
from scrapy.utils.misc import create_instance, load_object
from scrapy.utils.python import without_none_values
-
logger = logging.getLogger(__name__)
@@ -337,7 +336,9 @@ class FeedExporter:
slot.exporter.export_item(item)
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
- if self.storage_batch_item_count and slot.itemcount == self.storage_batch_item_count:
+ if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \
+ and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count',
+ self.storage_batch_item_count):
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
self._close_slot(slot, spider)
slots.append(self._start_new_batch(
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 578cd396b..3bc0c083c 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1327,8 +1327,9 @@ class BatchDeliveriesTest(FeedExportTestBase):
'<items>\n <item>\n <foo>FOO1</foo>\n </item>\n</items>'
).encode('latin-1')
],
- 'csv': ['bar,foo\r\nBAR,FOO\r\n'.encode('utf-8'),
- 'bar,foo\r\nBAR1,FOO1\r\n'.encode('utf-8')],
+ 'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),
+ 'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],
+ 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
}
settings = {
@@ -1348,8 +1349,15 @@ class BatchDeliveriesTest(FeedExportTestBase):
os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
'format': 'csv',
'indent': None,
- 'fields': ['bar', 'foo'],
+ 'fields': ['foo', 'bar'],
+ 'encoding': 'utf-8',
+ },
+ os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {
+ 'format': 'jsonlines',
+ 'indent': None,
+ 'fields': ['foo', 'bar'],
'encoding': 'utf-8',
+ 'batch_item_count': 0,
},
},
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
@@ -1359,6 +1367,31 @@ class BatchDeliveriesTest(FeedExportTestBase):
for expected_batch, got_batch in zip(expected, data[fmt]):
self.assertEqual(expected_batch, got_batch)
+ @defer.inlineCallbacks
+ def test_batch_item_count_feeds_setting(self):
+ items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
+
+ formats = {
+ 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'),
+ '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
+ }
+
+ settings = {
+ 'FEEDS': {
+ os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
+ 'format': 'jsonlines',
+ 'indent': None,
+ 'fields': ['foo', 'bar'],
+ 'encoding': 'utf-8',
+ 'batch_item_count': 1,
+ },
+ },
+ }
+ data = yield self.exported_data(items, settings)
+ for fmt, expected in formats.items():
+ for expected_batch, got_batch in zip(expected, data[fmt]):
+ self.assertEqual(expected_batch, got_batch)
+
@defer.inlineCallbacks
def test_batch_path_differ(self):
"""