Scrapy, a fast high-level web crawling & scraping framework for Python.
Rename FEED_STORAGE_BATCH_SIZE to FEED_STORAGE_BATCH_ITEM_COUNT
BroodingKangaroo2327ecead085
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 2106b41f5..917240d4d 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -220,7 +220,7 @@ These are the settings used for configuring the feed exports:* :setting:`FEED_STORAGE_FTP_ACTIVE`* :setting:`FEED_STORAGE_S3_ACL`* :setting:`FEED_EXPORTERS`- * :setting:`FEED_EXPORT_BATCH_SIZE`+ * :setting:`FEED_STORAGE_BATCH_ITEM_COUNT`.. currentmodule:: scrapy.extensions.feedexport@@ -431,9 +431,9 @@ format in :setting:`FEED_EXPORTERS`. E.g., to disable the built-in CSV exporter.. _botocore: https://github.com/boto/botocore.. _Canned ACL: https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#canned-acl-.. setting:: FEED_EXPORT_BATCH_SIZE+.. setting:: FEED_STORAGE_BATCH_ITEM_COUNT-FEED_EXPORT_BATCH_SIZE+FEED_STORAGE_BATCH_ITEM_COUNT----------------------Default: ``None``@@ -448,7 +448,7 @@ Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.For instance::- FEED_EXPORT_BATCH_SIZE=100+ FEED_STORAGE_BATCH_ITEM_COUNT=100Your request can be like::diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex a262f5d18..5bc946634 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -242,7 +242,7 @@ class FeedExporter:self.storages = self._load_components('FEED_STORAGES')self.exporters = self._load_components('FEED_EXPORTERS')- self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_SIZE', None)+ self.storage_batch_size = self.settings.get('FEED_STORAGE_BATCH_ITEM_COUNT', None)for uri, feed in self.feeds.items():if not self._storage_supported(uri):raise NotConfigured@@ -290,7 +290,7 @@ class FeedExporter:def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):"""Redirect the output data stream to a new file.- Execute multiple times if 'FEED_STORAGE_BATCH' setting is specified.+ Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.:param batch_id: sequence number of current batch:param uri: uri of the new batch to start:param feed: dict with parameters of feed@@ -326,7 +326,7 @@ class FeedExporter:slot.start_exporting()slot.exporter.export_item(item)slot.itemcount += 1- # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_SIZE and close the old one+ # create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old oneif self.storage_batch_size and slot.itemcount == self.storage_batch_size:uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)self._close_slot(slot, spider)@@ -358,12 +358,12 @@ class FeedExporter:def _batch_deliveries_supported(self, uri):"""- If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)s+ If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)sto distinguish different files of partial output"""if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:return True- logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')+ logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified')return Falsedef _storage_supported(self, uri):diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex c3463a505..5a7dc533e 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -146,7 +146,7 @@ FEED_STORAGES_BASE = {'s3': 'scrapy.extensions.feedexport.S3FeedStorage','ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',}-FEED_STORAGE_BATCH_SIZE = None+FEED_STORAGE_BATCH_ITEM_COUNT = NoneFEED_EXPORTERS = {}FEED_EXPORTERS_BASE = {'json': 'scrapy.exporters.JsonItemExporter',diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 2217bb4ed..1a21eeba9 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1025,7 +1025,7 @@ class PartialDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'jl', self._file_mark): {'format': 'jl'},},})- batch_size = settings['FEED_STORAGE_BATCH_SIZE']+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']rows = [{k: v for k, v in row.items() if v} for row in rows]data = yield self.exported_data(items, settings)for batch in data['jl']:@@ -1041,7 +1041,7 @@ class PartialDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'csv', self._file_mark): {'format': 'csv'},},})- batch_size = settings['FEED_STORAGE_BATCH_SIZE']+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']data = yield self.exported_data(items, settings)for batch in data['csv']:got_batch = csv.DictReader(to_unicode(batch).splitlines())@@ -1057,7 +1057,7 @@ class PartialDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'xml', self._file_mark): {'format': 'xml'},},})- batch_size = settings['FEED_STORAGE_BATCH_SIZE']+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']rows = [{k: v for k, v in row.items() if v} for row in rows]data = yield self.exported_data(items, settings)for batch in data['xml']:@@ -1075,7 +1075,7 @@ class PartialDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'json', self._file_mark): {'format': 'json'},},})- batch_size = settings['FEED_STORAGE_BATCH_SIZE']+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']rows = [{k: v for k, v in row.items() if v} for row in rows]data = yield self.exported_data(items, settings)# XML@@ -1100,7 +1100,7 @@ class PartialDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'pickle', self._file_mark): {'format': 'pickle'},},})- batch_size = settings['FEED_STORAGE_BATCH_SIZE']+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']rows = [{k: v for k, v in row.items() if v} for row in rows]data = yield self.exported_data(items, settings)import pickle@@ -1117,7 +1117,7 @@ class PartialDeliveriesTest(FeedExportTestBase):os.path.join(self._random_temp_filename(), 'marshal', self._file_mark): {'format': 'marshal'},},})- batch_size = settings['FEED_STORAGE_BATCH_SIZE']+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']rows = [{k: v for k, v in row.items() if v} for row in rows]data = yield self.exported_data(items, settings)import marshal@@ -1140,7 +1140,7 @@ class PartialDeliveriesTest(FeedExportTestBase):{'foo': 'bar3', 'baz': 'quux3', 'egg': ''}]settings = {- 'FEED_STORAGE_BATCH_SIZE': 2+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 2}header = self.MyItem.fields.keys()yield self.assertExported(items, header, rows, settings=settings)@@ -1151,7 +1151,7 @@ class PartialDeliveriesTest(FeedExportTestBase):'FEEDS': {self._random_temp_filename(): {'format': 'xml'},},- 'FEED_STORAGE_BATCH_SIZE': 1+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1}crawler = get_crawler(settings_dict=settings)self.assertRaises(NotConfigured, FeedExporter, crawler)@@ -1163,7 +1163,7 @@ class PartialDeliveriesTest(FeedExportTestBase):'FEEDS': {os.path.join(self._random_temp_filename(), fmt, self._file_mark): {'format': fmt},},- 'FEED_STORAGE_BATCH_SIZE': 1+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1}data = yield self.exported_no_data(settings)data = dict(data)@@ -1185,7 +1185,7 @@ class PartialDeliveriesTest(FeedExportTestBase):},'FEED_STORE_EMPTY': True,'FEED_EXPORT_INDENT': None,- 'FEED_STORAGE_BATCH_SIZE': 1,+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,}data = yield self.exported_no_data(settings)data = dict(data)@@ -1225,7 +1225,7 @@ class PartialDeliveriesTest(FeedExportTestBase):'encoding': 'utf-8',},},- 'FEED_STORAGE_BATCH_SIZE': 1,+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,}data = yield self.exported_data(items, settings)for fmt, expected in formats.items():@@ -1249,7 +1249,7 @@ class PartialDeliveriesTest(FeedExportTestBase):'format': 'json',},},- 'FEED_STORAGE_BATCH_SIZE': 1,+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,}data = yield self.exported_data(items, settings)self.assertEqual(len(items) + 1, len(data['json']))@@ -1291,7 +1291,7 @@ class PartialDeliveriesTest(FeedExportTestBase):'format': 'json',},},- 'FEED_STORAGE_BATCH_SIZE': 1,+ 'FEED_STORAGE_BATCH_ITEM_COUNT': 1,}items = [self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),@@ -1309,7 +1309,7 @@ class PartialDeliveriesTest(FeedExportTestBase):s3 = boto3.resource('s3')my_bucket = s3.Bucket(s3_test_bucket_name)- batch_size = settings['FEED_STORAGE_BATCH_SIZE']+ batch_size = settings['FEED_STORAGE_BATCH_ITEM_COUNT']with MockServer() as s:runner = CrawlerRunner(Settings(settings))