Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Minor updates

BroodingKangaroo7b1d3c35ea3b
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index 3da56821e..0b659f30e 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -442,7 +442,7 @@ If assigned an integer number higher than ``0``, Scrapy generates multiple outpu
storing up to the specified number of items in each output file.
When generating multiple output files, you must use at least one of the following
-placeholders in :setting:`FEED_URI` to indicate how the different output file names are
+placeholders in the feed URI to indicate how the different output file names are
generated:
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
@@ -457,7 +457,7 @@ For instance, if your settings include::
And your :command:`crawl` command line is::
- scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
+ scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
The command line above can generate a directory tree like::
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 2312c994e..5908987a3 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -242,7 +242,6 @@ class FeedExporter:
self.storages = self._load_components('FEED_STORAGES')
self.exporters = self._load_components('FEED_EXPORTERS')
- self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')
for uri, feed in self.feeds.items():
if not self._storage_supported(uri):
raise NotConfigured
@@ -253,7 +252,7 @@ class FeedExporter:
def open_spider(self, spider):
for uri, feed in self.feeds.items():
- uri_params = self._get_uri_params(spider, feed['uri_params'], None)
+ uri_params = self._get_uri_params(spider, feed['uri_params'])
self.slots.append(self._start_new_batch(
batch_id=1,
uri=uri % uri_params,
@@ -299,7 +298,7 @@ class FeedExporter:
def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
"""
Redirect the output data stream to a new file.
- Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
+ Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified
:param batch_id: sequence number of current batch
:param uri: uri of the new batch to start
:param feed: dict with parameters of feed
@@ -331,14 +330,15 @@ class FeedExporter:
def item_scraped(self, item, spider):
slots = []
- for idx, slot in enumerate(self.slots):
+ for slot in self.slots:
slot.start_exporting()
slot.exporter.export_item(item)
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
- if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \
- and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count',
- self.storage_batch_item_count):
+ if (
+ self.feeds[slot.uri_template]['batch_item_count']
+ and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']
+ ):
uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
self._close_slot(slot, spider)
slots.append(self._start_new_batch(
@@ -369,15 +369,17 @@ class FeedExporter:
def _settings_are_valid(self, uri):
"""
- If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s
- to distinguish different files of partial output
+ If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain
+ %(batch_time)s or %(batch_id)s to distinguish different files of partial output
"""
- if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:
- return True
- logger.error(
- '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified'
- )
- return False
+ for uri_template, values in self.feeds.items():
+ if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']):
+ logger.error(
+ '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting '
+ 'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template)
+ )
+ return False
+ return True
def _storage_supported(self, uri):
scheme = urlparse(uri).scheme
@@ -404,7 +406,7 @@ class FeedExporter:
def _get_storage(self, uri):
return self._get_instance(self.storages[urlparse(uri).scheme], uri)
- def _get_uri_params(self, spider, uri_params, slot):
+ def _get_uri_params(self, spider, uri_params, slot=None):
params = {}
for k in dir(spider):
params[k] = getattr(spider, k)
diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.py
index 5921f82bf..0e02f0f28 100644
--- a/scrapy/utils/conf.py
+++ b/scrapy/utils/conf.py
@@ -115,6 +115,10 @@ def feed_complete_default_values_from_settings(feed, settings):
out = feed.copy()
out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])
out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)
+ out.setdefault(
+ "batch_item_count",
+ out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT'))
+ )
out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))
out.setdefault("uri_params", settings["FEED_URI_PARAMS"])
if settings["FEED_EXPORT_INDENT"] is None:
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 542cce70f..db14b20b9 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):
yield self.assertExported(items, header, rows, settings=Settings(settings))
def test_wrong_path(self):
- """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """
+ """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """
settings = {
'FEEDS': {
self._random_temp_filename(): {'format': 'xml'},
@@ -1329,7 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
],
'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),
'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],
- 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
}
settings = {
@@ -1352,13 +1351,6 @@ class BatchDeliveriesTest(FeedExportTestBase):
'fields': ['foo', 'bar'],
'encoding': 'utf-8',
},
- os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
- 'format': 'jsonlines',
- 'indent': None,
- 'fields': ['foo', 'bar'],
- 'encoding': 'utf-8',
- 'batch_item_count': 0,
- },
},
'FEED_STORAGE_BATCH_ITEM_COUNT': 1,
}
@@ -1369,19 +1361,16 @@ class BatchDeliveriesTest(FeedExportTestBase):
@defer.inlineCallbacks
def test_batch_item_count_feeds_setting(self):
- items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]
-
+ items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]
formats = {
- 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'),
- '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],
+ 'json': ['[{"foo": "FOO"}]'.encode('utf-8'),
+ '[{"foo": "FOO1"}]'.encode('utf-8')],
}
-
settings = {
'FEEDS': {
- os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {
- 'format': 'jsonlines',
+ os.path.join(self._random_temp_filename(), 'json', self._file_mark): {
+ 'format': 'json',
'indent': None,
- 'fields': ['foo', 'bar'],
'encoding': 'utf-8',
'batch_item_count': 1,
},
diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.py
index e5d3ef582..95ec2b64a 100644
--- a/tests/test_utils_conf.py
+++ b/tests/test_utils_conf.py
@@ -149,6 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"FEED_EXPORT_INDENT": 42,
"FEED_STORE_EMPTY": True,
"FEED_URI_PARAMS": (1, 2, 3, 4),
+ "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
})
new_feed = feed_complete_default_values_from_settings(feed, settings)
self.assertEqual(new_feed, {
@@ -157,6 +158,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"indent": 42,
"store_empty": True,
"uri_params": (1, 2, 3, 4),
+ "batch_item_count": 2,
})
def test_feed_complete_default_values_from_settings_non_empty(self):
@@ -169,6 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],
"FEED_EXPORT_INDENT": 42,
"FEED_STORE_EMPTY": True,
+ "FEED_STORAGE_BATCH_ITEM_COUNT": 2,
})
new_feed = feed_complete_default_values_from_settings(feed, settings)
self.assertEqual(new_feed, {
@@ -177,6 +180,7 @@ class FeedExportConfigTestCase(unittest.TestCase):
"indent": 42,
"store_empty": True,
"uri_params": None,
+ "batch_item_count": 2,
})