Scrapy, a fast high-level web crawling & scraping framework for Python.
Minor fixes
BroodingKangaroo1cdcf8b08b8f
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex dfeea5b7f..638733b6a 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -437,17 +437,18 @@ FEED_STORAGE_BATCH_ITEM_COUNT-----------------------------Default: ``None``-An integer number that represents the number of scraped items stored in each output-file. Whenever the number of items exceeds this setting, a new file is-created and the output is redirected to it. The name of the new file will be selected-based on the timestamp when the feed is being created and/or on the batch sequence number.-Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.+If assigned an integer number higher than ``0``, Scrapy generates multiple output files+storing up to the specified number of items in each output file.++When generating multiple output files, you must use at least one of the following+placeholders in :setting:`FEED_URI` to indicate how the different output file names are+generated:* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created- (e.g. `2020-03-28T14-45-08.237134`)+ (e.g. ``2020-03-28T14-45-08.237134``)* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch- (e.g. `2` for the second file)+ (e.g. ``2`` for the second file)For instance, if your settings include::@@ -457,7 +458,7 @@ And your :command:`crawl` command line is::scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json-The resulting directory tree of above can be like::+The command line above can generate a directory tree like::->projectname-->dirname@@ -466,4 +467,4 @@ The resulting directory tree of above can be like::--->3-filename2020-03-28T14-45-10.046092.jsonWhere the first and second files contain exactly 100 items. The last one contains-<= 100 items.\ No newline at end of file+100 items or fever.diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 4c9362f3a..3d691c580 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -180,7 +180,7 @@ class FTPFeedStorage(BlockingFeedStorage):class _FeedSlot:- def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri):+ def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):self.file = fileself.exporter = exporterself.storage = storage@@ -188,7 +188,7 @@ class _FeedSlot:self.batch_id = batch_idself.format = formatself.store_empty = store_empty- self.template_uri = template_uri+ self.uri_template = uri_templateself.uri = uri# flagsself.itemcount = 0@@ -260,7 +260,7 @@ class FeedExporter:uri=uri % uri_params,feed=feed,spider=spider,- template_uri=uri,+ uri_template=uri,))def close_spider(self, spider):@@ -288,7 +288,7 @@ class FeedExporter:extra={'spider': spider}))return d- def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):+ def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):"""Redirect the output data stream to a new file.Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.@@ -296,7 +296,7 @@ class FeedExporter::param uri: uri of the new batch to start:param feed: dict with parameters of feed:param spider: user spider- :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri+ :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri"""storage = self._get_storage(uri)file = storage.open(spider)@@ -315,7 +315,7 @@ class FeedExporter:format=feed['format'],store_empty=feed['store_empty'],batch_id=batch_id,- template_uri=template_uri,+ uri_template=uri_template,)if slot.store_empty:slot.start_exporting()@@ -329,14 +329,14 @@ class FeedExporter:slot.itemcount += 1# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old oneif self.storage_batch_size and slot.itemcount == self.storage_batch_size:- uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)+ uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)self._close_slot(slot, spider)slots.append(self._start_new_batch(batch_id=slot.batch_id + 1,- uri=slot.template_uri % uri_params,- feed=self.feeds[slot.template_uri],+ uri=slot.uri_template % uri_params,+ feed=self.feeds[slot.uri_template],spider=spider,- template_uri=slot.template_uri,+ uri_template=slot.uri_template,))else:slots.append(slot)