Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Minor fixes

BroodingKangaroo1cdcf8b08b8f
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rst
index dfeea5b7f..638733b6a 100644
--- a/docs/topics/feed-exports.rst
+++ b/docs/topics/feed-exports.rst
@@ -437,17 +437,18 @@ FEED_STORAGE_BATCH_ITEM_COUNT
-----------------------------
Default: ``None``
-An integer number that represents the number of scraped items stored in each output
-file. Whenever the number of items exceeds this setting, a new file is
-created and the output is redirected to it. The name of the new file will be selected
-based on the timestamp when the feed is being created and/or on the batch sequence number.
-Therefore you must specify %(batch_time)s or %(batch_id)s or both in :setting:`FEED_URI`.
+If assigned an integer number higher than ``0``, Scrapy generates multiple output files
+storing up to the specified number of items in each output file.
+
+When generating multiple output files, you must use at least one of the following
+placeholders in :setting:`FEED_URI` to indicate how the different output file names are
+generated:
* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created
- (e.g. `2020-03-28T14-45-08.237134`)
+ (e.g. ``2020-03-28T14-45-08.237134``)
* ``%(batch_id)s`` - gets replaced by the batch sequence number of batch
- (e.g. `2` for the second file)
+ (e.g. ``2`` for the second file)
For instance, if your settings include::
@@ -457,7 +458,7 @@ And your :command:`crawl` command line is::
scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json
-The resulting directory tree of above can be like::
+The command line above can generate a directory tree like::
->projectname
-->dirname
@@ -466,4 +467,4 @@ The resulting directory tree of above can be like::
--->3-filename2020-03-28T14-45-10.046092.json
Where the first and second files contain exactly 100 items. The last one contains
-<= 100 items.
\ No newline at end of file
+100 items or fever.
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 4c9362f3a..3d691c580 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -180,7 +180,7 @@ class FTPFeedStorage(BlockingFeedStorage):
class _FeedSlot:
- def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, template_uri):
+ def __init__(self, file, exporter, storage, uri, format, store_empty, batch_id, uri_template):
self.file = file
self.exporter = exporter
self.storage = storage
@@ -188,7 +188,7 @@ class _FeedSlot:
self.batch_id = batch_id
self.format = format
self.store_empty = store_empty
- self.template_uri = template_uri
+ self.uri_template = uri_template
self.uri = uri
# flags
self.itemcount = 0
@@ -260,7 +260,7 @@ class FeedExporter:
uri=uri % uri_params,
feed=feed,
spider=spider,
- template_uri=uri,
+ uri_template=uri,
))
def close_spider(self, spider):
@@ -288,7 +288,7 @@ class FeedExporter:
extra={'spider': spider}))
return d
- def _start_new_batch(self, batch_id, uri, feed, spider, template_uri):
+ def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):
"""
Redirect the output data stream to a new file.
Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.
@@ -296,7 +296,7 @@ class FeedExporter:
:param uri: uri of the new batch to start
:param feed: dict with parameters of feed
:param spider: user spider
- :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri
+ :param uri_template: template of uri which contains %(batch_time)s or %(batch_id)s to create new uri
"""
storage = self._get_storage(uri)
file = storage.open(spider)
@@ -315,7 +315,7 @@ class FeedExporter:
format=feed['format'],
store_empty=feed['store_empty'],
batch_id=batch_id,
- template_uri=template_uri,
+ uri_template=uri_template,
)
if slot.store_empty:
slot.start_exporting()
@@ -329,14 +329,14 @@ class FeedExporter:
slot.itemcount += 1
# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one
if self.storage_batch_size and slot.itemcount == self.storage_batch_size:
- uri_params = self._get_uri_params(spider, self.feeds[slot.template_uri]['uri_params'], slot)
+ uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)
self._close_slot(slot, spider)
slots.append(self._start_new_batch(
batch_id=slot.batch_id + 1,
- uri=slot.template_uri % uri_params,
- feed=self.feeds[slot.template_uri],
+ uri=slot.uri_template % uri_params,
+ feed=self.feeds[slot.uri_template],
spider=spider,
- template_uri=slot.template_uri,
+ uri_template=slot.uri_template,
))
else:
slots.append(slot)