Scrapy, a fast high-level web crawling & scraping framework for Python.
Change time_id to batch_time
BroodingKangaroodad2ea75222d
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 6c463fc27..2106b41f5 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -441,9 +441,9 @@ An integer number which represent number of scraped items stored in each outputfile. Whenever the number of items exceeds this setting, a new file iscreated and output redirects to it. The name of the new file will be selectedbased on timestamp when the feed is being created and/or batch sequence number.-Therefore you must specify %(time_id)s or %(batch_id)s or both in FEED_URI.+Therefore you must specify %(batch_time)s or %(batch_id)s or both in FEED_URI.-* ``%(time_id)s`` - gets replaced by a timestamp when the feed is being created+* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created* ``%(batch_id)s`` - gets replaced by sequence number of batchFor instance::@@ -452,7 +452,7 @@ For instance::Your request can be like::- scrapy crawl spidername -o dirname/%(batch_id)s-filename%(time_id)s.json+ scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.jsonThe result directory tree of above can be like::diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex fe6061c33..a262f5d18 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -295,7 +295,7 @@ class FeedExporter::param uri: uri of the new batch to start:param feed: dict with parameters of feed:param spider: user spider- :param template_uri: template uri which contains %(time_id)s or %(batch_id)s to create new uri+ :param template_uri: template uri which contains %(batch_time)s or %(batch_id)s to create new uri"""storage = self._get_storage(uri)file = storage.open(spider)@@ -358,12 +358,12 @@ class FeedExporter:def _batch_deliveries_supported(self, uri):"""- If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(time_id)s or %(batch_id)s+ If FEED_STORAGE_BATCH_SIZE setting is specified uri has to contain %(batch_time)s or %(batch_id)sto distinguish different files of partial output"""- if self.storage_batch_size is None or '%(time_id)s' in uri or '%(batch_id)s' in uri:+ if self.storage_batch_size is None or '%(batch_time)s' in uri or '%(batch_id)s' in uri:return True- logger.warning('%(time_id)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')+ logger.warning('%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_SIZE setting is specified')return Falsedef _storage_supported(self, uri):@@ -396,7 +396,7 @@ class FeedExporter:for k in dir(spider):params[k] = getattr(spider, k)params['time'] = datetime.utcnow().replace(microsecond=0).isoformat().replace(':', '-')- params['time_id'] = datetime.utcnow().isoformat().replace(':', '-')+ params['batch_time'] = datetime.utcnow().isoformat().replace(':', '-')params['batch_id'] = slot.batch_id + 1 if slot is not None else 1uripar_function = load_object(uri_params) if uri_params else lambda x, y: Noneuripar_function(params, spider)diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 9fc39c3a6..2217bb4ed 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -989,7 +989,7 @@ class FeedExportTest(FeedExportTestBase):class PartialDeliveriesTest(FeedExportTestBase):__test__ = True- _file_mark = '_%(time_id)s_#%(batch_id)s_'+ _file_mark = '_%(batch_time)s_#%(batch_id)s_'@defer.inlineCallbacksdef run_and_export(self, spider_cls, settings):@@ -1146,7 +1146,7 @@ class PartialDeliveriesTest(FeedExportTestBase):yield self.assertExported(items, header, rows, settings=settings)def test_wrong_path(self):- """ If path is without %(time_id)s or %(batch_id)s an exception must be raised """+ """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """settings = {'FEEDS': {self._random_temp_filename(): {'format': 'xml'},@@ -1236,7 +1236,7 @@ class PartialDeliveriesTest(FeedExportTestBase):def test_batch_path_differ(self):"""Test that the name of all batch files differ from each other.- So %(time_id)s replaced with the current date.+ So %(batch_time)s replaced with the current date."""items = [self.MyItem({'foo': 'bar1', 'egg': 'spam1'}),@@ -1245,7 +1245,7 @@ class PartialDeliveriesTest(FeedExportTestBase):]settings = {'FEEDS': {- os.path.join(self._random_temp_filename(), '%(time_id)s'): {+ os.path.join(self._random_temp_filename(), '%(batch_time)s'): {'format': 'json',},},@@ -1281,7 +1281,7 @@ class PartialDeliveriesTest(FeedExportTestBase):chars = [random.choice(ascii_letters + digits) for _ in range(15)]filename = ''.join(chars)prefix = 'tmp/{filename}'.format(filename=filename)- s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(time_id)s.json'.format(+ s3_test_file_uri = 's3://{bucket_name}/{prefix}/%(batch_time)s.json'.format(bucket_name=s3_test_bucket_name, prefix=prefix)storage = S3FeedStorage(s3_test_bucket_name, access_key, secret_key)