Scrapy, a fast high-level web crawling & scraping framework for Python.
Minor updates
BroodingKangaroo7b1d3c35ea3b
diff --git a/docs/topics/feed-exports.rst b/docs/topics/feed-exports.rstindex 3da56821e..0b659f30e 100644--- a/docs/topics/feed-exports.rst+++ b/docs/topics/feed-exports.rst@@ -442,7 +442,7 @@ If assigned an integer number higher than ``0``, Scrapy generates multiple outpustoring up to the specified number of items in each output file.When generating multiple output files, you must use at least one of the following-placeholders in :setting:`FEED_URI` to indicate how the different output file names are+placeholders in the feed URI to indicate how the different output file names aregenerated:* ``%(batch_time)s`` - gets replaced by a timestamp when the feed is being created@@ -457,7 +457,7 @@ For instance, if your settings include::And your :command:`crawl` command line is::- scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.json+ scrapy crawl spidername -o dirname/%(batch_id)s-filename%(batch_time)s.jsonThe command line above can generate a directory tree like::diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 2312c994e..5908987a3 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -242,7 +242,6 @@ class FeedExporter:self.storages = self._load_components('FEED_STORAGES')self.exporters = self._load_components('FEED_EXPORTERS')- self.storage_batch_item_count = self.settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT')for uri, feed in self.feeds.items():if not self._storage_supported(uri):raise NotConfigured@@ -253,7 +252,7 @@ class FeedExporter:def open_spider(self, spider):for uri, feed in self.feeds.items():- uri_params = self._get_uri_params(spider, feed['uri_params'], None)+ uri_params = self._get_uri_params(spider, feed['uri_params'])self.slots.append(self._start_new_batch(batch_id=1,uri=uri % uri_params,@@ -299,7 +298,7 @@ class FeedExporter:def _start_new_batch(self, batch_id, uri, feed, spider, uri_template):"""Redirect the output data stream to a new file.- Execute multiple times if 'FEED_STORAGE_BATCH_ITEM_COUNT' setting is specified.+ Execute multiple times if FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified:param batch_id: sequence number of current batch:param uri: uri of the new batch to start:param feed: dict with parameters of feed@@ -331,14 +330,15 @@ class FeedExporter:def item_scraped(self, item, spider):slots = []- for idx, slot in enumerate(self.slots):+ for slot in self.slots:slot.start_exporting()slot.exporter.export_item(item)slot.itemcount += 1# create new slot for each slot with itemcount == FEED_STORAGE_BATCH_ITEM_COUNT and close the old one- if self.feeds[slot.uri_template].get('batch_item_count', self.storage_batch_item_count) \- and slot.itemcount == self.feeds[slot.uri_template].get('batch_item_count',- self.storage_batch_item_count):+ if (+ self.feeds[slot.uri_template]['batch_item_count']+ and slot.itemcount >= self.feeds[slot.uri_template]['batch_item_count']+ ):uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]['uri_params'], slot)self._close_slot(slot, spider)slots.append(self._start_new_batch(@@ -369,15 +369,17 @@ class FeedExporter:def _settings_are_valid(self, uri):"""- If FEED_STORAGE_BATCH_ITEM_COUNT setting is specified uri has to contain %(batch_time)s or %(batch_id)s- to distinguish different files of partial output+ If FEED_STORAGE_BATCH_ITEM_COUNT setting or FEEDS.batch_item_count is specified uri has to contain+ %(batch_time)s or %(batch_id)s to distinguish different files of partial output"""- if not self.storage_batch_item_count or '%(batch_time)s' in uri or '%(batch_id)s' in uri:- return True- logger.error(- '%(batch_time)s or %(batch_id)s must be in uri if FEED_STORAGE_BATCH_ITEM_COUNT setting is specified'- )- return False+ for uri_template, values in self.feeds.items():+ if values['batch_item_count'] and not any(s in uri_template for s in ['%(batch_time)s', '%(batch_id)s']):+ logger.error(+ '%(batch_time)s or %(batch_id)s must be in uri({}) if FEED_STORAGE_BATCH_ITEM_COUNT setting '+ 'or FEEDS.batch_item_count is specified and greater than 0.'.format(uri_template)+ )+ return False+ return Truedef _storage_supported(self, uri):scheme = urlparse(uri).scheme@@ -404,7 +406,7 @@ class FeedExporter:def _get_storage(self, uri):return self._get_instance(self.storages[urlparse(uri).scheme], uri)- def _get_uri_params(self, spider, uri_params, slot):+ def _get_uri_params(self, spider, uri_params, slot=None):params = {}for k in dir(spider):params[k] = getattr(spider, k)diff --git a/scrapy/utils/conf.py b/scrapy/utils/conf.pyindex 5921f82bf..0e02f0f28 100644--- a/scrapy/utils/conf.py+++ b/scrapy/utils/conf.py@@ -115,6 +115,10 @@ def feed_complete_default_values_from_settings(feed, settings):out = feed.copy()out.setdefault("encoding", settings["FEED_EXPORT_ENCODING"])out.setdefault("fields", settings.getlist("FEED_EXPORT_FIELDS") or None)+ out.setdefault(+ "batch_item_count",+ out.get('batch_item_count', settings.getint('FEED_STORAGE_BATCH_ITEM_COUNT'))+ )out.setdefault("store_empty", settings.getbool("FEED_STORE_EMPTY"))out.setdefault("uri_params", settings["FEED_URI_PARAMS"])if settings["FEED_EXPORT_INDENT"] is None:diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 542cce70f..db14b20b9 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -1265,7 +1265,7 @@ class BatchDeliveriesTest(FeedExportTestBase):yield self.assertExported(items, header, rows, settings=Settings(settings))def test_wrong_path(self):- """ If path is without %(batch_time)s or %(batch_id)s an exception must be raised """+ """ If path is without %(batch_time)s and %(batch_id)s an exception must be raised """settings = {'FEEDS': {self._random_temp_filename(): {'format': 'xml'},@@ -1329,7 +1329,6 @@ class BatchDeliveriesTest(FeedExportTestBase):],'csv': ['foo,bar\r\nFOO,BAR\r\n'.encode('utf-8'),'foo,bar\r\nFOO1,BAR1\r\n'.encode('utf-8')],- 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],}settings = {@@ -1352,13 +1351,6 @@ class BatchDeliveriesTest(FeedExportTestBase):'fields': ['foo', 'bar'],'encoding': 'utf-8',},- os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {- 'format': 'jsonlines',- 'indent': None,- 'fields': ['foo', 'bar'],- 'encoding': 'utf-8',- 'batch_item_count': 0,- },},'FEED_STORAGE_BATCH_ITEM_COUNT': 1,}@@ -1369,19 +1361,16 @@ class BatchDeliveriesTest(FeedExportTestBase):@defer.inlineCallbacksdef test_batch_item_count_feeds_setting(self):- items = [dict({'foo': u'FOO', 'bar': u'BAR'}), dict({'foo': u'FOO1', 'bar': u'BAR1'})]-+ items = [dict({'foo': u'FOO'}), dict({'foo': u'FOO1'})]formats = {- 'jsonlines': ['{"foo": "FOO", "bar": "BAR"}\n'.encode('utf-8'),- '{"foo": "FOO1", "bar": "BAR1"}\n'.encode('utf-8')],+ 'json': ['[{"foo": "FOO"}]'.encode('utf-8'),+ '[{"foo": "FOO1"}]'.encode('utf-8')],}-settings = {'FEEDS': {- os.path.join(self._random_temp_filename(), 'jsonlines', self._file_mark): {- 'format': 'jsonlines',+ os.path.join(self._random_temp_filename(), 'json', self._file_mark): {+ 'format': 'json','indent': None,- 'fields': ['foo', 'bar'],'encoding': 'utf-8','batch_item_count': 1,},diff --git a/tests/test_utils_conf.py b/tests/test_utils_conf.pyindex e5d3ef582..95ec2b64a 100644--- a/tests/test_utils_conf.py+++ b/tests/test_utils_conf.py@@ -149,6 +149,7 @@ class FeedExportConfigTestCase(unittest.TestCase):"FEED_EXPORT_INDENT": 42,"FEED_STORE_EMPTY": True,"FEED_URI_PARAMS": (1, 2, 3, 4),+ "FEED_STORAGE_BATCH_ITEM_COUNT": 2,})new_feed = feed_complete_default_values_from_settings(feed, settings)self.assertEqual(new_feed, {@@ -157,6 +158,7 @@ class FeedExportConfigTestCase(unittest.TestCase):"indent": 42,"store_empty": True,"uri_params": (1, 2, 3, 4),+ "batch_item_count": 2,})def test_feed_complete_default_values_from_settings_non_empty(self):@@ -169,6 +171,7 @@ class FeedExportConfigTestCase(unittest.TestCase):"FEED_EXPORT_FIELDS": ["f1", "f2", "f3"],"FEED_EXPORT_INDENT": 42,"FEED_STORE_EMPTY": True,+ "FEED_STORAGE_BATCH_ITEM_COUNT": 2,})new_feed = feed_complete_default_values_from_settings(feed, settings)self.assertEqual(new_feed, {@@ -177,6 +180,7 @@ class FeedExportConfigTestCase(unittest.TestCase):"indent": 42,"store_empty": True,"uri_params": None,+ "batch_item_count": 2,})