Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix FeedExporter
kenshi kikuchid311779887d5
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex cd26b5778..8a60bc528 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -274,8 +274,6 @@ class FTPFeedStorage(BlockingFeedStorage):class _FeedSlot:def __init__(self,- file,- exporter,storage,uri,format,@@ -283,9 +281,14 @@ class _FeedSlot:batch_id,uri_template,filter,+ feed_options,+ spider,+ exporters,+ settings,+ crawler,):- self.file = file- self.exporter = exporter+ self.file = None+ self.exporter = Noneself.storage = storage# feed paramsself.batch_id = batch_id@@ -294,15 +297,44 @@ class _FeedSlot:self.uri_template = uri_templateself.uri = uriself.filter = filter+ # exporter params+ self.feed_options = feed_options+ self.spider = spider+ self.exporters = exporters+ self.settings = settings+ self.crawler = crawler# flagsself.itemcount = 0self._exporting = False+ self._fileloaded = Falsedef start_exporting(self):+ if not self._fileloaded:+ self.file = self.storage.open(self.spider)+ if "postprocessing" in self.feed_options:+ self.file = PostProcessingManager(+ self.feed_options["postprocessing"], self.file, self.feed_options+ )+ self.exporter = self._get_exporter(+ file=self.file,+ format=self.feed_options["format"],+ fields_to_export=self.feed_options["fields"],+ encoding=self.feed_options["encoding"],+ indent=self.feed_options["indent"],+ **self.feed_options["item_export_kwargs"],+ )+ self._fileloaded = True+if not self._exporting:self.exporter.start_exporting()self._exporting = True+ def _get_instance(self, objcls, *args, **kwargs):+ return create_instance(objcls, self.settings, self.crawler, *args, **kwargs)++ def _get_exporter(self, file, format, *args, **kwargs):+ return self._get_instance(self.exporters[format], file, *args, **kwargs)+def finish_exporting(self):if self._exporting:self.exporter.finish_exporting()@@ -379,15 +411,22 @@ class FeedExporter:deferred_list = []for slot in self.slots:d = self._close_slot(slot, spider)- deferred_list.append(d)+ if d:+ deferred_list.append(d)return defer.DeferredList(deferred_list) if deferred_list else Nonedef _close_slot(self, slot, spider):- slot.finish_exporting()- if not slot.itemcount and not slot.store_empty:- # We need to call slot.storage.store nonetheless to get the file- # properly closed.- return defer.maybeDeferred(slot.storage.store, slot.file)+ if slot.itemcount:+ # Nomal case+ slot.finish_exporting()+ elif slot.store_empty and slot.batch_id == 1:+ # Need Store Empty+ slot.start_exporting()+ slot.finish_exporting()+ else:+ # In this case, the file is not stored, so no processing is required.+ return None+logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"d = defer.maybeDeferred(slot.storage.store, slot.file)@@ -423,23 +462,7 @@ class FeedExporter::param uri_template: template of uri which contains %(batch_time)s or %(batch_id)d to create new uri"""storage = self._get_storage(uri, feed_options)- file = storage.open(spider)- if "postprocessing" in feed_options:- file = PostProcessingManager(- feed_options["postprocessing"], file, feed_options- )-- exporter = self._get_exporter(- file=file,- format=feed_options["format"],- fields_to_export=feed_options["fields"],- encoding=feed_options["encoding"],- indent=feed_options["indent"],- **feed_options["item_export_kwargs"],- )slot = _FeedSlot(- file=file,- exporter=exporter,storage=storage,uri=uri,format=feed_options["format"],@@ -447,9 +470,12 @@ class FeedExporter:batch_id=batch_id,uri_template=uri_template,filter=self.filters[uri_template],+ feed_options=feed_options,+ spider=spider,+ exporters=self.exporters,+ settings=self.settings,+ crawler=getattr(self, "crawler", None),)- if slot.store_empty:- slot.start_exporting()return slotdef item_scraped(self, item, spider):@@ -533,14 +559,6 @@ class FeedExporter:else:logger.error("Unknown feed storage scheme: %(scheme)s", {"scheme": scheme})- def _get_instance(self, objcls, *args, **kwargs):- return create_instance(- objcls, self.settings, getattr(self, "crawler", None), *args, **kwargs- )-- def _get_exporter(self, file, format, *args, **kwargs):- return self._get_instance(self.exporters[format], file, *args, **kwargs)-def _get_storage(self, uri, feed_options):"""Fork of create_instance specific to feed storage classesdiff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 260ec1701..ea63d35c5 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -141,7 +141,7 @@ EXTENSIONS_BASE = {FEED_TEMPDIR = NoneFEEDS = {}FEED_URI_PARAMS = None # a function to extend uri arguments-FEED_STORE_EMPTY = False+FEED_STORE_EMPTY = TrueFEED_EXPORT_ENCODING = NoneFEED_EXPORT_FIELDS = NoneFEED_STORAGES = {}diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex eafe1b334..acdc39870 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -725,10 +725,9 @@ class FeedExportTest(FeedExportTestBase):yield crawler.crawl()for file_path, feed_options in FEEDS.items():- if not Path(file_path).exists():- continue-- content[feed_options["format"]] = Path(file_path).read_bytes()+ content[feed_options["format"]] = (+ Path(file_path).read_bytes() if Path(file_path).exists() else None+ )finally:for file_path in FEEDS.keys():@@ -945,9 +944,10 @@ class FeedExportTest(FeedExportTestBase):"FEEDS": {self._random_temp_filename(): {"format": fmt},},+ "FEED_STORE_EMPTY": False,}data = yield self.exported_no_data(settings)- self.assertEqual(b"", data[fmt])+ self.assertEqual(None, data[fmt])@defer.inlineCallbacksdef test_start_finish_exporting_items(self):@@ -1057,7 +1057,6 @@ class FeedExportTest(FeedExportTestBase):self._random_temp_filename(): {"format": "csv"},},"FEED_STORAGES": {"file": LogOnStoreFileStorage},- "FEED_STORE_EMPTY": False,}with LogCapture() as log:@@ -1680,10 +1679,9 @@ class FeedPostProcessedExportsTest(FeedExportTestBase):yield crawler.crawl()for file_path, feed_options in FEEDS.items():- if not Path(file_path).exists():- continue-- content[str(file_path)] = Path(file_path).read_bytes()+ content[str(file_path)] = (+ Path(file_path).read_bytes() if Path(file_path).exists() else None+ )finally:for file_path in FEEDS.keys():@@ -2184,6 +2182,9 @@ class BatchDeliveriesTest(FeedExportTestBase):for path, feed in FEEDS.items():dir_name = Path(path).parent+ if not dir_name.exists():+ content[feed["format"]] = []+ continuefor file in sorted(dir_name.iterdir()):content[feed["format"]].append(file.read_bytes())finally:@@ -2367,10 +2368,11 @@ class BatchDeliveriesTest(FeedExportTestBase):/ self._file_mark: {"format": fmt},},"FEED_EXPORT_BATCH_ITEM_COUNT": 1,+ "FEED_STORE_EMPTY": False,}data = yield self.exported_no_data(settings)data = dict(data)- self.assertEqual(b"", data[fmt][0])+ self.assertEqual(0, len(data[fmt]))@defer.inlineCallbacksdef test_export_no_items_store_empty(self):@@ -2484,9 +2486,6 @@ class BatchDeliveriesTest(FeedExportTestBase):for expected_batch, got_batch in zip(expected, data[fmt]):self.assertEqual(expected_batch, got_batch)- @pytest.mark.skipif(- sys.platform == "win32", reason="Odd behaviour on file creation/output"- )@defer.inlineCallbacksdef test_batch_path_differ(self):"""@@ -2508,7 +2507,7 @@ class BatchDeliveriesTest(FeedExportTestBase):"FEED_EXPORT_BATCH_ITEM_COUNT": 1,}data = yield self.exported_data(items, settings)- self.assertEqual(len(items), len([_ for _ in data["json"] if _]))+ self.assertEqual(len(items), len(data["json"]))@defer.inlineCallbacksdef test_stats_batch_file_success(self):@@ -2595,7 +2594,7 @@ class BatchDeliveriesTest(FeedExportTestBase):crawler = get_crawler(TestSpider, settings)yield crawler.crawl()- self.assertEqual(len(CustomS3FeedStorage.stubs), len(items) + 1)+ self.assertEqual(len(CustomS3FeedStorage.stubs), len(items))for stub in CustomS3FeedStorage.stubs[:-1]:stub.assert_no_pending_responses()