Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

adjustments

Felipe Boff Nunes0ffd1667ba66
diff --cc scrapy/extensions/feedexport.py
index 3e98a5a0b,cd26b5778..67ff26764
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@@ -350,19 -383,13 +384,20 @@@ class FeedExporter
return defer.DeferredList(deferred_list) if deferred_list else None
def _close_slot(self, slot, spider):
+
+ def get_file(slot_):
+ if isinstance(slot_.file, PostProcessingManager):
+ return slot_.file.file
+ return slot_.file
+
+ slot.finish_exporting()
if not slot.itemcount and not slot.store_empty:
# We need to call slot.storage.store nonetheless to get the file
# properly closed.
- return defer.maybeDeferred(slot.storage.store, slot.file)
+ return defer.maybeDeferred(slot.storage.store, get_file(slot))
- slot.finish_exporting()
++
logmsg = f"{slot.format} feed ({slot.itemcount} items) in: {slot.uri}"
- d = defer.maybeDeferred(slot.storage.store, slot.file)
+ d = defer.maybeDeferred(slot.storage.store, get_file(slot))
d.addCallback(
self._handle_store_success, logmsg, spider, type(slot.storage).__name__
diff --cc tests/test_feedexport.py
index a5fd1e467,eafe1b334..45027171f
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@@ -1515,63 -1636,8 +1636,63 @@@ class FeedExportTest(FeedExportTestBase
}
data = yield self.exported_data(items, settings)
- self.assertEqual(row['expected'], data[feed_options['format']])
+ self.assertEqual(row["expected"], data[feed_options["format"]])
+ @defer.inlineCallbacks
+ def test_storage_file_no_postprocessing(self):
+
+ @implementer(IFeedStorage)
+ class Storage:
+
+ def __init__(self, uri, *, feed_options=None):
+ pass
+
+ def open(self, spider):
+ Storage.open_file = tempfile.NamedTemporaryFile(prefix='feed-')
+ return Storage.open_file
+
+ def store(self, file):
+ Storage.store_file = file
+ file.close()
+
+ settings = {
+ 'FEEDS': {self._random_temp_filename(): {'format': 'jsonlines'}},
+ 'FEED_STORAGES': {'file': Storage},
+ }
+ yield self.exported_no_data(settings)
+ self.assertIs(Storage.open_file, Storage.store_file)
+
+ @defer.inlineCallbacks
+ def test_storage_file_postprocessing(self):
+
+ @implementer(IFeedStorage)
+ class Storage:
+
+ def __init__(self, uri, *, feed_options=None):
+ pass
+
+ def open(self, spider):
+ Storage.open_file = tempfile.NamedTemporaryFile(prefix='feed-')
+ return Storage.open_file
+
+ def store(self, file):
+ Storage.store_file = file
+ file.close()
+
+ settings = {
+ 'FEEDS': {
+ self._random_temp_filename(): {
+ 'format': 'jsonlines',
+ 'postprocessing': [
+ 'scrapy.extensions.postprocessing.GzipPlugin',
+ ],
+ },
+ },
+ 'FEED_STORAGES': {'file': Storage},
+ }
+ yield self.exported_no_data(settings)
+ self.assertIs(Storage.open_file, Storage.store_file)
+
class FeedPostProcessedExportsTest(FeedExportTestBase):
__test__ = True