Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix immediate delivery of full feed export batches (#7730) (#7733)
Fandub2d4eedea887
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 678a29e2e..c2997921d 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -13,7 +13,7 @@ import reimport sysimport warningsfrom abc import ABC, abstractmethod-from collections.abc import Callable, Coroutine+from collections.abc import Callablefrom datetime import datetime, timezonefrom pathlib import Path, PureWindowsPathfrom tempfile import NamedTemporaryFile@@ -475,7 +475,7 @@ class FeedExporter:self.feeds = {}self.slots: list[FeedSlot] = []self.filters: dict[str, ItemFilter] = {}- self._pending_close_coros: list[Coroutine[Any, Any, None]] = []+ self._pending_close_tasks: list[asyncio.Task[None] | Deferred[None]] = []if not self.settings["FEEDS"] and not self.settings["FEED_URI"]:raise NotConfigured@@ -539,23 +539,44 @@ class FeedExporter:)async def close_spider(self, spider: Spider) -> None:- self._pending_close_coros.extend(- self._close_slot(slot, spider) for slot in self.slots- )+ for slot in self.slots:+ self._schedule_slot_close(slot, spider)- if self._pending_close_coros:+ if self._pending_close_tasks:if is_asyncio_available():await asyncio.wait(- [asyncio.create_task(coro) for coro in self._pending_close_coros]+ cast("list[asyncio.Task[None]]", list(self._pending_close_tasks)))else:await DeferredList(- deferred_from_coro(coro) for coro in self._pending_close_coros+ cast("list[Deferred[None]]", list(self._pending_close_tasks)))# Send FEED_EXPORTER_CLOSED signalawait self.crawler.signals.send_catch_log_async(signals.feed_exporter_closed)+ def _schedule_slot_close(+ self, slot: FeedSlot, spider: Spider+ ) -> asyncio.Task[None] | Deferred[None]:+ """Start closing the slot without waiting for it to finish, keeping+ track of the pending work so that it can be awaited in+ :meth:`close_spider` if it hasn't finished by then."""+ aw: asyncio.Task[None] | Deferred[None]+ coro = self._close_slot(slot, spider)+ if is_asyncio_available():+ aw = asyncio.create_task(coro)+ self._pending_close_tasks.append(aw)+ aw.add_done_callback(self._pending_close_tasks.remove)+ else:+ aw = deferred_from_coro(coro)+ self._pending_close_tasks.append(aw)+ aw.addBoth(self._untrack_pending_close_task, aw)+ return aw++ def _untrack_pending_close_task(self, result: Any, aw: Deferred[None]) -> Any:+ self._pending_close_tasks.remove(aw)+ return result+@staticmethoddef _get_file(slot_: FeedSlot) -> IO[bytes]:assert slot_.file@@ -652,7 +673,7 @@ class FeedExporter:uri_params = self._get_uri_params(spider, self.feeds[slot.uri_template]["uri_params"], slot)- self._pending_close_coros.append(self._close_slot(slot, spider))+ self._schedule_slot_close(slot, spider)slots.append(self._start_new_batch(batch_id=slot.batch_id + 1,diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.pyindex 80ff6229b..4b0962c43 100644--- a/tests/test_feedexport_batch.py+++ b/tests/test_feedexport_batch.py@@ -210,6 +210,47 @@ class TestBatchDeliveries(TestFeedExportBase):header = MyItem.fields.keys()await self.assertExported(items, header, rows, settings=settings)+ @coroutine_test+ async def test_batch_delivered_when_full(self):+ """Full batches must be finalized and delivered as soon as they are+ full, instead of when the spider closes."""+ dir_path = self._random_temp_filename()+ batch1_path = Path(dir_path, "1.json")+ mockserver_url = self.mockserver.url("/")+ batch1_contents: list[bytes | None] = []++ class TestSpider(scrapy.Spider):+ name = "testspider"+ start_urls = [mockserver_url]++ def parse(self, response):+ yield {"foo": "bar1"}+ yield {"foo": "bar2"}+ yield scrapy.Request(+ mockserver_url, callback=self.parse2, dont_filter=True+ )++ def parse2(self, response):+ # the first batch was full after the second item, so it must+ # have been delivered by now+ batch1_contents.append(+ batch1_path.read_bytes() if batch1_path.exists() else None+ )+ yield {"foo": "bar3"}++ settings = {+ "FEEDS": {+ build_url(dir_path / "%(batch_id)d.json"): {"format": "json"},+ },+ "FEED_EXPORT_BATCH_ITEM_COUNT": 2,+ }+ crawler = get_crawler(TestSpider, settings)+ await crawler.crawl_async()++ assert batch1_contents, "the second request was not processed"+ assert batch1_contents[0] is not None, "batch 1 was not stored during the crawl"+ assert json.loads(batch1_contents[0]) == [{"foo": "bar1"}, {"foo": "bar2"}]+def test_wrong_path(self):"""If path is without %(batch_time)s and %(batch_id) an exception must be raised"""settings = {