Scrapy, a fast high-level web crawling & scraping framework for Python.
Let a stop finish while an item is in the item pipelines (#8348)
Adrian7df2b7099656
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 386b29f0b..92c1b2394 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -581,3 +581,4 @@ class Scraper:)finally:self.slot.itemproc_size -= 1+ self._check_if_closing()diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 103ee79b8..7088c17d4 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -22,6 +22,7 @@ from scrapy.exceptions import ()from scrapy.http import Requestfrom scrapy.spiders import Spider+from scrapy.utils.asyncio import sleepfrom scrapy.utils.defer import (_schedule_coro,deferred_from_coro,@@ -459,6 +460,29 @@ async def test_start_without_spider() -> None:await engine.close_async()+@coroutine_test+async def test_stop_while_item_in_pipelines() -> None:+ """A stop that starts while an item is going through the item pipelines+ completes once that item is done."""++ class StoppingPipeline:+ async def process_item(self, item: Any) -> Any:+ _schedule_coro(crawler.stop_async())+ await sleep(0.1)+ return item++ class ItemSpider(Spider):+ name = "item"++ async def start(self) -> AsyncIterator[Any]:+ yield {"a": "b"}++ crawler = get_crawler(ItemSpider, {"ITEM_PIPELINES": {StoppingPipeline: 1}})+ await crawler.crawl_async()+ assert crawler.stats.get_value("item_scraped_count") == 1+ assert crawler.stats.get_value("finish_reason") == "shutdown"++@coroutine_testasync def test_pause_unpause() -> None:engine = ExecutionEngine(get_crawler(DefaultSpider))