Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Let a stop finish while an item is in the item pipelines (#8348)

Adrian7df2b7099656
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 386b29f0b..92c1b2394 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -581,3 +581,4 @@ class Scraper:
)
finally:
self.slot.itemproc_size -= 1
+ self._check_if_closing()
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 103ee79b8..7088c17d4 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -22,6 +22,7 @@ from scrapy.exceptions import (
)
from scrapy.http import Request
from scrapy.spiders import Spider
+from scrapy.utils.asyncio import sleep
from scrapy.utils.defer import (
_schedule_coro,
deferred_from_coro,
@@ -459,6 +460,29 @@ async def test_start_without_spider() -> None:
await engine.close_async()
+@coroutine_test
+async def test_stop_while_item_in_pipelines() -> None:
+ """A stop that starts while an item is going through the item pipelines
+ completes once that item is done."""
+
+ class StoppingPipeline:
+ async def process_item(self, item: Any) -> Any:
+ _schedule_coro(crawler.stop_async())
+ await sleep(0.1)
+ return item
+
+ class ItemSpider(Spider):
+ name = "item"
+
+ async def start(self) -> AsyncIterator[Any]:
+ yield {"a": "b"}
+
+ crawler = get_crawler(ItemSpider, {"ITEM_PIPELINES": {StoppingPipeline: 1}})
+ await crawler.crawl_async()
+ assert crawler.stats.get_value("item_scraped_count") == 1
+ assert crawler.stats.get_value("finish_reason") == "shutdown"
+
+
@coroutine_test
async def test_pause_unpause() -> None:
engine = ExecutionEngine(get_crawler(DefaultSpider))