Scrapy, a fast high-level web crawling & scraping framework for Python.
Wait for the processing of start items (#7891)
Adrianeab4c09ca1ad
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.pyindex 104daf399..83e3381d6 100644--- a/scrapy/core/engine.py+++ b/scrapy/core/engine.py@@ -307,9 +307,7 @@ class ExecutionEngine:self.crawl(item_or_request)else:assert self._slot is not None- _schedule_coro(- self.scraper.start_itemproc_async(item_or_request, response=None)- )+ self.scraper._start_itemproc_nowait(item_or_request)self._slot.nextcall.schedule()async def _start_request_processing(self) -> None:diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.pyindex 351375d7b..8146f0e15 100644--- a/scrapy/core/scraper.py+++ b/scrapy/core/scraper.py@@ -69,7 +69,7 @@ class Slot:self.queue: deque[QueueTuple] = deque()self.active: set[Request] = set()self.active_size: int = 0- self.itemproc_size: int = 0 # just for scrapy.utils.engine.get_engine_status()+ self.itemproc_size: int = 0self.closing: Deferred[Spider] | None = Nonedef add_response_request(@@ -97,7 +97,7 @@ class Slot:self.active_size -= self.MIN_RESPONSE_SIZEdef is_idle(self) -> bool:- return not (self.queue or self.active)+ return not (self.queue or self.active or self.itemproc_size)def needs_backout(self) -> bool:return self.active_size > self.max_active_size@@ -476,6 +476,23 @@ class Scraper:)return deferred_from_coro(self.start_itemproc_async(item, response=response))+ def _start_itemproc_nowait(self, item: Any) -> None:+ """Send *item* to the item pipelines without waiting for the outcome.++ The scraper slot counts the item as being processed straight away, so+ that the spider is not considered idle before the item pipelines have+ had a chance to run.+ """+ assert self.slot is not None # typing+ self.slot.itemproc_size += 1+ _schedule_coro(self._start_itemproc_nowait_async(item))++ async def _start_itemproc_nowait_async(self, item: Any) -> None:+ assert self.slot is not None # typing+ # start_itemproc_async() takes over the count of this item.+ self.slot.itemproc_size -= 1+ await self.start_itemproc_async(item, response=None)+async def start_itemproc_async(self, item: Any, *, response: Response | Failure | None) -> None:diff --git a/tests/test_spider_start.py b/tests/test_spider_start.pyindex 482b42d68..cf8d3804d 100644--- a/tests/test_spider_start.py+++ b/tests/test_spider_start.py@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Anyimport pytestfrom scrapy import Spider, signals+from scrapy.utils.asyncio import sleep as scrapy_sleepfrom scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.test import get_crawler@@ -22,7 +23,9 @@ ITEM_B = {"id": "b"}async def _test_spider(- spider: type[Spider], expected_items: list[Any] | None = None+ spider: type[Spider],+ expected_items: list[Any] | None = None,+ settings: dict[str, Any] | None = None,) -> None:actual_items = []expected_items = [] if expected_items is None else expected_items@@ -30,7 +33,7 @@ async def _test_spider(def track_item(item, response, spider):actual_items.append(item)- crawler = get_crawler(spider)+ crawler = get_crawler(spider, settings)crawler.signals.connect(track_item, signals.item_scraped)await crawler.crawl_async()assert crawler.stats.get_value("finish_reason") == "finished"@@ -101,3 +104,24 @@ async def test_twisted_delayed() -> None:yield ITEM_Aawait _test_start(start, [ITEM_A])+++@coroutine_test+async def test_slow_pipeline() -> None:+ class SlowPipeline:+ async def process_item(self, item):+ await scrapy_sleep(SLEEP_SECONDS)+ return item++ class TestSpider(Spider):+ name = "test"++ async def start(self):+ yield ITEM_A+ yield ITEM_B++ await _test_spider(+ TestSpider,+ [ITEM_A, ITEM_B],+ {"ITEM_PIPELINES": {SlowPipeline: 0}},+ )