Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Wait for the processing of start items (#7891)

Adrianeab4c09ca1ad
diff --git a/scrapy/core/engine.py b/scrapy/core/engine.py
index 104daf399..83e3381d6 100644
--- a/scrapy/core/engine.py
+++ b/scrapy/core/engine.py
@@ -307,9 +307,7 @@ class ExecutionEngine:
self.crawl(item_or_request)
else:
assert self._slot is not None
- _schedule_coro(
- self.scraper.start_itemproc_async(item_or_request, response=None)
- )
+ self.scraper._start_itemproc_nowait(item_or_request)
self._slot.nextcall.schedule()
async def _start_request_processing(self) -> None:
diff --git a/scrapy/core/scraper.py b/scrapy/core/scraper.py
index 351375d7b..8146f0e15 100644
--- a/scrapy/core/scraper.py
+++ b/scrapy/core/scraper.py
@@ -69,7 +69,7 @@ class Slot:
self.queue: deque[QueueTuple] = deque()
self.active: set[Request] = set()
self.active_size: int = 0
- self.itemproc_size: int = 0 # just for scrapy.utils.engine.get_engine_status()
+ self.itemproc_size: int = 0
self.closing: Deferred[Spider] | None = None
def add_response_request(
@@ -97,7 +97,7 @@ class Slot:
self.active_size -= self.MIN_RESPONSE_SIZE
def is_idle(self) -> bool:
- return not (self.queue or self.active)
+ return not (self.queue or self.active or self.itemproc_size)
def needs_backout(self) -> bool:
return self.active_size > self.max_active_size
@@ -476,6 +476,23 @@ class Scraper:
)
return deferred_from_coro(self.start_itemproc_async(item, response=response))
+ def _start_itemproc_nowait(self, item: Any) -> None:
+ """Send *item* to the item pipelines without waiting for the outcome.
+
+ The scraper slot counts the item as being processed straight away, so
+ that the spider is not considered idle before the item pipelines have
+ had a chance to run.
+ """
+ assert self.slot is not None # typing
+ self.slot.itemproc_size += 1
+ _schedule_coro(self._start_itemproc_nowait_async(item))
+
+ async def _start_itemproc_nowait_async(self, item: Any) -> None:
+ assert self.slot is not None # typing
+ # start_itemproc_async() takes over the count of this item.
+ self.slot.itemproc_size -= 1
+ await self.start_itemproc_async(item, response=None)
+
async def start_itemproc_async(
self, item: Any, *, response: Response | Failure | None
) -> None:
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index 482b42d68..cf8d3804d 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -6,6 +6,7 @@ from typing import TYPE_CHECKING, Any
import pytest
from scrapy import Spider, signals
+from scrapy.utils.asyncio import sleep as scrapy_sleep
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.test import get_crawler
@@ -22,7 +23,9 @@ ITEM_B = {"id": "b"}
async def _test_spider(
- spider: type[Spider], expected_items: list[Any] | None = None
+ spider: type[Spider],
+ expected_items: list[Any] | None = None,
+ settings: dict[str, Any] | None = None,
) -> None:
actual_items = []
expected_items = [] if expected_items is None else expected_items
@@ -30,7 +33,7 @@ async def _test_spider(
def track_item(item, response, spider):
actual_items.append(item)
- crawler = get_crawler(spider)
+ crawler = get_crawler(spider, settings)
crawler.signals.connect(track_item, signals.item_scraped)
await crawler.crawl_async()
assert crawler.stats.get_value("finish_reason") == "finished"
@@ -101,3 +104,24 @@ async def test_twisted_delayed() -> None:
yield ITEM_A
await _test_start(start, [ITEM_A])
+
+
+@coroutine_test
+async def test_slow_pipeline() -> None:
+ class SlowPipeline:
+ async def process_item(self, item):
+ await scrapy_sleep(SLEEP_SECONDS)
+ return item
+
+ class TestSpider(Spider):
+ name = "test"
+
+ async def start(self):
+ yield ITEM_A
+ yield ITEM_B
+
+ await _test_spider(
+ TestSpider,
+ [ITEM_A, ITEM_B],
+ {"ITEM_PIPELINES": {SlowPipeline: 0}},
+ )