Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

fix flaky test_slow_pipeline by comparing items unordered (#8008)

Anand685abd6dbb87
diff --git a/tests/test_spider_start.py b/tests/test_spider_start.py
index cf8d3804d..34f78b4d0 100644
--- a/tests/test_spider_start.py
+++ b/tests/test_spider_start.py
@@ -26,17 +26,23 @@ async def _test_spider(
spider: type[Spider],
expected_items: list[Any] | None = None,
settings: dict[str, Any] | None = None,
+ *,
+ ordered: bool = True,
) -> None:
- actual_items = []
+ actual_items: list[Any] = []
expected_items = [] if expected_items is None else expected_items
- def track_item(item, response, spider):
+ def track_item(item: Any, response: Any, spider: Spider) -> None:
actual_items.append(item)
crawler = get_crawler(spider, settings)
crawler.signals.connect(track_item, signals.item_scraped)
await crawler.crawl_async()
assert crawler.stats.get_value("finish_reason") == "finished"
+ if not ordered:
+ actual_items = sorted(actual_items, key=repr)
+ expected_items = sorted(expected_items, key=repr)
+
assert actual_items == expected_items
@@ -124,4 +130,5 @@ async def test_slow_pipeline() -> None:
TestSpider,
[ITEM_A, ITEM_B],
{"ITEM_PIPELINES": {SlowPipeline: 0}},
+ ordered=False,
)