Scrapy, a fast high-level web crawling & scraping framework for Python.
Change default priority queue to DownloaderAwarePriorityQueue (#6940)
Thalison Fernandesd8583a89c7f9
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rstindex b4d058754..cace1f883 100644--- a/docs/topics/broad-crawls.rst+++ b/docs/topics/broad-crawls.rst@@ -41,19 +41,6 @@ efficient broad crawl... _broad-crawls-scheduler-priority-queue:-Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`-=================================================--Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.-It works best during single-domain crawl. It does not work well with crawling-many different domains in parallel--To apply the recommended priority queue use:--.. code-block:: python-- SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue"-.. _broad-crawls-concurrency:Increase concurrencydiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex a92070918..48ba573d7 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1739,10 +1739,10 @@ Type of in-memory queue used by the scheduler. Other available type is:SCHEDULER_PRIORITY_QUEUE-------------------------Default: ``'scrapy.pqueues.ScrapyPriorityQueue'``+Default: ``'scrapy.pqueues.DownloaderAwarePriorityQueue'``Type of priority queue used by the scheduler. Another available type is-``scrapy.pqueues.DownloaderAwarePriorityQueue``.+``scrapy.pqueues.ScrapyPriorityQueue``.``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many differentdomains in parallel.diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex e66f78c47..d741bf3ac 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -479,7 +479,7 @@ SCHEDULER = "scrapy.core.scheduler.Scheduler"SCHEDULER_DEBUG = FalseSCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue"SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue"-SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue"+SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue"SCHEDULER_START_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue"SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.pyindex 1d76ee611..ff9604c36 100644--- a/tests/test_downloaderslotssettings.py+++ b/tests/test_downloaderslotssettings.py@@ -1,11 +1,13 @@import timefrom typing import Any+import pytestfrom twisted.internet.defer import inlineCallbacksfrom scrapy import Requestfrom scrapy.core.downloader import Downloader, Slotfrom scrapy.crawler import CrawlerRunner+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_futurefrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_crawlerfrom tests.mockserver.http import MockServer@@ -104,3 +106,33 @@ def test_params():assert getattr(expected, param) == getattr(actual, param), (f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}")+++ "priority_queue_class",+ [+ "scrapy.pqueues.ScrapyPriorityQueue",+ "scrapy.pqueues.DownloaderAwarePriorityQueue",+ ],+)+@deferred_f_from_coro_f+async def test_none_slot_with_priority_queue(+ mockserver: MockServer, priority_queue_class: str+) -> None:+ """Test specific cases for None slot handling with different priority queues."""+ crawler = get_crawler(+ DownloaderSlotsSettingsTestSpider,+ settings_dict={"SCHEDULER_PRIORITY_QUEUE": priority_queue_class},+ )+ await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))+ assert isinstance(crawler.spider, DownloaderSlotsSettingsTestSpider)++ assert hasattr(crawler.spider, "times")+ assert None not in crawler.spider.times+ assert crawler.spider.default_slot in crawler.spider.times+ assert len(crawler.spider.times[crawler.spider.default_slot]) == 2++ assert crawler.stats+ stats = crawler.stats+ assert stats.get_value("spider_exceptions", 0) == 0+ assert stats.get_value("downloader/exception_count", 0) == 0diff --git a/tests/test_engine.py b/tests/test_engine.pyindex 7e550d661..dc98364fc 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -438,6 +438,7 @@ class TestEngine(TestEngineBase):crawler = get_crawler(DefaultSpider)crawler.spider = crawler._create_spider()e = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = eyield deferred_from_coro(e.open_spider_async())_schedule_coro(e.start_async())with pytest.raises(RuntimeError, match="Engine already running"):@@ -450,6 +451,7 @@ class TestEngine(TestEngineBase):crawler = get_crawler(DefaultSpider)crawler.spider = crawler._create_spider()e = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = eawait e.open_spider_async()with pytest.raises(RuntimeError, match="Engine already running"):await asyncio.gather(e.start_async(), e.start_async())@@ -645,6 +647,7 @@ class TestEngineCloseSpider:@deferred_f_from_coro_fasync def test_no_slot(self, crawler: Crawler) -> None:engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engineawait engine.open_spider_async()slot = engine._slotengine._slot = None@@ -666,6 +669,7 @@ class TestEngineCloseSpider:self, crawler: Crawler, caplog: pytest.LogCaptureFixture) -> None:engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engineawait engine.open_spider_async()assert engine._slotdel engine._slot.heartbeat@@ -677,6 +681,7 @@ class TestEngineCloseSpider:self, crawler: Crawler, caplog: pytest.LogCaptureFixture) -> None:engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engineawait engine.open_spider_async()del engine.downloader.slotsawait engine.close_spider_async()@@ -687,6 +692,7 @@ class TestEngineCloseSpider:self, crawler: Crawler, caplog: pytest.LogCaptureFixture) -> None:engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engineawait engine.open_spider_async()engine.scraper.slot = Noneawait engine.close_spider_async()@@ -697,6 +703,7 @@ class TestEngineCloseSpider:self, crawler: Crawler, caplog: pytest.LogCaptureFixture) -> None:engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engineawait engine.open_spider_async()assert engine._slotdel cast("Scheduler", engine._slot.scheduler).dqs@@ -708,6 +715,7 @@ class TestEngineCloseSpider:self, crawler: Crawler, caplog: pytest.LogCaptureFixture) -> None:engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engineawait engine.open_spider_async()signal_manager = engine.signalsdel engine.signals@@ -725,6 +733,7 @@ class TestEngineCloseSpider:self, crawler: Crawler, caplog: pytest.LogCaptureFixture) -> None:engine = ExecutionEngine(crawler, lambda _: None)+ crawler.engine = engineawait engine.open_spider_async()del cast("MemoryStatsCollector", crawler.stats).spider_statsawait engine.close_spider_async()@@ -735,6 +744,7 @@ class TestEngineCloseSpider:self, crawler: Crawler, caplog: pytest.LogCaptureFixture) -> None:engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))+ crawler.engine = engineawait engine.open_spider_async()await engine.close_spider_async()assert "Error running spider_closed_callback" in caplog.text@@ -747,6 +757,7 @@ class TestEngineCloseSpider:raise ValueErrorengine = ExecutionEngine(crawler, cb)+ crawler.engine = engineawait engine.open_spider_async()await engine.close_spider_async()assert "Error running spider_closed_callback" in caplog.text