Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Change default priority queue to DownloaderAwarePriorityQueue (#6940)

Thalison Fernandesd8583a89c7f9
diff --git a/docs/topics/broad-crawls.rst b/docs/topics/broad-crawls.rst
index b4d058754..cace1f883 100644
--- a/docs/topics/broad-crawls.rst
+++ b/docs/topics/broad-crawls.rst
@@ -41,19 +41,6 @@ efficient broad crawl.
.. _broad-crawls-scheduler-priority-queue:
-Use the right :setting:`SCHEDULER_PRIORITY_QUEUE`
-=================================================
-
-Scrapy’s default scheduler priority queue is ``'scrapy.pqueues.ScrapyPriorityQueue'``.
-It works best during single-domain crawl. It does not work well with crawling
-many different domains in parallel
-
-To apply the recommended priority queue use:
-
-.. code-block:: python
-
- SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue"
-
.. _broad-crawls-concurrency:
Increase concurrency
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index a92070918..48ba573d7 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1739,10 +1739,10 @@ Type of in-memory queue used by the scheduler. Other available type is:
SCHEDULER_PRIORITY_QUEUE
------------------------
-Default: ``'scrapy.pqueues.ScrapyPriorityQueue'``
+Default: ``'scrapy.pqueues.DownloaderAwarePriorityQueue'``
Type of priority queue used by the scheduler. Another available type is
-``scrapy.pqueues.DownloaderAwarePriorityQueue``.
+``scrapy.pqueues.ScrapyPriorityQueue``.
``scrapy.pqueues.DownloaderAwarePriorityQueue`` works better than
``scrapy.pqueues.ScrapyPriorityQueue`` when you crawl many different
domains in parallel.
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index e66f78c47..d741bf3ac 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -479,7 +479,7 @@ SCHEDULER = "scrapy.core.scheduler.Scheduler"
SCHEDULER_DEBUG = False
SCHEDULER_DISK_QUEUE = "scrapy.squeues.PickleLifoDiskQueue"
SCHEDULER_MEMORY_QUEUE = "scrapy.squeues.LifoMemoryQueue"
-SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.ScrapyPriorityQueue"
+SCHEDULER_PRIORITY_QUEUE = "scrapy.pqueues.DownloaderAwarePriorityQueue"
SCHEDULER_START_DISK_QUEUE = "scrapy.squeues.PickleFifoDiskQueue"
SCHEDULER_START_MEMORY_QUEUE = "scrapy.squeues.FifoMemoryQueue"
diff --git a/tests/test_downloaderslotssettings.py b/tests/test_downloaderslotssettings.py
index 1d76ee611..ff9604c36 100644
--- a/tests/test_downloaderslotssettings.py
+++ b/tests/test_downloaderslotssettings.py
@@ -1,11 +1,13 @@
import time
from typing import Any
+import pytest
from twisted.internet.defer import inlineCallbacks
from scrapy import Request
from scrapy.core.downloader import Downloader, Slot
from scrapy.crawler import CrawlerRunner
+from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_crawler
from tests.mockserver.http import MockServer
@@ -104,3 +106,33 @@ def test_params():
assert getattr(expected, param) == getattr(actual, param), (
f"Slot.{param}: {getattr(expected, param)!r} != {getattr(actual, param)!r}"
)
+
+
+ "priority_queue_class",
+ [
+ "scrapy.pqueues.ScrapyPriorityQueue",
+ "scrapy.pqueues.DownloaderAwarePriorityQueue",
+ ],
+)
+@deferred_f_from_coro_f
+async def test_none_slot_with_priority_queue(
+ mockserver: MockServer, priority_queue_class: str
+) -> None:
+ """Test specific cases for None slot handling with different priority queues."""
+ crawler = get_crawler(
+ DownloaderSlotsSettingsTestSpider,
+ settings_dict={"SCHEDULER_PRIORITY_QUEUE": priority_queue_class},
+ )
+ await maybe_deferred_to_future(crawler.crawl(mockserver=mockserver))
+ assert isinstance(crawler.spider, DownloaderSlotsSettingsTestSpider)
+
+ assert hasattr(crawler.spider, "times")
+ assert None not in crawler.spider.times
+ assert crawler.spider.default_slot in crawler.spider.times
+ assert len(crawler.spider.times[crawler.spider.default_slot]) == 2
+
+ assert crawler.stats
+ stats = crawler.stats
+ assert stats.get_value("spider_exceptions", 0) == 0
+ assert stats.get_value("downloader/exception_count", 0) == 0
diff --git a/tests/test_engine.py b/tests/test_engine.py
index 7e550d661..dc98364fc 100644
--- a/tests/test_engine.py
+++ b/tests/test_engine.py
@@ -438,6 +438,7 @@ class TestEngine(TestEngineBase):
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
e = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = e
yield deferred_from_coro(e.open_spider_async())
_schedule_coro(e.start_async())
with pytest.raises(RuntimeError, match="Engine already running"):
@@ -450,6 +451,7 @@ class TestEngine(TestEngineBase):
crawler = get_crawler(DefaultSpider)
crawler.spider = crawler._create_spider()
e = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = e
await e.open_spider_async()
with pytest.raises(RuntimeError, match="Engine already running"):
await asyncio.gather(e.start_async(), e.start_async())
@@ -645,6 +647,7 @@ class TestEngineCloseSpider:
@deferred_f_from_coro_f
async def test_no_slot(self, crawler: Crawler) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
await engine.open_spider_async()
slot = engine._slot
engine._slot = None
@@ -666,6 +669,7 @@ class TestEngineCloseSpider:
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
await engine.open_spider_async()
assert engine._slot
del engine._slot.heartbeat
@@ -677,6 +681,7 @@ class TestEngineCloseSpider:
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
await engine.open_spider_async()
del engine.downloader.slots
await engine.close_spider_async()
@@ -687,6 +692,7 @@ class TestEngineCloseSpider:
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
await engine.open_spider_async()
engine.scraper.slot = None
await engine.close_spider_async()
@@ -697,6 +703,7 @@ class TestEngineCloseSpider:
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
await engine.open_spider_async()
assert engine._slot
del cast("Scheduler", engine._slot.scheduler).dqs
@@ -708,6 +715,7 @@ class TestEngineCloseSpider:
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
await engine.open_spider_async()
signal_manager = engine.signals
del engine.signals
@@ -725,6 +733,7 @@ class TestEngineCloseSpider:
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: None)
+ crawler.engine = engine
await engine.open_spider_async()
del cast("MemoryStatsCollector", crawler.stats).spider_stats
await engine.close_spider_async()
@@ -735,6 +744,7 @@ class TestEngineCloseSpider:
self, crawler: Crawler, caplog: pytest.LogCaptureFixture
) -> None:
engine = ExecutionEngine(crawler, lambda _: defer.fail(ValueError()))
+ crawler.engine = engine
await engine.open_spider_async()
await engine.close_spider_async()
assert "Error running spider_closed_callback" in caplog.text
@@ -747,6 +757,7 @@ class TestEngineCloseSpider:
raise ValueError
engine = ExecutionEngine(crawler, cb)
+ crawler.engine = engine
await engine.open_spider_async()
await engine.close_spider_async()
assert "Error running spider_closed_callback" in caplog.text