Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Improve test coverage for pqueues.py (#7640)

Adrian7b3f88f8abcb
diff --git a/tests/test_pqueues.py b/tests/test_pqueues.py
index 7be9241b9..6c6a6584a 100644
--- a/tests/test_pqueues.py
+++ b/tests/test_pqueues.py
@@ -8,7 +8,7 @@ from scrapy.core.downloader import Downloader
from scrapy.http.request import Request
from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue
from scrapy.spiders import Spider
-from scrapy.squeues import FifoMemoryQueue
+from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueue
from scrapy.utils.misc import build_from_crawler, load_object
from scrapy.utils.test import get_crawler
from tests.test_scheduler import MockDownloader
@@ -76,6 +76,29 @@ class TestPriorityQueue:
assert queue.pop().url == req3.url
assert not queue.close()
+ def test_init_prios_with_start_queue(self):
+ temp_dir = tempfile.mkdtemp()
+ queue = ScrapyPriorityQueue.from_crawler(
+ self.crawler,
+ PickleFifoDiskQueue,
+ temp_dir,
+ start_queue_cls=PickleFifoDiskQueue,
+ )
+ req = Request("https://example.org/", meta={"is_start_request": True})
+ queue.push(req)
+ startprios = queue.close()
+
+ queue2 = ScrapyPriorityQueue.from_crawler(
+ self.crawler,
+ PickleFifoDiskQueue,
+ temp_dir,
+ startprios,
+ start_queue_cls=PickleFifoDiskQueue,
+ )
+ assert len(queue2) == 1
+ assert queue2.pop().url == req.url
+ queue2.close()
+
def test_queue_push_pop_priorities(self):
temp_dir = tempfile.mkdtemp()
queue = ScrapyPriorityQueue.from_crawler(
@@ -207,6 +230,33 @@ class TestDownloaderAwarePriorityQueue:
assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"]
+ def test_pop_prefers_slot_with_fewer_active_downloads(self):
+ downloader = self.queue._downloader_interface.downloader
+
+ req_a = Request("https://example.org/a")
+ req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"
+ req_b = Request("https://example.org/b")
+ req_b.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"
+ req_c = Request("https://example.org/c")
+ req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"
+
+ for req in (req_a, req_b, req_c):
+ self.queue.push(req)
+
+ downloader.increment("slot-a")
+ downloader.increment("slot-c")
+
+ popped = self.queue.pop()
+ assert popped.url == req_b.url
+
+ def test_contains(self):
+ req = Request("https://example.org/")
+ req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot"
+ assert "example-slot" not in self.queue
+ self.queue.push(req)
+ assert "example-slot" in self.queue
+ assert "other-slot" not in self.queue
+
@pytest.mark.parametrize(
("input_", "output"),