Scrapy, a fast high-level web crawling & scraping framework for Python.
Improve test coverage for pqueues.py (#7640)
Adrian7b3f88f8abcb
diff --git a/tests/test_pqueues.py b/tests/test_pqueues.pyindex 7be9241b9..6c6a6584a 100644--- a/tests/test_pqueues.py+++ b/tests/test_pqueues.py@@ -8,7 +8,7 @@ from scrapy.core.downloader import Downloaderfrom scrapy.http.request import Requestfrom scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueuefrom scrapy.spiders import Spider-from scrapy.squeues import FifoMemoryQueue+from scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueuefrom scrapy.utils.misc import build_from_crawler, load_objectfrom scrapy.utils.test import get_crawlerfrom tests.test_scheduler import MockDownloader@@ -76,6 +76,29 @@ class TestPriorityQueue:assert queue.pop().url == req3.urlassert not queue.close()+ def test_init_prios_with_start_queue(self):+ temp_dir = tempfile.mkdtemp()+ queue = ScrapyPriorityQueue.from_crawler(+ self.crawler,+ PickleFifoDiskQueue,+ temp_dir,+ start_queue_cls=PickleFifoDiskQueue,+ )+ req = Request("https://example.org/", meta={"is_start_request": True})+ queue.push(req)+ startprios = queue.close()++ queue2 = ScrapyPriorityQueue.from_crawler(+ self.crawler,+ PickleFifoDiskQueue,+ temp_dir,+ startprios,+ start_queue_cls=PickleFifoDiskQueue,+ )+ assert len(queue2) == 1+ assert queue2.pop().url == req.url+ queue2.close()+def test_queue_push_pop_priorities(self):temp_dir = tempfile.mkdtemp()queue = ScrapyPriorityQueue.from_crawler(@@ -207,6 +230,33 @@ class TestDownloaderAwarePriorityQueue:assert slots == ["slot-a", "slot-b", "slot-c", "slot-a"]+ def test_pop_prefers_slot_with_fewer_active_downloads(self):+ downloader = self.queue._downloader_interface.downloader++ req_a = Request("https://example.org/a")+ req_a.meta[Downloader.DOWNLOAD_SLOT] = "slot-a"+ req_b = Request("https://example.org/b")+ req_b.meta[Downloader.DOWNLOAD_SLOT] = "slot-b"+ req_c = Request("https://example.org/c")+ req_c.meta[Downloader.DOWNLOAD_SLOT] = "slot-c"++ for req in (req_a, req_b, req_c):+ self.queue.push(req)++ downloader.increment("slot-a")+ downloader.increment("slot-c")++ popped = self.queue.pop()+ assert popped.url == req_b.url++ def test_contains(self):+ req = Request("https://example.org/")+ req.meta[Downloader.DOWNLOAD_SLOT] = "example-slot"+ assert "example-slot" not in self.queue+ self.queue.push(req)+ assert "example-slot" in self.queue+ assert "other-slot" not in self.queue+@pytest.mark.parametrize(("input_", "output"),