Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove the job directory of a download slot once it drains (#7955)
Adrian63485522b619
diff --git a/scrapy/pqueues.py b/scrapy/pqueues.pyindex 41411ceaf..8e9783f5a 100644--- a/scrapy/pqueues.py+++ b/scrapy/pqueues.py@@ -2,6 +2,8 @@ from __future__ import annotationsimport hashlibimport logging+from contextlib import suppress+from pathlib import Pathfrom typing import TYPE_CHECKING, Protocol, castfrom scrapy.utils.misc import build_from_crawler@@ -409,6 +411,11 @@ class DownloaderAwarePriorityQueue:request = queue.pop()if len(queue) == 0:del self.pqueues[slot]+ if self.key:+ # Reclaim the slot directory; rmdir leaves it alone if the+ # downstream queues did not remove all their files.+ with suppress(OSError):+ Path(self.key, _path_safe(slot)).rmdir()return requestdef push(self, request: Request) -> None:diff --git a/tests/test_pqueues.py b/tests/test_pqueues.pyindex 85fefd172..6ecbb0721 100644--- a/tests/test_pqueues.py+++ b/tests/test_pqueues.py@@ -6,7 +6,7 @@ import queuelibfrom scrapy.core.downloader import Downloaderfrom scrapy.http.request import Request-from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue+from scrapy.pqueues import DownloaderAwarePriorityQueue, ScrapyPriorityQueue, _path_safefrom scrapy.spiders import Spiderfrom scrapy.squeues import FifoMemoryQueue, PickleFifoDiskQueuefrom scrapy.utils.misc import build_from_crawler, load_object@@ -258,6 +258,29 @@ class TestDownloaderAwarePriorityQueue:assert "other-slot" not in self.queue+def test_slot_directory_removed_when_slot_drains(tmp_path):+ crawler = get_crawler(Spider)+ crawler.spider = crawler._create_spider("foo")+ crawler.engine = Mock(downloader=MockDownloader())+ queue = DownloaderAwarePriorityQueue.from_crawler(+ crawler=crawler,+ downstream_queue_cls=PickleFifoDiskQueue,+ key=str(tmp_path),+ )+ request = Request("https://example.org/1")+ slot_dir = tmp_path / _path_safe("example.org")++ queue.push(request)+ assert slot_dir.is_dir()++ assert queue.pop().url == request.url+ assert not slot_dir.exists()++ queue.push(request)+ assert slot_dir.is_dir()+ queue.close()++@pytest.mark.parametrize(("input_", "output"),[