Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: set min required free space even if psutil is not installed (#124)
NTFSvolumed6010ff27bd9
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 4ea503bc8..634644aee 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -15,6 +15,7 @@ from cyberdrop_dl.exceptions import (DownloadError,DurationError,ErrorLogMessage,+ InsufficientFreeSpaceError,InvalidContentTypeError,RestrictedDateRangeError,RestrictedFiletypeError,@@ -290,7 +291,8 @@ class Downloader:async def check_file_can_download(self, media_item: MediaItem) -> None:"""Checks if the file can be downloaded."""- await storage.check(media_item)+ if not await storage.has_sufficient_space(media_item.download_folder):+ raise InsufficientFreeSpaceError(media_item)if not self.manager.client_manager.is_allowed_filetype(media_item):raise RestrictedFiletypeError(origin=media_item)if not await self.manager.client_manager.check_file_duration(media_item):@@ -387,7 +389,7 @@ class Downloader:ClientConnectorError,) as e:ui_message = getattr(e, "status", type(e).__name__)- if size := await aio.get_size(media_item.partial_file):+ if media_item.partial_file and (size := await aio.get_size(media_item.partial_file)):if self._current_attempt_filesize.get(media_item.filename, 0) >= size:raise DownloadError(ui_message, message=f"{self.log_prefix} failed", retry=True) from Nonediff --git a/cyberdrop_dl/storage/__init__.py b/cyberdrop_dl/storage/__init__.pyindex 1be567151..6fab91ce3 100644--- a/cyberdrop_dl/storage/__init__.py+++ b/cyberdrop_dl/storage/__init__.py@@ -19,15 +19,31 @@ if TYPE_CHECKING:_required_free_space: ContextVar[int] = ContextVar("_required_free_space")++def _disk_usage(folder: Path) -> int:+ path = folder+ while True:+ try:+ return shutil.disk_usage(path).free+ except FileNotFoundError:+ if path.parent == path:+ return 0+ path = path.parent++try:- from ._psutil import has_sufficient_space+ from ._psutil import get_free_space as _get_free_spacefrom ._psutil import start_loop as _psutil_loopexcept ImportError:_psutil_loop = None- async def has_sufficient_space(folder: Path, /, required_free_space: int) -> bool:- usage = await asyncio.to_thread(shutil.disk_usage, folder)- return usage.free > required_free_space+ async def _get_free_space(folder: Path) -> int:+ return await asyncio.to_thread(_disk_usage, folder)+++async def has_sufficient_space(folder: Path, /, required_free_space: int | None = None) -> bool:+ free_space = await _get_free_space(folder)+ return free_space == -1 or free_space > (required_free_space or _required_free_space.get())def create_free_space_checker(media_item: MediaItem, *, frecuency: int = 5) -> Callable[[], Awaitable[None]]:@@ -36,35 +52,30 @@ def create_free_space_checker(media_item: MediaItem, *, frecuency: int = 5) -> Casync def checker() -> None:nonlocal current_chunkif current_chunk % frecuency == 0:- await check(media_item)+ if not await has_sufficient_space(media_item.download_folder):+ raise InsufficientFreeSpaceError(media_item)+current_chunk += 1return checker-async def check(media_item: MediaItem) -> None:- """Checks if there is enough free space to download this item."""-- if not await has_sufficient_space(media_item.download_folder, _required_free_space.get()):- raise InsufficientFreeSpaceError(media_item)--@contextlib.asynccontextmanagerasync def monitor(required_free_space: int) -> AsyncGenerator[None]:+ token = _required_free_space.set(required_free_space)if _psutil_loop is None:logger.warning("psutil is not available on this system. Falling back to eager checks for free space")- yield- return-- loop = asyncio.create_task(_psutil_loop(), name="storage monitor")- token = _required_free_space.set(required_free_space)- await asyncio.sleep(0)+ loop = None+ else:+ loop = asyncio.create_task(_psutil_loop(), name="storage monitor")+ await asyncio.sleep(0)try:yieldfinally:_required_free_space.reset(token)- try:- _ = loop.cancel("On monitor exit")- await loop- except asyncio.CancelledError:- pass+ if loop is not None:+ try:+ _ = loop.cancel("On monitor exit")+ await loop+ except asyncio.CancelledError:+ passdiff --git a/cyberdrop_dl/storage/_psutil.py b/cyberdrop_dl/storage/_psutil.pyindex c8ba694f9..5be6ad437 100644--- a/cyberdrop_dl/storage/_psutil.py+++ b/cyberdrop_dl/storage/_psutil.py@@ -45,13 +45,12 @@ class _Stats:return str(self.__json__())-async def _get_free_space(path: Path) -> int:+def _disk_usage(path: Path) -> int:unsupported = Nonefree_space = 0try:- usage = await asyncio.to_thread(psutil.disk_usage, str(path))- free_space = usage.free+ free_space = psutil.disk_usage(str(path)).freeexcept OSError as e:if "operation not supported" not in str(e).casefold():raise@@ -68,11 +67,10 @@ async def _get_free_space(path: Path) -> int:return free_space-async def has_sufficient_space(folder: Path, /, required_free_space: int) -> bool:- await _check_nt_network_drive(folder)+async def get_free_space(folder: Path) -> int:mount = _get_mount_point(folder)if not mount:- return False+ return 0free_space = _free_space.get(mount)if free_space is None:@@ -81,11 +79,11 @@ async def has_sufficient_space(folder: Path, /, required_free_space: int) -> booif free_space is None:# Manually query this mount now. Next time it will be part of the loop- free_space = _free_space[mount] = await _get_free_space(mount)+ free_space = _free_space[mount] = await asyncio.to_thread(_disk_usage, mount)logger.info(f"A new mountpoint ('{mount!s}') will be used for '{folder}'")logger.info("Storage status \n%s", _Stats())- return free_space == -1 or free_space > required_free_space+ return free_spacedef _find_partition(path: Path) -> DiskPartition | None:@@ -134,13 +132,16 @@ def clear_cache() -> None:async def start_loop() -> None:"""Infinite loop to get free space of all used mounts and update internal dict"""- async def update():+ async def update() -> None:mountpoints = sorted(mount for mount, free_space in _free_space.items() if free_space != -1)if not mountpoints:return- results = await asyncio.gather(*map(_get_free_space, mountpoints))- _free_space.update(zip(mountpoints, results, strict=True))+ usage = await asyncio.gather(+ *(asyncio.to_thread(_disk_usage, mount) for mount in mountpoints),+ return_exceptions=False,+ )+ _free_space.update(zip(mountpoints, usage, strict=True))last_check = -1while True:diff --git a/tests/test_storage.py b/tests/test_storage.pyindex 1e338406b..15cc52a11 100644--- a/tests/test_storage.py+++ b/tests/test_storage.py@@ -7,66 +7,64 @@ from unittest import mockimport pytest-from cyberdrop_dl import aio-from cyberdrop_dl.storage import _psutil as storage+from cyberdrop_dl import storage+from cyberdrop_dl.storage import _psutildef create_partition(path: str):- return storage.DiskPartition(Path(path), Path(path), "", "")+ return _psutil.DiskPartition(Path(path), Path(path), "", "")def find_partition(path: str):- return storage._find_partition(Path(path))+ return _psutil._find_partition(Path(path))-async def test_unsupported_fs_should_not_return_zero() -> None:- cwd = await aio.resolve(Path())- free_space = await storage._get_free_space(cwd)+async def test_unsupported_fs_should_not_return_zero(tmp_path: Path) -> None:+ free_space = _psutil._disk_usage(tmp_path)assert free_space > 0with mock.patch("psutil.disk_usage", side_effect=OSError(None, "operation not supported")):- free_space = await storage._get_free_space(cwd)+ free_space = _psutil._disk_usage(tmp_path)assert free_space == -1with mock.patch("psutil.disk_usage", side_effect=OSError(None, "another error")):with pytest.raises(OSError):- _ = await storage._get_free_space(cwd)+ _ = _psutil._disk_usage(tmp_path)-async def test_fuse_filesystem_should_not_return_zero() -> None:- cwd = await aio.resolve(Path())- partition = storage._find_partition(cwd)+def test_fuse_filesystem_should_not_return_zero(tmp_path: Path) -> None:+ partition = _psutil._find_partition(tmp_path)assert partition- assert not storage._is_fuse_fs(cwd)- storage._PARTITIONS = [dataclasses.replace(partition, fstype="fuse")] # pyright: ignore[reportPrivateUsage]- assert storage._is_fuse_fs(cwd)+ assert not _psutil._is_fuse_fs(tmp_path)+ _psutil._PARTITIONS = [dataclasses.replace(partition, fstype="fuse")] # pyright: ignore[reportPrivateUsage]+ assert _psutil._is_fuse_fs(tmp_path)- free_space = await storage._get_free_space(cwd)+ free_space = _psutil._disk_usage(tmp_path)assert free_space > 0class NullUsage:free = 0with mock.patch("psutil.disk_usage", return_value=NullUsage()):- free_space = await storage._get_free_space(cwd)+ free_space = _psutil._disk_usage(tmp_path)assert free_space == -1def test_storage_only_work_with_abs_paths() -> None:cwd = Path()with pytest.raises(ValueError):- _ = storage._find_partition(cwd)+ _ = _psutil._find_partition(cwd)- assert storage._find_partition(cwd.resolve())+ assert _psutil._find_partition(cwd.resolve())@pytest.mark.skipif(os.name == "nt", reason="Test paths are only posix")-async def test_find_partition_finds_the_correct_partition() -> None:+def test_find_partition_finds_the_correct_partition() -> None:root, home, usb, external_ssd = partitions = [create_partition(path) for path in ("/", "/home", "/mnt/USB", "/home/external_SSD")]- storage._PARTITIONS = partitions # pyright: ignore[reportPrivateUsage]+ _psutil._PARTITIONS = partitions # pyright: ignore[reportPrivateUsage]assert find_partition("/swap_file") is rootassert find_partition("/home/user/.bash_rc") is home@@ -76,12 +74,38 @@ async def test_find_partition_finds_the_correct_partition() -> None:@pytest.mark.skipif(os.name != "nt", reason="Test paths are only for windows")-async def test_find_partition_finds_the_correct_partition_windows() -> None:+def test_find_partition_finds_the_correct_partition_windows() -> None:c_drive, d_drive = partitions = [create_partition(path) for path in ("C:/", "D:/")]- storage._PARTITIONS = partitions # pyright: ignore[reportPrivateUsage]+ _psutil._PARTITIONS = partitions # pyright: ignore[reportPrivateUsage]assert find_partition("C:/pagefile") is c_driveassert find_partition("C:/") is c_driveassert find_partition("D:/music/song.mp3") is d_driveassert find_partition("Z:/") is None+++async def test_no_psutil_check_does_not_raise_exception(tmp_path: Path) -> None:+ with mock.patch.object(storage, "_psutil_loop", None):+ async with storage.monitor(100):+ assert await storage.has_sufficient_space(tmp_path)+++def test_no_psutil_returns_size_of_closest_parent_on_file_that_does_not_exists(tmp_path: Path) -> None:+ folder = tmp_path / "folder_abc/that/does/not/exists"+ result = storage._disk_usage(folder)+ assert result > 0+ assert result == storage._disk_usage(tmp_path)+++async def test_psutil_returns_size_of_closest_parent_on_file_that_does_not_exists(tmp_path: Path) -> None:+ folder = tmp_path / "folder_abc/that/does/not/exists"+ result = await _psutil.get_free_space(folder)+ assert result > 0+ assert result == await _psutil.get_free_space(tmp_path)+++async def test_psutil_raw_raises_file_not_found_error_on_file_that_does_not_exists(tmp_path: Path) -> None:+ folder = tmp_path / "folder_abc/that/does/not/exists"+ with pytest.raises(FileNotFoundError):+ _ = _psutil._disk_usage(folder)