Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: set min required free space even if psutil is not installed (#124)

NTFSvolumed6010ff27bd9
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 4ea503bc8..634644aee 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -15,6 +15,7 @@ from cyberdrop_dl.exceptions import (
DownloadError,
DurationError,
ErrorLogMessage,
+ InsufficientFreeSpaceError,
InvalidContentTypeError,
RestrictedDateRangeError,
RestrictedFiletypeError,
@@ -290,7 +291,8 @@ class Downloader:
async def check_file_can_download(self, media_item: MediaItem) -> None:
"""Checks if the file can be downloaded."""
- await storage.check(media_item)
+ if not await storage.has_sufficient_space(media_item.download_folder):
+ raise InsufficientFreeSpaceError(media_item)
if not self.manager.client_manager.is_allowed_filetype(media_item):
raise RestrictedFiletypeError(origin=media_item)
if not await self.manager.client_manager.check_file_duration(media_item):
@@ -387,7 +389,7 @@ class Downloader:
ClientConnectorError,
) as e:
ui_message = getattr(e, "status", type(e).__name__)
- if size := await aio.get_size(media_item.partial_file):
+ if media_item.partial_file and (size := await aio.get_size(media_item.partial_file)):
if self._current_attempt_filesize.get(media_item.filename, 0) >= size:
raise DownloadError(ui_message, message=f"{self.log_prefix} failed", retry=True) from None
diff --git a/cyberdrop_dl/storage/__init__.py b/cyberdrop_dl/storage/__init__.py
index 1be567151..6fab91ce3 100644
--- a/cyberdrop_dl/storage/__init__.py
+++ b/cyberdrop_dl/storage/__init__.py
@@ -19,15 +19,31 @@ if TYPE_CHECKING:
_required_free_space: ContextVar[int] = ContextVar("_required_free_space")
+
+def _disk_usage(folder: Path) -> int:
+ path = folder
+ while True:
+ try:
+ return shutil.disk_usage(path).free
+ except FileNotFoundError:
+ if path.parent == path:
+ return 0
+ path = path.parent
+
+
try:
- from ._psutil import has_sufficient_space
+ from ._psutil import get_free_space as _get_free_space
from ._psutil import start_loop as _psutil_loop
except ImportError:
_psutil_loop = None
- async def has_sufficient_space(folder: Path, /, required_free_space: int) -> bool:
- usage = await asyncio.to_thread(shutil.disk_usage, folder)
- return usage.free > required_free_space
+ async def _get_free_space(folder: Path) -> int:
+ return await asyncio.to_thread(_disk_usage, folder)
+
+
+async def has_sufficient_space(folder: Path, /, required_free_space: int | None = None) -> bool:
+ free_space = await _get_free_space(folder)
+ return free_space == -1 or free_space > (required_free_space or _required_free_space.get())
def create_free_space_checker(media_item: MediaItem, *, frecuency: int = 5) -> Callable[[], Awaitable[None]]:
@@ -36,35 +52,30 @@ def create_free_space_checker(media_item: MediaItem, *, frecuency: int = 5) -> C
async def checker() -> None:
nonlocal current_chunk
if current_chunk % frecuency == 0:
- await check(media_item)
+ if not await has_sufficient_space(media_item.download_folder):
+ raise InsufficientFreeSpaceError(media_item)
+
current_chunk += 1
return checker
-async def check(media_item: MediaItem) -> None:
- """Checks if there is enough free space to download this item."""
-
- if not await has_sufficient_space(media_item.download_folder, _required_free_space.get()):
- raise InsufficientFreeSpaceError(media_item)
-
-
@contextlib.asynccontextmanager
async def monitor(required_free_space: int) -> AsyncGenerator[None]:
+ token = _required_free_space.set(required_free_space)
if _psutil_loop is None:
logger.warning("psutil is not available on this system. Falling back to eager checks for free space")
- yield
- return
-
- loop = asyncio.create_task(_psutil_loop(), name="storage monitor")
- token = _required_free_space.set(required_free_space)
- await asyncio.sleep(0)
+ loop = None
+ else:
+ loop = asyncio.create_task(_psutil_loop(), name="storage monitor")
+ await asyncio.sleep(0)
try:
yield
finally:
_required_free_space.reset(token)
- try:
- _ = loop.cancel("On monitor exit")
- await loop
- except asyncio.CancelledError:
- pass
+ if loop is not None:
+ try:
+ _ = loop.cancel("On monitor exit")
+ await loop
+ except asyncio.CancelledError:
+ pass
diff --git a/cyberdrop_dl/storage/_psutil.py b/cyberdrop_dl/storage/_psutil.py
index c8ba694f9..5be6ad437 100644
--- a/cyberdrop_dl/storage/_psutil.py
+++ b/cyberdrop_dl/storage/_psutil.py
@@ -45,13 +45,12 @@ class _Stats:
return str(self.__json__())
-async def _get_free_space(path: Path) -> int:
+def _disk_usage(path: Path) -> int:
unsupported = None
free_space = 0
try:
- usage = await asyncio.to_thread(psutil.disk_usage, str(path))
- free_space = usage.free
+ free_space = psutil.disk_usage(str(path)).free
except OSError as e:
if "operation not supported" not in str(e).casefold():
raise
@@ -68,11 +67,10 @@ async def _get_free_space(path: Path) -> int:
return free_space
-async def has_sufficient_space(folder: Path, /, required_free_space: int) -> bool:
- await _check_nt_network_drive(folder)
+async def get_free_space(folder: Path) -> int:
mount = _get_mount_point(folder)
if not mount:
- return False
+ return 0
free_space = _free_space.get(mount)
if free_space is None:
@@ -81,11 +79,11 @@ async def has_sufficient_space(folder: Path, /, required_free_space: int) -> boo
if free_space is None:
# Manually query this mount now. Next time it will be part of the loop
- free_space = _free_space[mount] = await _get_free_space(mount)
+ free_space = _free_space[mount] = await asyncio.to_thread(_disk_usage, mount)
logger.info(f"A new mountpoint ('{mount!s}') will be used for '{folder}'")
logger.info("Storage status \n%s", _Stats())
- return free_space == -1 or free_space > required_free_space
+ return free_space
def _find_partition(path: Path) -> DiskPartition | None:
@@ -134,13 +132,16 @@ def clear_cache() -> None:
async def start_loop() -> None:
"""Infinite loop to get free space of all used mounts and update internal dict"""
- async def update():
+ async def update() -> None:
mountpoints = sorted(mount for mount, free_space in _free_space.items() if free_space != -1)
if not mountpoints:
return
- results = await asyncio.gather(*map(_get_free_space, mountpoints))
- _free_space.update(zip(mountpoints, results, strict=True))
+ usage = await asyncio.gather(
+ *(asyncio.to_thread(_disk_usage, mount) for mount in mountpoints),
+ return_exceptions=False,
+ )
+ _free_space.update(zip(mountpoints, usage, strict=True))
last_check = -1
while True:
diff --git a/tests/test_storage.py b/tests/test_storage.py
index 1e338406b..15cc52a11 100644
--- a/tests/test_storage.py
+++ b/tests/test_storage.py
@@ -7,66 +7,64 @@ from unittest import mock
import pytest
-from cyberdrop_dl import aio
-from cyberdrop_dl.storage import _psutil as storage
+from cyberdrop_dl import storage
+from cyberdrop_dl.storage import _psutil
def create_partition(path: str):
- return storage.DiskPartition(Path(path), Path(path), "", "")
+ return _psutil.DiskPartition(Path(path), Path(path), "", "")
def find_partition(path: str):
- return storage._find_partition(Path(path))
+ return _psutil._find_partition(Path(path))
-async def test_unsupported_fs_should_not_return_zero() -> None:
- cwd = await aio.resolve(Path())
- free_space = await storage._get_free_space(cwd)
+async def test_unsupported_fs_should_not_return_zero(tmp_path: Path) -> None:
+ free_space = _psutil._disk_usage(tmp_path)
assert free_space > 0
with mock.patch("psutil.disk_usage", side_effect=OSError(None, "operation not supported")):
- free_space = await storage._get_free_space(cwd)
+ free_space = _psutil._disk_usage(tmp_path)
assert free_space == -1
with mock.patch("psutil.disk_usage", side_effect=OSError(None, "another error")):
with pytest.raises(OSError):
- _ = await storage._get_free_space(cwd)
+ _ = _psutil._disk_usage(tmp_path)
-async def test_fuse_filesystem_should_not_return_zero() -> None:
- cwd = await aio.resolve(Path())
- partition = storage._find_partition(cwd)
+def test_fuse_filesystem_should_not_return_zero(tmp_path: Path) -> None:
+ partition = _psutil._find_partition(tmp_path)
assert partition
- assert not storage._is_fuse_fs(cwd)
- storage._PARTITIONS = [dataclasses.replace(partition, fstype="fuse")] # pyright: ignore[reportPrivateUsage]
- assert storage._is_fuse_fs(cwd)
+ assert not _psutil._is_fuse_fs(tmp_path)
+ _psutil._PARTITIONS = [dataclasses.replace(partition, fstype="fuse")] # pyright: ignore[reportPrivateUsage]
+ assert _psutil._is_fuse_fs(tmp_path)
- free_space = await storage._get_free_space(cwd)
+ free_space = _psutil._disk_usage(tmp_path)
assert free_space > 0
class NullUsage:
free = 0
with mock.patch("psutil.disk_usage", return_value=NullUsage()):
- free_space = await storage._get_free_space(cwd)
+ free_space = _psutil._disk_usage(tmp_path)
assert free_space == -1
def test_storage_only_work_with_abs_paths() -> None:
cwd = Path()
with pytest.raises(ValueError):
- _ = storage._find_partition(cwd)
+ _ = _psutil._find_partition(cwd)
- assert storage._find_partition(cwd.resolve())
+ assert _psutil._find_partition(cwd.resolve())
@pytest.mark.skipif(os.name == "nt", reason="Test paths are only posix")
-async def test_find_partition_finds_the_correct_partition() -> None:
+def test_find_partition_finds_the_correct_partition() -> None:
root, home, usb, external_ssd = partitions = [
create_partition(path) for path in ("/", "/home", "/mnt/USB", "/home/external_SSD")
]
- storage._PARTITIONS = partitions # pyright: ignore[reportPrivateUsage]
+ _psutil._PARTITIONS = partitions # pyright: ignore[reportPrivateUsage]
assert find_partition("/swap_file") is root
assert find_partition("/home/user/.bash_rc") is home
@@ -76,12 +74,38 @@ async def test_find_partition_finds_the_correct_partition() -> None:
@pytest.mark.skipif(os.name != "nt", reason="Test paths are only for windows")
-async def test_find_partition_finds_the_correct_partition_windows() -> None:
+def test_find_partition_finds_the_correct_partition_windows() -> None:
c_drive, d_drive = partitions = [create_partition(path) for path in ("C:/", "D:/")]
- storage._PARTITIONS = partitions # pyright: ignore[reportPrivateUsage]
+ _psutil._PARTITIONS = partitions # pyright: ignore[reportPrivateUsage]
assert find_partition("C:/pagefile") is c_drive
assert find_partition("C:/") is c_drive
assert find_partition("D:/music/song.mp3") is d_drive
assert find_partition("Z:/") is None
+
+
+async def test_no_psutil_check_does_not_raise_exception(tmp_path: Path) -> None:
+ with mock.patch.object(storage, "_psutil_loop", None):
+ async with storage.monitor(100):
+ assert await storage.has_sufficient_space(tmp_path)
+
+
+def test_no_psutil_returns_size_of_closest_parent_on_file_that_does_not_exists(tmp_path: Path) -> None:
+ folder = tmp_path / "folder_abc/that/does/not/exists"
+ result = storage._disk_usage(folder)
+ assert result > 0
+ assert result == storage._disk_usage(tmp_path)
+
+
+async def test_psutil_returns_size_of_closest_parent_on_file_that_does_not_exists(tmp_path: Path) -> None:
+ folder = tmp_path / "folder_abc/that/does/not/exists"
+ result = await _psutil.get_free_space(folder)
+ assert result > 0
+ assert result == await _psutil.get_free_space(tmp_path)
+
+
+async def test_psutil_raw_raises_file_not_found_error_on_file_that_does_not_exists(tmp_path: Path) -> None:
+ folder = tmp_path / "folder_abc/that/does/not/exists"
+ with pytest.raises(FileNotFoundError):
+ _ = _psutil._disk_usage(folder)