Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: sorting ignoring files at root of input dir (#101)

NTFSvolume2eb2e0414f57
diff --git a/cyberdrop_dl/ui/program_ui.py b/cyberdrop_dl/ui/program_ui.py
index 4d78b0a44..20dfb228d 100644
--- a/cyberdrop_dl/ui/program_ui.py
+++ b/cyberdrop_dl/ui/program_ui.py
@@ -87,7 +87,13 @@ class ProgramUI:
def _sort_files(self) -> None:
"""Sort files in download folder"""
sorter = Sorter.from_manager(self.manager)
- asyncio.run(sorter.run())
+ console.print(
+ f"You are about to sort files from '{sorter.input_dir}' to '{sorter.output_dir}'", style="bold red"
+ )
+ answer = input("Type 'YES' to proceed")
+ if answer.strip().casefold() == "yes":
+ asyncio.run(sorter.run())
+ enter_to_continue()
def _view_changelog(self) -> None:
clear_term()
diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py
index b7889d59d..86c0cf2cb 100644
--- a/cyberdrop_dl/utils/sorting.py
+++ b/cyberdrop_dl/utils/sorting.py
@@ -19,24 +19,22 @@ from cyberdrop_dl.utils import strings
from cyberdrop_dl.utils.utilities import purge_dir_tree as delete_empty_files_and_folders
if TYPE_CHECKING:
- from collections.abc import Iterable
-
from cyberdrop_dl.managers.manager import Manager
logger = logging.getLogger(__name__)
[email protected](slots=True)
[email protected](slots=True, kw_only=True)
class Sorter:
input_dir: Path
output_dir: Path
- incrementer_format: str
audio_format: str | None
image_format: str | None
video_format: str | None
other_format: str | None
+ incrementer_format: str = "{i}"
tui: SortingUI = dataclasses.field(init=False)
@@ -56,27 +54,32 @@ class Sorter:
other_format=settings.sorted_other,
)
- async def run(self) -> None:
+ async def run(self, show_tui: bool = True) -> None:
if not await asyncio.to_thread(self.input_dir.is_dir):
- logger.error(f"Sort directory ('{self.input_dir}' does not exist", extra={"color": "red"})
+ logger.error(f"Sort directory '{self.input_dir}' does not exist", extra={"color": "red"})
return
logger.info("Sorting downloads...", extra={"color": "cyan"})
await asyncio.to_thread(self.output_dir.mkdir, parents=True, exist_ok=True)
+ self.tui._progress.disable = show_tui
with self.tui:
- subfolders = await asyncio.to_thread(_subfolders, self.input_dir)
- await self._sort_folders(subfolders)
- logger.info("DONE!", extra={"color": "green"})
- _ = delete_empty_files_and_folders(self.input_dir)
+ await self._run()
- async def _sort_folders(self, folders: Iterable[Path]) -> None:
+ async def _run(self) -> None:
async with asyncio.TaskGroup() as tg:
- for fut in asyncio.as_completed(asyncio.to_thread(_files, folders) for folders in folders):
- folder, files = await fut
- for file in files:
- _ = tg.create_task(self._sort_file(folder.name, file))
+ async def _sort_folder(folder: Path) -> None:
+ for path in await asyncio.to_thread(lambda: folder.glob("*")):
+ if await asyncio.to_thread(path.is_file):
+ _ = tg.create_task(self._sort_file(folder.name, path))
+ else:
+ _ = tg.create_task(_sort_folder(path))
+
+ await _sort_folder(self.input_dir)
+
+ logger.info("DONE!", extra={"color": "green"})
+ _ = delete_empty_files_and_folders(self.input_dir)
async def _sort_file(self, folder_name: str, file: Path) -> None:
ext = file.suffix.lower()
@@ -93,7 +96,7 @@ class Sorter:
await self.sort_other(file, folder_name)
except Exception:
- logger.exception("Unknown error while sorting {}", file)
+ logger.exception("Unknown error while sorting '{}'", file)
self.tui.stats.errors += 1
async def sort_audio(self, file: Path, base_name: str) -> None:
@@ -185,7 +188,7 @@ class Sorter:
self.tui.stats.others += 1
async def _move_file(self, file: Path, base_name: str, format_str: str, /, **kwargs: object) -> bool:
- success = _format_dest(
+ dest = _format_dest(
file,
base_name,
format_str,
@@ -193,12 +196,12 @@ class Sorter:
sort_dir=self.output_dir,
**kwargs,
)
- success = bool(await asyncio.to_thread(_move_file, file, success, self.incrementer_format))
- if success:
- logger.debug("Moved '{}' to '{}'", file, success)
+ dest = await asyncio.to_thread(_move_file, file, dest, self.incrementer_format)
+ if dest:
+ logger.warning("Moved '{}' to '{}'", file, dest)
else:
self.tui.stats.errors += 1
- return success
+ return bool(dest)
def _format_dest(
@@ -228,14 +231,6 @@ def _format_dest(
return Path(dest)
-def _subfolders(directory: Path) -> tuple[Path, ...]:
- return tuple(path for path in directory.resolve().iterdir() if path.is_dir())
-
-
-def _files(directory: Path) -> tuple[Path, tuple[Path, ...]]:
- return directory, tuple(path for path in directory.resolve().rglob("*") if path.is_file())
-
-
def _move_but_not_overwrite(source: Path, dest: Path) -> Path:
if dest.exists():
raise FileExistsError(dest)
@@ -301,4 +296,4 @@ async def _try_probe(kind: str, file: Path) -> ffmpeg.FFprobeResult | None:
try:
return await ffmpeg.probe(file)
except (RuntimeError, CalledProcessError, OSError):
- logger.exception(f"Unable to get {kind} properties of '{file}'")
+ logger.exception("Unable to get {} properties of '{}'", kind, file)
diff --git a/tests/test_sorter.py b/tests/test_sorter.py
index 62b292afa..c3760f549 100644
--- a/tests/test_sorter.py
+++ b/tests/test_sorter.py
@@ -1,10 +1,11 @@
import datetime
+import itertools
import shutil
from pathlib import Path
import pytest
-from cyberdrop_dl.utils.sorting import _format_dest, _have_same_content, _move_file
+from cyberdrop_dl.utils.sorting import Sorter, _format_dest, _have_same_content, _move_file
DOWNLOADS = Path("/mnt/home/user/downloads/cdl/")
SORT_DIR = DOWNLOADS.parent / "cdl_sorted"
@@ -146,3 +147,58 @@ class TestMoveFile:
monkeypatch.setattr(shutil, "move", boom)
assert _move_file(src, dst) is None
assert src.exists()
+
+
+class Files:
+ VIDEOS = "video.mp4", "movies/4K/a_movie.mkv"
+ OTHERS = (
+ "docker-compose.yml",
+ "index.html",
+ "styles.css",
+ "tests/scripts/notes.txt",
+ "tests/scripts/script.js",
+ "tests/setup.sh",
+ "tests/utils.py",
+ )
+ AUDIOS = ("music/song.mp3",)
+ IMAGES = ("logo.png", "tests/logo.jpg")
+ TEMP = ("tmp/download.part",)
+
+ ALL = *VIDEOS, *OTHERS, *AUDIOS, *IMAGES, *TEMP
+
+
+async def test_sorter(tmp_path: Path) -> None:
+ assert len(Files.ALL) == len(set(Files.ALL))
+ input_dir = tmp_path / "cdl_downloads"
+ output_dir = tmp_path / "cdl_sorted_downloads"
+ sorter = Sorter(
+ input_dir=input_dir,
+ output_dir=output_dir,
+ audio_format="{sort_dir}/audio/{filename}{ext}",
+ image_format="{sort_dir}/image/{filename}{ext}",
+ video_format="{sort_dir}/video/{filename}{ext}",
+ other_format="{sort_dir}/other/{filename}{ext}",
+ )
+
+ for idx, file in enumerate(Files.ALL):
+ path = input_dir / file
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(str(idx))
+
+ await sorter.run(show_tui=False)
+ stats = sorter.tui.stats
+ assert stats.total == len(Files.ALL) - len(Files.TEMP)
+ assert stats.videos == len(Files.VIDEOS)
+ assert stats.others == len(Files.OTHERS)
+
+ assert sorted(f for f in output_dir.rglob("*") if f.is_file()) == sorted(
+ itertools.chain.from_iterable(
+ (output_dir / kind / file.name for file in map(Path, files))
+ for kind, files in [
+ ("audio", Files.AUDIOS),
+ ("image", Files.IMAGES),
+ ("video", Files.VIDEOS),
+ ("other", Files.OTHERS),
+ ]
+ )
+ )