Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: sorting ignoring files at root of input dir (#101)
NTFSvolume2eb2e0414f57
diff --git a/cyberdrop_dl/ui/program_ui.py b/cyberdrop_dl/ui/program_ui.pyindex 4d78b0a44..20dfb228d 100644--- a/cyberdrop_dl/ui/program_ui.py+++ b/cyberdrop_dl/ui/program_ui.py@@ -87,7 +87,13 @@ class ProgramUI:def _sort_files(self) -> None:"""Sort files in download folder"""sorter = Sorter.from_manager(self.manager)- asyncio.run(sorter.run())+ console.print(+ f"You are about to sort files from '{sorter.input_dir}' to '{sorter.output_dir}'", style="bold red"+ )+ answer = input("Type 'YES' to proceed")+ if answer.strip().casefold() == "yes":+ asyncio.run(sorter.run())+ enter_to_continue()def _view_changelog(self) -> None:clear_term()diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.pyindex b7889d59d..86c0cf2cb 100644--- a/cyberdrop_dl/utils/sorting.py+++ b/cyberdrop_dl/utils/sorting.py@@ -19,24 +19,22 @@ from cyberdrop_dl.utils import stringsfrom cyberdrop_dl.utils.utilities import purge_dir_tree as delete_empty_files_and_foldersif TYPE_CHECKING:- from collections.abc import Iterable-from cyberdrop_dl.managers.manager import Managerlogger = logging.getLogger(__name__)[email protected](slots=True)[email protected](slots=True, kw_only=True)class Sorter:input_dir: Pathoutput_dir: Path- incrementer_format: straudio_format: str | Noneimage_format: str | Nonevideo_format: str | Noneother_format: str | None+ incrementer_format: str = "{i}"tui: SortingUI = dataclasses.field(init=False)@@ -56,27 +54,32 @@ class Sorter:other_format=settings.sorted_other,)- async def run(self) -> None:+ async def run(self, show_tui: bool = True) -> None:if not await asyncio.to_thread(self.input_dir.is_dir):- logger.error(f"Sort directory ('{self.input_dir}' does not exist", extra={"color": "red"})+ logger.error(f"Sort directory '{self.input_dir}' does not exist", extra={"color": "red"})returnlogger.info("Sorting downloads...", extra={"color": "cyan"})await asyncio.to_thread(self.output_dir.mkdir, parents=True, exist_ok=True)+ self.tui._progress.disable = show_tuiwith self.tui:- subfolders = await asyncio.to_thread(_subfolders, self.input_dir)- await self._sort_folders(subfolders)- logger.info("DONE!", extra={"color": "green"})- _ = delete_empty_files_and_folders(self.input_dir)+ await self._run()- async def _sort_folders(self, folders: Iterable[Path]) -> None:+ async def _run(self) -> None:async with asyncio.TaskGroup() as tg:- for fut in asyncio.as_completed(asyncio.to_thread(_files, folders) for folders in folders):- folder, files = await fut- for file in files:- _ = tg.create_task(self._sort_file(folder.name, file))+ async def _sort_folder(folder: Path) -> None:+ for path in await asyncio.to_thread(lambda: folder.glob("*")):+ if await asyncio.to_thread(path.is_file):+ _ = tg.create_task(self._sort_file(folder.name, path))+ else:+ _ = tg.create_task(_sort_folder(path))++ await _sort_folder(self.input_dir)++ logger.info("DONE!", extra={"color": "green"})+ _ = delete_empty_files_and_folders(self.input_dir)async def _sort_file(self, folder_name: str, file: Path) -> None:ext = file.suffix.lower()@@ -93,7 +96,7 @@ class Sorter:await self.sort_other(file, folder_name)except Exception:- logger.exception("Unknown error while sorting {}", file)+ logger.exception("Unknown error while sorting '{}'", file)self.tui.stats.errors += 1async def sort_audio(self, file: Path, base_name: str) -> None:@@ -185,7 +188,7 @@ class Sorter:self.tui.stats.others += 1async def _move_file(self, file: Path, base_name: str, format_str: str, /, **kwargs: object) -> bool:- success = _format_dest(+ dest = _format_dest(file,base_name,format_str,@@ -193,12 +196,12 @@ class Sorter:sort_dir=self.output_dir,**kwargs,)- success = bool(await asyncio.to_thread(_move_file, file, success, self.incrementer_format))- if success:- logger.debug("Moved '{}' to '{}'", file, success)+ dest = await asyncio.to_thread(_move_file, file, dest, self.incrementer_format)+ if dest:+ logger.warning("Moved '{}' to '{}'", file, dest)else:self.tui.stats.errors += 1- return success+ return bool(dest)def _format_dest(@@ -228,14 +231,6 @@ def _format_dest(return Path(dest)-def _subfolders(directory: Path) -> tuple[Path, ...]:- return tuple(path for path in directory.resolve().iterdir() if path.is_dir())---def _files(directory: Path) -> tuple[Path, tuple[Path, ...]]:- return directory, tuple(path for path in directory.resolve().rglob("*") if path.is_file())--def _move_but_not_overwrite(source: Path, dest: Path) -> Path:if dest.exists():raise FileExistsError(dest)@@ -301,4 +296,4 @@ async def _try_probe(kind: str, file: Path) -> ffmpeg.FFprobeResult | None:try:return await ffmpeg.probe(file)except (RuntimeError, CalledProcessError, OSError):- logger.exception(f"Unable to get {kind} properties of '{file}'")+ logger.exception("Unable to get {} properties of '{}'", kind, file)diff --git a/tests/test_sorter.py b/tests/test_sorter.pyindex 62b292afa..c3760f549 100644--- a/tests/test_sorter.py+++ b/tests/test_sorter.py@@ -1,10 +1,11 @@import datetime+import itertoolsimport shutilfrom pathlib import Pathimport pytest-from cyberdrop_dl.utils.sorting import _format_dest, _have_same_content, _move_file+from cyberdrop_dl.utils.sorting import Sorter, _format_dest, _have_same_content, _move_fileDOWNLOADS = Path("/mnt/home/user/downloads/cdl/")SORT_DIR = DOWNLOADS.parent / "cdl_sorted"@@ -146,3 +147,58 @@ class TestMoveFile:monkeypatch.setattr(shutil, "move", boom)assert _move_file(src, dst) is Noneassert src.exists()+++class Files:+ VIDEOS = "video.mp4", "movies/4K/a_movie.mkv"+ OTHERS = (+ "docker-compose.yml",+ "index.html",+ "styles.css",+ "tests/scripts/notes.txt",+ "tests/scripts/script.js",+ "tests/setup.sh",+ "tests/utils.py",+ )+ AUDIOS = ("music/song.mp3",)+ IMAGES = ("logo.png", "tests/logo.jpg")+ TEMP = ("tmp/download.part",)++ ALL = *VIDEOS, *OTHERS, *AUDIOS, *IMAGES, *TEMP+++async def test_sorter(tmp_path: Path) -> None:+ assert len(Files.ALL) == len(set(Files.ALL))+ input_dir = tmp_path / "cdl_downloads"+ output_dir = tmp_path / "cdl_sorted_downloads"+ sorter = Sorter(+ input_dir=input_dir,+ output_dir=output_dir,+ audio_format="{sort_dir}/audio/{filename}{ext}",+ image_format="{sort_dir}/image/{filename}{ext}",+ video_format="{sort_dir}/video/{filename}{ext}",+ other_format="{sort_dir}/other/{filename}{ext}",+ )++ for idx, file in enumerate(Files.ALL):+ path = input_dir / file+ path.parent.mkdir(parents=True, exist_ok=True)+ path.write_text(str(idx))++ await sorter.run(show_tui=False)+ stats = sorter.tui.stats+ assert stats.total == len(Files.ALL) - len(Files.TEMP)+ assert stats.videos == len(Files.VIDEOS)+ assert stats.others == len(Files.OTHERS)++ assert sorted(f for f in output_dir.rglob("*") if f.is_file()) == sorted(+ itertools.chain.from_iterable(+ (output_dir / kind / file.name for file in map(Path, files))+ for kind, files in [+ ("audio", Files.AUDIOS),+ ("image", Files.IMAGES),+ ("video", Files.VIDEOS),+ ("other", Files.OTHERS),+ ]+ )+ )