Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
perf: make all file system calls async (#951)
NTFSvolume7ea85d42f86e
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex a354aeca3..ec1d8dee0 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -230,8 +230,8 @@ class DownloadClient:return False- if resp.status != HTTPStatus.PARTIAL_CONTENT and media_item.partial_file.is_file():- media_item.partial_file.unlink()+ if resp.status != HTTPStatus.PARTIAL_CONTENT:+ await asyncio.to_thread(media_item.partial_file.unlink, missing_ok=True)if not media_item.datetime and (last_modified := get_last_modified(resp.headers)):msg = f"Unable to parse upload date for {media_item.url}, using `Last-Modified` header as file datetime"@@ -298,9 +298,12 @@ class DownloadClient:check_free_space = partial(self.manager.storage_manager.check_free_space, media_item)await check_free_space()- media_item.partial_file.parent.mkdir(parents=True, exist_ok=True)- if not media_item.partial_file.is_file():- media_item.partial_file.touch()+ def prepare():+ media_item.partial_file.parent.mkdir(parents=True, exist_ok=True)+ if not media_item.partial_file.is_file():+ media_item.partial_file.touch()++ await asyncio.to_thread(prepare)last_slow_speed_read = None@@ -381,7 +384,7 @@ class DownloadClient:async def add_file_size(self, domain: str, media_item: MediaItem) -> None:if not media_item.complete_file:media_item.complete_file = self.get_file_location(media_item)- if media_item.complete_file.is_file():+ if await asyncio.to_thread(media_item.complete_file.is_file):await self.manager.db_manager.history_table.add_filesize(domain, media_item)async def handle_media_item_completion(self, media_item: MediaItem, downloaded: bool = False) -> None:diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.pyindex dc24d9052..af6ebd408 100644--- a/cyberdrop_dl/clients/hash_client.py+++ b/cyberdrop_dl/clients/hash_client.py@@ -12,6 +12,7 @@ from send2trash import send2trashfrom cyberdrop_dl.ui.prompts.basic_prompts import enter_to_continuefrom cyberdrop_dl.utils.data_enums_classes.hash import Hashingfrom cyberdrop_dl.utils.logger import log+from cyberdrop_dl.utils.utilities import get_size_or_noneif TYPE_CHECKING:from yarl import URL@@ -50,7 +51,7 @@ class HashClient:async def hash_directory(self, path: Path) -> None:path = Path(path)with self.manager.live_manager.get_hash_live(stop=True):- if not path.is_dir():+ if not await asyncio.to_thread(path.is_dir):raise NotADirectoryErrorfor file in path.rglob("*"):await self.update_db_and_retrive_hash(file)@@ -61,7 +62,7 @@ class HashClient:hash = await self.update_db_and_retrive_hash(media_item.complete_file, media_item.original_filename, media_item.referer)- self.save_hash_data(media_item, hash)+ await self.save_hash_data(media_item, hash)async def hash_item_during_download(self, media_item: MediaItem) -> None:if media_item.is_segment:@@ -73,7 +74,7 @@ class HashClient:hash = await self.update_db_and_retrive_hash(media_item.complete_file, media_item.original_filename, media_item.referer)- self.save_hash_data(media_item, hash)+ await self.save_hash_data(media_item, hash)except Exception as e:log(f"After hash processing failed: {media_item.complete_file} with error {e}", 40, exc_info=True)@@ -81,11 +82,9 @@ class HashClient:self, file: Path | str, original_filename: str | None = None, referer: URL | None = None):file = Path(file)- if not file.is_file():+ if file.suffix == ".part":return- elif file.stat().st_size == 0:- return- elif file.suffix == ".part":+ if not await asyncio.to_thread(get_size_or_none, file):returnhash = await self._update_db_and_retrive_hash_helper(file, original_filename, referer, hash_type=self.xxhash)if self.manager.config_manager.settings_data.dupe_cleanup_options.add_md5_hash:@@ -128,9 +127,10 @@ class HashClient:log(f"Error hashing {file} : {e}", 40, exc_info=True)return hash- def save_hash_data(self, media_item: MediaItem, hash: str):- absolute_path = media_item.complete_file.resolve()- size = media_item.complete_file.stat().st_size+ async def save_hash_data(self, media_item: MediaItem, hash: str):+ absolute_path = await asyncio.to_thread(media_item.complete_file.resolve)+ size = await asyncio.to_thread(get_size_or_none, media_item.complete_file)+ assert sizeself.hashed_media_items.add(media_item)if hash:media_item.hash = hashdiff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 545252b75..e3ed22def 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -21,7 +21,7 @@ from cyberdrop_dl.clients.errors import (from cyberdrop_dl.utils.constants import CustomHTTPStatusfrom cyberdrop_dl.utils.data_enums_classes.url_objects import HlsSegment, MediaItemfrom cyberdrop_dl.utils.logger import log-from cyberdrop_dl.utils.utilities import error_handling_wrapper+from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_noneif TYPE_CHECKING:from collections.abc import Callable, Coroutine, Generator@@ -290,8 +290,7 @@ class Downloader:ClientError,) as e:ui_message = getattr(e, "status", type(e).__name__)- if media_item.partial_file and media_item.partial_file.is_file():- size = media_item.partial_file.stat().st_size+ if size := await asyncio.to_thread(get_size_or_none, media_item.partial_file):if (media_item.filename in self._current_attempt_filesizeand self._current_attempt_filesize[media_item.filename] >= sizediff --git a/cyberdrop_dl/managers/log_manager.py b/cyberdrop_dl/managers/log_manager.pyindex b74aeeeef..167c0a510 100644--- a/cyberdrop_dl/managers/log_manager.py+++ b/cyberdrop_dl/managers/log_manager.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport csvfrom asyncio import Lockfrom pathlib import Path@@ -38,7 +39,7 @@ class LogManager:"""Write to the specified csv file. kwargs are columns for the CSV."""self._csv_locks[file] = self._csv_locks.get(file, Lock())async with self._csv_locks[file]:- write_headers = not file.is_file()+ write_headers = not await asyncio.to_thread(file.is_file)async with aiofiles.open(file, "a", encoding="utf8", newline="") as csv_file:writer = csv.DictWriter(csv_file, fieldnames=kwargs.keys(), delimiter=CSV_DELIMITER, quoting=csv.QUOTE_ALL@@ -71,7 +72,11 @@ class LogManager:async def update_last_forum_post(self) -> None:"""Updates the last forum post."""input_file = self.manager.path_manager.input_file- if not input_file.is_file() or not self.last_post_log.is_file():++ def proceed():+ return input_file.is_file() or self.last_post_log.is_file()++ if not await asyncio.to_thread(proceed):returnlog_spacer(20)diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex 403a6570c..b06983b4a 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport refrom dataclasses import Fieldfrom datetime import date, datetime@@ -114,6 +115,10 @@ class ScrapeMapper:async def parse_input_file_groups(self) -> AsyncGenerator[tuple[str, list[URL]]]:"""Split URLs from input file by their groups."""input_file = self.manager.path_manager.input_file+ if not await asyncio.to_thread(input_file.is_file):+ yield ("", [])+ return+block_quote = Falsecurrent_group_name = ""async with aiofiles.open(input_file, encoding="utf8") as f:@@ -136,10 +141,6 @@ class ScrapeMapper:async def load_links(self) -> AsyncGenerator[ScrapeItem]:"""Loads links from args / input file."""- input_file = self.manager.path_manager.input_file- # we need to touch the file just in case, purge_tree deletes it- if not input_file.is_file():- input_file.touch(exist_ok=True)if not self.manager.parsed_args.cli_only_args.links:self.using_input_file = Truediff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.pyindex e5191e667..d5b3ce1dd 100644--- a/cyberdrop_dl/utils/sorting.py+++ b/cyberdrop_dl/utils/sorting.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport contextlibimport itertoolsfrom fractions import Fraction@@ -17,6 +18,7 @@ from cyberdrop_dl.utils.logger import log_with_colorfrom cyberdrop_dl.utils.utilities import purge_dir_treeif TYPE_CHECKING:+ from collections.abc import AsyncGeneratorfrom datetime import datetimefrom cyberdrop_dl.managers.manager import Manager@@ -40,9 +42,16 @@ class Sorter:self.video_format: str | None = settings.sorted_videoself.other_format: str | None = settings.sorted_other- def _get_files(self, directory: Path) -> list[Path]:+ async def _get_files(self, directory: Path) -> AsyncGenerator[Path]:"""Finds all files in a directory and returns them in a list."""- return [file.resolve() for file in directory.rglob("*") if file.is_file()]++ def resolve_if_file(path: Path) -> Path | None:+ if path.is_file():+ return path.resolve()++ for file in directory.rglob("*"):+ if file := await asyncio.to_thread(resolve_if_file, file):+ yield filedef _move_file(self, old_path: Path, new_path: Path) -> bool:"""Moves a file to a destination folder."""@@ -74,19 +83,20 @@ class Sorter:async def run(self) -> None:"""Sorts the files in the download directory into their respective folders."""- if not self.download_folder.is_dir():+ if not asyncio.to_thread(self.download_folder.is_dir):log_with_color(f"Download directory ({self.download_folder}) does not exist", "red", 40)returnlog_with_color("\nSorting downloads, please wait", "cyan", 20)- self.sorted_folder.mkdir(parents=True, exist_ok=True)+ await asyncio.to_thread(self.sorted_folder.mkdir, parents=True, exist_ok=True)files_to_sort: dict[str, list[Path]] = {}with self.manager.live_manager.get_sort_live(stop=True):- subfolders = [f for f in self.download_folder.iterdir() if f.is_dir()]- for folder in subfolders:- files_to_sort[folder.name] = self._get_files(folder)+ for subfolder in self.download_folder.iterdir():+ if not await asyncio.to_thread(subfolder.is_dir):+ continue+ files_to_sort[subfolder.name] = [file async for file in self._get_files(subfolder)]await self._sort_files(files_to_sort)log_with_color("DONE!", "green", 20)diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex 6f7493d97..bb77b021e 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -1,5 +1,6 @@from __future__ import annotations+import asyncioimport contextlibimport jsonimport os@@ -10,6 +11,7 @@ import subprocessfrom dataclasses import dataclass, fieldsfrom functools import lru_cache, partial, wrapsfrom pathlib import Path+from stat import S_ISREGfrom typing import TYPE_CHECKING, Any, ClassVar, ParamSpec, Protocol, TypeVarimport aiofiles@@ -311,8 +313,8 @@ async def send_webhook_message(manager: Manager) -> None:form = FormData()- if "attach_logs" in webhook.tags and main_log.is_file():- if main_log.stat().st_size <= 25 * 1024 * 1024:+ if "attach_logs" in webhook.tags and (size := await asyncio.to_thread(get_size_or_none, main_log)):+ if size <= 25 * 1024 * 1024: # 25MBasync with aiofiles.open(main_log, "rb") as f:form.add_field("file", await f.read(), filename=main_log.name)@@ -498,6 +500,19 @@ def get_filename_from_headers(headers: Mapping[str, Any]) -> str | None:return matches[0] or matches[1]+def get_size_or_none(path: Path) -> int | None:+ """Checks if this is a file and returns its size with a single system call.++ Returns `None` otherwise"""++ try:+ stat = path.stat()+ if S_ISREG(stat.st_mode):+ return stat.st_size+ except (OSError, ValueError):+ return None++log_cyan = partial(log_with_color, style="cyan", level=20)log_yellow = partial(log_with_color, style="yellow", level=20)log_green = partial(log_with_color, style="green", level=20)