Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove redundant utils (#109)
NTFSvolume29e1aa679ae3
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 8649361c6..df46926c2 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -17,7 +17,6 @@ from cyberdrop_dl.constants import FileExtfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, SlowDownloadErrorfrom cyberdrop_dl.utils import dates-from cyberdrop_dl.utils.utilities import get_size_or_noneif TYPE_CHECKING:from collections.abc import AsyncGenerator, Callable, Coroutine, Generator, Mapping@@ -104,11 +103,7 @@ class DownloadClient:media_item.partial_file = download_dir / f"{downloaded_filename}{constants.TempExt.PART}"resume_point = 0- if (- self._supports_ranges- and media_item.partial_file- and (size := await asyncio.to_thread(get_size_or_none, media_item.partial_file))- ):+ if self._supports_ranges and media_item.partial_file and (size := await aio.get_size(media_item.partial_file)):resume_point = sizedownload_headers["Range"] = f"bytes={size}-"diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.pyindex 21aad844e..6b47a0375 100644--- a/cyberdrop_dl/clients/hash_client.py+++ b/cyberdrop_dl/clients/hash_client.py@@ -8,9 +8,8 @@ from typing import TYPE_CHECKING, Literalfrom send2trash import send2trash-from cyberdrop_dl import constants+from cyberdrop_dl import aio, constantsfrom cyberdrop_dl.constants import Hashing-from cyberdrop_dl.utils.utilities import get_size_or_noneif TYPE_CHECKING:from yarl import URL@@ -94,7 +93,7 @@ class HashClient:if file.suffix in constants.TempExt:return- if not await asyncio.to_thread(get_size_or_none, file):+ if not await aio.get_size(file):returnhash = await self._update_db_and_retrive_hash_helper(file, original_filename, referer, hash_type=self.xxhash)@@ -143,7 +142,7 @@ class HashClient:if not hash:returnabsolute_path = await asyncio.to_thread(media_item.path.resolve)- size = await asyncio.to_thread(get_size_or_none, media_item.path)+ size = await aio.get_size(media_item.path)assert sizeself.hashed_media_items.append(media_item)if hash:diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.pyindex a03ffeedf..3480bb786 100755--- a/cyberdrop_dl/config/config_model.py+++ b/cyberdrop_dl/config/config_model.py@@ -21,7 +21,7 @@ from cyberdrop_dl.models.types import ()from cyberdrop_dl.models.validators import falsy_as, to_timedeltafrom cyberdrop_dl.utils.strings import validate_format_string-from cyberdrop_dl.utils.utilities import purge_dir_tree+from cyberdrop_dl.utils.utilities import delete_empty_files_and_foldersfrom ._common import ConfigModel@@ -123,7 +123,7 @@ class Logs(AliasModel):if (now - datetime.fromtimestamp(file.stat().st_ctime)) > self.logs_expire_after:file.unlink()- purge_dir_tree(self.log_folder)+ delete_empty_files_and_folders(self.log_folder)class FileSizeLimits(BaseModel):diff --git a/cyberdrop_dl/crawlers/discourse/_discourse.py b/cyberdrop_dl/crawlers/discourse/_discourse.pyindex 3fb957ffd..29859fd01 100644--- a/cyberdrop_dl/crawlers/discourse/_discourse.py+++ b/cyberdrop_dl/crawlers/discourse/_discourse.py@@ -15,7 +15,7 @@ from cyberdrop_dl.crawlers._forum import MessageBoardCrawlerfrom cyberdrop_dl.exceptions import MaxChildrenErrorfrom cyberdrop_dl.utils import cssfrom cyberdrop_dl.utils.dates import to_timestamp-from cyberdrop_dl.utils.utilities import error_handling_wrapper, unique+from cyberdrop_dl.utils.utilities import error_handling_wrapperfrom .models import AvailablePost, PostStream, Topic@@ -138,14 +138,15 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):images = css.iselect(soup, *css.images)links = css.iselect(soup, *css.links)external_links = (ref.url for ref in post.link_counts)- for link_str in unique(itertools.chain(external_links, images, links)):++ for link_str in dict.fromkeys(itertools.chain(external_links, images, links)):try:if link_str:yield self.parse_url(link_str)except Exception:continue- return unique(iter_links())+ return iter_links()@classmethoddef parse_url(diff --git a/cyberdrop_dl/crawlers/wordpress/_wordpress.py b/cyberdrop_dl/crawlers/wordpress/_wordpress.pyindex 21bee138a..61a01d851 100644--- a/cyberdrop_dl/crawlers/wordpress/_wordpress.py+++ b/cyberdrop_dl/crawlers/wordpress/_wordpress.py@@ -21,7 +21,7 @@ from cyberdrop_dl.data_structures.url_objects import QueryDatetimeRangefrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.utils import css, open_graphfrom cyberdrop_dl.utils.dates import to_timestamp-from cyberdrop_dl.utils.utilities import error_handling_wrapper, unique+from cyberdrop_dl.utils.utilities import error_handling_wrapperfrom .models import HTML, Category, CategorySequence, ColletionType, Post, PostSequence, Tag, TagSequence@@ -183,7 +183,7 @@ class WordPressBaseCrawler(Crawler, is_abc=True):@finaldef iter_parse_url(self, iterable: Iterable[str]) -> Iterable[AbsoluteHttpURL]:- for link_str in unique(iterable):+ for link_str in dict.fromkeys(iterable):try:yield self.parse_url(link_str)except Exception:diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.pyindex e8642cf94..663fe2e78 100644--- a/cyberdrop_dl/utils/sorting.py+++ b/cyberdrop_dl/utils/sorting.py@@ -16,7 +16,7 @@ from cyberdrop_dl import ffmpegfrom cyberdrop_dl.constants import FileExt, TempExtfrom cyberdrop_dl.progress.sorting import SortingUIfrom cyberdrop_dl.utils import strings-from cyberdrop_dl.utils.utilities import purge_dir_tree as delete_empty_files_and_folders+from cyberdrop_dl.utils.utilities import delete_empty_files_and_foldersif TYPE_CHECKING:from cyberdrop_dl.managers.manager import Managerdiff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.pyindex 77f4c7e4e..e3dd61dd0 100644--- a/cyberdrop_dl/utils/utilities.py+++ b/cyberdrop_dl/utils/utilities.py@@ -10,32 +10,17 @@ import osimport platformimport reimport sys-from collections.abc import Generator-from functools import wrapsfrom http import HTTPStatusfrom pathlib import Path-from stat import S_ISREG-from typing import (- TYPE_CHECKING,- Any,- ClassVar,- Concatenate,- ParamSpec,- Protocol,- Self,- TypeGuard,- TypeVar,- cast,- overload,-)+from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, ParamSpec, Protocol, Self, TypeVar, cast, overload+import yarlfrom aiohttp import ClientConnectorError, TooManyRedirectsfrom mega.errors import MegaNzErrorfrom pydantic import ValidationError-from yarl import URL+from typing_extensions import TypeIs-from cyberdrop_dl import constants-from cyberdrop_dl.data_structures import AbsoluteHttpURL+from cyberdrop_dl.constants import TempExtfrom cyberdrop_dl.exceptions import (CDLBaseError,ErrorLogMessage,@@ -44,10 +29,9 @@ from cyberdrop_dl.exceptions import (create_error_msg,get_origin,)-from cyberdrop_dl.utils import jsonif TYPE_CHECKING:- from collections.abc import Callable, Coroutine, Generator, Iterable+ from collections.abc import Callable, Coroutine, Generatorfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItemfrom cyberdrop_dl.downloader.downloader import Downloader@@ -57,10 +41,9 @@ if TYPE_CHECKING:manager: Manager_HasManagerT = TypeVar("_HasManagerT", bound=_HasManager)- _Origin = TypeVar("_Origin", bound=ScrapeItem | MediaItem | URL)+ _Origin = TypeVar("_Origin", bound=ScrapeItem | MediaItem | yarl.URL)_P = ParamSpec("_P")-_T = TypeVar("_T")_R = TypeVar("_R")@@ -93,14 +76,11 @@ class DictDataclass(Dataclass, Protocol):return cls(**cls.filter_dict(data))-_BLOB_OR_SVG = ("data:", "blob:", "javascript:")--@contextlib.contextmanager-def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | URL) -> Generator[None]:- link: URL = item if isinstance(item, URL) else item.url+def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yarl.URL) -> Generator[None]:+ link: yarl.URL = item if isinstance(item, yarl.URL) else item.urlerror_log_msg = origin = exc_info = None- link_to_show: URL | str = ""+ link_to_show: yarl.URL | str = ""is_segment: bool = getattr(item, "is_segment", False)is_downloader: bool = bool(getattr(self, "log_prefix", False))try:@@ -110,13 +90,16 @@ def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | URLexcept CDLBaseError as e:error_log_msg = ErrorLogMessage(e.ui_failure, str(e))origin = e.origin- link_to_show: URL | str = getattr(e, "url", None) or link_to_show+ link_to_show = getattr(e, "url", None) or link_to_showexcept NotImplementedError as e:error_log_msg = ErrorLogMessage("NotImplemented")exc_info = eexcept TooManyRedirects as e:ui_failure = "Too Many Redirects"- info = json.dumps({"url": e.request_info.real_url, "history": [r.real_url for r in e.history]}, indent=4)+ info = {+ "url": str(e.request_info.real_url),+ "history": tuple(str(r.real_url) for r in e.history),+ }error_log_msg = ErrorLogMessage(ui_failure, f"{ui_failure}\n{info}")except MegaNzError as e:if code := getattr(e, "code", None):@@ -184,14 +167,14 @@ def error_handling_wrapper(if inspect.iscoroutinefunction(func):- @wraps(func)+ @functools.wraps(func)async def async_wrapper(self: _HasManagerT, item: _Origin, *args: _P.args, **kwargs: _P.kwargs) -> _R:with error_handling_context(self, item):return await func(self, item, *args, **kwargs)return async_wrapper- @wraps(func)+ @functools.wraps(func)def wrapper(self: _HasManagerT, item: _Origin, *args: _P.args, **kwargs: _P.kwargs) -> _R:with error_handling_context(self, item):result = func(self, item, *args, **kwargs)@@ -208,17 +191,14 @@ def get_download_path(manager: Manager, scrape_item: ScrapeItem, domain: str) ->return download_dir / scrape_item.create_download_path(domain)-"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""---def get_size(path: os.DirEntry[str]) -> int | None:+def _get_size(path: os.DirEntry[str]) -> int | None:try:return path.stat(follow_symlinks=False).st_sizeexcept (OSError, ValueError):return-def purge_dir_tree(dirname: Path | str) -> bool:+def delete_empty_files_and_folders(dirname: Path | str) -> bool:"""walks and removes in place"""has_non_empty_files = False@@ -231,15 +211,17 @@ def purge_dir_tree(dirname: Path | str) -> bool:except OSError:is_dir = Falseif is_dir:- deleted = purge_dir_tree(entry.path)+ deleted = delete_empty_files_and_folders(entry.path)if not deleted:has_non_empty_subfolders = True- elif get_size(entry) == 0:+ elif _get_size(entry) == 0:+ logger.debug(f"Deleting '{entry.path}'")os.unlink(entry) # noqa: PTH108else:has_non_empty_files = True- except (OSError, PermissionError):+ except OSError:+ logger.exception(f"Unknown errot while walking '{dirname}'")passif has_non_empty_files or has_non_empty_subfolders:@@ -252,14 +234,25 @@ def purge_dir_tree(dirname: Path | str) -> bool:def check_partials_and_empty_folders(manager: Manager) -> None:- """Checks for partial downloads, deletes partial files and empty folders."""+ download_folder = manager.config.files.download_folder++ _check_for_partial_files(download_folder)settings = manager.config_manager.settings_data.runtime_optionsif settings.delete_partial_files:- delete_partial_files(manager)- if not settings.skip_check_for_partial_files:- check_for_partial_files(manager)- if not settings.skip_check_for_empty_folders:- delete_empty_folders(manager)+ logger.info("Deleting partial downloads...")+ for file in _partial_files(download_folder):+ logger.debug(f"Deleting '{file}'")+ file.unlink(missing_ok=True)++ if settings.skip_check_for_empty_folders:+ return++ logger.info("Deleting empty files and folders...")+ _ = delete_empty_files_and_folders(download_folder)++ sorted_folder = manager.config.sorting.sort_folder+ if sorted_folder and manager.config_manager.settings_data.sorting.sort_downloads:+ _ = delete_empty_files_and_folders(sorted_folder)def _partial_files(dir: Path | str) -> Generator[Path]:@@ -273,37 +266,19 @@ def _partial_files(dir: Path | str) -> Generator[Path]:passsuffix = entry.name.rpartition(".")[-1]- if f".{suffix}" in constants.TempExt:+ if f".{suffix}" in TempExt:yield Path(entry.path)except OSError:return-def delete_partial_files(manager: Manager) -> None:- """Deletes partial download files recursively."""- logger.info("Deleting partial downloads...")- for file in _partial_files(manager.config.files.download_folder):- file.unlink(missing_ok=True)---def check_for_partial_files(manager: Manager) -> None:- """Checks if there are partial downloads in any subdirectory and logs if found."""+def _check_for_partial_files(path: Path) -> None:logger.info("Checking for partial downloads...")- has_partial_files = next(_partial_files(manager.config.files.download_folder), None)+ has_partial_files = next(_partial_files(path), None)if has_partial_files:logger.warning("There are partial downloads in the downloads folder")-def delete_empty_folders(manager: Manager) -> None:- """Deletes empty folders efficiently."""- logger.info("Checking for empty folders...")- purge_dir_tree(manager.config.files.download_folder)-- sorted_folder = manager.config.sorting.sort_folder- if sorted_folder and manager.config_manager.settings_data.sorting.sort_downloads:- purge_dir_tree(sorted_folder)--def get_text_between(original_text: str, start: str, end: str) -> str:"""Extracts the text between two strings in a larger text. Result will be stripped"""start_index = original_text.index(start) + len(start)@@ -311,7 +286,10 @@ def get_text_between(original_text: str, start: str, end: str) -> str:return original_text[start_index:end_index].strip()-def _str_to_url(link_str: str) -> URL:+def _str_to_url(link_str: str) -> yarl.URL:+ if not link_str:+ raise InvalidURLError("link_str is empty", url=link_str)+def fix_query_params_encoding(link: str) -> str:if "?" not in link:return link@@ -322,19 +300,16 @@ def _str_to_url(link_str: str) -> URL:def fix_multiple_slashes(link_str: str) -> str:return re.sub(r"(?:https?)?:?(\/{3,})", "//", link_str)- if not link_str:- raise InvalidURLError("link_str is empty", url=link_str)-try:clean_link_str = fix_multiple_slashes(fix_query_params_encoding(link_str))- return URL(clean_link_str, encoded="%" in clean_link_str)+ return yarl.URL(clean_link_str, encoded="%" in clean_link_str)except (AttributeError, ValueError, TypeError) as e:raise InvalidURLError(str(e), url=link_str) from edef parse_url(- link_str: AbsoluteHttpURL | URL | str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True+ link_str: AbsoluteHttpURL | yarl.URL | str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True) -> AbsoluteHttpURL:"""Parse a string into an absolute URL, handling relative URLs, encoding and optionally removes trailing slash (trimming).Raises:@@ -355,7 +330,7 @@ def parse_url(return remove_trailing_slash(url)-def is_absolute_http_url(url: URL) -> TypeGuard[AbsoluteHttpURL]:+def is_absolute_http_url(url: yarl.URL) -> TypeIs[AbsoluteHttpURL]:return url.absolute and url.scheme.startswith("http")@@ -374,19 +349,6 @@ def remove_parts(return url.with_path("/".join(new_parts), keep_fragment=keep_fragment, keep_query=keep_query)-def get_size_or_none(path: Path) -> int | None:- """Checks if this is a file and returns its size with a single system call.-- Returns `None` otherwise"""-- try:- stat = path.stat()- if S_ISREG(stat.st_mode):- return stat.st_size- except (OSError, ValueError):- return None--@functools.cachedef get_system_information() -> dict[str, Any]:def get_common_name() -> str:@@ -421,22 +383,7 @@ def get_system_information() -> dict[str, Any]:def is_blob_or_svg(link: str) -> bool:- return any(link.startswith(x) for x in _BLOB_OR_SVG)---def unique(iterable: Iterable[_T], *, hashable: bool = True) -> Iterable[_T]:- """Yields unique values from iterable, keeping original order"""- if hashable:- seen: set[_T] | list[_T] = set()- add: Callable[[_T], None] = seen.add- else:- seen = []- add = seen.append-- for value in iterable:- if value not in seen:- add(value)- yield value+ return link.startswith(("data:", "blob:", "javascript:"))def xor_decrypt(encrypted_data: bytes, key: bytes) -> str: