Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove redundant utils (#109)

NTFSvolume29e1aa679ae3
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.py
index 8649361c6..df46926c2 100644
--- a/cyberdrop_dl/clients/download_client.py
+++ b/cyberdrop_dl/clients/download_client.py
@@ -17,7 +17,6 @@ from cyberdrop_dl.constants import FileExt
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import DDOSGuardError, DownloadError, InvalidContentTypeError, SlowDownloadError
from cyberdrop_dl.utils import dates
-from cyberdrop_dl.utils.utilities import get_size_or_none
if TYPE_CHECKING:
from collections.abc import AsyncGenerator, Callable, Coroutine, Generator, Mapping
@@ -104,11 +103,7 @@ class DownloadClient:
media_item.partial_file = download_dir / f"{downloaded_filename}{constants.TempExt.PART}"
resume_point = 0
- if (
- self._supports_ranges
- and media_item.partial_file
- and (size := await asyncio.to_thread(get_size_or_none, media_item.partial_file))
- ):
+ if self._supports_ranges and media_item.partial_file and (size := await aio.get_size(media_item.partial_file)):
resume_point = size
download_headers["Range"] = f"bytes={size}-"
diff --git a/cyberdrop_dl/clients/hash_client.py b/cyberdrop_dl/clients/hash_client.py
index 21aad844e..6b47a0375 100644
--- a/cyberdrop_dl/clients/hash_client.py
+++ b/cyberdrop_dl/clients/hash_client.py
@@ -8,9 +8,8 @@ from typing import TYPE_CHECKING, Literal
from send2trash import send2trash
-from cyberdrop_dl import constants
+from cyberdrop_dl import aio, constants
from cyberdrop_dl.constants import Hashing
-from cyberdrop_dl.utils.utilities import get_size_or_none
if TYPE_CHECKING:
from yarl import URL
@@ -94,7 +93,7 @@ class HashClient:
if file.suffix in constants.TempExt:
return
- if not await asyncio.to_thread(get_size_or_none, file):
+ if not await aio.get_size(file):
return
hash = await self._update_db_and_retrive_hash_helper(file, original_filename, referer, hash_type=self.xxhash)
@@ -143,7 +142,7 @@ class HashClient:
if not hash:
return
absolute_path = await asyncio.to_thread(media_item.path.resolve)
- size = await asyncio.to_thread(get_size_or_none, media_item.path)
+ size = await aio.get_size(media_item.path)
assert size
self.hashed_media_items.append(media_item)
if hash:
diff --git a/cyberdrop_dl/config/config_model.py b/cyberdrop_dl/config/config_model.py
index a03ffeedf..3480bb786 100755
--- a/cyberdrop_dl/config/config_model.py
+++ b/cyberdrop_dl/config/config_model.py
@@ -21,7 +21,7 @@ from cyberdrop_dl.models.types import (
)
from cyberdrop_dl.models.validators import falsy_as, to_timedelta
from cyberdrop_dl.utils.strings import validate_format_string
-from cyberdrop_dl.utils.utilities import purge_dir_tree
+from cyberdrop_dl.utils.utilities import delete_empty_files_and_folders
from ._common import ConfigModel
@@ -123,7 +123,7 @@ class Logs(AliasModel):
if (now - datetime.fromtimestamp(file.stat().st_ctime)) > self.logs_expire_after:
file.unlink()
- purge_dir_tree(self.log_folder)
+ delete_empty_files_and_folders(self.log_folder)
class FileSizeLimits(BaseModel):
diff --git a/cyberdrop_dl/crawlers/discourse/_discourse.py b/cyberdrop_dl/crawlers/discourse/_discourse.py
index 3fb957ffd..29859fd01 100644
--- a/cyberdrop_dl/crawlers/discourse/_discourse.py
+++ b/cyberdrop_dl/crawlers/discourse/_discourse.py
@@ -15,7 +15,7 @@ from cyberdrop_dl.crawlers._forum import MessageBoardCrawler
from cyberdrop_dl.exceptions import MaxChildrenError
from cyberdrop_dl.utils import css
from cyberdrop_dl.utils.dates import to_timestamp
-from cyberdrop_dl.utils.utilities import error_handling_wrapper, unique
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
from .models import AvailablePost, PostStream, Topic
@@ -138,14 +138,15 @@ class DiscourseCrawler(MessageBoardCrawler, is_generic=True):
images = css.iselect(soup, *css.images)
links = css.iselect(soup, *css.links)
external_links = (ref.url for ref in post.link_counts)
- for link_str in unique(itertools.chain(external_links, images, links)):
+
+ for link_str in dict.fromkeys(itertools.chain(external_links, images, links)):
try:
if link_str:
yield self.parse_url(link_str)
except Exception:
continue
- return unique(iter_links())
+ return iter_links()
@classmethod
def parse_url(
diff --git a/cyberdrop_dl/crawlers/wordpress/_wordpress.py b/cyberdrop_dl/crawlers/wordpress/_wordpress.py
index 21bee138a..61a01d851 100644
--- a/cyberdrop_dl/crawlers/wordpress/_wordpress.py
+++ b/cyberdrop_dl/crawlers/wordpress/_wordpress.py
@@ -21,7 +21,7 @@ from cyberdrop_dl.data_structures.url_objects import QueryDatetimeRange
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.utils import css, open_graph
from cyberdrop_dl.utils.dates import to_timestamp
-from cyberdrop_dl.utils.utilities import error_handling_wrapper, unique
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
from .models import HTML, Category, CategorySequence, ColletionType, Post, PostSequence, Tag, TagSequence
@@ -183,7 +183,7 @@ class WordPressBaseCrawler(Crawler, is_abc=True):
@final
def iter_parse_url(self, iterable: Iterable[str]) -> Iterable[AbsoluteHttpURL]:
- for link_str in unique(iterable):
+ for link_str in dict.fromkeys(iterable):
try:
yield self.parse_url(link_str)
except Exception:
diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py
index e8642cf94..663fe2e78 100644
--- a/cyberdrop_dl/utils/sorting.py
+++ b/cyberdrop_dl/utils/sorting.py
@@ -16,7 +16,7 @@ from cyberdrop_dl import ffmpeg
from cyberdrop_dl.constants import FileExt, TempExt
from cyberdrop_dl.progress.sorting import SortingUI
from cyberdrop_dl.utils import strings
-from cyberdrop_dl.utils.utilities import purge_dir_tree as delete_empty_files_and_folders
+from cyberdrop_dl.utils.utilities import delete_empty_files_and_folders
if TYPE_CHECKING:
from cyberdrop_dl.managers.manager import Manager
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index 77f4c7e4e..e3dd61dd0 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -10,32 +10,17 @@ import os
import platform
import re
import sys
-from collections.abc import Generator
-from functools import wraps
from http import HTTPStatus
from pathlib import Path
-from stat import S_ISREG
-from typing import (
- TYPE_CHECKING,
- Any,
- ClassVar,
- Concatenate,
- ParamSpec,
- Protocol,
- Self,
- TypeGuard,
- TypeVar,
- cast,
- overload,
-)
+from typing import TYPE_CHECKING, Any, ClassVar, Concatenate, ParamSpec, Protocol, Self, TypeVar, cast, overload
+import yarl
from aiohttp import ClientConnectorError, TooManyRedirects
from mega.errors import MegaNzError
from pydantic import ValidationError
-from yarl import URL
+from typing_extensions import TypeIs
-from cyberdrop_dl import constants
-from cyberdrop_dl.data_structures import AbsoluteHttpURL
+from cyberdrop_dl.constants import TempExt
from cyberdrop_dl.exceptions import (
CDLBaseError,
ErrorLogMessage,
@@ -44,10 +29,9 @@ from cyberdrop_dl.exceptions import (
create_error_msg,
get_origin,
)
-from cyberdrop_dl.utils import json
if TYPE_CHECKING:
- from collections.abc import Callable, Coroutine, Generator, Iterable
+ from collections.abc import Callable, Coroutine, Generator
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
from cyberdrop_dl.downloader.downloader import Downloader
@@ -57,10 +41,9 @@ if TYPE_CHECKING:
manager: Manager
_HasManagerT = TypeVar("_HasManagerT", bound=_HasManager)
- _Origin = TypeVar("_Origin", bound=ScrapeItem | MediaItem | URL)
+ _Origin = TypeVar("_Origin", bound=ScrapeItem | MediaItem | yarl.URL)
_P = ParamSpec("_P")
-_T = TypeVar("_T")
_R = TypeVar("_R")
@@ -93,14 +76,11 @@ class DictDataclass(Dataclass, Protocol):
return cls(**cls.filter_dict(data))
-_BLOB_OR_SVG = ("data:", "blob:", "javascript:")
-
-
@contextlib.contextmanager
-def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | URL) -> Generator[None]:
- link: URL = item if isinstance(item, URL) else item.url
+def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | yarl.URL) -> Generator[None]:
+ link: yarl.URL = item if isinstance(item, yarl.URL) else item.url
error_log_msg = origin = exc_info = None
- link_to_show: URL | str = ""
+ link_to_show: yarl.URL | str = ""
is_segment: bool = getattr(item, "is_segment", False)
is_downloader: bool = bool(getattr(self, "log_prefix", False))
try:
@@ -110,13 +90,16 @@ def error_handling_context(self: _HasManager, item: ScrapeItem | MediaItem | URL
except CDLBaseError as e:
error_log_msg = ErrorLogMessage(e.ui_failure, str(e))
origin = e.origin
- link_to_show: URL | str = getattr(e, "url", None) or link_to_show
+ link_to_show = getattr(e, "url", None) or link_to_show
except NotImplementedError as e:
error_log_msg = ErrorLogMessage("NotImplemented")
exc_info = e
except TooManyRedirects as e:
ui_failure = "Too Many Redirects"
- info = json.dumps({"url": e.request_info.real_url, "history": [r.real_url for r in e.history]}, indent=4)
+ info = {
+ "url": str(e.request_info.real_url),
+ "history": tuple(str(r.real_url) for r in e.history),
+ }
error_log_msg = ErrorLogMessage(ui_failure, f"{ui_failure}\n{info}")
except MegaNzError as e:
if code := getattr(e, "code", None):
@@ -184,14 +167,14 @@ def error_handling_wrapper(
if inspect.iscoroutinefunction(func):
- @wraps(func)
+ @functools.wraps(func)
async def async_wrapper(self: _HasManagerT, item: _Origin, *args: _P.args, **kwargs: _P.kwargs) -> _R:
with error_handling_context(self, item):
return await func(self, item, *args, **kwargs)
return async_wrapper
- @wraps(func)
+ @functools.wraps(func)
def wrapper(self: _HasManagerT, item: _Origin, *args: _P.args, **kwargs: _P.kwargs) -> _R:
with error_handling_context(self, item):
result = func(self, item, *args, **kwargs)
@@ -208,17 +191,14 @@ def get_download_path(manager: Manager, scrape_item: ScrapeItem, domain: str) ->
return download_dir / scrape_item.create_download_path(domain)
-"""~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~"""
-
-
-def get_size(path: os.DirEntry[str]) -> int | None:
+def _get_size(path: os.DirEntry[str]) -> int | None:
try:
return path.stat(follow_symlinks=False).st_size
except (OSError, ValueError):
return
-def purge_dir_tree(dirname: Path | str) -> bool:
+def delete_empty_files_and_folders(dirname: Path | str) -> bool:
"""walks and removes in place"""
has_non_empty_files = False
@@ -231,15 +211,17 @@ def purge_dir_tree(dirname: Path | str) -> bool:
except OSError:
is_dir = False
if is_dir:
- deleted = purge_dir_tree(entry.path)
+ deleted = delete_empty_files_and_folders(entry.path)
if not deleted:
has_non_empty_subfolders = True
- elif get_size(entry) == 0:
+ elif _get_size(entry) == 0:
+ logger.debug(f"Deleting '{entry.path}'")
os.unlink(entry) # noqa: PTH108
else:
has_non_empty_files = True
- except (OSError, PermissionError):
+ except OSError:
+ logger.exception(f"Unknown errot while walking '{dirname}'")
pass
if has_non_empty_files or has_non_empty_subfolders:
@@ -252,14 +234,25 @@ def purge_dir_tree(dirname: Path | str) -> bool:
def check_partials_and_empty_folders(manager: Manager) -> None:
- """Checks for partial downloads, deletes partial files and empty folders."""
+ download_folder = manager.config.files.download_folder
+
+ _check_for_partial_files(download_folder)
settings = manager.config_manager.settings_data.runtime_options
if settings.delete_partial_files:
- delete_partial_files(manager)
- if not settings.skip_check_for_partial_files:
- check_for_partial_files(manager)
- if not settings.skip_check_for_empty_folders:
- delete_empty_folders(manager)
+ logger.info("Deleting partial downloads...")
+ for file in _partial_files(download_folder):
+ logger.debug(f"Deleting '{file}'")
+ file.unlink(missing_ok=True)
+
+ if settings.skip_check_for_empty_folders:
+ return
+
+ logger.info("Deleting empty files and folders...")
+ _ = delete_empty_files_and_folders(download_folder)
+
+ sorted_folder = manager.config.sorting.sort_folder
+ if sorted_folder and manager.config_manager.settings_data.sorting.sort_downloads:
+ _ = delete_empty_files_and_folders(sorted_folder)
def _partial_files(dir: Path | str) -> Generator[Path]:
@@ -273,37 +266,19 @@ def _partial_files(dir: Path | str) -> Generator[Path]:
pass
suffix = entry.name.rpartition(".")[-1]
- if f".{suffix}" in constants.TempExt:
+ if f".{suffix}" in TempExt:
yield Path(entry.path)
except OSError:
return
-def delete_partial_files(manager: Manager) -> None:
- """Deletes partial download files recursively."""
- logger.info("Deleting partial downloads...")
- for file in _partial_files(manager.config.files.download_folder):
- file.unlink(missing_ok=True)
-
-
-def check_for_partial_files(manager: Manager) -> None:
- """Checks if there are partial downloads in any subdirectory and logs if found."""
+def _check_for_partial_files(path: Path) -> None:
logger.info("Checking for partial downloads...")
- has_partial_files = next(_partial_files(manager.config.files.download_folder), None)
+ has_partial_files = next(_partial_files(path), None)
if has_partial_files:
logger.warning("There are partial downloads in the downloads folder")
-def delete_empty_folders(manager: Manager) -> None:
- """Deletes empty folders efficiently."""
- logger.info("Checking for empty folders...")
- purge_dir_tree(manager.config.files.download_folder)
-
- sorted_folder = manager.config.sorting.sort_folder
- if sorted_folder and manager.config_manager.settings_data.sorting.sort_downloads:
- purge_dir_tree(sorted_folder)
-
-
def get_text_between(original_text: str, start: str, end: str) -> str:
"""Extracts the text between two strings in a larger text. Result will be stripped"""
start_index = original_text.index(start) + len(start)
@@ -311,7 +286,10 @@ def get_text_between(original_text: str, start: str, end: str) -> str:
return original_text[start_index:end_index].strip()
-def _str_to_url(link_str: str) -> URL:
+def _str_to_url(link_str: str) -> yarl.URL:
+ if not link_str:
+ raise InvalidURLError("link_str is empty", url=link_str)
+
def fix_query_params_encoding(link: str) -> str:
if "?" not in link:
return link
@@ -322,19 +300,16 @@ def _str_to_url(link_str: str) -> URL:
def fix_multiple_slashes(link_str: str) -> str:
return re.sub(r"(?:https?)?:?(\/{3,})", "//", link_str)
- if not link_str:
- raise InvalidURLError("link_str is empty", url=link_str)
-
try:
clean_link_str = fix_multiple_slashes(fix_query_params_encoding(link_str))
- return URL(clean_link_str, encoded="%" in clean_link_str)
+ return yarl.URL(clean_link_str, encoded="%" in clean_link_str)
except (AttributeError, ValueError, TypeError) as e:
raise InvalidURLError(str(e), url=link_str) from e
def parse_url(
- link_str: AbsoluteHttpURL | URL | str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True
+ link_str: AbsoluteHttpURL | yarl.URL | str, relative_to: AbsoluteHttpURL | None = None, *, trim: bool = True
) -> AbsoluteHttpURL:
"""Parse a string into an absolute URL, handling relative URLs, encoding and optionally removes trailing slash (trimming).
Raises:
@@ -355,7 +330,7 @@ def parse_url(
return remove_trailing_slash(url)
-def is_absolute_http_url(url: URL) -> TypeGuard[AbsoluteHttpURL]:
+def is_absolute_http_url(url: yarl.URL) -> TypeIs[AbsoluteHttpURL]:
return url.absolute and url.scheme.startswith("http")
@@ -374,19 +349,6 @@ def remove_parts(
return url.with_path("/".join(new_parts), keep_fragment=keep_fragment, keep_query=keep_query)
-def get_size_or_none(path: Path) -> int | None:
- """Checks if this is a file and returns its size with a single system call.
-
- Returns `None` otherwise"""
-
- try:
- stat = path.stat()
- if S_ISREG(stat.st_mode):
- return stat.st_size
- except (OSError, ValueError):
- return None
-
-
@functools.cache
def get_system_information() -> dict[str, Any]:
def get_common_name() -> str:
@@ -421,22 +383,7 @@ def get_system_information() -> dict[str, Any]:
def is_blob_or_svg(link: str) -> bool:
- return any(link.startswith(x) for x in _BLOB_OR_SVG)
-
-
-def unique(iterable: Iterable[_T], *, hashable: bool = True) -> Iterable[_T]:
- """Yields unique values from iterable, keeping original order"""
- if hashable:
- seen: set[_T] | list[_T] = set()
- add: Callable[[_T], None] = seen.add
- else:
- seen = []
- add = seen.append
-
- for value in iterable:
- if value not in seen:
- add(value)
- yield value
+ return link.startswith(("data:", "blob:", "javascript:"))
def xor_decrypt(encrypted_data: bytes, key: bytes) -> str: