Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add new helper base methods
NTFSvolumea08ac7efbac5
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 9e8640b58..68d3059cc 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -26,7 +26,16 @@ from cyberdrop_dl.downloader.http import Downloaderfrom cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeErrorfrom cyberdrop_dl.mediaprops import ISO639Subtitle, Resolutionfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem-from cyberdrop_dl.utils import css, dates, error_handling_context, is_absolute_http_url, is_blob_or_svg, m3u8, parse_url+from cyberdrop_dl.utils import (+ css,+ dates,+ error_handling_context,+ is_absolute_http_url,+ is_blob_or_svg,+ m3u8,+ parse_url,+ unique,+)from cyberdrop_dl.utils.filepath import (check_dangerous_filename,check_path_traversal,@@ -756,6 +765,23 @@ class Crawler(HTTPMixin, HLSMixin, ABC):yield thumb, link+ @final+ @classmethod+ def iter_urls(+ cls, tag: Tag, selector: str, attribute: str = "href", origin: AbsoluteHttpURL | None = None+ ) -> Generator[AbsoluteHttpURL]:+ for url in unique(css.iselect(tag, selector, attribute)):+ yield cls.parse_url(url, origin)++ @final+ async def make_album_checker(self, album_id: str) -> Callable[[AbsoluteHttpURL], bool]:+ results = await self.get_album_results(album_id)++ def should_download(url: AbsoluteHttpURL) -> bool:+ return not self.check_album_results(url, results)++ return should_download+@finaldef iter_children(self,@@ -976,7 +1002,7 @@ class API(HTTPMixin, ABC):def __post_init__(self) -> None: ...def __repr__(self) -> str:- return f"<{type(self).__name__}(PRIMARY_URL={self.PRIMARY_URL!r})>"+ return f"<{type(self).__name__}({self.PRIMARY_URL!r})>"@final@propertydiff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.pyindex b489590a2..339bbe77f 100644--- a/cyberdrop_dl/utils/__init__.py+++ b/cyberdrop_dl/utils/__init__.py@@ -26,7 +26,7 @@ from cyberdrop_dl.utils._dataclasses import type_adapter as type_adapterfrom cyberdrop_dl.utils._path_traverse import has_partial_files, partial_filesif TYPE_CHECKING:- from collections.abc import Callable, Coroutine, Generator+ from collections.abc import Callable, Coroutine, Generator, Iterablefrom pathlib import Pathfrom cyberdrop_dl.downloader.http import Downloader@@ -369,3 +369,11 @@ def truncated_preview(content: str, max_len: int = 100) -> str:def basic_auth(username: str, password: str) -> str:token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")return f"Basic {token}"+++def unique(itr: Iterable[_T], /) -> Generator[_T]:+ seen: set[_T] = set()+ for ele in itr:+ if ele not in seen:+ seen.add(ele)+ yield ele