Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add new helper base methods

NTFSvolumea08ac7efbac5
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 9e8640b58..68d3059cc 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -26,7 +26,16 @@ from cyberdrop_dl.downloader.http import Downloader
from cyberdrop_dl.exceptions import MaxChildrenError, NoExtensionError, ScrapeError
from cyberdrop_dl.mediaprops import ISO639Subtitle, Resolution
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
-from cyberdrop_dl.utils import css, dates, error_handling_context, is_absolute_http_url, is_blob_or_svg, m3u8, parse_url
+from cyberdrop_dl.utils import (
+ css,
+ dates,
+ error_handling_context,
+ is_absolute_http_url,
+ is_blob_or_svg,
+ m3u8,
+ parse_url,
+ unique,
+)
from cyberdrop_dl.utils.filepath import (
check_dangerous_filename,
check_path_traversal,
@@ -756,6 +765,23 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
yield thumb, link
+ @final
+ @classmethod
+ def iter_urls(
+ cls, tag: Tag, selector: str, attribute: str = "href", origin: AbsoluteHttpURL | None = None
+ ) -> Generator[AbsoluteHttpURL]:
+ for url in unique(css.iselect(tag, selector, attribute)):
+ yield cls.parse_url(url, origin)
+
+ @final
+ async def make_album_checker(self, album_id: str) -> Callable[[AbsoluteHttpURL], bool]:
+ results = await self.get_album_results(album_id)
+
+ def should_download(url: AbsoluteHttpURL) -> bool:
+ return not self.check_album_results(url, results)
+
+ return should_download
+
@final
def iter_children(
self,
@@ -976,7 +1002,7 @@ class API(HTTPMixin, ABC):
def __post_init__(self) -> None: ...
def __repr__(self) -> str:
- return f"<{type(self).__name__}(PRIMARY_URL={self.PRIMARY_URL!r})>"
+ return f"<{type(self).__name__}({self.PRIMARY_URL!r})>"
@final
@property
diff --git a/cyberdrop_dl/utils/__init__.py b/cyberdrop_dl/utils/__init__.py
index b489590a2..339bbe77f 100644
--- a/cyberdrop_dl/utils/__init__.py
+++ b/cyberdrop_dl/utils/__init__.py
@@ -26,7 +26,7 @@ from cyberdrop_dl.utils._dataclasses import type_adapter as type_adapter
from cyberdrop_dl.utils._path_traverse import has_partial_files, partial_files
if TYPE_CHECKING:
- from collections.abc import Callable, Coroutine, Generator
+ from collections.abc import Callable, Coroutine, Generator, Iterable
from pathlib import Path
from cyberdrop_dl.downloader.http import Downloader
@@ -369,3 +369,11 @@ def truncated_preview(content: str, max_len: int = 100) -> str:
def basic_auth(username: str, password: str) -> str:
token = base64.b64encode(f"{username}:{password}".encode()).decode("ascii")
return f"Basic {token}"
+
+
+def unique(itr: Iterable[_T], /) -> Generator[_T]:
+ seen: set[_T] = set()
+ for ele in itr:
+ if ele not in seen:
+ seen.add(ele)
+ yield ele