Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add bandcamp config options
NTFSvolumecc67a530cd5d
diff --git a/cyberdrop_dl/config/crawlers.py b/cyberdrop_dl/config/crawlers.pyindex 3a6734499..5b95c7aca 100644--- a/cyberdrop_dl/config/crawlers.py+++ b/cyberdrop_dl/config/crawlers.py@@ -1,3 +1,5 @@+from typing import Literal+from pydantic import Fieldfrom cyberdrop_dl.models import ConfigGroup, ConfigModel@@ -16,6 +18,22 @@ class TikTokConfig(ConfigModel):"Download videos in original quality (slower)"+class BandcampConfig(ConfigModel):+ formats: RemoveDuplicates[+ tuple[Literal["mp3-320", "mp3", "aac-hi", "wav", "flac", "vorbis", "aiff", "alas"], ...],+ ] = (+ "mp3-320",+ "mp3",+ "aac-hi",+ "wav",+ "flac",+ "vorbis",+ "aiff",+ "alas",+ )+ "Format to choose for downloads (if available), ordered by preference"++class GenericCrawlers(ConfigModel):wordpress_media: FalsyAsTuple[HttpURL] = ()wordpress_html: FalsyAsTuple[HttpURL] = ()@@ -27,6 +45,7 @@ class Crawlers(ConfigGroup, name=None):disabled: RemoveDuplicates[FalsyAsTuple[NonEmptyStr]] = ()"Name of crawlers to disable for the current run"+ bandcamp: BandcampConfig = Field(default_factory=BandcampConfig)generic: GenericCrawlers = Field(default_factory=GenericCrawlers)kemono: KemonoConfig = Field(default_factory=KemonoConfig)coomer: KemonoConfig = Field(default_factory=KemonoConfig)diff --git a/cyberdrop_dl/crawlers/bandcamp.py b/cyberdrop_dl/crawlers/bandcamp.pyindex 9bdd16924..58845a484 100644--- a/cyberdrop_dl/crawlers/bandcamp.py+++ b/cyberdrop_dl/crawlers/bandcamp.py@@ -6,7 +6,6 @@ import randomimport timefrom typing import TYPE_CHECKING, Any, ClassVar-from cyberdrop_dl import envfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_idfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -14,13 +13,10 @@ from cyberdrop_dl.utils import css, datesfrom cyberdrop_dl.utils.errors import error_handling_wrapperif TYPE_CHECKING:- from collections.abc import Generator+ from collections.abc import Callable, Generatorfrom cyberdrop_dl.url_objects import ScrapeItem-SUPPORTED_FORMATS = "mp3-320", "mp3", "aac-hi", "wav", "flac", "vorbis", "aiff", "alas" # Ordered by compatibility-USE_FORMATS = tuple(dict.fromkeys((env.BANDCAMP_FORMATS).split(","))) if env.BANDCAMP_FORMATS else SUPPORTED_FORMATS-@dataclasses.dataclass(slots=True, frozen=True)class Format:@@ -34,14 +30,15 @@ class BandcampCrawler(Crawler, db_path="path_qs_frag"):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Album": "/album/<slug>","Song": "/track/<slug>",- "**NOTE**": (- f"You can set 'CDL_BANDCAMP_FORMATS' env var to a comma separated list of formats to download (Ordered by preference)"- f" [Default = {','.join(SUPPORTED_FORMATS)!r}]"- ),}DOMAIN: ClassVar[str] = "bandcamp"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://bandcamp.com")+ def __post_init__(self) -> None:+ formats = self.config.crawlers.bandcamp.formats+ get_scores = _create_score_fn(formats)+ self._best_score: Callable[[str], int] = lambda x: max(get_scores(x))+async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case ["track", _, *_]:@@ -110,7 +107,7 @@ class BandcampCrawler(Crawler, db_path="path_qs_frag"):if not file_info:raise ScrapeError(402, "No streaming or download formats available (purchase only)")- return max(self._parse_formats(file_info), key=lambda x: _score(x.codec))+ return max(self._parse_formats(file_info), key=lambda x: self._best_score(x.codec))def _parse_formats(self, file_info: dict[str, str]) -> Generator[Format]:for name, format_url in file_info.items():@@ -126,7 +123,7 @@ class BandcampCrawler(Crawler, db_path="path_qs_frag"):blob = await self._get_page_info(free_download_url, "blob")downloads: dict[str, dict[str, str]] = blob["download_items"][0]["downloads"]- name = max(downloads, key=_score)+ name = max(downloads, key=self._best_score)download_url = downloads[name]["url"]ext_map: dict[str, str] = {fmt["name"]: fmt["file_extension"] for fmt in blob["download_formats"]}@@ -145,11 +142,11 @@ class BandcampCrawler(Crawler, db_path="path_qs_frag"):return json.loads(stat[stat.find("{") : stat.rfind("}") + 1])-def _score(name: str) -> int:- def scores() -> Generator[int]:- for idx, fmt in enumerate(reversed(USE_FORMATS)):+def _create_score_fn(formats: tuple[str, ...]) -> Callable[[str], Generator[int]]:+ def scores(name: str) -> Generator[int]:+ for idx, fmt in enumerate(reversed(formats)):if fmt in name.casefold():yield idxyield -1- return max(scores())+ return scoresdiff --git a/cyberdrop_dl/models/types.py b/cyberdrop_dl/models/types.pyindex 74e2c0362..606bb102d 100644--- a/cyberdrop_dl/models/types.py+++ b/cyberdrop_dl/models/types.py@@ -44,7 +44,7 @@ type Timedelta = Annotated[str, return_type=str, when_used="json"), # Serialize as str to save it as sexageximal (hh:mm:ss) instead of pydantic's ISO duration (PT1H5M26S)]-type RemoveDuplicates[T: tuple[str, ...]] = Annotated[T, AfterValidator(remove_duplicates)]+type RemoveDuplicates[T] = Annotated[T, AfterValidator(remove_duplicates)]# Only use this for config validation. To parse URLs internally while scraping, call `parse_url` directlydiff --git a/cyberdrop_dl/models/validators/__init__.py b/cyberdrop_dl/models/validators/__init__.pyindex ca3c2fd2e..bdeb186ad 100644--- a/cyberdrop_dl/models/validators/__init__.py+++ b/cyberdrop_dl/models/validators/__init__.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport datetimeimport re-from typing import TYPE_CHECKING, Literal, SupportsIndex, SupportsInt+from typing import TYPE_CHECKING, Literal, SupportsIndex, SupportsInt, overloadfrom pydantic import ByteSize, TypeAdapter@@ -110,5 +110,13 @@ def falsy_as_none[T](value: T | Literal[""] | None) -> T | None:return falsy_as(value, None)-def remove_duplicates[T: str](value: tuple[T, ...]) -> tuple[T, ...]:- return tuple(sorted(set(value)))+@overload+def remove_duplicates[T](values: list[T]) -> list[T]: ...+++@overload+def remove_duplicates[T](values: tuple[T, ...]) -> tuple[T, ...]: ...+++def remove_duplicates[T](values: list[T] | tuple[T, ...]) -> list[T] | tuple[T, ...]:+ return type(values)(dict.fromkeys(values))