Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add bandcamp config options

NTFSvolumecc67a530cd5d
diff --git a/cyberdrop_dl/config/crawlers.py b/cyberdrop_dl/config/crawlers.py
index 3a6734499..5b95c7aca 100644
--- a/cyberdrop_dl/config/crawlers.py
+++ b/cyberdrop_dl/config/crawlers.py
@@ -1,3 +1,5 @@
+from typing import Literal
+
from pydantic import Field
from cyberdrop_dl.models import ConfigGroup, ConfigModel
@@ -16,6 +18,22 @@ class TikTokConfig(ConfigModel):
"Download videos in original quality (slower)"
+class BandcampConfig(ConfigModel):
+ formats: RemoveDuplicates[
+ tuple[Literal["mp3-320", "mp3", "aac-hi", "wav", "flac", "vorbis", "aiff", "alas"], ...],
+ ] = (
+ "mp3-320",
+ "mp3",
+ "aac-hi",
+ "wav",
+ "flac",
+ "vorbis",
+ "aiff",
+ "alas",
+ )
+ "Format to choose for downloads (if available), ordered by preference"
+
+
class GenericCrawlers(ConfigModel):
wordpress_media: FalsyAsTuple[HttpURL] = ()
wordpress_html: FalsyAsTuple[HttpURL] = ()
@@ -27,6 +45,7 @@ class Crawlers(ConfigGroup, name=None):
disabled: RemoveDuplicates[FalsyAsTuple[NonEmptyStr]] = ()
"Name of crawlers to disable for the current run"
+ bandcamp: BandcampConfig = Field(default_factory=BandcampConfig)
generic: GenericCrawlers = Field(default_factory=GenericCrawlers)
kemono: KemonoConfig = Field(default_factory=KemonoConfig)
coomer: KemonoConfig = Field(default_factory=KemonoConfig)
diff --git a/cyberdrop_dl/crawlers/bandcamp.py b/cyberdrop_dl/crawlers/bandcamp.py
index 9bdd16924..58845a484 100644
--- a/cyberdrop_dl/crawlers/bandcamp.py
+++ b/cyberdrop_dl/crawlers/bandcamp.py
@@ -6,7 +6,6 @@ import random
import time
from typing import TYPE_CHECKING, Any, ClassVar
-from cyberdrop_dl import env
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths, auto_task_id
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -14,13 +13,10 @@ from cyberdrop_dl.utils import css, dates
from cyberdrop_dl.utils.errors import error_handling_wrapper
if TYPE_CHECKING:
- from collections.abc import Generator
+ from collections.abc import Callable, Generator
from cyberdrop_dl.url_objects import ScrapeItem
-SUPPORTED_FORMATS = "mp3-320", "mp3", "aac-hi", "wav", "flac", "vorbis", "aiff", "alas" # Ordered by compatibility
-USE_FORMATS = tuple(dict.fromkeys((env.BANDCAMP_FORMATS).split(","))) if env.BANDCAMP_FORMATS else SUPPORTED_FORMATS
-
@dataclasses.dataclass(slots=True, frozen=True)
class Format:
@@ -34,14 +30,15 @@ class BandcampCrawler(Crawler, db_path="path_qs_frag"):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Album": "/album/<slug>",
"Song": "/track/<slug>",
- "**NOTE**": (
- f"You can set 'CDL_BANDCAMP_FORMATS' env var to a comma separated list of formats to download (Ordered by preference)"
- f" [Default = {','.join(SUPPORTED_FORMATS)!r}]"
- ),
}
DOMAIN: ClassVar[str] = "bandcamp"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://bandcamp.com")
+ def __post_init__(self) -> None:
+ formats = self.config.crawlers.bandcamp.formats
+ get_scores = _create_score_fn(formats)
+ self._best_score: Callable[[str], int] = lambda x: max(get_scores(x))
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case ["track", _, *_]:
@@ -110,7 +107,7 @@ class BandcampCrawler(Crawler, db_path="path_qs_frag"):
if not file_info:
raise ScrapeError(402, "No streaming or download formats available (purchase only)")
- return max(self._parse_formats(file_info), key=lambda x: _score(x.codec))
+ return max(self._parse_formats(file_info), key=lambda x: self._best_score(x.codec))
def _parse_formats(self, file_info: dict[str, str]) -> Generator[Format]:
for name, format_url in file_info.items():
@@ -126,7 +123,7 @@ class BandcampCrawler(Crawler, db_path="path_qs_frag"):
blob = await self._get_page_info(free_download_url, "blob")
downloads: dict[str, dict[str, str]] = blob["download_items"][0]["downloads"]
- name = max(downloads, key=_score)
+ name = max(downloads, key=self._best_score)
download_url = downloads[name]["url"]
ext_map: dict[str, str] = {fmt["name"]: fmt["file_extension"] for fmt in blob["download_formats"]}
@@ -145,11 +142,11 @@ class BandcampCrawler(Crawler, db_path="path_qs_frag"):
return json.loads(stat[stat.find("{") : stat.rfind("}") + 1])
-def _score(name: str) -> int:
- def scores() -> Generator[int]:
- for idx, fmt in enumerate(reversed(USE_FORMATS)):
+def _create_score_fn(formats: tuple[str, ...]) -> Callable[[str], Generator[int]]:
+ def scores(name: str) -> Generator[int]:
+ for idx, fmt in enumerate(reversed(formats)):
if fmt in name.casefold():
yield idx
yield -1
- return max(scores())
+ return scores
diff --git a/cyberdrop_dl/models/types.py b/cyberdrop_dl/models/types.py
index 74e2c0362..606bb102d 100644
--- a/cyberdrop_dl/models/types.py
+++ b/cyberdrop_dl/models/types.py
@@ -44,7 +44,7 @@ type Timedelta = Annotated[
str, return_type=str, when_used="json"
), # Serialize as str to save it as sexageximal (hh:mm:ss) instead of pydantic's ISO duration (PT1H5M26S)
]
-type RemoveDuplicates[T: tuple[str, ...]] = Annotated[T, AfterValidator(remove_duplicates)]
+type RemoveDuplicates[T] = Annotated[T, AfterValidator(remove_duplicates)]
# Only use this for config validation. To parse URLs internally while scraping, call `parse_url` directly
diff --git a/cyberdrop_dl/models/validators/__init__.py b/cyberdrop_dl/models/validators/__init__.py
index ca3c2fd2e..bdeb186ad 100644
--- a/cyberdrop_dl/models/validators/__init__.py
+++ b/cyberdrop_dl/models/validators/__init__.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import datetime
import re
-from typing import TYPE_CHECKING, Literal, SupportsIndex, SupportsInt
+from typing import TYPE_CHECKING, Literal, SupportsIndex, SupportsInt, overload
from pydantic import ByteSize, TypeAdapter
@@ -110,5 +110,13 @@ def falsy_as_none[T](value: T | Literal[""] | None) -> T | None:
return falsy_as(value, None)
-def remove_duplicates[T: str](value: tuple[T, ...]) -> tuple[T, ...]:
- return tuple(sorted(set(value)))
+@overload
+def remove_duplicates[T](values: list[T]) -> list[T]: ...
+
+
+@overload
+def remove_duplicates[T](values: tuple[T, ...]) -> tuple[T, ...]: ...
+
+
+def remove_duplicates[T](values: list[T] | tuple[T, ...]) -> list[T] | tuple[T, ...]:
+ return type(values)(dict.fromkeys(values))