Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add clypit and onepace config
NTFSvolumef67dd71ecf7f
diff --git a/cyberdrop_dl/config/crawlers.py b/cyberdrop_dl/config/crawlers.pyindex 5b95c7aca..5dd38dee8 100644--- a/cyberdrop_dl/config/crawlers.py+++ b/cyberdrop_dl/config/crawlers.py@@ -34,6 +34,16 @@ class BandcampConfig(ConfigModel):"Format to choose for downloads (if available), ordered by preference"+class ClypitConfig(ConfigModel):+ prefer_mp3: bool = False+ """Download audios as .mp3 files even if WAV (high quality) versions are available"""+++class OnePaceConfig(ConfigModel):+ prefer_dub: bool = False+ """Download episodes with english audio tracks instead of japanase (if available)"""++class GenericCrawlers(ConfigModel):wordpress_media: FalsyAsTuple[HttpURL] = ()wordpress_html: FalsyAsTuple[HttpURL] = ()@@ -46,8 +56,9 @@ class Crawlers(ConfigGroup, name=None):"Name of crawlers to disable for the current run"bandcamp: BandcampConfig = Field(default_factory=BandcampConfig)+ clypit: ClypitConfig = Field(default_factory=ClypitConfig)generic: GenericCrawlers = Field(default_factory=GenericCrawlers)kemono: KemonoConfig = Field(default_factory=KemonoConfig)coomer: KemonoConfig = Field(default_factory=KemonoConfig)- nekohouse: KemonoConfig = Field(default_factory=KemonoConfig)+ one_pace: OnePaceConfig = Field(default_factory=OnePaceConfig)tiktok: TikTokConfig = Field(default_factory=TikTokConfig)diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yamlindex 976d3596b..837784972 100644--- a/cyberdrop_dl/config/default.yaml+++ b/cyberdrop_dl/config/default.yaml@@ -14,6 +14,18 @@ auth:api_key: nullcookies: nullcrawlers:+ bandcamp:+ formats:+ - mp3-320+ - mp3+ - aac-hi+ - wav+ - flac+ - vorbis+ - aiff+ - alas+ clypit:+ prefer_mp3: falsecoomer:ignore_ads: falseignore_post_content: true@@ -26,9 +38,8 @@ crawlers:kemono:ignore_ads: falseignore_post_content: true- nekohouse:- ignore_ads: false- ignore_post_content: true+ one_pace:+ prefer_dub: falsetiktok:original: falsedeep_scrape: falsediff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.jsonindex 3234575c5..f63b624ac 100644--- a/cyberdrop_dl/config/schema.json+++ b/cyberdrop_dl/config/schema.json@@ -66,6 +66,26 @@"title": "Authentication","type": "object"},+ "BandcampConfig": {+ "additionalProperties": false,+ "properties": {+ "formats": {+ "$ref": "#/$defs/RemoveDuplicates_tuple_Literal__mp3-320____mp3____aac-hi____wav____flac____vorbis____aiff____alas____________",+ "default": [+ "mp3-320",+ "mp3",+ "aac-hi",+ "wav",+ "flac",+ "vorbis",+ "aiff",+ "alas"+ ]+ }+ },+ "title": "BandcampConfig",+ "type": "object"+ },"ByteSizeSerilized": {"anyOf": [{@@ -82,6 +102,18 @@"format": "path","type": "string"},+ "ClypitConfig": {+ "additionalProperties": false,+ "properties": {+ "prefer_mp3": {+ "default": false,+ "title": "Prefer Mp3",+ "type": "boolean"+ }+ },+ "title": "ClypitConfig",+ "type": "object"+ },"Crawlers": {"additionalProperties": false,"properties": {@@ -89,6 +121,12 @@"$ref": "#/$defs/RemoveDuplicates_FalsyAsTuple_NonEmptyStr__","default": []},+ "bandcamp": {+ "$ref": "#/$defs/BandcampConfig"+ },+ "clypit": {+ "$ref": "#/$defs/ClypitConfig"+ },"generic": {"$ref": "#/$defs/GenericCrawlers"},@@ -98,8 +136,8 @@"coomer": {"$ref": "#/$defs/KemonoConfig"},- "nekohouse": {- "$ref": "#/$defs/KemonoConfig"+ "one_pace": {+ "$ref": "#/$defs/OnePaceConfig"},"tiktok": {"$ref": "#/$defs/TikTokConfig"@@ -718,9 +756,37 @@"title": "Notifications","type": "object"},+ "OnePaceConfig": {+ "additionalProperties": false,+ "properties": {+ "prefer_dub": {+ "default": false,+ "title": "Prefer Dub",+ "type": "boolean"+ }+ },+ "title": "OnePaceConfig",+ "type": "object"+ },"RemoveDuplicates_FalsyAsTuple_NonEmptyStr__": {"$ref": "#/$defs/FalsyAsTuple_NonEmptyStr_"},+ "RemoveDuplicates_tuple_Literal__mp3-320____mp3____aac-hi____wav____flac____vorbis____aiff____alas____________": {+ "items": {+ "enum": [+ "mp3-320",+ "mp3",+ "aac-hi",+ "wav",+ "flac",+ "vorbis",+ "aiff",+ "alas"+ ],+ "type": "string"+ },+ "type": "array"+ },"Sort": {"additionalProperties": false,"properties": {diff --git a/cyberdrop_dl/crawlers/clyp_it.py b/cyberdrop_dl/crawlers/clyp_it.pyindex 1516865ca..ce54bf543 100644--- a/cyberdrop_dl/crawlers/clyp_it.py+++ b/cyberdrop_dl/crawlers/clyp_it.py@@ -4,7 +4,6 @@ import dataclassesimport datetimefrom typing import TYPE_CHECKING, Any, ClassVar-from cyberdrop_dl import envfrom cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, auto_task_idfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import extr_text, parse_url@@ -90,7 +89,7 @@ class ClypItAPI(API):return next(a for a in audios if a.id == audio_id)async def wav(self, audio_id: str, token: str | None) -> AbsoluteHttpURL | None:- if env.CLYPIT_PREFER_MP3:+ if self.config.crawlers.clypit.prefer_mp3:return Noneurl = self.PRIMARY_URL / audio_idif token:diff --git a/cyberdrop_dl/crawlers/onepace.py b/cyberdrop_dl/crawlers/onepace.pyindex 04513f7c7..1aececa2a 100644--- a/cyberdrop_dl/crawlers/onepace.py+++ b/cyberdrop_dl/crawlers/onepace.py@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, ClassVarfrom pydantic import dataclasses-from cyberdrop_dl import envfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedPathsfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import next_js@@ -24,11 +23,6 @@ class PlayGroup:dub: strplaylists: list[Playlist]- @property- def score(self) -> tuple[int, int]:- langs = ("ja", "en") if env.ONEPACE_PREFER_DUB else ("en", "ja")- return langs.index(self.dub), (None, "en").index(self.sub)-@dataclasses.dataclass(slots=True)class Playlist:@@ -53,6 +47,9 @@ class OnePaceCrawler(Crawler):FOLDER_DOMAIN: ClassVar[str] = "OnePace"PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://onepace.net")+ def __post_init__(self) -> None:+ self._langs: tuple[str, str] = ("ja", "en") if self.config.crawlers.one_pace.prefer_dub else ("en", "ja")+async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:case [*_, "watch"]:@@ -75,7 +72,13 @@ class OnePaceCrawler(Crawler):@error_handling_wrapperdef _episode(self, scrape_item: ScrapeItem, ep: Episode) -> None:- best_group = max(ep.playlistGroups, key=lambda x: x.score)++ def score(group: PlayGroup) -> tuple[int, int]:+ dub_score = self._langs.index(group.dub)+ sub_score = (None, "en").index(group.sub)+ return dub_score, sub_score++ best_group = max(ep.playlistGroups, key=score)self.log.info("Downloading %s with subs=%s and lang=%s",scrape_item.url,diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.pyindex a4733d549..5906f9a82 100644--- a/cyberdrop_dl/env.py+++ b/cyberdrop_dl/env.py@@ -39,11 +39,8 @@ EDITOR = os.getenv("EDITOR")# CRAWLERS-BANDCAMP_FORMATS = _env("BANDCAMP_FORMATS")ENABLE_TWITTER = bool(_env("ENABLE_TWITTER"))GOFILE_SALT = _env("GOFILE_SALT")-ONEPACE_PREFER_DUB = bool(_env("ONEPACE_PREFER_DUB"))-CLYPIT_PREFER_MP3 = bool(_env("CLYPIT_PREFER_MP3"))ALL_VARS = dict(sorted(ALL_VARS.items())) # pyright: ignore[reportConstantRedefinition]ALL_VARS_RESOLVED = dict(