Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add clypit and onepace config

NTFSvolumef67dd71ecf7f
diff --git a/cyberdrop_dl/config/crawlers.py b/cyberdrop_dl/config/crawlers.py
index 5b95c7aca..5dd38dee8 100644
--- a/cyberdrop_dl/config/crawlers.py
+++ b/cyberdrop_dl/config/crawlers.py
@@ -34,6 +34,16 @@ class BandcampConfig(ConfigModel):
"Format to choose for downloads (if available), ordered by preference"
+class ClypitConfig(ConfigModel):
+ prefer_mp3: bool = False
+ """Download audios as .mp3 files even if WAV (high quality) versions are available"""
+
+
+class OnePaceConfig(ConfigModel):
+ prefer_dub: bool = False
+ """Download episodes with english audio tracks instead of japanase (if available)"""
+
+
class GenericCrawlers(ConfigModel):
wordpress_media: FalsyAsTuple[HttpURL] = ()
wordpress_html: FalsyAsTuple[HttpURL] = ()
@@ -46,8 +56,9 @@ class Crawlers(ConfigGroup, name=None):
"Name of crawlers to disable for the current run"
bandcamp: BandcampConfig = Field(default_factory=BandcampConfig)
+ clypit: ClypitConfig = Field(default_factory=ClypitConfig)
generic: GenericCrawlers = Field(default_factory=GenericCrawlers)
kemono: KemonoConfig = Field(default_factory=KemonoConfig)
coomer: KemonoConfig = Field(default_factory=KemonoConfig)
- nekohouse: KemonoConfig = Field(default_factory=KemonoConfig)
+ one_pace: OnePaceConfig = Field(default_factory=OnePaceConfig)
tiktok: TikTokConfig = Field(default_factory=TikTokConfig)
diff --git a/cyberdrop_dl/config/default.yaml b/cyberdrop_dl/config/default.yaml
index 976d3596b..837784972 100644
--- a/cyberdrop_dl/config/default.yaml
+++ b/cyberdrop_dl/config/default.yaml
@@ -14,6 +14,18 @@ auth:
api_key: null
cookies: null
crawlers:
+ bandcamp:
+ formats:
+ - mp3-320
+ - mp3
+ - aac-hi
+ - wav
+ - flac
+ - vorbis
+ - aiff
+ - alas
+ clypit:
+ prefer_mp3: false
coomer:
ignore_ads: false
ignore_post_content: true
@@ -26,9 +38,8 @@ crawlers:
kemono:
ignore_ads: false
ignore_post_content: true
- nekohouse:
- ignore_ads: false
- ignore_post_content: true
+ one_pace:
+ prefer_dub: false
tiktok:
original: false
deep_scrape: false
diff --git a/cyberdrop_dl/config/schema.json b/cyberdrop_dl/config/schema.json
index 3234575c5..f63b624ac 100644
--- a/cyberdrop_dl/config/schema.json
+++ b/cyberdrop_dl/config/schema.json
@@ -66,6 +66,26 @@
"title": "Authentication",
"type": "object"
},
+ "BandcampConfig": {
+ "additionalProperties": false,
+ "properties": {
+ "formats": {
+ "$ref": "#/$defs/RemoveDuplicates_tuple_Literal__mp3-320____mp3____aac-hi____wav____flac____vorbis____aiff____alas____________",
+ "default": [
+ "mp3-320",
+ "mp3",
+ "aac-hi",
+ "wav",
+ "flac",
+ "vorbis",
+ "aiff",
+ "alas"
+ ]
+ }
+ },
+ "title": "BandcampConfig",
+ "type": "object"
+ },
"ByteSizeSerilized": {
"anyOf": [
{
@@ -82,6 +102,18 @@
"format": "path",
"type": "string"
},
+ "ClypitConfig": {
+ "additionalProperties": false,
+ "properties": {
+ "prefer_mp3": {
+ "default": false,
+ "title": "Prefer Mp3",
+ "type": "boolean"
+ }
+ },
+ "title": "ClypitConfig",
+ "type": "object"
+ },
"Crawlers": {
"additionalProperties": false,
"properties": {
@@ -89,6 +121,12 @@
"$ref": "#/$defs/RemoveDuplicates_FalsyAsTuple_NonEmptyStr__",
"default": []
},
+ "bandcamp": {
+ "$ref": "#/$defs/BandcampConfig"
+ },
+ "clypit": {
+ "$ref": "#/$defs/ClypitConfig"
+ },
"generic": {
"$ref": "#/$defs/GenericCrawlers"
},
@@ -98,8 +136,8 @@
"coomer": {
"$ref": "#/$defs/KemonoConfig"
},
- "nekohouse": {
- "$ref": "#/$defs/KemonoConfig"
+ "one_pace": {
+ "$ref": "#/$defs/OnePaceConfig"
},
"tiktok": {
"$ref": "#/$defs/TikTokConfig"
@@ -718,9 +756,37 @@
"title": "Notifications",
"type": "object"
},
+ "OnePaceConfig": {
+ "additionalProperties": false,
+ "properties": {
+ "prefer_dub": {
+ "default": false,
+ "title": "Prefer Dub",
+ "type": "boolean"
+ }
+ },
+ "title": "OnePaceConfig",
+ "type": "object"
+ },
"RemoveDuplicates_FalsyAsTuple_NonEmptyStr__": {
"$ref": "#/$defs/FalsyAsTuple_NonEmptyStr_"
},
+ "RemoveDuplicates_tuple_Literal__mp3-320____mp3____aac-hi____wav____flac____vorbis____aiff____alas____________": {
+ "items": {
+ "enum": [
+ "mp3-320",
+ "mp3",
+ "aac-hi",
+ "wav",
+ "flac",
+ "vorbis",
+ "aiff",
+ "alas"
+ ],
+ "type": "string"
+ },
+ "type": "array"
+ },
"Sort": {
"additionalProperties": false,
"properties": {
diff --git a/cyberdrop_dl/crawlers/clyp_it.py b/cyberdrop_dl/crawlers/clyp_it.py
index 1516865ca..ce54bf543 100644
--- a/cyberdrop_dl/crawlers/clyp_it.py
+++ b/cyberdrop_dl/crawlers/clyp_it.py
@@ -4,7 +4,6 @@ import dataclasses
import datetime
from typing import TYPE_CHECKING, Any, ClassVar
-from cyberdrop_dl import env
from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, auto_task_id
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import extr_text, parse_url
@@ -90,7 +89,7 @@ class ClypItAPI(API):
return next(a for a in audios if a.id == audio_id)
async def wav(self, audio_id: str, token: str | None) -> AbsoluteHttpURL | None:
- if env.CLYPIT_PREFER_MP3:
+ if self.config.crawlers.clypit.prefer_mp3:
return None
url = self.PRIMARY_URL / audio_id
if token:
diff --git a/cyberdrop_dl/crawlers/onepace.py b/cyberdrop_dl/crawlers/onepace.py
index 04513f7c7..1aececa2a 100644
--- a/cyberdrop_dl/crawlers/onepace.py
+++ b/cyberdrop_dl/crawlers/onepace.py
@@ -4,7 +4,6 @@ from typing import TYPE_CHECKING, Any, ClassVar
from pydantic import dataclasses
-from cyberdrop_dl import env
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import next_js
@@ -24,11 +23,6 @@ class PlayGroup:
dub: str
playlists: list[Playlist]
- @property
- def score(self) -> tuple[int, int]:
- langs = ("ja", "en") if env.ONEPACE_PREFER_DUB else ("en", "ja")
- return langs.index(self.dub), (None, "en").index(self.sub)
-
@dataclasses.dataclass(slots=True)
class Playlist:
@@ -53,6 +47,9 @@ class OnePaceCrawler(Crawler):
FOLDER_DOMAIN: ClassVar[str] = "OnePace"
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://onepace.net")
+ def __post_init__(self) -> None:
+ self._langs: tuple[str, str] = ("ja", "en") if self.config.crawlers.one_pace.prefer_dub else ("en", "ja")
+
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
case [*_, "watch"]:
@@ -75,7 +72,13 @@ class OnePaceCrawler(Crawler):
@error_handling_wrapper
def _episode(self, scrape_item: ScrapeItem, ep: Episode) -> None:
- best_group = max(ep.playlistGroups, key=lambda x: x.score)
+
+ def score(group: PlayGroup) -> tuple[int, int]:
+ dub_score = self._langs.index(group.dub)
+ sub_score = (None, "en").index(group.sub)
+ return dub_score, sub_score
+
+ best_group = max(ep.playlistGroups, key=score)
self.log.info(
"Downloading %s with subs=%s and lang=%s",
scrape_item.url,
diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.py
index a4733d549..5906f9a82 100644
--- a/cyberdrop_dl/env.py
+++ b/cyberdrop_dl/env.py
@@ -39,11 +39,8 @@ EDITOR = os.getenv("EDITOR")
# CRAWLERS
-BANDCAMP_FORMATS = _env("BANDCAMP_FORMATS")
ENABLE_TWITTER = bool(_env("ENABLE_TWITTER"))
GOFILE_SALT = _env("GOFILE_SALT")
-ONEPACE_PREFER_DUB = bool(_env("ONEPACE_PREFER_DUB"))
-CLYPIT_PREFER_MP3 = bool(_env("CLYPIT_PREFER_MP3"))
ALL_VARS = dict(sorted(ALL_VARS.items())) # pyright: ignore[reportConstantRedefinition]
ALL_VARS_RESOLVED = dict(