Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add onepace support (#153)

NTFSvolume9fe7eb39ffb7
diff --git a/.gitignore b/.gitignore
index 3e4bf1d6b..6a55d8b00 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,8 @@
*.csv
.coverage
.config.yaml
+debug_run.test
+URLs.txt
/build
# Python cache
diff --git a/cyberdrop_dl/crawlers/onepace.py b/cyberdrop_dl/crawlers/onepace.py
new file mode 100644
index 000000000..e8d073530
--- /dev/null
+++ b/cyberdrop_dl/crawlers/onepace.py
@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from pydantic import dataclasses
+
+from cyberdrop_dl import env
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import next_js
+from cyberdrop_dl.utils.utilities import DictDataclass, error_handling_wrapper
+
+if TYPE_CHECKING:
+ from bs4 import BeautifulSoup
+
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+
[email protected](slots=True)
+class PlayGroup:
+ sub: str | None
+ dub: str
+ playlists: list[Playlist]
+
+ @property
+ def score(self) -> tuple[int, int]:
+ langs = ("ja", "en") if env.ONEPACE_PREFER_DUB else ("en", "ja")
+ return langs.index(self.dub), (None, "en").index(self.sub)
+
+
[email protected](slots=True)
+class Playlist:
+ id: str
+ resolution: int
+ url: AbsoluteHttpURL
+
+
[email protected](slots=True)
+class Episode(DictDataclass):
+ title: str
+ slug: str
+ playGroups: list[PlayGroup] # noqa: N815
+
+
+_PD_BASE = AbsoluteHttpURL("https://pixeldrain.com/l/")
+
+
+class OnePaceCrawler(Crawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"All episodes": "/watch"}
+ DOMAIN: ClassVar[str] = "onepace.net"
+ FOLDER_DOMAIN: ClassVar[str] = "OnePace"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://onepace.net")
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case [*_, "watch"]:
+ return await self.all_episodes(scrape_item)
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def all_episodes(self, scrape_item: ScrapeItem) -> None:
+ soup = await self.request_soup(self.PRIMARY_URL / "en/watch")
+ episodes = _extract_episodes(soup)
+
+ await self.write_metadata(scrape_item, "one_pace_episodes", {"episodes": episodes})
+ scrape_item.setup_as_profile(self.FOLDER_DOMAIN)
+ for episode in episodes:
+ self._episode(scrape_item.copy(), Episode.from_dict(episode))
+ scrape_item.add_children()
+
+ def _episode(self, scrape_item: ScrapeItem, ep: Episode) -> None:
+ scrape_item.url = scrape_item.url.with_fragment(ep.slug)
+ best_group = max(ep.playGroups, key=lambda x: x.score)
+ self.log.info(
+ "Downloading %s with subs=%s and lang=%s",
+ scrape_item.url,
+ best_group.sub,
+ best_group.dub,
+ )
+ best_playlist = max(best_group.playlists, key=lambda x: x.resolution)
+ self.handle_embed(scrape_item.create_child(best_playlist.url))
+
+
+def _extract_episodes(soup: BeautifulSoup) -> tuple[dict[str, Any], ...]:
+ episodes: tuple[dict[str, Any], ...] = tuple(next_js.ifind(next_js.extract(soup), "slug", "title", "playGroups"))
+
+ for ep in episodes:
+ ep["backdrops"] = [
+ {name: v for name, v in backdrop.items() if not name.startswith("blur")} for backdrop in ep["backdrops"]
+ ]
+ for group in ep["playGroups"]:
+ for playlist in group["playlists"]:
+ playlist["url"] = _PD_BASE / playlist["id"]
+
+ return episodes
diff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.py
index ee54532bb..6d5b118fd 100644
--- a/cyberdrop_dl/env.py
+++ b/cyberdrop_dl/env.py
@@ -21,3 +21,4 @@ PIXELDRAIN_PROXY = os.getenv("CDL_PIXELDRAIN_PROXY")
BANDCAMP_FORMATS = os.getenv("CDL_BANDCAMP_FORMATS")
EPORNER_PREFER_H264 = os.getenv("CDL_EPORNER_PREFER_H264")
ENABLE_TWITTER = bool(os.getenv("CDL_ENABLE_TWITTER"))
+ONEPACE_PREFER_DUB = bool(os.getenv("CDL_ONEPACE_PREFER_DUB"))