Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add onepace support (#153)
NTFSvolume9fe7eb39ffb7
diff --git a/.gitignore b/.gitignoreindex 3e4bf1d6b..6a55d8b00 100644--- a/.gitignore+++ b/.gitignore@@ -9,6 +9,8 @@*.csv.coverage.config.yaml+debug_run.test+URLs.txt/build# Python cachediff --git a/cyberdrop_dl/crawlers/onepace.py b/cyberdrop_dl/crawlers/onepace.pynew file mode 100644index 000000000..e8d073530--- /dev/null+++ b/cyberdrop_dl/crawlers/onepace.py@@ -0,0 +1,96 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Any, ClassVar++from pydantic import dataclasses++from cyberdrop_dl import env+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import next_js+from cyberdrop_dl.utils.utilities import DictDataclass, error_handling_wrapper++if TYPE_CHECKING:+ from bs4 import BeautifulSoup++ from cyberdrop_dl.data_structures.url_objects import ScrapeItem++[email protected](slots=True)+class PlayGroup:+ sub: str | None+ dub: str+ playlists: list[Playlist]++ @property+ def score(self) -> tuple[int, int]:+ langs = ("ja", "en") if env.ONEPACE_PREFER_DUB else ("en", "ja")+ return langs.index(self.dub), (None, "en").index(self.sub)++[email protected](slots=True)+class Playlist:+ id: str+ resolution: int+ url: AbsoluteHttpURL++[email protected](slots=True)+class Episode(DictDataclass):+ title: str+ slug: str+ playGroups: list[PlayGroup] # noqa: N815+++_PD_BASE = AbsoluteHttpURL("https://pixeldrain.com/l/")+++class OnePaceCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"All episodes": "/watch"}+ DOMAIN: ClassVar[str] = "onepace.net"+ FOLDER_DOMAIN: ClassVar[str] = "OnePace"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://onepace.net")++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case [*_, "watch"]:+ return await self.all_episodes(scrape_item)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def all_episodes(self, scrape_item: ScrapeItem) -> None:+ soup = await self.request_soup(self.PRIMARY_URL / "en/watch")+ episodes = _extract_episodes(soup)++ await self.write_metadata(scrape_item, "one_pace_episodes", {"episodes": episodes})+ scrape_item.setup_as_profile(self.FOLDER_DOMAIN)+ for episode in episodes:+ self._episode(scrape_item.copy(), Episode.from_dict(episode))+ scrape_item.add_children()++ def _episode(self, scrape_item: ScrapeItem, ep: Episode) -> None:+ scrape_item.url = scrape_item.url.with_fragment(ep.slug)+ best_group = max(ep.playGroups, key=lambda x: x.score)+ self.log.info(+ "Downloading %s with subs=%s and lang=%s",+ scrape_item.url,+ best_group.sub,+ best_group.dub,+ )+ best_playlist = max(best_group.playlists, key=lambda x: x.resolution)+ self.handle_embed(scrape_item.create_child(best_playlist.url))+++def _extract_episodes(soup: BeautifulSoup) -> tuple[dict[str, Any], ...]:+ episodes: tuple[dict[str, Any], ...] = tuple(next_js.ifind(next_js.extract(soup), "slug", "title", "playGroups"))++ for ep in episodes:+ ep["backdrops"] = [+ {name: v for name, v in backdrop.items() if not name.startswith("blur")} for backdrop in ep["backdrops"]+ ]+ for group in ep["playGroups"]:+ for playlist in group["playlists"]:+ playlist["url"] = _PD_BASE / playlist["id"]++ return episodesdiff --git a/cyberdrop_dl/env.py b/cyberdrop_dl/env.pyindex ee54532bb..6d5b118fd 100644--- a/cyberdrop_dl/env.py+++ b/cyberdrop_dl/env.py@@ -21,3 +21,4 @@ PIXELDRAIN_PROXY = os.getenv("CDL_PIXELDRAIN_PROXY")BANDCAMP_FORMATS = os.getenv("CDL_BANDCAMP_FORMATS")EPORNER_PREFER_H264 = os.getenv("CDL_EPORNER_PREFER_H264")ENABLE_TWITTER = bool(os.getenv("CDL_ENABLE_TWITTER"))+ONEPACE_PREFER_DUB = bool(os.getenv("CDL_ONEPACE_PREFER_DUB"))