Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add series support (Pluto.tv) (#2069)
NTFSvolumebc8cf3c9bc7c
diff --git a/cyberdrop_dl/crawlers/pluto_tv.py b/cyberdrop_dl/crawlers/pluto_tv.pyindex 6b9c8c487..c5e2c2406 100644--- a/cyberdrop_dl/crawlers/pluto_tv.py+++ b/cyberdrop_dl/crawlers/pluto_tv.py@@ -3,68 +3,99 @@ from __future__ import annotationsimport dataclassesimport uuidfrom contextvars import ContextVar-from typing import TYPE_CHECKING, Any, ClassVar, override+from typing import TYPE_CHECKING, Any, ClassVar, TypedDictimport yarlfrom cyberdrop_dl.cache import cached_method-from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, compose_ep_name+from cyberdrop_dl.clients.http import HTTPConfig+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, auto_task_id, compose_ep_namefrom cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem-from cyberdrop_dl.utils import css, next_js+from cyberdrop_dl.utils import css, extr_text, next_jsfrom cyberdrop_dl.utils.dataclass import Deserializerfrom cyberdrop_dl.utils.errors import error_handling_wrappersession_token: ContextVar[str] = ContextVar("session_token")if TYPE_CHECKING:+ from collections.abc import Generator+from cyberdrop_dl.utils.m3u8 import Rendition+FIREFOX = "Mozilla/5.0 (X11; Linux x86_64; rv:151.0) Gecko/20100101 Firefox/151.0"+[email protected]_headers(user_agent=FIREFOX)class PlutoCrawler(Crawler):SUPPORTED_PATHS: ClassVar[SupportedPaths] = {- "Episode": (".../shows/<series_id>/episode/<episode_id>",),+ "Episode": "<region>/shows/<show_id>/episode/<episode_id>",+ "Show": (+ "<region>/shows/<show_slug>",+ "<region>/shows/<show_slug>/season/<season>",+ ),}PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pluto.tv")DOMAIN: ClassVar[str] = "pluto.tv"- @staticmethod- @override- def __db_path__(url: AbsoluteHttpURL, /) -> str:- _region, sep, rest = url.path.partition("/shows/")- assert sep- return sep + rest-def __post_init__(self) -> None:self.api: PlutoAPI = PlutoAPI.from_crawler(self)async def fetch(self, scrape_item: ScrapeItem) -> None:match scrape_item.url.parts[1:]:- case [*_, "shows", series_id, "episode", episode_id]:- await self.episode(scrape_item, series_id, episode_id)- case [*_, "shows", series_id, "season", _season]:- raise ValueError-+ case [*_, "shows", show_id, "episode", episode_id]:+ await self.episode(scrape_item, show_id, episode_id)+ case [*_, "shows", show_id]:+ await self.show(scrape_item, show_id)+ case [*_, "shows", show_id, "season", season]:+ await self.show(scrape_item, show_id, int(season))case _:raise ValueError+ @error_handling_wrapper+ async def show(self, scrape_item: ScrapeItem, show_slug: str, season: int | None = None) -> None:+ scrape_item.setup_as_album("")++ async with self.request(scrape_item.url) as resp:+ text = await resp.text()+ scrape_item.url = resp.url++ series_id = extr_text(text, "/ptvm/series/", "/")+ series = await self.api.series(series_id)+ downloaded = await self.get_album_results(series.id)+ scrape_item.setup_as_album(self.create_title(series.title, series.id), album_id=series.id)++ base_url = self.PRIMARY_URL.with_path(scrape_item.url.path.partition(show_slug)[0]) / show_slug+ for ep in series.episodes():+ if season is not None and ep.season != season:+ continue++ url = base_url / "episode" / ep.id+ if self.check_album_results(url, downloaded):+ continue++ new_item = scrape_item.create_child(url)+ self.create_task(self._episode_task(new_item, ep))+ scrape_item.add_children()+@error_handling_wrapperasync def episode(self, scrape_item: ScrapeItem, series_id: str, episode_id: str) -> None:if await self.check_complete(scrape_item.url):return- soup = await self.request_soup(scrape_item.url)- data = next_js.data(soup)- ep = data["props"]["pageProps"]["episodeMetadata"]+ async with self.request(scrape_item.url) as resp:+ soup = await resp.soup()+ scrape_item.url = resp.url++ props = next_js.data(soup)["props"]["pageProps"]+ ep = props["episodeMetadata"]episode = _deserialize(Episode, ep, id=episode_id)scrape_item.setup_as_album(self.create_title(ep["seriesTitle"], series_id), album_id=series_id)await self._episode(scrape_item, episode)- @error_handling_wrapperasync def _episode(self, scrape_item: ScrapeItem, ep: Episode) -> None:+ scrape_item.uploaded_at = self.parse_iso_date(ep.airDateISO)m3u8_url = await self.api.stream(ep.id)- m3u8, info = await self.request_m3u8_playlist(- m3u8_url, headers={"User-Agent": self.api.FIREFOX}, keep_query=True- )+ m3u8, info = await self.request_m3u8_playlist(m3u8_url, keep_query=True)_remove_ads_segments(m3u8)filename = self.create_custom_filename(compose_ep_name(ep.season, ep.number, ep.title),@@ -85,11 +116,13 @@ class PlutoCrawler(Crawler):metadata=ep,)+ _episode_task = auto_task_id(error_handling_wrapper(_episode))+[email protected]_headers(user_agent=FIREFOX)class PlutoAPI(API):- BOOT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://boot.pluto.tv/v4/start")- FIREFOX: ClassVar[str] = "Mozilla/5.0 (X11; Linux x86_64; rv:151.0) Gecko/20100101 Firefox/151.0"GRAPHQL_ENDPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pluto.tv/api/tn/app-shell/graphql/")+ SERIES: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://service-vod.clusters.pluto.tv/v4/vod/series/")M3U8: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://cfd-v4-service-channel-stitcher-use1-1.prd.pluto.tv/v2/stitch/hls/episode")@@ -106,6 +139,7 @@ class PlutoAPI(API):soup = await self.request_soup(self.PRIMARY_URL)return css.select(soup, "meta[name=appVersion], meta[name=app_version]", "content")+ @cached_method(ttl=1800)async def start(self) -> Session:resp = await self.request_gql("PtvStart",@@ -128,7 +162,6 @@ class PlutoAPI(API):resp = await self.request_json(self.GRAPHQL_ENDPOINT,method="POST",- headers={"User-Agent": self.FIREFOX},json={"query": globals()[operation],"variables": variables,@@ -140,7 +173,6 @@ class PlutoAPI(API):async def stream(self, episode_id: str) -> AbsoluteHttpURL:session = await self.start()url = self.M3U8 / episode_id / "master.m3u8"- sid = str(uuid.uuid1())return url.with_query({"advertisingId": "",@@ -154,14 +186,20 @@ class PlutoAPI(API):"deviceType": "web","deviceVersion": "151.0","serverSideAds": "false",- "sessionID": sid,- "sid": sid,+ "sessionID": session.id,+ "sid": session.id,"userId": "","jwt": session.jwt,"includeExtendedEvents": "true",})+ async def series(self, series_id: str) -> Series:+ session = await self.start()+ url = (self.SERIES / series_id / "seasons").with_query(offset=0)+ resp = await self.request_json(url, headers={"Authorization": f"Bearer {session.jwt}"})+ return _deserialize(Series, resp)+@dataclasses.dataclass(slots=True, frozen=True)class Session:@@ -170,7 +208,7 @@ class Session:_deserialize = Deserializer(- {"season": "seasonNum", "number": "episodeNum"},+ {"season": "seasonNum", "number": "episodeNum", "title": "name", "id": "_id"},{"season": int, "number": int},)@@ -182,6 +220,25 @@ class Episode:season: intnumber: inttitle: str+ airDateISO: str # noqa: N815+++class Season(TypedDict):+ number: int+ episodes: list[dict[str, Any]]++[email protected](slots=True)+class Series:+ id: str+ title: str+ slug: str+ seasons: list[Season] = dataclasses.field(default_factory=list)++ def episodes(self) -> Generator[Episode]:+ for season in self.seasons:+ for ep in season["episodes"]:+ yield _deserialize(Episode, ep)def _remove_ads_segments(rendition: Rendition) -> None:diff --git a/cyberdrop_dl/downloader/hls.py b/cyberdrop_dl/downloader/hls.pyindex c3173a4ba..0673188f7 100644--- a/cyberdrop_dl/downloader/hls.py+++ b/cyberdrop_dl/downloader/hls.py@@ -7,12 +7,11 @@ from contextvars import ContextVarfrom http import HTTPStatusfrom typing import TYPE_CHECKING, NamedTuple, Protocol-from Crypto.Cipher import AES-from cyberdrop_dl import aio, constants, ffmpegfrom cyberdrop_dl.exceptions import DownloadErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItemfrom cyberdrop_dl.utils import parse_url+from cyberdrop_dl.utils.crypto import aes_cbc_decrypt, aes_unpadfrom cyberdrop_dl.utils.m3u8 import HLSKeyif TYPE_CHECKING:@@ -75,7 +74,7 @@ class AESHLSDecrypter(HLSDecrypter):async def __call__(self, content: bytes, key: HLSKey, headers: dict[str, str] | None = None) -> bytes:aes_key = await self.get_key(key.uri, headers=headers)- return AES.new(aes_key, AES.MODE_CBC, key.iv).encrypt(content)+ return aes_unpad(aes_cbc_decrypt(content, aes_key, key.iv))def _parse_segments(segments: Sequence[Segment | InitializationSection]) -> Generator[HLSSegment]:diff --git a/cyberdrop_dl/utils/crypto.py b/cyberdrop_dl/utils/crypto.pyindex 50a94202b..4108781cc 100644--- a/cyberdrop_dl/utils/crypto.py+++ b/cyberdrop_dl/utils/crypto.py@@ -1,6 +1,7 @@from __future__ import annotationsfrom Crypto.Cipher import AES+from Crypto.Util.Padding import unpadEMPTY_IV = b"\0" * AES.block_size@@ -11,3 +12,7 @@ def aes_cbc_encrypt(data: bytes, key: bytes, iv: bytes = EMPTY_IV) -> bytes:def aes_cbc_decrypt(data: bytes, key: bytes, iv: bytes = EMPTY_IV) -> bytes:return AES.new(key, AES.MODE_CBC, iv).decrypt(data)+++def aes_unpad(data: bytes) -> bytes:+ return unpad(data, AES.block_size)diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.pyindex 0a6ab12f1..b7898e072 100644--- a/cyberdrop_dl/utils/m3u8.py+++ b/cyberdrop_dl/utils/m3u8.py@@ -85,7 +85,7 @@ class Rendition:subtitle: M3U8 | Nonedef __iter__(self) -> Iterator[M3U8 | None]:- return iter(dataclasses.astuple(self))+ yield from (self.video, self.audio, self.subtitle)@dataclasses.dataclass(frozen=True, slots=True, order=True)diff --git a/tests/crawlers/test_cases/pluto_tv.py b/tests/crawlers/test_cases/pluto_tv.pynew file mode 100644index 000000000..4ff2aed07--- /dev/null+++ b/tests/crawlers/test_cases/pluto_tv.py@@ -0,0 +1,19 @@+DOMAIN = "pluto.tv"+TEST_CASES = [+ {+ "url": "https://pluto.tv/shows/3020012/episode/611bc94773019a0013eba3ed",+ "results": [+ {+ "url": "re:https://pluto.tv/.+/shows/3020012/episode/611bc94773019a0013eba3ed",+ "referer": "re:https://pluto.tv/.+/shows/3020012/episode/611bc94773019a0013eba3ed",+ "download_folder": "re:Drift (Pluto.tv)",+ "filename": "S01E001 - Episodio 1 [611bc94773019a0013eba3ed][avc1].mp4",+ "original_filename": "Episodio 1",+ "download_filename": "S01E001 - Episodio 1 [611bc94773019a0013eba3ed][avc1].mp4",+ "debrid_url": None,+ "album_id": "3020012",+ "uploaded_at": int,+ },+ ],+ }+]