Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add series support (Pluto.tv) (#2069)

NTFSvolumebc8cf3c9bc7c
diff --git a/cyberdrop_dl/crawlers/pluto_tv.py b/cyberdrop_dl/crawlers/pluto_tv.py
index 6b9c8c487..c5e2c2406 100644
--- a/cyberdrop_dl/crawlers/pluto_tv.py
+++ b/cyberdrop_dl/crawlers/pluto_tv.py
@@ -3,68 +3,99 @@ from __future__ import annotations
import dataclasses
import uuid
from contextvars import ContextVar
-from typing import TYPE_CHECKING, Any, ClassVar, override
+from typing import TYPE_CHECKING, Any, ClassVar, TypedDict
import yarl
from cyberdrop_dl.cache import cached_method
-from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, compose_ep_name
+from cyberdrop_dl.clients.http import HTTPConfig
+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, auto_task_id, compose_ep_name
from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
-from cyberdrop_dl.utils import css, next_js
+from cyberdrop_dl.utils import css, extr_text, next_js
from cyberdrop_dl.utils.dataclass import Deserializer
from cyberdrop_dl.utils.errors import error_handling_wrapper
session_token: ContextVar[str] = ContextVar("session_token")
if TYPE_CHECKING:
+ from collections.abc import Generator
+
from cyberdrop_dl.utils.m3u8 import Rendition
+FIREFOX = "Mozilla/5.0 (X11; Linux x86_64; rv:151.0) Gecko/20100101 Firefox/151.0"
+
[email protected]_headers(user_agent=FIREFOX)
class PlutoCrawler(Crawler):
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
- "Episode": (".../shows/<series_id>/episode/<episode_id>",),
+ "Episode": "<region>/shows/<show_id>/episode/<episode_id>",
+ "Show": (
+ "<region>/shows/<show_slug>",
+ "<region>/shows/<show_slug>/season/<season>",
+ ),
}
PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pluto.tv")
DOMAIN: ClassVar[str] = "pluto.tv"
- @staticmethod
- @override
- def __db_path__(url: AbsoluteHttpURL, /) -> str:
- _region, sep, rest = url.path.partition("/shows/")
- assert sep
- return sep + rest
-
def __post_init__(self) -> None:
self.api: PlutoAPI = PlutoAPI.from_crawler(self)
async def fetch(self, scrape_item: ScrapeItem) -> None:
match scrape_item.url.parts[1:]:
- case [*_, "shows", series_id, "episode", episode_id]:
- await self.episode(scrape_item, series_id, episode_id)
- case [*_, "shows", series_id, "season", _season]:
- raise ValueError
-
+ case [*_, "shows", show_id, "episode", episode_id]:
+ await self.episode(scrape_item, show_id, episode_id)
+ case [*_, "shows", show_id]:
+ await self.show(scrape_item, show_id)
+ case [*_, "shows", show_id, "season", season]:
+ await self.show(scrape_item, show_id, int(season))
case _:
raise ValueError
+ @error_handling_wrapper
+ async def show(self, scrape_item: ScrapeItem, show_slug: str, season: int | None = None) -> None:
+ scrape_item.setup_as_album("")
+
+ async with self.request(scrape_item.url) as resp:
+ text = await resp.text()
+ scrape_item.url = resp.url
+
+ series_id = extr_text(text, "/ptvm/series/", "/")
+ series = await self.api.series(series_id)
+ downloaded = await self.get_album_results(series.id)
+ scrape_item.setup_as_album(self.create_title(series.title, series.id), album_id=series.id)
+
+ base_url = self.PRIMARY_URL.with_path(scrape_item.url.path.partition(show_slug)[0]) / show_slug
+ for ep in series.episodes():
+ if season is not None and ep.season != season:
+ continue
+
+ url = base_url / "episode" / ep.id
+ if self.check_album_results(url, downloaded):
+ continue
+
+ new_item = scrape_item.create_child(url)
+ self.create_task(self._episode_task(new_item, ep))
+ scrape_item.add_children()
+
@error_handling_wrapper
async def episode(self, scrape_item: ScrapeItem, series_id: str, episode_id: str) -> None:
if await self.check_complete(scrape_item.url):
return
- soup = await self.request_soup(scrape_item.url)
- data = next_js.data(soup)
- ep = data["props"]["pageProps"]["episodeMetadata"]
+ async with self.request(scrape_item.url) as resp:
+ soup = await resp.soup()
+ scrape_item.url = resp.url
+
+ props = next_js.data(soup)["props"]["pageProps"]
+ ep = props["episodeMetadata"]
episode = _deserialize(Episode, ep, id=episode_id)
scrape_item.setup_as_album(self.create_title(ep["seriesTitle"], series_id), album_id=series_id)
await self._episode(scrape_item, episode)
- @error_handling_wrapper
async def _episode(self, scrape_item: ScrapeItem, ep: Episode) -> None:
+ scrape_item.uploaded_at = self.parse_iso_date(ep.airDateISO)
m3u8_url = await self.api.stream(ep.id)
- m3u8, info = await self.request_m3u8_playlist(
- m3u8_url, headers={"User-Agent": self.api.FIREFOX}, keep_query=True
- )
+ m3u8, info = await self.request_m3u8_playlist(m3u8_url, keep_query=True)
_remove_ads_segments(m3u8)
filename = self.create_custom_filename(
compose_ep_name(ep.season, ep.number, ep.title),
@@ -85,11 +116,13 @@ class PlutoCrawler(Crawler):
metadata=ep,
)
+ _episode_task = auto_task_id(error_handling_wrapper(_episode))
+
[email protected]_headers(user_agent=FIREFOX)
class PlutoAPI(API):
- BOOT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://boot.pluto.tv/v4/start")
- FIREFOX: ClassVar[str] = "Mozilla/5.0 (X11; Linux x86_64; rv:151.0) Gecko/20100101 Firefox/151.0"
GRAPHQL_ENDPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pluto.tv/api/tn/app-shell/graphql/")
+ SERIES: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://service-vod.clusters.pluto.tv/v4/vod/series/")
M3U8: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL(
"https://cfd-v4-service-channel-stitcher-use1-1.prd.pluto.tv/v2/stitch/hls/episode"
)
@@ -106,6 +139,7 @@ class PlutoAPI(API):
soup = await self.request_soup(self.PRIMARY_URL)
return css.select(soup, "meta[name=appVersion], meta[name=app_version]", "content")
+ @cached_method(ttl=1800)
async def start(self) -> Session:
resp = await self.request_gql(
"PtvStart",
@@ -128,7 +162,6 @@ class PlutoAPI(API):
resp = await self.request_json(
self.GRAPHQL_ENDPOINT,
method="POST",
- headers={"User-Agent": self.FIREFOX},
json={
"query": globals()[operation],
"variables": variables,
@@ -140,7 +173,6 @@ class PlutoAPI(API):
async def stream(self, episode_id: str) -> AbsoluteHttpURL:
session = await self.start()
url = self.M3U8 / episode_id / "master.m3u8"
- sid = str(uuid.uuid1())
return url.with_query(
{
"advertisingId": "",
@@ -154,14 +186,20 @@ class PlutoAPI(API):
"deviceType": "web",
"deviceVersion": "151.0",
"serverSideAds": "false",
- "sessionID": sid,
- "sid": sid,
+ "sessionID": session.id,
+ "sid": session.id,
"userId": "",
"jwt": session.jwt,
"includeExtendedEvents": "true",
}
)
+ async def series(self, series_id: str) -> Series:
+ session = await self.start()
+ url = (self.SERIES / series_id / "seasons").with_query(offset=0)
+ resp = await self.request_json(url, headers={"Authorization": f"Bearer {session.jwt}"})
+ return _deserialize(Series, resp)
+
@dataclasses.dataclass(slots=True, frozen=True)
class Session:
@@ -170,7 +208,7 @@ class Session:
_deserialize = Deserializer(
- {"season": "seasonNum", "number": "episodeNum"},
+ {"season": "seasonNum", "number": "episodeNum", "title": "name", "id": "_id"},
{"season": int, "number": int},
)
@@ -182,6 +220,25 @@ class Episode:
season: int
number: int
title: str
+ airDateISO: str # noqa: N815
+
+
+class Season(TypedDict):
+ number: int
+ episodes: list[dict[str, Any]]
+
+
[email protected](slots=True)
+class Series:
+ id: str
+ title: str
+ slug: str
+ seasons: list[Season] = dataclasses.field(default_factory=list)
+
+ def episodes(self) -> Generator[Episode]:
+ for season in self.seasons:
+ for ep in season["episodes"]:
+ yield _deserialize(Episode, ep)
def _remove_ads_segments(rendition: Rendition) -> None:
diff --git a/cyberdrop_dl/downloader/hls.py b/cyberdrop_dl/downloader/hls.py
index c3173a4ba..0673188f7 100644
--- a/cyberdrop_dl/downloader/hls.py
+++ b/cyberdrop_dl/downloader/hls.py
@@ -7,12 +7,11 @@ from contextvars import ContextVar
from http import HTTPStatus
from typing import TYPE_CHECKING, NamedTuple, Protocol
-from Crypto.Cipher import AES
-
from cyberdrop_dl import aio, constants, ffmpeg
from cyberdrop_dl.exceptions import DownloadError
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem
from cyberdrop_dl.utils import parse_url
+from cyberdrop_dl.utils.crypto import aes_cbc_decrypt, aes_unpad
from cyberdrop_dl.utils.m3u8 import HLSKey
if TYPE_CHECKING:
@@ -75,7 +74,7 @@ class AESHLSDecrypter(HLSDecrypter):
async def __call__(self, content: bytes, key: HLSKey, headers: dict[str, str] | None = None) -> bytes:
aes_key = await self.get_key(key.uri, headers=headers)
- return AES.new(aes_key, AES.MODE_CBC, key.iv).encrypt(content)
+ return aes_unpad(aes_cbc_decrypt(content, aes_key, key.iv))
def _parse_segments(segments: Sequence[Segment | InitializationSection]) -> Generator[HLSSegment]:
diff --git a/cyberdrop_dl/utils/crypto.py b/cyberdrop_dl/utils/crypto.py
index 50a94202b..4108781cc 100644
--- a/cyberdrop_dl/utils/crypto.py
+++ b/cyberdrop_dl/utils/crypto.py
@@ -1,6 +1,7 @@
from __future__ import annotations
from Crypto.Cipher import AES
+from Crypto.Util.Padding import unpad
EMPTY_IV = b"\0" * AES.block_size
@@ -11,3 +12,7 @@ def aes_cbc_encrypt(data: bytes, key: bytes, iv: bytes = EMPTY_IV) -> bytes:
def aes_cbc_decrypt(data: bytes, key: bytes, iv: bytes = EMPTY_IV) -> bytes:
return AES.new(key, AES.MODE_CBC, iv).decrypt(data)
+
+
+def aes_unpad(data: bytes) -> bytes:
+ return unpad(data, AES.block_size)
diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.py
index 0a6ab12f1..b7898e072 100644
--- a/cyberdrop_dl/utils/m3u8.py
+++ b/cyberdrop_dl/utils/m3u8.py
@@ -85,7 +85,7 @@ class Rendition:
subtitle: M3U8 | None
def __iter__(self) -> Iterator[M3U8 | None]:
- return iter(dataclasses.astuple(self))
+ yield from (self.video, self.audio, self.subtitle)
@dataclasses.dataclass(frozen=True, slots=True, order=True)
diff --git a/tests/crawlers/test_cases/pluto_tv.py b/tests/crawlers/test_cases/pluto_tv.py
new file mode 100644
index 000000000..4ff2aed07
--- /dev/null
+++ b/tests/crawlers/test_cases/pluto_tv.py
@@ -0,0 +1,19 @@
+DOMAIN = "pluto.tv"
+TEST_CASES = [
+ {
+ "url": "https://pluto.tv/shows/3020012/episode/611bc94773019a0013eba3ed",
+ "results": [
+ {
+ "url": "re:https://pluto.tv/.+/shows/3020012/episode/611bc94773019a0013eba3ed",
+ "referer": "re:https://pluto.tv/.+/shows/3020012/episode/611bc94773019a0013eba3ed",
+ "download_folder": "re:Drift (Pluto.tv)",
+ "filename": "S01E001 - Episodio 1 [611bc94773019a0013eba3ed][avc1].mp4",
+ "original_filename": "Episodio 1",
+ "download_filename": "S01E001 - Episodio 1 [611bc94773019a0013eba3ed][avc1].mp4",
+ "debrid_url": None,
+ "album_id": "3020012",
+ "uploaded_at": int,
+ },
+ ],
+ }
+]