Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add Pluto.tv support (#2016)

NTFSvolume40e43cf702ad
diff --git a/cyberdrop_dl/crawlers/_hls.py b/cyberdrop_dl/crawlers/_hls.py
index 424566be2..7acafca1a 100644
--- a/cyberdrop_dl/crawlers/_hls.py
+++ b/cyberdrop_dl/crawlers/_hls.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import dataclasses
import logging
import os
from abc import ABC, abstractmethod
@@ -50,12 +51,16 @@ class HLSMixin(ABC):
method: HttpMethod = "GET",
only: Iterable[str] = (),
exclude: Iterable[str] = ("vp09",),
+ keep_query: bool = False, # noqa: FBT001, FBT002
**kwargs: Unpack[RequestParams],
) -> tuple[m3u8.Rendition, m3u8.RenditionDetails | None]:
m3u8_obj = await self._request_m3u8(url, method, **kwargs)
if m3u8_obj.is_variant:
logger.info("Selecting best rendition from %s", url)
rendition = m3u8.select_best_rendition(m3u8_obj, only=only, exclude=exclude)
+ if keep_query:
+ rendition = dataclasses.replace(rendition, urls=rendition.urls.with_query(url.query))
+
logger.info("Selected best rendition for %s:\n%s", url, rendition)
return await self._resolve_rendition(rendition, method, **kwargs)
m3u8_obj.media_type = "video"
@@ -108,10 +113,13 @@ class HLSMixin(ABC):
method: HttpMethod = "GET",
only: Iterable[str] = (),
exclude: Iterable[str] = ("vp09",),
+ keep_query: bool = False, # noqa: FBT001, FBT002
**kwargs: Unpack[RequestParams],
) -> tuple[m3u8.Rendition, m3u8.RenditionDetails]:
"""Get m3u8 rendition group from a playlist m3u8 (variant m3u8), selecting the best format"""
- playlist, info = await self.request_m3u8(url, method, only=only, exclude=exclude, **kwargs)
+ playlist, info = await self.request_m3u8(
+ url, method, only=only, exclude=exclude, keep_query=keep_query, **kwargs
+ )
if info is None:
raise ScrapeError(422, "Not a variant m3u8", origin=url)
return playlist, info
diff --git a/cyberdrop_dl/crawlers/pluto_tv.py b/cyberdrop_dl/crawlers/pluto_tv.py
new file mode 100644
index 000000000..6b9c8c487
--- /dev/null
+++ b/cyberdrop_dl/crawlers/pluto_tv.py
@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+import dataclasses
+import uuid
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, Any, ClassVar, override
+
+import yarl
+
+from cyberdrop_dl.cache import cached_method
+from cyberdrop_dl.crawlers.crawler import API, Crawler, SupportedPaths, compose_ep_name
+from cyberdrop_dl.url_objects import AbsoluteHttpURL, ScrapeItem
+from cyberdrop_dl.utils import css, next_js
+from cyberdrop_dl.utils.dataclass import Deserializer
+from cyberdrop_dl.utils.errors import error_handling_wrapper
+
+session_token: ContextVar[str] = ContextVar("session_token")
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.utils.m3u8 import Rendition
+
+
+class PlutoCrawler(Crawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Episode": (".../shows/<series_id>/episode/<episode_id>",),
+ }
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pluto.tv")
+ DOMAIN: ClassVar[str] = "pluto.tv"
+
+ @staticmethod
+ @override
+ def __db_path__(url: AbsoluteHttpURL, /) -> str:
+ _region, sep, rest = url.path.partition("/shows/")
+ assert sep
+ return sep + rest
+
+ def __post_init__(self) -> None:
+ self.api: PlutoAPI = PlutoAPI.from_crawler(self)
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case [*_, "shows", series_id, "episode", episode_id]:
+ await self.episode(scrape_item, series_id, episode_id)
+ case [*_, "shows", series_id, "season", _season]:
+ raise ValueError
+
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def episode(self, scrape_item: ScrapeItem, series_id: str, episode_id: str) -> None:
+ if await self.check_complete(scrape_item.url):
+ return
+
+ soup = await self.request_soup(scrape_item.url)
+ data = next_js.data(soup)
+ ep = data["props"]["pageProps"]["episodeMetadata"]
+ episode = _deserialize(Episode, ep, id=episode_id)
+ scrape_item.setup_as_album(self.create_title(ep["seriesTitle"], series_id), album_id=series_id)
+ await self._episode(scrape_item, episode)
+
+ @error_handling_wrapper
+ async def _episode(self, scrape_item: ScrapeItem, ep: Episode) -> None:
+ m3u8_url = await self.api.stream(ep.id)
+ m3u8, info = await self.request_m3u8_playlist(
+ m3u8_url, headers={"User-Agent": self.api.FIREFOX}, keep_query=True
+ )
+ _remove_ads_segments(m3u8)
+ filename = self.create_custom_filename(
+ compose_ep_name(ep.season, ep.number, ep.title),
+ ext := ".mp4",
+ file_id=ep.id,
+ resolution=info.resolution,
+ video_codec=info.codecs.video or "avc1",
+ audio_codec=info.codecs.audio,
+ fps=info.stream_info.frame_rate,
+ )
+ await self.handle_file(
+ scrape_item.url,
+ scrape_item,
+ ep.title,
+ ext,
+ m3u8=m3u8,
+ custom_filename=filename,
+ metadata=ep,
+ )
+
+
+class PlutoAPI(API):
+ BOOT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://boot.pluto.tv/v4/start")
+ FIREFOX: ClassVar[str] = "Mozilla/5.0 (X11; Linux x86_64; rv:151.0) Gecko/20100101 Firefox/151.0"
+ GRAPHQL_ENDPOINT: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://pluto.tv/api/tn/app-shell/graphql/")
+ M3U8: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL(
+ "https://cfd-v4-service-channel-stitcher-use1-1.prd.pluto.tv/v2/stitch/hls/episode"
+ )
+
+ def __post_init__(self) -> None:
+ self._device_version: str = "151.0.0" # from firefox's UA
+ self._client_id: str = str(uuid.uuid4())
+
+ def __repr__(self) -> str:
+ return f"<{type(self).__name__}(app_version={self.app_version.get()}, client_id={self._client_id}, device_version={self._device_version})>"
+
+ @cached_method()
+ async def app_version(self) -> str:
+ soup = await self.request_soup(self.PRIMARY_URL)
+ return css.select(soup, "meta[name=appVersion], meta[name=app_version]", "content")
+
+ async def start(self) -> Session:
+ resp = await self.request_gql(
+ "PtvStart",
+ {
+ "params": {
+ "deviceModel": "web",
+ "drmCapabilities": "widevine:L3",
+ "isClientDNT": True,
+ "deviceId": self._client_id,
+ "ptvAppName": "web",
+ "cmAudienceID": "",
+ "updateType": "v1v2",
+ }
+ },
+ )
+
+ return _deserialize(Session, resp["ptvStart"]["session"])
+
+ async def request_gql(self, operation: str, variables: dict[str, Any]) -> dict[str, Any]:
+ resp = await self.request_json(
+ self.GRAPHQL_ENDPOINT,
+ method="POST",
+ headers={"User-Agent": self.FIREFOX},
+ json={
+ "query": globals()[operation],
+ "variables": variables,
+ "operationName": operation,
+ },
+ )
+ return resp["data"]
+
+ async def stream(self, episode_id: str) -> AbsoluteHttpURL:
+ session = await self.start()
+ url = self.M3U8 / episode_id / "master.m3u8"
+ sid = str(uuid.uuid1())
+ return url.with_query(
+ {
+ "advertisingId": "",
+ "appName": "web",
+ "appVersion": await self.app_version(),
+ "app_name": "web",
+ "clientID": self._client_id,
+ "deviceId": self._client_id,
+ "deviceMake": "firefox",
+ "deviceModel": "web",
+ "deviceType": "web",
+ "deviceVersion": "151.0",
+ "serverSideAds": "false",
+ "sessionID": sid,
+ "sid": sid,
+ "userId": "",
+ "jwt": session.jwt,
+ "includeExtendedEvents": "true",
+ }
+ )
+
+
[email protected](slots=True, frozen=True)
+class Session:
+ id: str
+ jwt: str
+
+
+_deserialize = Deserializer(
+ {"season": "seasonNum", "number": "episodeNum"},
+ {"season": int, "number": int},
+)
+
+
[email protected](slots=True, frozen=True)
+class Episode:
+ id: str
+ description: str
+ season: int
+ number: int
+ title: str
+
+
+def _remove_ads_segments(rendition: Rendition) -> None:
+ for m3u8 in rendition:
+ if not m3u8:
+ continue
+
+ m3u8.data["segments"] = [s for s in m3u8.data["segments"] if not _is_ad(s["uri"])]
+ m3u8._initialize_attributes() # pyright: ignore[reportUnknownMemberType, reportAttributeAccessIssue]
+
+
+def _is_ad(uri: str) -> bool:
+ path = yarl.URL(uri).path.casefold()
+ return any(ad_name in path for ad_name in ("_ad%2f", "_ad/", "_ad_bumper", "plutotv_filler"))
+
+
+PtvStart = """
+query PtvStart($params: StartParameters!) {
+ ptvStart(params: $params) {
+ deviceId
+ session {
+ id
+ jwt
+ }
+ refreshInSec
+ }
+}
+"""
diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.py
index f0c995aa7..0a6ab12f1 100644
--- a/cyberdrop_dl/utils/m3u8.py
+++ b/cyberdrop_dl/utils/m3u8.py
@@ -70,6 +70,13 @@ class MediaURLs(NamedTuple):
audio: AbsoluteHttpURL | None
subtitle: AbsoluteHttpURL | None
+ def with_query(self, query: Mapping[str, Any]) -> MediaURLs:
+ return MediaURLs(
+ video=self.video.with_query(query),
+ audio=self.audio and self.audio.with_query(query),
+ subtitle=self.subtitle and self.subtitle.with_query(query),
+ )
+
@dataclasses.dataclass(slots=True)
class Rendition:
@@ -178,12 +185,12 @@ class HLSKey:
@classmethod
def parse(cls, seg: m3u8.model.Segment | m3u8.model.InitializationSection) -> Self | None:
- if isinstance(seg, m3u8.model.Segment) and seg.key:
+ if isinstance(seg, m3u8.model.Segment) and seg.key and seg.key.method and seg.key.method != "NONE":
assert seg.key.iv
assert seg.key.uri
return cls(
method=Encryption(seg.key.method),
- iv=bytes.fromhex(seg.key.iv),
+ iv=bytes.fromhex(seg.key.iv.removeprefix("0x")),
uri=parse_url(seg.key.uri, trim=False),
)
@@ -192,7 +199,9 @@ def _validate_keys(stream: M3U8) -> None:
if not stream.keys:
return
for key in stream.keys:
- if key is None: # First one could be None # pyright: ignore[reportUnnecessaryComparison]
+ if (
+ key is None or key.method == "NONE"
+ ): # First one could be None # pyright: ignore[reportUnnecessaryComparison]
continue
if key.method != Encryption.AES_128:
raise DownloadError(
diff --git a/cyberdrop_dl/utils/next_js.py b/cyberdrop_dl/utils/next_js.py
index c43d38e1c..1ebeadc1c 100644
--- a/cyberdrop_dl/utils/next_js.py
+++ b/cyberdrop_dl/utils/next_js.py
@@ -254,6 +254,10 @@ def remove_unused_tags(file: Path) -> None:
file.write_text(soup.prettify())
+def data(tag: BeautifulSoup) -> Any:
+ return json.loads(css.select_text(tag, "#__NEXT_DATA__"))
+
+
if __name__ == "__main__":
import sys
from pathlib import Path