Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add support for AES-128 encryption (HLS) (#2068)
NTFSvolumebb7c1564d127
diff --git a/cyberdrop_dl/downloader/hls.py b/cyberdrop_dl/downloader/hls.pyindex c1e0679d0..c3173a4ba 100644--- a/cyberdrop_dl/downloader/hls.py+++ b/cyberdrop_dl/downloader/hls.py@@ -1,15 +1,19 @@from __future__ import annotationsimport asyncio+import dataclassesimport loggingfrom contextvars import ContextVarfrom http import HTTPStatus-from typing import TYPE_CHECKING, NamedTuple+from typing import TYPE_CHECKING, NamedTuple, Protocol++from Crypto.Cipher import AESfrom cyberdrop_dl import aio, constants, ffmpegfrom cyberdrop_dl.exceptions import DownloadErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItemfrom cyberdrop_dl.utils import parse_url+from cyberdrop_dl.utils.m3u8 import HLSKeyif TYPE_CHECKING:from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence@@ -17,12 +21,14 @@ if TYPE_CHECKING:from m3u8.model import InitializationSection, Segment+ from cyberdrop_dl.clients.http import HTTPClientfrom cyberdrop_dl.utils.m3u8 import M3U8, RenditionDownloadFn = Callable[[MediaItem], Awaitable[bool]]CONCURRENT_SEGMENTS: ContextVar[int] = ContextVar("CONCURRENT_SEGMENTS")+_DECRYPTER: ContextVar[AESHLSDecrypter] = ContextVar("_DECRYPTER")logger = logging.getLogger(__name__)@@ -41,6 +47,35 @@ class HLSSegment(NamedTuple):idx: intname: strurl: AbsoluteHttpURL+ decrypt_info: HLSKey | None = None+++class HLSDecrypter(Protocol):+ async def __call__(self, content: bytes, key: HLSKey) -> bytes: ...++[email protected](slots=True, frozen=True)+class AESHLSDecrypter(HLSDecrypter):+ client: HTTPClient+ _lock: asyncio.Lock = dataclasses.field(default_factory=asyncio.Lock)+ _cache: dict[AbsoluteHttpURL, bytes] = dataclasses.field(default_factory=dict)++ async def get_key(self, uri: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> bytes:+ if aes_key := self._cache.get(uri):+ return aes_key++ async with self._lock:+ if aes_key := self._cache.get(uri):+ return aes_key++ async with self.client.raw_request(uri, headers=headers or {}) as resp:+ aes_key = self._cache[uri] = await resp.read()++ return aes_key++ async def __call__(self, content: bytes, key: HLSKey, headers: dict[str, str] | None = None) -> bytes:+ aes_key = await self.get_key(key.uri, headers=headers)+ return AES.new(aes_key, AES.MODE_CBC, key.iv).encrypt(content)def _parse_segments(segments: Sequence[Segment | InitializationSection]) -> Generator[HLSSegment]:@@ -50,7 +85,8 @@ def _parse_segments(segments: Sequence[Segment | InitializationSection]) -> Geneyield HLSSegment(idx=index - 1,name=f"{index:0{padding}d}{constants.TempExt.HLS}",- url=parse_url(segment.absolute_uri),+ url=parse_url(segment.absolute_uri, trim=False),+ decrypt_info=HLSKey.parse(segment),)@@ -77,6 +113,9 @@ def _create_media_segments(is_segment=True,)seg_media_item.headers = media_item.headers.copy()+ if segment.decrypt_info:+ segment.decrypt_info(seg_media_item.extra_info)+yield seg_media_item@@ -118,10 +157,23 @@ async def _download_m3u8(m3u8.source,)results = await _download_segments(m_segments, m3u8.total_segments, download, sem)+ await _decrypt_segments((r.item for r in results), sem)await _merge_segments(tuple(result.item.path for result in results), output)return output+async def _decrypt_segments(items: Iterable[MediaItem], sem: asyncio.BoundedSemaphore) -> None:+ decrypter = _DECRYPTER.get()++ async def decrypt(item: MediaItem) -> None:+ if key := HLSKey.get(item.extra_info):+ logger.debug(f"Decrypting '{item.path}' with {key}")+ content = await decrypter(await aio.read_bytes(item.path), key, item.headers)+ await aio.write_bytes(item.path, content)++ await aio.map(decrypt, items, task_limit=sem)++async def _download_segments(segments: Iterable[MediaItem],count: int,@@ -160,13 +212,14 @@ def _prepare_output_path(m3u8: M3U8, output: Path) -> Path:return output.with_suffix(suffix)-async def download(media_item: MediaItem, rendition: Rendition, download_fn: DownloadFn) -> Streams:+async def download(media_item: MediaItem, rendition: Rendition, download_fn: DownloadFn, client: HTTPClient) -> Streams:"""Download a rendition group"""temp_dir = media_item.path.with_suffix(constants.TempExt.HLS)sem = asyncio.BoundedSemaphore(CONCURRENT_SEGMENTS.get())async def download(m3u8: M3U8) -> Path:+ _DECRYPTER.set(AESHLSDecrypter(client))return await _download_m3u8(m3u8, temp_dir, media_item, download_fn, sem)async def download_subs() -> Path | None:diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.pyindex b1a0ca166..a93f753ca 100644--- a/cyberdrop_dl/downloader/http.py+++ b/cyberdrop_dl/downloader/http.py@@ -281,7 +281,7 @@ class Downloader:await self._hls_download(media_item, rendition)async def _hls_download(self, media_item: MediaItem, rendition: Rendition) -> None:- streams = await hls.download(media_item, rendition, self._download)+ streams = await hls.download(media_item, rendition, self._download, self.client.http_client)if not streams.audio:await aio.move(streams.video, media_item.path)diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.pyindex 3963d78ef..f0c995aa7 100644--- a/cyberdrop_dl/utils/m3u8.py+++ b/cyberdrop_dl/utils/m3u8.py@@ -4,18 +4,19 @@ import dataclassesimport datetimeimport loggingfrom enum import StrEnum-from typing import TYPE_CHECKING, Any, Literal, NamedTuple+from typing import TYPE_CHECKING, Any, ClassVar, Literal, NamedTuple, Selfimport m3u8.modelfrom m3u8 import M3U8 as _M3U8from m3u8 import Media, Playlist+from cyberdrop_dl.exceptions import DownloadErrorfrom cyberdrop_dl.mediaprops import Codecs, Resolutionfrom cyberdrop_dl.signature import simple_reprfrom cyberdrop_dl.utils import parse_urlif TYPE_CHECKING:- from collections.abc import Generator, Iterable, Iterator+ from collections.abc import Generator, Iterable, Iterator, Mappingfrom pathlib import Pathfrom cyberdrop_dl.url_objects import AbsoluteHttpURL@@ -153,6 +154,57 @@ def _parse_stream_resolution(stream_info: StreamInfo, video_url: AbsoluteHttpURLreturn Resolution.unknown()+class Encryption(StrEnum):+ AES_128 = "AES-128"++[email protected](slots=True, frozen=True)+class HLSKey:+ __attr__: ClassVar[str] = "HLS_DECRYPT_INFO"+ method: Encryption+ uri: AbsoluteHttpURL+ iv: bytes++ @classmethod+ def get(cls, data: Mapping[str, Any]) -> Self | None:+ key = data.get(cls.__attr__)+ if type(key) is cls:+ return key++ def __call__(self, data: dict[str, Any]) -> Self | None:+ key = data.setdefault(self.__attr__, self)+ if key is not self:+ raise RuntimeError("Another key is already set")++ @classmethod+ def parse(cls, seg: m3u8.model.Segment | m3u8.model.InitializationSection) -> Self | None:+ if isinstance(seg, m3u8.model.Segment) and seg.key:+ assert seg.key.iv+ assert seg.key.uri+ return cls(+ method=Encryption(seg.key.method),+ iv=bytes.fromhex(seg.key.iv),+ uri=parse_url(seg.key.uri, trim=False),+ )+++def _validate_keys(stream: M3U8) -> None:+ if not stream.keys:+ return+ for key in stream.keys:+ if key is None: # First one could be None # pyright: ignore[reportUnnecessaryComparison]+ continue+ if key.method != Encryption.AES_128:+ raise DownloadError(+ "Encrypted", f"M3U8 playlist {stream.source} has an unsupported encryption algorithm: \n{key!s}"+ )+ if not key.iv or not key.uri:+ raise DownloadError(+ "Encrypted",+ f"M3U8 playlist {stream.source} is encrypted but we have to IV to decrypted it: \n{key!s}",+ )++class M3U8(_M3U8):def __init__(self,@@ -166,6 +218,7 @@ class M3U8(_M3U8):self.media_type: Literal["video", "audio", "subtitle"] | None = media_typeself.source: AbsoluteHttpURL | Path | None = sourcesuper().__init__(content, base_uri=str(base_uri) if base_uri else None)+ _validate_keys(self)@propertydef total_segments(self) -> int: