Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add support for AES-128 encryption (HLS) (#2068)

NTFSvolumebb7c1564d127
diff --git a/cyberdrop_dl/downloader/hls.py b/cyberdrop_dl/downloader/hls.py
index c1e0679d0..c3173a4ba 100644
--- a/cyberdrop_dl/downloader/hls.py
+++ b/cyberdrop_dl/downloader/hls.py
@@ -1,15 +1,19 @@
from __future__ import annotations
import asyncio
+import dataclasses
import logging
from contextvars import ContextVar
from http import HTTPStatus
-from typing import TYPE_CHECKING, NamedTuple
+from typing import TYPE_CHECKING, NamedTuple, Protocol
+
+from Crypto.Cipher import AES
from cyberdrop_dl import aio, constants, ffmpeg
from cyberdrop_dl.exceptions import DownloadError
from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem
from cyberdrop_dl.utils import parse_url
+from cyberdrop_dl.utils.m3u8 import HLSKey
if TYPE_CHECKING:
from collections.abc import Awaitable, Callable, Generator, Iterable, Sequence
@@ -17,12 +21,14 @@ if TYPE_CHECKING:
from m3u8.model import InitializationSection, Segment
+ from cyberdrop_dl.clients.http import HTTPClient
from cyberdrop_dl.utils.m3u8 import M3U8, Rendition
DownloadFn = Callable[[MediaItem], Awaitable[bool]]
CONCURRENT_SEGMENTS: ContextVar[int] = ContextVar("CONCURRENT_SEGMENTS")
+_DECRYPTER: ContextVar[AESHLSDecrypter] = ContextVar("_DECRYPTER")
logger = logging.getLogger(__name__)
@@ -41,6 +47,35 @@ class HLSSegment(NamedTuple):
idx: int
name: str
url: AbsoluteHttpURL
+ decrypt_info: HLSKey | None = None
+
+
+class HLSDecrypter(Protocol):
+ async def __call__(self, content: bytes, key: HLSKey) -> bytes: ...
+
+
[email protected](slots=True, frozen=True)
+class AESHLSDecrypter(HLSDecrypter):
+ client: HTTPClient
+ _lock: asyncio.Lock = dataclasses.field(default_factory=asyncio.Lock)
+ _cache: dict[AbsoluteHttpURL, bytes] = dataclasses.field(default_factory=dict)
+
+ async def get_key(self, uri: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> bytes:
+ if aes_key := self._cache.get(uri):
+ return aes_key
+
+ async with self._lock:
+ if aes_key := self._cache.get(uri):
+ return aes_key
+
+ async with self.client.raw_request(uri, headers=headers or {}) as resp:
+ aes_key = self._cache[uri] = await resp.read()
+
+ return aes_key
+
+ async def __call__(self, content: bytes, key: HLSKey, headers: dict[str, str] | None = None) -> bytes:
+ aes_key = await self.get_key(key.uri, headers=headers)
+ return AES.new(aes_key, AES.MODE_CBC, key.iv).encrypt(content)
def _parse_segments(segments: Sequence[Segment | InitializationSection]) -> Generator[HLSSegment]:
@@ -50,7 +85,8 @@ def _parse_segments(segments: Sequence[Segment | InitializationSection]) -> Gene
yield HLSSegment(
idx=index - 1,
name=f"{index:0{padding}d}{constants.TempExt.HLS}",
- url=parse_url(segment.absolute_uri),
+ url=parse_url(segment.absolute_uri, trim=False),
+ decrypt_info=HLSKey.parse(segment),
)
@@ -77,6 +113,9 @@ def _create_media_segments(
is_segment=True,
)
seg_media_item.headers = media_item.headers.copy()
+ if segment.decrypt_info:
+ segment.decrypt_info(seg_media_item.extra_info)
+
yield seg_media_item
@@ -118,10 +157,23 @@ async def _download_m3u8(
m3u8.source,
)
results = await _download_segments(m_segments, m3u8.total_segments, download, sem)
+ await _decrypt_segments((r.item for r in results), sem)
await _merge_segments(tuple(result.item.path for result in results), output)
return output
+async def _decrypt_segments(items: Iterable[MediaItem], sem: asyncio.BoundedSemaphore) -> None:
+ decrypter = _DECRYPTER.get()
+
+ async def decrypt(item: MediaItem) -> None:
+ if key := HLSKey.get(item.extra_info):
+ logger.debug(f"Decrypting '{item.path}' with {key}")
+ content = await decrypter(await aio.read_bytes(item.path), key, item.headers)
+ await aio.write_bytes(item.path, content)
+
+ await aio.map(decrypt, items, task_limit=sem)
+
+
async def _download_segments(
segments: Iterable[MediaItem],
count: int,
@@ -160,13 +212,14 @@ def _prepare_output_path(m3u8: M3U8, output: Path) -> Path:
return output.with_suffix(suffix)
-async def download(media_item: MediaItem, rendition: Rendition, download_fn: DownloadFn) -> Streams:
+async def download(media_item: MediaItem, rendition: Rendition, download_fn: DownloadFn, client: HTTPClient) -> Streams:
"""Download a rendition group"""
temp_dir = media_item.path.with_suffix(constants.TempExt.HLS)
sem = asyncio.BoundedSemaphore(CONCURRENT_SEGMENTS.get())
async def download(m3u8: M3U8) -> Path:
+ _DECRYPTER.set(AESHLSDecrypter(client))
return await _download_m3u8(m3u8, temp_dir, media_item, download_fn, sem)
async def download_subs() -> Path | None:
diff --git a/cyberdrop_dl/downloader/http.py b/cyberdrop_dl/downloader/http.py
index b1a0ca166..a93f753ca 100644
--- a/cyberdrop_dl/downloader/http.py
+++ b/cyberdrop_dl/downloader/http.py
@@ -281,7 +281,7 @@ class Downloader:
await self._hls_download(media_item, rendition)
async def _hls_download(self, media_item: MediaItem, rendition: Rendition) -> None:
- streams = await hls.download(media_item, rendition, self._download)
+ streams = await hls.download(media_item, rendition, self._download, self.client.http_client)
if not streams.audio:
await aio.move(streams.video, media_item.path)
diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.py
index 3963d78ef..f0c995aa7 100644
--- a/cyberdrop_dl/utils/m3u8.py
+++ b/cyberdrop_dl/utils/m3u8.py
@@ -4,18 +4,19 @@ import dataclasses
import datetime
import logging
from enum import StrEnum
-from typing import TYPE_CHECKING, Any, Literal, NamedTuple
+from typing import TYPE_CHECKING, Any, ClassVar, Literal, NamedTuple, Self
import m3u8.model
from m3u8 import M3U8 as _M3U8
from m3u8 import Media, Playlist
+from cyberdrop_dl.exceptions import DownloadError
from cyberdrop_dl.mediaprops import Codecs, Resolution
from cyberdrop_dl.signature import simple_repr
from cyberdrop_dl.utils import parse_url
if TYPE_CHECKING:
- from collections.abc import Generator, Iterable, Iterator
+ from collections.abc import Generator, Iterable, Iterator, Mapping
from pathlib import Path
from cyberdrop_dl.url_objects import AbsoluteHttpURL
@@ -153,6 +154,57 @@ def _parse_stream_resolution(stream_info: StreamInfo, video_url: AbsoluteHttpURL
return Resolution.unknown()
+class Encryption(StrEnum):
+ AES_128 = "AES-128"
+
+
[email protected](slots=True, frozen=True)
+class HLSKey:
+ __attr__: ClassVar[str] = "HLS_DECRYPT_INFO"
+ method: Encryption
+ uri: AbsoluteHttpURL
+ iv: bytes
+
+ @classmethod
+ def get(cls, data: Mapping[str, Any]) -> Self | None:
+ key = data.get(cls.__attr__)
+ if type(key) is cls:
+ return key
+
+ def __call__(self, data: dict[str, Any]) -> Self | None:
+ key = data.setdefault(self.__attr__, self)
+ if key is not self:
+ raise RuntimeError("Another key is already set")
+
+ @classmethod
+ def parse(cls, seg: m3u8.model.Segment | m3u8.model.InitializationSection) -> Self | None:
+ if isinstance(seg, m3u8.model.Segment) and seg.key:
+ assert seg.key.iv
+ assert seg.key.uri
+ return cls(
+ method=Encryption(seg.key.method),
+ iv=bytes.fromhex(seg.key.iv),
+ uri=parse_url(seg.key.uri, trim=False),
+ )
+
+
+def _validate_keys(stream: M3U8) -> None:
+ if not stream.keys:
+ return
+ for key in stream.keys:
+ if key is None: # First one could be None # pyright: ignore[reportUnnecessaryComparison]
+ continue
+ if key.method != Encryption.AES_128:
+ raise DownloadError(
+ "Encrypted", f"M3U8 playlist {stream.source} has an unsupported encryption algorithm: \n{key!s}"
+ )
+ if not key.iv or not key.uri:
+ raise DownloadError(
+ "Encrypted",
+ f"M3U8 playlist {stream.source} is encrypted but we have to IV to decrypted it: \n{key!s}",
+ )
+
+
class M3U8(_M3U8):
def __init__(
self,
@@ -166,6 +218,7 @@ class M3U8(_M3U8):
self.media_type: Literal["video", "audio", "subtitle"] | None = media_type
self.source: AbsoluteHttpURL | Path | None = source
super().__init__(content, base_uri=str(base_uri) if base_uri else None)
+ _validate_keys(self)
@property
def total_segments(self) -> int: