Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: download of encrypted segments with no IV (HLS)

NTFSvolume1b280fdccce1
diff --git a/cyberdrop_dl/downloader/hls.py b/cyberdrop_dl/downloader/hls.py
index 9194681ac..f18bf2395 100644
--- a/cyberdrop_dl/downloader/hls.py
+++ b/cyberdrop_dl/downloader/hls.py
@@ -56,16 +56,20 @@ class HLSDecrypter(Protocol):
@dataclasses.dataclass(slots=True, frozen=True)
class AESHLSDecrypter(HLSDecrypter):
client: HTTPClient
- _lock: asyncio.Lock = dataclasses.field(default_factory=asyncio.Lock)
+ _locks: aio.WeakAsyncLocks[AbsoluteHttpURL] = dataclasses.field(default_factory=aio.WeakAsyncLocks)
_cache: dict[AbsoluteHttpURL, bytes] = dataclasses.field(default_factory=dict)
async def get_key(self, uri: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> bytes:
- if aes_key := self._cache.get(uri):
- return aes_key
+ try:
+ return self._cache[uri]
+ except LookupError:
+ pass
- async with self._lock:
- if aes_key := self._cache.get(uri):
- return aes_key
+ async with self._locks[uri]:
+ try:
+ return self._cache[uri]
+ except LookupError:
+ pass
async with self.client.raw_request(uri, headers=headers or {}) as resp:
aes_key = self._cache[uri] = await resp.read()
@@ -85,7 +89,7 @@ def _create_segments(segments: Iterable[Segment | InitializationSection], count:
idx=index - 1,
name=f"{index:0{padding}d}{constants.TempExt.HLS}",
url=parse_url(segment.absolute_uri, trim=False),
- decrypt_info=HLSKey.parse(segment),
+ decrypt_info=HLSKey.parse(segment, idx=index - 1),
)
diff --git a/cyberdrop_dl/signature.py b/cyberdrop_dl/signature.py
index 1eaa1d183..fec41d41e 100644
--- a/cyberdrop_dl/signature.py
+++ b/cyberdrop_dl/signature.py
@@ -38,3 +38,8 @@ def simple_repr(*names: str) -> Callable[..., str]:
return f"<{type(self).__name__}({', '.join(fields)}>"
return repr_
+
+
+def vars_repr(self: object) -> str:
+ fields = (f"{name}={val!r}" for name, val in vars(self).items())
+ return f"<{type(self).__name__}({', '.join(fields)}>"
diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.py
index 8fd2f4a38..b62907f43 100644
--- a/cyberdrop_dl/utils/m3u8.py
+++ b/cyberdrop_dl/utils/m3u8.py
@@ -3,16 +3,18 @@ from __future__ import annotations
import dataclasses
import datetime as dt
import logging
+import struct
from enum import StrEnum
from typing import TYPE_CHECKING, Any, ClassVar, Literal, NamedTuple, Self
import m3u8.model
from m3u8 import M3U8 as _M3U8
from m3u8 import Media, Playlist
+from m3u8.mixins import BasePathMixin, GroupedBasePathMixin
from cyberdrop_dl.exceptions import DownloadError
from cyberdrop_dl.mediaprops import Codecs, Resolution
-from cyberdrop_dl.signature import simple_repr
+from cyberdrop_dl.signature import simple_repr, vars_repr
from cyberdrop_dl.utils import parse_url
from cyberdrop_dl.utils.dataclass import DictDataclass
@@ -25,6 +27,9 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
+BasePathMixin.__repr__ = GroupedBasePathMixin.__repr__ = vars_repr
+
+
class MediaType(StrEnum):
audio = "AUDIO"
video = "VIDEO"
@@ -186,14 +191,14 @@ class HLSKey:
raise RuntimeError("Another key is already set")
@classmethod
- def parse(cls, seg: m3u8.model.Segment | m3u8.model.InitializationSection) -> Self | None:
+ def parse(cls, seg: m3u8.model.Segment | m3u8.model.InitializationSection, idx: int) -> Self | None:
if isinstance(seg, m3u8.model.Segment) and seg.key and seg.key.method and seg.key.method != "NONE":
- assert seg.key.iv
+ iv = bytes.fromhex(seg.key.iv.removeprefix("0x")) if seg.key.iv else struct.pack(">8xq", idx)
assert seg.key.uri
return cls(
method=Encryption(seg.key.method),
- iv=bytes.fromhex(seg.key.iv.removeprefix("0x")),
- uri=parse_url(seg.key.uri, trim=False),
+ iv=iv,
+ uri=parse_url(seg.key.absolute_uri, trim=False),
)
@@ -207,10 +212,10 @@ def _validate_keys(stream: M3U8) -> None:
raise DownloadError(
"Encrypted", f"M3U8 playlist {stream.source} has an unsupported encryption algorithm: \n{key!s}"
)
- if not key.iv or not key.uri:
+ if not key.uri:
raise DownloadError(
"Encrypted",
- f"M3U8 playlist {stream.source} is encrypted but we have no IV to decrypted it: \n{key!s}",
+ f"M3U8 playlist {stream.source} is encrypted but we have no URI to fetch to decrypted it: \n{key!s}",
)