Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: download of encrypted segments with no IV (HLS)
NTFSvolume1b280fdccce1
diff --git a/cyberdrop_dl/downloader/hls.py b/cyberdrop_dl/downloader/hls.pyindex 9194681ac..f18bf2395 100644--- a/cyberdrop_dl/downloader/hls.py+++ b/cyberdrop_dl/downloader/hls.py@@ -56,16 +56,20 @@ class HLSDecrypter(Protocol):@dataclasses.dataclass(slots=True, frozen=True)class AESHLSDecrypter(HLSDecrypter):client: HTTPClient- _lock: asyncio.Lock = dataclasses.field(default_factory=asyncio.Lock)+ _locks: aio.WeakAsyncLocks[AbsoluteHttpURL] = dataclasses.field(default_factory=aio.WeakAsyncLocks)_cache: dict[AbsoluteHttpURL, bytes] = dataclasses.field(default_factory=dict)async def get_key(self, uri: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> bytes:- if aes_key := self._cache.get(uri):- return aes_key+ try:+ return self._cache[uri]+ except LookupError:+ pass- async with self._lock:- if aes_key := self._cache.get(uri):- return aes_key+ async with self._locks[uri]:+ try:+ return self._cache[uri]+ except LookupError:+ passasync with self.client.raw_request(uri, headers=headers or {}) as resp:aes_key = self._cache[uri] = await resp.read()@@ -85,7 +89,7 @@ def _create_segments(segments: Iterable[Segment | InitializationSection], count:idx=index - 1,name=f"{index:0{padding}d}{constants.TempExt.HLS}",url=parse_url(segment.absolute_uri, trim=False),- decrypt_info=HLSKey.parse(segment),+ decrypt_info=HLSKey.parse(segment, idx=index - 1),)diff --git a/cyberdrop_dl/signature.py b/cyberdrop_dl/signature.pyindex 1eaa1d183..fec41d41e 100644--- a/cyberdrop_dl/signature.py+++ b/cyberdrop_dl/signature.py@@ -38,3 +38,8 @@ def simple_repr(*names: str) -> Callable[..., str]:return f"<{type(self).__name__}({', '.join(fields)}>"return repr_+++def vars_repr(self: object) -> str:+ fields = (f"{name}={val!r}" for name, val in vars(self).items())+ return f"<{type(self).__name__}({', '.join(fields)}>"diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.pyindex 8fd2f4a38..b62907f43 100644--- a/cyberdrop_dl/utils/m3u8.py+++ b/cyberdrop_dl/utils/m3u8.py@@ -3,16 +3,18 @@ from __future__ import annotationsimport dataclassesimport datetime as dtimport logging+import structfrom enum import StrEnumfrom typing import TYPE_CHECKING, Any, ClassVar, Literal, NamedTuple, Selfimport m3u8.modelfrom m3u8 import M3U8 as _M3U8from m3u8 import Media, Playlist+from m3u8.mixins import BasePathMixin, GroupedBasePathMixinfrom cyberdrop_dl.exceptions import DownloadErrorfrom cyberdrop_dl.mediaprops import Codecs, Resolution-from cyberdrop_dl.signature import simple_repr+from cyberdrop_dl.signature import simple_repr, vars_reprfrom cyberdrop_dl.utils import parse_urlfrom cyberdrop_dl.utils.dataclass import DictDataclass@@ -25,6 +27,9 @@ if TYPE_CHECKING:logger = logging.getLogger(__name__)+BasePathMixin.__repr__ = GroupedBasePathMixin.__repr__ = vars_repr++class MediaType(StrEnum):audio = "AUDIO"video = "VIDEO"@@ -186,14 +191,14 @@ class HLSKey:raise RuntimeError("Another key is already set")@classmethod- def parse(cls, seg: m3u8.model.Segment | m3u8.model.InitializationSection) -> Self | None:+ def parse(cls, seg: m3u8.model.Segment | m3u8.model.InitializationSection, idx: int) -> Self | None:if isinstance(seg, m3u8.model.Segment) and seg.key and seg.key.method and seg.key.method != "NONE":- assert seg.key.iv+ iv = bytes.fromhex(seg.key.iv.removeprefix("0x")) if seg.key.iv else struct.pack(">8xq", idx)assert seg.key.urireturn cls(method=Encryption(seg.key.method),- iv=bytes.fromhex(seg.key.iv.removeprefix("0x")),- uri=parse_url(seg.key.uri, trim=False),+ iv=iv,+ uri=parse_url(seg.key.absolute_uri, trim=False),)@@ -207,10 +212,10 @@ def _validate_keys(stream: M3U8) -> None:raise DownloadError("Encrypted", f"M3U8 playlist {stream.source} has an unsupported encryption algorithm: \n{key!s}")- if not key.iv or not key.uri:+ if not key.uri:raise DownloadError("Encrypted",- f"M3U8 playlist {stream.source} is encrypted but we have no IV to decrypted it: \n{key!s}",+ f"M3U8 playlist {stream.source} is encrypted but we have no URI to fetch to decrypted it: \n{key!s}",)