Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: add m3u8 module (#981)
NTFSvolumed7fb19cac473
diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.pynew file mode 100644index 000000000..f1ec9dbe4--- /dev/null+++ b/cyberdrop_dl/utils/m3u8.py@@ -0,0 +1,251 @@+from __future__ import annotations++import re+from dataclasses import dataclass+from datetime import timedelta+from enum import StrEnum+from fractions import Fraction+from functools import cached_property+from typing import TYPE_CHECKING, NamedTuple++from m3u8 import M3U8 as _M3U8+from m3u8 import Media, Playlist++from cyberdrop_dl.utils.utilities import parse_url++if TYPE_CHECKING:+ from collections.abc import Generator, Iterable++ from m3u8.model import StreamInfo+ from yarl import URL++RESOLUTION_REGEX = [+ re.compile(regex)+ for regex in [+ r"(?P<width>\d+)x(?P<height>\d+)", # ex: 1920x1080+ r"(?P<height_p>\d+)p", # ex: 1080p, 720p+ ]+]+++VIDEO_CODECS = "avc1", "avc2", "avc3", "avc4", "av1", "hevc", "hev1", "hev2", "hvc1", "hvc2", "vp8", "vp9", "vp10"+AUDIO_CODECS = "ac-3", "ec-3", "mp3", "mp4a", "opus", "vorbis"+++class Codecs(NamedTuple):+ video: str+ audio: str | None++ @staticmethod+ def parse(codecs: str) -> Codecs:+ video_codec = audio_codec = None++ def codec_or_none(codec: str, lookup_array: Iterable[str]) -> str | None:+ codec, *_ = codec.split(".")+ clean_codec = codec[:-1].replace("0", "") + codec[-1]+ return next((key for key in lookup_array if clean_codec.startswith(key)), None)++ for codec in codecs.split(","):+ if not video_codec and (found := codec_or_none(codec, VIDEO_CODECS)):+ video_codec = found+ elif not audio_codec and (found := codec_or_none(codec, AUDIO_CODECS)):+ audio_codec = found+ if video_codec and audio_codec:+ break++ assert video_codec+ if "avc" in video_codec:+ video_codec = "avc1"+ elif "hev" in video_codec or "hvc" in video_codec:+ video_codec = "hevc"+ return Codecs(video_codec, audio_codec)+++class Resolution(NamedTuple):+ width: int+ height: int++ def name(self) -> str:+ if 7600 < self.width < 8200:+ return "8K"+ if 3800 < self.width < 4100:+ return "4K"+ return f"{self.height}p"++ def aspect_ratio(self) -> Fraction:+ return Fraction(self.width, self.height)+++RESOLUTIONS = [+ Resolution(*resolution) # Best to worst+ for resolution in [+ (7680, 4320),+ (3840, 2160),+ (2560, 1440),+ (1920, 1080),+ (1280, 720),+ (640, 480),+ (640, 360),+ (480, 320),+ (426, 240),+ (320, 240),+ (256, 144),+ ]+]+++class MediaType(StrEnum):+ audio = "AUDIO"+ video = "VIDEO"+ subtitles = "SUBTITLES"+ closed_captions = "CLOSED-CAPTIONS"+++class MediaList(list[Media]):+ def filter_by(+ self,+ type: MediaType | str | None = None,+ group_id: str | None = None,+ language: str | None = None,+ name: str | None = None,+ ) -> Generator[Media]:+ assert any(value is not None for value in (type, group_id, language, name))+ attrs = {name: value for name, value in locals().items() if value is not None and name != "self"}+ for media in self:+ if all(getattr(media, name) == value for name, value in attrs.items()):+ yield media++ def filter(+ self,+ type: MediaType | str | None = None,+ group_id: str | None = None,+ language: str | None = None,+ name: str | None = None,+ ) -> MediaList:+ return MediaList(self.filter_by(type, group_id, language, name))++ def get_default(self) -> Media | None:+ if self:+ for media in self:+ if media.default == "YES":+ return media++ for media in self:+ if media.autoselect == "YES":+ return media+++class MediaURLs(NamedTuple):+ video: URL+ audio: URL | None+ subtitle: URL | None+++@dataclass(frozen=True, slots=True, order=True)+class RenditionGroup:+ resolution: Resolution+ codecs: Codecs+ stream_info: StreamInfo+ media: MediaList+ urls: MediaURLs++ @staticmethod+ def new(playlist: Playlist) -> RenditionGroup:+ assert playlist.uri+ assert playlist.stream_info.codecs++ def get_url(m3u8_obj: Playlist | Media) -> URL:+ return parse_url(m3u8_obj.absolute_uri, trim=False)++ video_url = get_url(playlist)+ subtitle_url = audio_url = None++ if playlist.stream_info.resolution is not None:+ resolution: Resolution = Resolution(*playlist.stream_info.resolution)+ else:+ resolution = get_resolution_from_url(video_url)++ codecs = Codecs.parse(playlist.stream_info.codecs)+ media = MediaList(playlist.media)++ if audio_group_id := playlist.stream_info.audio:+ audio = next(media.filter_by(type=MediaType.audio, group_id=audio_group_id))+ audio_url: URL | None = get_url(audio)++ if subtitle_group_id := playlist.stream_info.subtitles:+ subtitle = next(media.filter_by(type=MediaType.subtitles, group_id=subtitle_group_id))+ subtitle_url: URL | None = get_url(subtitle)++ media_urls = MediaURLs(video_url, audio_url, subtitle_url)+ return RenditionGroup(resolution, codecs, playlist.stream_info, media, media_urls)+++class M3U8(_M3U8):+ @cached_property+ def total_duration(self) -> timedelta:+ total_duration: float = sum(duration for segment in self.segments if (duration := segment.duration))+ return timedelta(seconds=total_duration)++ @staticmethod+ def new(m3u8_content: str, url: URL) -> M3U8:+ return M3U8(m3u8_content, base_uri=str(url.parent))++ @property+ def is_master_playlist(self):+ return self.is_variant or not self.segments++ def as_variant(self) -> VariantM3U8:+ return VariantM3U8(self)+++class VariantM3U8:+ """M3U8 that has links to other M3U8s inside"""++ def __init__(self, m3u8: M3U8):+ assert m3u8.is_master_playlist+ self._m3u8 = m3u8+ self.groups = sorted((RenditionGroup.new(playlist) for playlist in m3u8.playlists), reverse=True)++ def get_rendition_groups(+ self, only: Iterable[str] = (), *, exclude: Iterable[str] = ()+ ) -> Generator[RenditionGroup]:+ """Yields M3U8 options, best to worst"""+ assert not (only and exclude)+ if isinstance(exclude, str):+ exclude = (exclude,)+ if isinstance(only, str):+ only = (only,)+ for group in self.groups:+ if only and group.codecs.video not in only:+ continue+ if group.codecs.video not in exclude:+ yield group++ def get_best_group(self, only: Iterable[str] = (), *, exclude: Iterable[str] = ()) -> RenditionGroup:+ return next(self.get_rendition_groups(only=only, exclude=exclude))+++def get_resolution_from_url(url: URL | str) -> Resolution:+ if isinstance(url, str):+ url = parse_url(url, trim=False) # Raises Invalid URL error for relative URLs++ for resolution in RESOLUTIONS:+ if str(resolution.height) in url.parts:+ return resolution++ for pattern in RESOLUTION_REGEX:+ if match := re.search(pattern, url.path):+ try:+ width = int(match.group("width"))+ height = int(match.group("height"))+ return Resolution(width, height)+ except (IndexError, KeyError):+ height_ = match.groupdict().get("height_p")+ if not height_:+ continue++ height = int(height_)+ width = round(height * 16 / 9) # Assume 16:9 aspect ratio+ return Resolution(width, height)++ raise RuntimeError("Unable to parse resolution")diff --git a/poetry.lock b/poetry.lockindex 47c96a124..7bed70233 100644--- a/poetry.lock+++ b/poetry.lock@@ -1,4 +1,4 @@-# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.+# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand.[[package]]name = "aiofiles"@@ -1040,6 +1040,18 @@ docs = ["sphinx (>=1.6.0)", "sphinx_bootstrap_theme"]flake8 = ["flake8"]tests = ["psutil", "pytest (!=3.3.0)", "pytest-cov"]+[[package]]+name = "m3u8"+version = "6.0.0"+description = "Python m3u8 parser"+optional = false+python-versions = ">=3.7"+groups = ["main"]+files = [+ {file = "m3u8-6.0.0-py3-none-any.whl", hash = "sha256:566d0748739c552dad10f8c87150078de6a0ec25071fa48e6968e96fc6dcba5d"},+ {file = "m3u8-6.0.0.tar.gz", hash = "sha256:7ade990a1667d7a653bcaf9413b16c3eb5cd618982ff46aaff57fe6d9fa9c0fd"},+]+[[package]]name = "markdown"version = "3.8"@@ -2721,4 +2733,4 @@ propcache = ">=0.2.1"[metadata]lock-version = "2.1"python-versions = ">=3.11,<4"-content-hash = "9ccc9f73a004c0f5f1fc9ff5ce4ccd8f3239c059e970542791c16765449d95c0"+content-hash = "44476db2ce4d92ff818203b0506dc312595d32b17674b4d78e9b2c568b5f0d67"diff --git a/pyproject.toml b/pyproject.tomlindex dd4786326..395edc666 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -16,6 +16,7 @@ dependencies = ["curl-cffi >=0.10.0","get-video-properties >=0.1.1","inquirerpy >=0.3.4",+ "m3u8 >=6.0.0","mediafire >=0.6.1","myjdapi >=1.1.9","packaging >=24.2",diff --git a/tests/test_m3u8.py b/tests/test_m3u8.pynew file mode 100644index 000000000..a5c763c3b--- /dev/null+++ b/tests/test_m3u8.py@@ -0,0 +1,193 @@+from datetime import timedelta++import pytest+from yarl import URL++from cyberdrop_dl.clients.errors import InvalidURLError+from cyberdrop_dl.utils import m3u8+++def m3u8_content():+ return """+ #EXTM3U+ #EXT-X-VERSION:3+ #EXT-X-TARGETDURATION:10+ #EXT-X-MEDIA-SEQUENCE:0++ #EXTINF:10.000,+ media-segment-00001.ts+ #EXTINF:10.000,+ media-segment-00002.ts+ #EXTINF:8.500,+ media-segment-00003.ts+ #EXT-X-ENDLIST+ """+++def m3u8_master_content():+ return """+ #EXTM3U+ #EXT-X-VERSION:3++ #EXT-X-STREAM-INF:BANDWIDTH=1500000,RESOLUTION=640x360,CODECS="avc1.4d401f,mp4a.40.2"+ low/stream.m3u8++ #EXT-X-STREAM-INF:BANDWIDTH=3000000,RESOLUTION=1280x720,CODECS="avc1.4d402a,mp4a.40.2"+ medium/stream.m3u8++ #EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=1920x1080,CODECS="avc1.4d402j,mp4a.40.2"+ high/stream.m3u8+ """+++def m3u8_master_content2():+ return """+ #EXTM3U+ #EXT-X-VERSION:4++ #EXT-X-MEDIA:AUTOSELECT=YES,DEFAULT=NO,GROUP-ID="audio",LANGUAGE="en",TYPE=AUDIO,NAME="audio",URI="audio/audio.m3u8"++ #EXT-X-STREAM-INF:BANDWIDTH=4960946,AVERAGE-BANDWIDTH=4461993,CODECS="vp09.00.40.08.00.02.02.02.00,mp4a.40.2",RESOLUTION=1080x1920,FRAME-RATE=60.000,AUDIO="audio"+ vp9_1080p/video.m3u8+ #EXT-X-STREAM-INF:BANDWIDTH=2803971,AVERAGE-BANDWIDTH=2509229,CODECS="vp09.00.31.08.00.02.02.02.00,mp4a.40.2",RESOLUTION=720x1280,FRAME-RATE=60.000,AUDIO="audio"+ vp9_720p/video.m3u8+ #EXT-X-STREAM-INF:BANDWIDTH=1230307,AVERAGE-BANDWIDTH=1127509,CODECS="vp09.00.30.08.00.02.02.02.00,mp4a.40.2",RESOLUTION=480x854,FRAME-RATE=30.000,AUDIO="audio"+ vp9_480p/video.m3u8+ #EXT-X-STREAM-INF:BANDWIDTH=901303,AVERAGE-BANDWIDTH=811965,CODECS="vp09.00.21.08.00.02.02.02.00,mp4a.40.2",RESOLUTION=360x640,FRAME-RATE=30.000,AUDIO="audio"+ vp9_360p/video.m3u8+ #EXT-X-STREAM-INF:BANDWIDTH=629922,AVERAGE-BANDWIDTH=571504,CODECS="vp09.00.20.08.00.02.02.02.00,mp4a.40.2",RESOLUTION=240x426,FRAME-RATE=30.000,AUDIO="audio"+ vp9_240p/video.m3u8+ #EXT-X-STREAM-INF:BANDWIDTH=8245366,AVERAGE-BANDWIDTH=7656815,CODECS="avc1.64002a,mp4a.40.2",RESOLUTION=1080x1920,FRAME-RATE=60.000,AUDIO="audio"+ 1080p/video.m3u8+ #EXT-X-STREAM-INF:BANDWIDTH=3878384,AVERAGE-BANDWIDTH=3558624,CODECS="avc1.640020,mp4a.40.2",RESOLUTION=720x1280,FRAME-RATE=60.000,AUDIO="audio"+ 720p/video.m3u8+ #EXT-X-STREAM-INF:BANDWIDTH=1716380,AVERAGE-BANDWIDTH=1564076,CODECS="avc1.64001f,mp4a.40.2",RESOLUTION=480x854,FRAME-RATE=30.000,AUDIO="audio"+ 480p/video.m3u8+ #EXT-X-STREAM-INF:BANDWIDTH=1181122,AVERAGE-BANDWIDTH=1072652,CODECS="avc1.64001e,mp4a.40.2",RESOLUTION=360x640,FRAME-RATE=30.000,AUDIO="audio"+ 360p/video.m3u8+ #EXT-X-STREAM-INF:BANDWIDTH=738423,AVERAGE-BANDWIDTH=671683,CODECS="avc1.640015,mp4a.40.2",RESOLUTION=240x426,FRAME-RATE=30.000,AUDIO="audio"+ 240p/video.m3u8"""+++ "url, resolution, name",+ [+ ("https://example.com/4b4ef277/720p/video.m3u8", (1280, 720), "720p"),+ ("https://example.com/4b4ef277/1920x1080p/video.m3u8", (1920, 1080), "1080p"),+ ("https://example.com/4b4ef277/1920x1234p/video.m3u8", (1920, 1234), "1234p"),+ ("https://example.com/4b4ef277/4096x2160/video.m3u8", (4096, 2160), "4K"),+ ("https://example.com/480/playlist.m3u8", (640, 480), "480p"),+ ],+)+def test_get_resolution_from_url(url: str, resolution: tuple[int, int], name: str) -> None:+ result = m3u8.get_resolution_from_url(url)+ assert result == resolution+ assert result.name() == name+++ "url, exception",+ [+ ("https://example.com/780/playlist.m3u8", RuntimeError),+ ("https://example.com", RuntimeError),+ ("/example.com", AttributeError),+ ("", InvalidURLError),+ ],+)+def test_get_resolution_from_url_invalid_url(url: str, exception: type[Exception]) -> None:+ with pytest.raises(exception):+ m3u8.get_resolution_from_url(url)+++ "codecs, result",+ [+ ("avc1.4d401f,mp4a.40.2", ("avc1", "mp4a")),+ ("hvc1.1.6.L93.B0,mp4a.40.2", ("hevc", "mp4a")),+ ("hevc1.1.6.L93.B0,mp4a.40.2", ("hevc", "mp4a")),+ ("vp09.00.10.08,opus", ("vp9", "opus")),+ ("vp10.00.10.08,opus", ("vp10", "opus")),+ ("vp9.00.10.08,opus", ("vp9", "opus")),+ ("av01.0.04M.08,opus", ("av1", "opus")),+ ("avc1.4d401f,ac-3", ("avc1", "ac-3")),+ ("avc01.4d401f,ec-3", ("avc1", "ec-3")),+ ("avc1.4d401f", ("avc1", None)),+ ],+)+def test_codecs_parse(codecs: str, result: m3u8.Codecs) -> None:+ assert m3u8.Codecs.parse(codecs) == result+++def test_m3u8(m3u8_content: str) -> None:+ m3u8_obj = m3u8.M3U8(m3u8_content)+ assert m3u8_obj.total_duration == timedelta(seconds=28.5)+ assert not m3u8_obj.is_master_playlist+ with pytest.raises(AssertionError):+ m3u8_obj.as_variant()+++def test_m3u8_master(m3u8_master_content: str) -> None:+ m3u8_obj = m3u8.M3U8.new(m3u8_master_content, URL("https://example.com/4b4ef277/playlist.m3u8"))+ assert m3u8_obj.is_master_playlist+ variant = m3u8_obj.as_variant()+ assert len(variant.groups) == 3+ best = variant.get_best_group()+ assert best.urls.video == URL("https://example.com/4b4ef277/high/stream.m3u8")+ assert best.urls.audio is None+ assert best.urls.subtitle is None+ assert best.resolution == (1920, 1080)+ assert best.codecs == ("avc1", "mp4a")+++def test_m3u8_master_exclude_codec(m3u8_master_content: str) -> None:+ variant = m3u8.M3U8.new(m3u8_master_content, URL("https://example.com/4b4ef277/playlist.m3u8")).as_variant()+ assert variant.get_best_group(exclude="hevc")+ with pytest.raises(StopIteration):+ variant.get_best_group(exclude="avc1")+ with pytest.raises(StopIteration):+ variant.get_best_group(only="vp9")+ with pytest.raises(AssertionError):+ variant.get_best_group(only="vp9", exclude="avc1")+++def test_m3u8_master2(m3u8_master_content2: str) -> None:+ m3u8_obj = m3u8.M3U8.new(m3u8_master_content2, URL("https://example.com/4b4ef277/playlist.m3u8"))+ assert m3u8_obj.is_master_playlist+ variant = m3u8_obj.as_variant()+ assert len(variant.groups) == 10+ best = variant.get_best_group()+ assert best.urls.video == URL("https://example.com/4b4ef277/vp9_1080p/video.m3u8")+ assert best.urls.audio == URL("https://example.com/4b4ef277/audio/audio.m3u8")+ assert best.urls.subtitle is None+ assert best.resolution == (1080, 1920)+ assert best.codecs == ("vp9", "mp4a")+++def test_m3u8_master2_exclude_codec(m3u8_master_content2: str) -> None:+ variant = m3u8.M3U8.new(m3u8_master_content2, URL("https://example.com/4b4ef277/playlist.m3u8")).as_variant()+ best = variant.get_best_group(exclude="vp9")+ assert best.codecs.video == "avc1"+ assert best.urls.video == URL("https://example.com/4b4ef277/1080p/video.m3u8")+ best = variant.get_best_group(only="vp9")+ assert best.codecs.video == "vp9"+ with pytest.raises(StopIteration):+ best = variant.get_best_group(exclude=["vp9", "avc1"])+ with pytest.raises(StopIteration):+ best = variant.get_best_group(only="hevc")+ with pytest.raises(AssertionError):+ best = variant.get_best_group(only="vp9", exclude="avc1")+++def test_m3u8_master2_audio(m3u8_master_content2: str) -> None:+ variant = m3u8.M3U8.new(m3u8_master_content2, URL("https://example.com/4b4ef277/playlist.m3u8")).as_variant()+ best = variant.get_best_group()+ assert best.media.filter(group_id="audio")+ assert not best.media.filter(group_id="AUDIO")+ audios = best.media.filter(type="AUDIO")+ assert len(audios) == 1+ assert audios.get_default()+ assert best.media.filter(language="en")+ assert not best.media.filter(language="jpn")