Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: add m3u8 module (#981)

NTFSvolumed7fb19cac473
diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.py
new file mode 100644
index 000000000..f1ec9dbe4
--- /dev/null
+++ b/cyberdrop_dl/utils/m3u8.py
@@ -0,0 +1,251 @@
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from datetime import timedelta
+from enum import StrEnum
+from fractions import Fraction
+from functools import cached_property
+from typing import TYPE_CHECKING, NamedTuple
+
+from m3u8 import M3U8 as _M3U8
+from m3u8 import Media, Playlist
+
+from cyberdrop_dl.utils.utilities import parse_url
+
+if TYPE_CHECKING:
+ from collections.abc import Generator, Iterable
+
+ from m3u8.model import StreamInfo
+ from yarl import URL
+
+RESOLUTION_REGEX = [
+ re.compile(regex)
+ for regex in [
+ r"(?P<width>\d+)x(?P<height>\d+)", # ex: 1920x1080
+ r"(?P<height_p>\d+)p", # ex: 1080p, 720p
+ ]
+]
+
+
+VIDEO_CODECS = "avc1", "avc2", "avc3", "avc4", "av1", "hevc", "hev1", "hev2", "hvc1", "hvc2", "vp8", "vp9", "vp10"
+AUDIO_CODECS = "ac-3", "ec-3", "mp3", "mp4a", "opus", "vorbis"
+
+
+class Codecs(NamedTuple):
+ video: str
+ audio: str | None
+
+ @staticmethod
+ def parse(codecs: str) -> Codecs:
+ video_codec = audio_codec = None
+
+ def codec_or_none(codec: str, lookup_array: Iterable[str]) -> str | None:
+ codec, *_ = codec.split(".")
+ clean_codec = codec[:-1].replace("0", "") + codec[-1]
+ return next((key for key in lookup_array if clean_codec.startswith(key)), None)
+
+ for codec in codecs.split(","):
+ if not video_codec and (found := codec_or_none(codec, VIDEO_CODECS)):
+ video_codec = found
+ elif not audio_codec and (found := codec_or_none(codec, AUDIO_CODECS)):
+ audio_codec = found
+ if video_codec and audio_codec:
+ break
+
+ assert video_codec
+ if "avc" in video_codec:
+ video_codec = "avc1"
+ elif "hev" in video_codec or "hvc" in video_codec:
+ video_codec = "hevc"
+ return Codecs(video_codec, audio_codec)
+
+
+class Resolution(NamedTuple):
+ width: int
+ height: int
+
+ def name(self) -> str:
+ if 7600 < self.width < 8200:
+ return "8K"
+ if 3800 < self.width < 4100:
+ return "4K"
+ return f"{self.height}p"
+
+ def aspect_ratio(self) -> Fraction:
+ return Fraction(self.width, self.height)
+
+
+RESOLUTIONS = [
+ Resolution(*resolution) # Best to worst
+ for resolution in [
+ (7680, 4320),
+ (3840, 2160),
+ (2560, 1440),
+ (1920, 1080),
+ (1280, 720),
+ (640, 480),
+ (640, 360),
+ (480, 320),
+ (426, 240),
+ (320, 240),
+ (256, 144),
+ ]
+]
+
+
+class MediaType(StrEnum):
+ audio = "AUDIO"
+ video = "VIDEO"
+ subtitles = "SUBTITLES"
+ closed_captions = "CLOSED-CAPTIONS"
+
+
+class MediaList(list[Media]):
+ def filter_by(
+ self,
+ type: MediaType | str | None = None,
+ group_id: str | None = None,
+ language: str | None = None,
+ name: str | None = None,
+ ) -> Generator[Media]:
+ assert any(value is not None for value in (type, group_id, language, name))
+ attrs = {name: value for name, value in locals().items() if value is not None and name != "self"}
+ for media in self:
+ if all(getattr(media, name) == value for name, value in attrs.items()):
+ yield media
+
+ def filter(
+ self,
+ type: MediaType | str | None = None,
+ group_id: str | None = None,
+ language: str | None = None,
+ name: str | None = None,
+ ) -> MediaList:
+ return MediaList(self.filter_by(type, group_id, language, name))
+
+ def get_default(self) -> Media | None:
+ if self:
+ for media in self:
+ if media.default == "YES":
+ return media
+
+ for media in self:
+ if media.autoselect == "YES":
+ return media
+
+
+class MediaURLs(NamedTuple):
+ video: URL
+ audio: URL | None
+ subtitle: URL | None
+
+
+@dataclass(frozen=True, slots=True, order=True)
+class RenditionGroup:
+ resolution: Resolution
+ codecs: Codecs
+ stream_info: StreamInfo
+ media: MediaList
+ urls: MediaURLs
+
+ @staticmethod
+ def new(playlist: Playlist) -> RenditionGroup:
+ assert playlist.uri
+ assert playlist.stream_info.codecs
+
+ def get_url(m3u8_obj: Playlist | Media) -> URL:
+ return parse_url(m3u8_obj.absolute_uri, trim=False)
+
+ video_url = get_url(playlist)
+ subtitle_url = audio_url = None
+
+ if playlist.stream_info.resolution is not None:
+ resolution: Resolution = Resolution(*playlist.stream_info.resolution)
+ else:
+ resolution = get_resolution_from_url(video_url)
+
+ codecs = Codecs.parse(playlist.stream_info.codecs)
+ media = MediaList(playlist.media)
+
+ if audio_group_id := playlist.stream_info.audio:
+ audio = next(media.filter_by(type=MediaType.audio, group_id=audio_group_id))
+ audio_url: URL | None = get_url(audio)
+
+ if subtitle_group_id := playlist.stream_info.subtitles:
+ subtitle = next(media.filter_by(type=MediaType.subtitles, group_id=subtitle_group_id))
+ subtitle_url: URL | None = get_url(subtitle)
+
+ media_urls = MediaURLs(video_url, audio_url, subtitle_url)
+ return RenditionGroup(resolution, codecs, playlist.stream_info, media, media_urls)
+
+
+class M3U8(_M3U8):
+ @cached_property
+ def total_duration(self) -> timedelta:
+ total_duration: float = sum(duration for segment in self.segments if (duration := segment.duration))
+ return timedelta(seconds=total_duration)
+
+ @staticmethod
+ def new(m3u8_content: str, url: URL) -> M3U8:
+ return M3U8(m3u8_content, base_uri=str(url.parent))
+
+ @property
+ def is_master_playlist(self):
+ return self.is_variant or not self.segments
+
+ def as_variant(self) -> VariantM3U8:
+ return VariantM3U8(self)
+
+
+class VariantM3U8:
+ """M3U8 that has links to other M3U8s inside"""
+
+ def __init__(self, m3u8: M3U8):
+ assert m3u8.is_master_playlist
+ self._m3u8 = m3u8
+ self.groups = sorted((RenditionGroup.new(playlist) for playlist in m3u8.playlists), reverse=True)
+
+ def get_rendition_groups(
+ self, only: Iterable[str] = (), *, exclude: Iterable[str] = ()
+ ) -> Generator[RenditionGroup]:
+ """Yields M3U8 options, best to worst"""
+ assert not (only and exclude)
+ if isinstance(exclude, str):
+ exclude = (exclude,)
+ if isinstance(only, str):
+ only = (only,)
+ for group in self.groups:
+ if only and group.codecs.video not in only:
+ continue
+ if group.codecs.video not in exclude:
+ yield group
+
+ def get_best_group(self, only: Iterable[str] = (), *, exclude: Iterable[str] = ()) -> RenditionGroup:
+ return next(self.get_rendition_groups(only=only, exclude=exclude))
+
+
+def get_resolution_from_url(url: URL | str) -> Resolution:
+ if isinstance(url, str):
+ url = parse_url(url, trim=False) # Raises Invalid URL error for relative URLs
+
+ for resolution in RESOLUTIONS:
+ if str(resolution.height) in url.parts:
+ return resolution
+
+ for pattern in RESOLUTION_REGEX:
+ if match := re.search(pattern, url.path):
+ try:
+ width = int(match.group("width"))
+ height = int(match.group("height"))
+ return Resolution(width, height)
+ except (IndexError, KeyError):
+ height_ = match.groupdict().get("height_p")
+ if not height_:
+ continue
+
+ height = int(height_)
+ width = round(height * 16 / 9) # Assume 16:9 aspect ratio
+ return Resolution(width, height)
+
+ raise RuntimeError("Unable to parse resolution")
diff --git a/poetry.lock b/poetry.lock
index 47c96a124..7bed70233 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -1,4 +1,4 @@
-# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand.
+# This file is automatically @generated by Poetry 2.1.3 and should not be changed by hand.
[[package]]
name = "aiofiles"
@@ -1040,6 +1040,18 @@ docs = ["sphinx (>=1.6.0)", "sphinx_bootstrap_theme"]
flake8 = ["flake8"]
tests = ["psutil", "pytest (!=3.3.0)", "pytest-cov"]
+[[package]]
+name = "m3u8"
+version = "6.0.0"
+description = "Python m3u8 parser"
+optional = false
+python-versions = ">=3.7"
+groups = ["main"]
+files = [
+ {file = "m3u8-6.0.0-py3-none-any.whl", hash = "sha256:566d0748739c552dad10f8c87150078de6a0ec25071fa48e6968e96fc6dcba5d"},
+ {file = "m3u8-6.0.0.tar.gz", hash = "sha256:7ade990a1667d7a653bcaf9413b16c3eb5cd618982ff46aaff57fe6d9fa9c0fd"},
+]
+
[[package]]
name = "markdown"
version = "3.8"
@@ -2721,4 +2733,4 @@ propcache = ">=0.2.1"
[metadata]
lock-version = "2.1"
python-versions = ">=3.11,<4"
-content-hash = "9ccc9f73a004c0f5f1fc9ff5ce4ccd8f3239c059e970542791c16765449d95c0"
+content-hash = "44476db2ce4d92ff818203b0506dc312595d32b17674b4d78e9b2c568b5f0d67"
diff --git a/pyproject.toml b/pyproject.toml
index dd4786326..395edc666 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -16,6 +16,7 @@ dependencies = [
"curl-cffi >=0.10.0",
"get-video-properties >=0.1.1",
"inquirerpy >=0.3.4",
+ "m3u8 >=6.0.0",
"mediafire >=0.6.1",
"myjdapi >=1.1.9",
"packaging >=24.2",
diff --git a/tests/test_m3u8.py b/tests/test_m3u8.py
new file mode 100644
index 000000000..a5c763c3b
--- /dev/null
+++ b/tests/test_m3u8.py
@@ -0,0 +1,193 @@
+from datetime import timedelta
+
+import pytest
+from yarl import URL
+
+from cyberdrop_dl.clients.errors import InvalidURLError
+from cyberdrop_dl.utils import m3u8
+
+
+def m3u8_content():
+ return """
+ #EXTM3U
+ #EXT-X-VERSION:3
+ #EXT-X-TARGETDURATION:10
+ #EXT-X-MEDIA-SEQUENCE:0
+
+ #EXTINF:10.000,
+ media-segment-00001.ts
+ #EXTINF:10.000,
+ media-segment-00002.ts
+ #EXTINF:8.500,
+ media-segment-00003.ts
+ #EXT-X-ENDLIST
+ """
+
+
+def m3u8_master_content():
+ return """
+ #EXTM3U
+ #EXT-X-VERSION:3
+
+ #EXT-X-STREAM-INF:BANDWIDTH=1500000,RESOLUTION=640x360,CODECS="avc1.4d401f,mp4a.40.2"
+ low/stream.m3u8
+
+ #EXT-X-STREAM-INF:BANDWIDTH=3000000,RESOLUTION=1280x720,CODECS="avc1.4d402a,mp4a.40.2"
+ medium/stream.m3u8
+
+ #EXT-X-STREAM-INF:BANDWIDTH=5000000,RESOLUTION=1920x1080,CODECS="avc1.4d402j,mp4a.40.2"
+ high/stream.m3u8
+ """
+
+
+def m3u8_master_content2():
+ return """
+ #EXTM3U
+ #EXT-X-VERSION:4
+
+ #EXT-X-MEDIA:AUTOSELECT=YES,DEFAULT=NO,GROUP-ID="audio",LANGUAGE="en",TYPE=AUDIO,NAME="audio",URI="audio/audio.m3u8"
+
+ #EXT-X-STREAM-INF:BANDWIDTH=4960946,AVERAGE-BANDWIDTH=4461993,CODECS="vp09.00.40.08.00.02.02.02.00,mp4a.40.2",RESOLUTION=1080x1920,FRAME-RATE=60.000,AUDIO="audio"
+ vp9_1080p/video.m3u8
+ #EXT-X-STREAM-INF:BANDWIDTH=2803971,AVERAGE-BANDWIDTH=2509229,CODECS="vp09.00.31.08.00.02.02.02.00,mp4a.40.2",RESOLUTION=720x1280,FRAME-RATE=60.000,AUDIO="audio"
+ vp9_720p/video.m3u8
+ #EXT-X-STREAM-INF:BANDWIDTH=1230307,AVERAGE-BANDWIDTH=1127509,CODECS="vp09.00.30.08.00.02.02.02.00,mp4a.40.2",RESOLUTION=480x854,FRAME-RATE=30.000,AUDIO="audio"
+ vp9_480p/video.m3u8
+ #EXT-X-STREAM-INF:BANDWIDTH=901303,AVERAGE-BANDWIDTH=811965,CODECS="vp09.00.21.08.00.02.02.02.00,mp4a.40.2",RESOLUTION=360x640,FRAME-RATE=30.000,AUDIO="audio"
+ vp9_360p/video.m3u8
+ #EXT-X-STREAM-INF:BANDWIDTH=629922,AVERAGE-BANDWIDTH=571504,CODECS="vp09.00.20.08.00.02.02.02.00,mp4a.40.2",RESOLUTION=240x426,FRAME-RATE=30.000,AUDIO="audio"
+ vp9_240p/video.m3u8
+ #EXT-X-STREAM-INF:BANDWIDTH=8245366,AVERAGE-BANDWIDTH=7656815,CODECS="avc1.64002a,mp4a.40.2",RESOLUTION=1080x1920,FRAME-RATE=60.000,AUDIO="audio"
+ 1080p/video.m3u8
+ #EXT-X-STREAM-INF:BANDWIDTH=3878384,AVERAGE-BANDWIDTH=3558624,CODECS="avc1.640020,mp4a.40.2",RESOLUTION=720x1280,FRAME-RATE=60.000,AUDIO="audio"
+ 720p/video.m3u8
+ #EXT-X-STREAM-INF:BANDWIDTH=1716380,AVERAGE-BANDWIDTH=1564076,CODECS="avc1.64001f,mp4a.40.2",RESOLUTION=480x854,FRAME-RATE=30.000,AUDIO="audio"
+ 480p/video.m3u8
+ #EXT-X-STREAM-INF:BANDWIDTH=1181122,AVERAGE-BANDWIDTH=1072652,CODECS="avc1.64001e,mp4a.40.2",RESOLUTION=360x640,FRAME-RATE=30.000,AUDIO="audio"
+ 360p/video.m3u8
+ #EXT-X-STREAM-INF:BANDWIDTH=738423,AVERAGE-BANDWIDTH=671683,CODECS="avc1.640015,mp4a.40.2",RESOLUTION=240x426,FRAME-RATE=30.000,AUDIO="audio"
+ 240p/video.m3u8"""
+
+
+ "url, resolution, name",
+ [
+ ("https://example.com/4b4ef277/720p/video.m3u8", (1280, 720), "720p"),
+ ("https://example.com/4b4ef277/1920x1080p/video.m3u8", (1920, 1080), "1080p"),
+ ("https://example.com/4b4ef277/1920x1234p/video.m3u8", (1920, 1234), "1234p"),
+ ("https://example.com/4b4ef277/4096x2160/video.m3u8", (4096, 2160), "4K"),
+ ("https://example.com/480/playlist.m3u8", (640, 480), "480p"),
+ ],
+)
+def test_get_resolution_from_url(url: str, resolution: tuple[int, int], name: str) -> None:
+ result = m3u8.get_resolution_from_url(url)
+ assert result == resolution
+ assert result.name() == name
+
+
+ "url, exception",
+ [
+ ("https://example.com/780/playlist.m3u8", RuntimeError),
+ ("https://example.com", RuntimeError),
+ ("/example.com", AttributeError),
+ ("", InvalidURLError),
+ ],
+)
+def test_get_resolution_from_url_invalid_url(url: str, exception: type[Exception]) -> None:
+ with pytest.raises(exception):
+ m3u8.get_resolution_from_url(url)
+
+
+ "codecs, result",
+ [
+ ("avc1.4d401f,mp4a.40.2", ("avc1", "mp4a")),
+ ("hvc1.1.6.L93.B0,mp4a.40.2", ("hevc", "mp4a")),
+ ("hevc1.1.6.L93.B0,mp4a.40.2", ("hevc", "mp4a")),
+ ("vp09.00.10.08,opus", ("vp9", "opus")),
+ ("vp10.00.10.08,opus", ("vp10", "opus")),
+ ("vp9.00.10.08,opus", ("vp9", "opus")),
+ ("av01.0.04M.08,opus", ("av1", "opus")),
+ ("avc1.4d401f,ac-3", ("avc1", "ac-3")),
+ ("avc01.4d401f,ec-3", ("avc1", "ec-3")),
+ ("avc1.4d401f", ("avc1", None)),
+ ],
+)
+def test_codecs_parse(codecs: str, result: m3u8.Codecs) -> None:
+ assert m3u8.Codecs.parse(codecs) == result
+
+
+def test_m3u8(m3u8_content: str) -> None:
+ m3u8_obj = m3u8.M3U8(m3u8_content)
+ assert m3u8_obj.total_duration == timedelta(seconds=28.5)
+ assert not m3u8_obj.is_master_playlist
+ with pytest.raises(AssertionError):
+ m3u8_obj.as_variant()
+
+
+def test_m3u8_master(m3u8_master_content: str) -> None:
+ m3u8_obj = m3u8.M3U8.new(m3u8_master_content, URL("https://example.com/4b4ef277/playlist.m3u8"))
+ assert m3u8_obj.is_master_playlist
+ variant = m3u8_obj.as_variant()
+ assert len(variant.groups) == 3
+ best = variant.get_best_group()
+ assert best.urls.video == URL("https://example.com/4b4ef277/high/stream.m3u8")
+ assert best.urls.audio is None
+ assert best.urls.subtitle is None
+ assert best.resolution == (1920, 1080)
+ assert best.codecs == ("avc1", "mp4a")
+
+
+def test_m3u8_master_exclude_codec(m3u8_master_content: str) -> None:
+ variant = m3u8.M3U8.new(m3u8_master_content, URL("https://example.com/4b4ef277/playlist.m3u8")).as_variant()
+ assert variant.get_best_group(exclude="hevc")
+ with pytest.raises(StopIteration):
+ variant.get_best_group(exclude="avc1")
+ with pytest.raises(StopIteration):
+ variant.get_best_group(only="vp9")
+ with pytest.raises(AssertionError):
+ variant.get_best_group(only="vp9", exclude="avc1")
+
+
+def test_m3u8_master2(m3u8_master_content2: str) -> None:
+ m3u8_obj = m3u8.M3U8.new(m3u8_master_content2, URL("https://example.com/4b4ef277/playlist.m3u8"))
+ assert m3u8_obj.is_master_playlist
+ variant = m3u8_obj.as_variant()
+ assert len(variant.groups) == 10
+ best = variant.get_best_group()
+ assert best.urls.video == URL("https://example.com/4b4ef277/vp9_1080p/video.m3u8")
+ assert best.urls.audio == URL("https://example.com/4b4ef277/audio/audio.m3u8")
+ assert best.urls.subtitle is None
+ assert best.resolution == (1080, 1920)
+ assert best.codecs == ("vp9", "mp4a")
+
+
+def test_m3u8_master2_exclude_codec(m3u8_master_content2: str) -> None:
+ variant = m3u8.M3U8.new(m3u8_master_content2, URL("https://example.com/4b4ef277/playlist.m3u8")).as_variant()
+ best = variant.get_best_group(exclude="vp9")
+ assert best.codecs.video == "avc1"
+ assert best.urls.video == URL("https://example.com/4b4ef277/1080p/video.m3u8")
+ best = variant.get_best_group(only="vp9")
+ assert best.codecs.video == "vp9"
+ with pytest.raises(StopIteration):
+ best = variant.get_best_group(exclude=["vp9", "avc1"])
+ with pytest.raises(StopIteration):
+ best = variant.get_best_group(only="hevc")
+ with pytest.raises(AssertionError):
+ best = variant.get_best_group(only="vp9", exclude="avc1")
+
+
+def test_m3u8_master2_audio(m3u8_master_content2: str) -> None:
+ variant = m3u8.M3U8.new(m3u8_master_content2, URL("https://example.com/4b4ef277/playlist.m3u8")).as_variant()
+ best = variant.get_best_group()
+ assert best.media.filter(group_id="audio")
+ assert not best.media.filter(group_id="AUDIO")
+ audios = best.media.filter(type="AUDIO")
+ assert len(audios) == 1
+ assert audios.get_default()
+ assert best.media.filter(language="en")
+ assert not best.media.filter(language="jpn")