Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: move HLS parsing logic to its own class (#92)

NTFSvolume488a01e6550c
diff --git a/cyberdrop_dl/crawlers/_hls.py b/cyberdrop_dl/crawlers/_hls.py
new file mode 100644
index 000000000..7caa36872
--- /dev/null
+++ b/cyberdrop_dl/crawlers/_hls.py
@@ -0,0 +1,95 @@
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING, Literal
+
+from cyberdrop_dl.utils import m3u8
+
+if TYPE_CHECKING:
+ from collections.abc import Iterable, Mapping
+
+ import aiohttp
+
+ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+
+
+class HLSParser(ABC):
+ """Class to fetch and parse HTTP live streams
+
+ For multi variant m3u8, the best resolution will be automatically selected"""
+
+ @abstractmethod
+ async def request_text(
+ self,
+ url: AbsoluteHttpURL,
+ /,
+ method: Literal["GET"] = "GET",
+ headers: Mapping[str, str] | None = None,
+ ) -> str: ...
+
+ async def request_m3u8(
+ self,
+ url: AbsoluteHttpURL,
+ /,
+ headers: Mapping[str, str] | None = None,
+ *,
+ only: Iterable[str] = (),
+ exclude: Iterable[str] = ("vp09",),
+ ) -> tuple[m3u8.Rendition, m3u8.RenditionDetails | None]:
+ m3u8_obj = await self._request_m3u8(url, headers)
+ if m3u8_obj.is_variant:
+ rendition = m3u8.select_best_rendition(m3u8_obj, only=only, exclude=exclude)
+ return await self._resolve_rendition(rendition, headers)
+ m3u8_obj.media_type = "video"
+ return m3u8.Rendition(m3u8_obj), None
+
+ async def _resolve_rendition(
+ self,
+ rendition: m3u8.RenditionDetails,
+ /,
+ headers: Mapping[str, str] | None = None,
+ ):
+ video, *audio_and_subs = await asyncio.gather(
+ *(
+ self._request_m3u8(url, headers, name)
+ for name, url in zip(
+ ("video", "audio", "subtitle"),
+ rendition.urls,
+ strict=True,
+ )
+ if url
+ )
+ )
+
+ return m3u8.Rendition(video, *audio_and_subs), rendition
+
+ async def _request_m3u8(
+ self,
+ url: AbsoluteHttpURL,
+ /,
+ headers: Mapping[str, str] | None = None,
+ media_type: Literal["video", "audio", "subtitle"] | None = None,
+ ) -> m3u8.M3U8:
+ content = await self.request_text(url, headers=headers)
+ return m3u8.M3U8(content, url.parent, media_type)
+
+
[email protected](slots=True)
+class SimpleHLSParser(HLSParser):
+ """A simple parser that does not depend on the manager.
+
+ DO NOT USE. This is only for testing"""
+
+ _session: aiohttp.ClientSession
+
+ async def request_text(
+ self,
+ url: AbsoluteHttpURL,
+ /,
+ method: Literal["GET"] = "GET",
+ headers: Mapping[str, str] | None = None,
+ ) -> str:
+ async with self._session.get(url, headers=headers) as resp:
+ return await resp.text()
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 72af539e1..e22d11f79 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -29,9 +29,11 @@ from typing import (
)
from aiolimiter import AsyncLimiter
+from typing_extensions import deprecated
from cyberdrop_dl import constants
from cyberdrop_dl.clients import HTTPClient, HTTPClientProxy
+from cyberdrop_dl.crawlers._hls import HLSParser
from cyberdrop_dl.data_structures.mediaprops import ISO639Subtitle, Resolution
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItem
from cyberdrop_dl.downloader.downloader import Downloader
@@ -54,7 +56,7 @@ from cyberdrop_dl.utils.utilities import (
)
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator, Callable, Coroutine, Generator, Iterable
+ from collections.abc import AsyncGenerator, Callable, Coroutine, Generator, Iterable, Mapping
from http.cookies import BaseCookie
from types import ModuleType
@@ -81,15 +83,15 @@ VALID_RESOLUTION_NAMES = "4K", "8K", "HQ", "Unknown"
@dataclasses.dataclass(slots=True, frozen=True)
-class PlaceHolderConfig:
- include_file_id: bool = True
- include_video_codec: bool = True
- include_audio_codec: bool = True
- include_resolution: bool = True
- include_hash: bool = True
+class _PlaceHolderConfigInclude:
+ file_id: bool = True
+ video_codec: bool = True
+ audio_codec: bool = True
+ resolution: bool = True
+ hash: bool = True
-_placeholder_config = PlaceHolderConfig()
+_include = _PlaceHolderConfigInclude()
_DB_PATH_BUILDERS: dict[str, Callable[[AbsoluteHttpURL], str]] = {
"url": lambda url: str(url),
@@ -144,7 +146,7 @@ class Registry:
cls._import_from(sub_module)
-class Crawler(HTTPClientProxy, ABC):
+class Crawler(HTTPClientProxy, HLSParser, ABC):
OLD_DOMAINS: ClassVar[tuple[str, ...]] = ()
SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ()
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {}
@@ -216,7 +218,6 @@ class Crawler(HTTPClientProxy, ABC):
cls,
is_abc: bool = False,
is_generic: bool = False,
- generic_name: str = "",
db_path: Literal["url", "name", "path", "path_qs", "path_qs_frag", "path_frag"] | None = None,
**kwargs,
) -> None:
@@ -238,9 +239,8 @@ class Crawler(HTTPClientProxy, ABC):
cls.__db_path__ = staticmethod(_DB_PATH_BUILDERS[db_path])
if cls.IS_GENERIC:
- cls.GENERIC_NAME: str = generic_name or cls.NAME
cls.SCRAPE_MAPPER_KEYS = ()
- cls.INFO: CrawlerInfo = CrawlerInfo.generic(cls.GENERIC_NAME, cls.SUPPORTED_PATHS)
+ cls.INFO: CrawlerInfo = CrawlerInfo.generic(cls.NAME, cls.SUPPORTED_PATHS)
Registry.generic.add(cls)
return
@@ -415,7 +415,7 @@ class Crawler(HTTPClientProxy, ABC):
*,
custom_filename: str | None = None,
debrid_link: AbsoluteHttpURL | None = None,
- m3u8: m3u8.RenditionGroup | None = None,
+ m3u8: m3u8.Rendition | None = None,
metadata: object = None,
referer: AbsoluteHttpURL | None = None,
) -> None:
@@ -444,7 +444,7 @@ class Crawler(HTTPClientProxy, ABC):
await self.handle_media_item(media_item, m3u8)
@final
- async def _download(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None) -> None:
+ async def _download(self, media_item: MediaItem, m3u8: m3u8.Rendition | None) -> None:
try:
if m3u8:
await self.downloader.download_hls(media_item, m3u8)
@@ -473,7 +473,7 @@ class Crawler(HTTPClientProxy, ABC):
self.manager.progress_manager.download_progress.add_previously_completed()
return check_complete
- async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:
+ async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:
await self.manager.states.RUNNING.wait()
check_complete = await self.check_complete(media_item.url, media_item.referer)
if check_complete:
@@ -807,41 +807,41 @@ class Crawler(HTTPClientProxy, ABC):
scrape_item.url = redirect
self.create_task(self.run(scrape_item))
+ @deprecated("Use self.request_m3u8 instead")
async def get_m3u8_from_playlist_url(
self,
m3u8_playlist_url: AbsoluteHttpURL,
/,
- headers: dict[str, str] | None = None,
+ headers: Mapping[str, str] | None = None,
*,
only: Iterable[str] = (),
exclude: Iterable[str] = ("vp09",),
- ) -> tuple[m3u8.RenditionGroup, m3u8.RenditionGroupDetails]:
+ ) -> tuple[m3u8.Rendition, m3u8.RenditionDetails]:
"""Get m3u8 rendition group from a playlist m3u8 (variant m3u8), selecting the best format"""
- m3u8_playlist = await self._get_m3u8(m3u8_playlist_url, headers)
- rendition_group_info = m3u8.get_best_group_from_playlist(m3u8_playlist, only=only, exclude=exclude)
- renditions_urls = rendition_group_info.urls
- video = await self._get_m3u8(renditions_urls.video, headers, "video")
- audio = await self._get_m3u8(renditions_urls.audio, headers, "audio") if renditions_urls.audio else None
- subtitle = (
- await self._get_m3u8(renditions_urls.subtitle, headers, "subtitles") if renditions_urls.subtitle else None
- )
- return m3u8.RenditionGroup(video, audio, subtitle), rendition_group_info
+ playlist, info = await self.request_m3u8(m3u8_playlist_url, headers, only=only, exclude=exclude)
+ if info is None:
+ raise ScrapeError(422, "Not a variant m3u8", origin=m3u8_playlist_url)
+ return playlist, info
+ @deprecated("Use self.request_m3u8 instead")
async def get_m3u8_from_index_url(
- self, url: AbsoluteHttpURL, /, headers: dict[str, str] | None = None
- ) -> m3u8.RenditionGroup:
+ self, url: AbsoluteHttpURL, /, headers: Mapping[str, str] | None = None
+ ) -> m3u8.Rendition:
"""Get m3u8 rendition group from an index that only has 1 rendition, a video (non variant m3u8)"""
- return m3u8.RenditionGroup(await self._get_m3u8(url, headers, "video"))
+ playlist, info = await self.request_m3u8(url, headers)
+ if info is not None:
+ raise ScrapeError(422, "This is a variant m3u8", origin=url)
+ return playlist
+ @deprecated("Use self._request_m3u8 instead")
async def _get_m3u8(
self,
url: AbsoluteHttpURL,
/,
- headers: dict[str, str] | None = None,
- media_type: Literal["video", "audio", "subtitles"] | None = None,
+ headers: Mapping[str, str] | None = None,
+ media_type: Literal["video", "audio", "subtitle"] | None = None,
) -> m3u8.M3U8:
- content = await self.request_text(url, headers=headers)
- return m3u8.M3U8(content, url.parent, media_type)
+ return await self._request_m3u8(url, headers, media_type)
def create_custom_filename(
self,
@@ -859,30 +859,25 @@ class Crawler(HTTPClientProxy, ABC):
calling_args = {name: value for name, value in locals().items() if value is not None and name not in ("self",)}
# remove OS separators (if any)
stem = sanitize_filename(Path(name).as_posix().replace("/", "-")).strip().removesuffix(ext).strip()
- extra_info: list[str] = []
-
- if _placeholder_config.include_file_id and file_id:
- extra_info.append(file_id)
- if _placeholder_config.include_video_codec and video_codec:
- extra_info.append(video_codec)
- if _placeholder_config.include_audio_codec and audio_codec:
- extra_info.append(audio_codec)
-
- if (
- _placeholder_config.include_resolution
- and resolution
- and resolution not in [Resolution.highest(), Resolution.unknown()]
- ):
- if not isinstance(resolution, Resolution):
- resolution = Resolution.parse(resolution)
- extra_info.append(resolution.name)
- if _placeholder_config.include_hash and hash_string:
- assert any(hash_string.startswith(x) for x in HASH_PREFIXES), f"Invalid: {hash_string = }"
- extra_info.append(hash_string)
+ def extra_info() -> Generator[str]:
+ if _include.file_id and file_id:
+ yield file_id
+ if _include.video_codec and video_codec:
+ yield video_codec
+ if _include.audio_codec and audio_codec:
+ yield audio_codec
+
+ if _include.resolution and resolution and resolution not in (Resolution.highest(), Resolution.unknown()):
+ res = resolution if type(resolution) is Resolution else Resolution.parse(resolution)
+ yield res.name
+
+ if _include.hash and hash_string:
+ assert any(hash_string.startswith(x) for x in HASH_PREFIXES), f"Invalid: {hash_string = }"
+ yield hash_string
- filename, extra_info_had_invalid_chars = _make_custom_filename(stem, ext, extra_info, only_truncate_stem)
- if extra_info_had_invalid_chars:
+ filename, had_invalid_chars = _make_custom_filename(stem, ext, list(extra_info()), only_truncate_stem)
+ if had_invalid_chars:
msg = (
f"Custom filename creation for {self.FOLDER_DOMAIN} seems to be broken. "
f"Important information was removed while creating a filename. "
diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.py
index 4cab3ec00..27e9030e8 100644
--- a/cyberdrop_dl/crawlers/rumble.py
+++ b/cyberdrop_dl/crawlers/rumble.py
@@ -45,7 +45,7 @@ class Format(NamedTuple):
size: int
type: FormatType # On formats where everything else is the same, choose mp4 over webm
url: AbsoluteHttpURL
- m3u8: m3u8.RenditionGroup | None = None
+ m3u8: m3u8.Rendition | None = None
@dataclasses.dataclass(slots=True, frozen=True)
diff --git a/cyberdrop_dl/crawlers/tiktok.py b/cyberdrop_dl/crawlers/tiktok.py
index a3c48f82a..a720bc39f 100644
--- a/cyberdrop_dl/crawlers/tiktok.py
+++ b/cyberdrop_dl/crawlers/tiktok.py
@@ -263,7 +263,7 @@ class TikTokCrawler(Crawler):
)
scrape_item.add_children()
- async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:
+ async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:
if media_item.ext == ".mp3":
media_item.download_folder = media_item.download_folder / "Audios"
diff --git a/cyberdrop_dl/crawlers/twitter_images.py b/cyberdrop_dl/crawlers/twitter_images.py
index 87ef71596..012898966 100644
--- a/cyberdrop_dl/crawlers/twitter_images.py
+++ b/cyberdrop_dl/crawlers/twitter_images.py
@@ -41,7 +41,7 @@ class TwimgCrawler(Crawler):
filename, ext = self.get_filename_and_ext(filename)
await self.handle_file(link, scrape_item, filename, ext)
- async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:
+ async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:
_, *media_item.fallbacks = list(_make_download_urls(media_item.url))
if media_item.referer == media_item.url and media_item.parents:
media_item.referer = media_item.parents[0]
diff --git a/cyberdrop_dl/crawlers/voe_sx.py b/cyberdrop_dl/crawlers/voe_sx.py
index 62a6f0fb2..02b14e9ea 100644
--- a/cyberdrop_dl/crawlers/voe_sx.py
+++ b/cyberdrop_dl/crawlers/voe_sx.py
@@ -133,9 +133,7 @@ class VoeSxCrawler(Crawler):
VoeSxCrawler._handle_video(self, scrape_item, video, m3u8)
- def _handle_video(
- self: Crawler, scrape_item: ScrapeItem, video: VoeVideo, m3u8: m3u8.RenditionGroup | None
- ) -> None:
+ def _handle_video(self: Crawler, scrape_item: ScrapeItem, video: VoeVideo, m3u8: m3u8.Rendition | None) -> None:
custom_filename = self.create_custom_filename(
video.title,
video.url.suffix,
diff --git a/cyberdrop_dl/data_structures/mediaprops.py b/cyberdrop_dl/data_structures/mediaprops.py
index 5fde9562c..0a4b26673 100644
--- a/cyberdrop_dl/data_structures/mediaprops.py
+++ b/cyberdrop_dl/data_structures/mediaprops.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import re
from fractions import Fraction
-from typing import TYPE_CHECKING, Final, NamedTuple
+from typing import TYPE_CHECKING, Final, NamedTuple, final
from cyberdrop_dl.exceptions import ScrapeError
@@ -49,6 +49,7 @@ class Codecs(NamedTuple):
return Codecs(video_codec, audio_codec)
+@final
class Resolution(NamedTuple):
width: int
height: int
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 06638b226..2bd7839a1 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -64,7 +64,7 @@ if TYPE_CHECKING:
from cyberdrop_dl.clients.download_client import DownloadClient
from cyberdrop_dl.managers.manager import Manager
- from cyberdrop_dl.utils.m3u8 import M3U8, RenditionGroup
+ from cyberdrop_dl.utils.m3u8 import M3U8, Rendition
P = ParamSpec("P")
R = TypeVar("R")
@@ -189,7 +189,7 @@ class Downloader:
return await self.start_download(media_item)
@error_handling_wrapper
- async def download_hls(self, media_item: MediaItem, m3u8_group: RenditionGroup) -> None:
+ async def download_hls(self, media_item: MediaItem, m3u8_group: Rendition) -> None:
if media_item.url.path in self.processed_items and not self._ignore_history:
return
@@ -202,7 +202,7 @@ class Downloader:
async with self._download_context(media_item):
await self._start_hls_download(media_item, m3u8_group)
- async def _start_hls_download(self, media_item: MediaItem, m3u8_group: RenditionGroup) -> None:
+ async def _start_hls_download(self, media_item: MediaItem, m3u8_group: Rendition) -> None:
media_item.path = media_item.download_folder / media_item.filename
# TODO: register database duration from m3u8 info
# TODO: compute approx size for UI from the m3u8 info
@@ -227,7 +227,7 @@ class Downloader:
await self.finalize_download(media_item, downloaded=True)
async def _download_rendition_group(
- self, media_item: MediaItem, m3u8_group: RenditionGroup
+ self, media_item: MediaItem, m3u8_group: Rendition
) -> tuple[Path, Path | None, Path | None]:
temp_dir = media_item.path.with_suffix(constants.TempExt.HLS)
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index 4d361a3d4..08ff846a8 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -396,7 +396,7 @@ def register_crawler(
from_user: bool | Literal["raise"] = False,
) -> None:
if crawler.IS_GENERIC and include_generics:
- keys = (crawler.GENERIC_NAME,)
+ keys = (crawler.NAME,)
else:
keys = crawler.SCRAPE_MAPPER_KEYS
@@ -407,7 +407,7 @@ def register_crawler(
other = match
if other:
msg = (
- f"Unable to assign {crawler.PRIMARY_URL} to generic crawler {crawler.GENERIC_NAME}. "
+ f"Unable to assign {crawler.PRIMARY_URL} to generic crawler {crawler.NAME}. "
f"URL conflicts with URL format of builtin crawler {other.NAME}. "
"URL will be ignored"
)
@@ -416,7 +416,7 @@ def register_crawler(
logger.error(msg)
continue
else:
- logger.info(f"Successfully mapped {crawler.PRIMARY_URL} to generic crawler {crawler.GENERIC_NAME}")
+ logger.info(f"Successfully mapped {crawler.PRIMARY_URL} to crawler {crawler.NAME}")
elif other:
msg = f"{domain} from {crawler.NAME} already registered by {other}"
diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.py
index df4f2a10e..181b546eb 100644
--- a/cyberdrop_dl/utils/m3u8.py
+++ b/cyberdrop_dl/utils/m3u8.py
@@ -66,7 +66,7 @@ class MediaURLs(NamedTuple):
subtitle: AbsoluteHttpURL | None
-class RenditionGroup(NamedTuple):
+class Rendition(NamedTuple):
video: M3U8
audio: M3U8 | None = None
subtitle: M3U8 | None = None
@@ -96,7 +96,7 @@ class StreamInfo:
@dataclass(frozen=True, slots=True, order=True)
-class RenditionGroupDetails:
+class RenditionDetails:
resolution: Resolution
codecs: Codecs
stream_info: StreamInfo
@@ -104,7 +104,7 @@ class RenditionGroupDetails:
urls: MediaURLs
@staticmethod
- def new(playlist: Playlist) -> RenditionGroupDetails:
+ def new(playlist: Playlist) -> RenditionDetails:
assert playlist.uri
def get_url(m3u8_obj: Playlist | Media) -> AbsoluteHttpURL:
@@ -131,7 +131,7 @@ class RenditionGroupDetails:
media_urls = MediaURLs(video_url, audio_url, subtitle_url)
stream_info = StreamInfo(**vars(playlist.stream_info))
- return RenditionGroupDetails(resolution, codecs, stream_info, media, media_urls)
+ return RenditionDetails(resolution, codecs, stream_info, media, media_urls)
class M3U8(_M3U8):
@@ -139,11 +139,11 @@ class M3U8(_M3U8):
self,
content: str,
base_uri: AbsoluteHttpURL | None = None,
- media_type: Literal["video", "audio", "subtitles"] | None = None,
+ media_type: Literal["video", "audio", "subtitle"] | None = None,
) -> None:
if base_uri and base_uri.suffix.casefold() == ".m3u8":
base_uri = base_uri.parent
- self.media_type: Literal["video", "audio", "subtitles"] | None = media_type
+ self.media_type: Literal["video", "audio", "subtitle"] | None = media_type
super().__init__(content, base_uri=str(base_uri) if base_uri else None)
def __repr__(self) -> str:
@@ -163,11 +163,11 @@ class VariantM3U8Parser:
def __init__(self, m3u8: M3U8) -> None:
assert m3u8.is_variant
self._m3u8 = m3u8
- self.groups = sorted((RenditionGroupDetails.new(playlist) for playlist in m3u8.playlists), reverse=True)
+ self.groups = sorted((RenditionDetails.new(playlist) for playlist in m3u8.playlists), reverse=True)
def get_rendition_groups(
self, only: Iterable[str] = (), *, exclude: Iterable[str] = ()
- ) -> Generator[RenditionGroupDetails]:
+ ) -> Generator[RenditionDetails]:
"""Yields M3U8 options, best to worst"""
assert not (only and exclude), "only one of `only` or `exclude` can be supplied, not both"
if isinstance(exclude, str):
@@ -183,13 +183,13 @@ class VariantM3U8Parser:
continue
yield group
- def get_best_group(self, only: Iterable[str] = (), *, exclude: Iterable[str] = ()) -> RenditionGroupDetails:
+ def get_best_group(self, only: Iterable[str] = (), *, exclude: Iterable[str] = ()) -> RenditionDetails:
return next(self.get_rendition_groups(only=only, exclude=exclude))
-def get_best_group_from_playlist(
+def select_best_rendition(
m3u8_playlist: M3U8, only: Iterable[str] = (), *, exclude: Iterable[str] = ()
-) -> RenditionGroupDetails:
+) -> RenditionDetails:
return VariantM3U8Parser(m3u8_playlist).get_best_group(only=only, exclude=exclude)