Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: move HLS parsing logic to its own class (#92)
NTFSvolume488a01e6550c
diff --git a/cyberdrop_dl/crawlers/_hls.py b/cyberdrop_dl/crawlers/_hls.pynew file mode 100644index 000000000..7caa36872--- /dev/null+++ b/cyberdrop_dl/crawlers/_hls.py@@ -0,0 +1,95 @@+from __future__ import annotations++import asyncio+import dataclasses+from abc import ABC, abstractmethod+from typing import TYPE_CHECKING, Literal++from cyberdrop_dl.utils import m3u8++if TYPE_CHECKING:+ from collections.abc import Iterable, Mapping++ import aiohttp++ from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+++class HLSParser(ABC):+ """Class to fetch and parse HTTP live streams++ For multi variant m3u8, the best resolution will be automatically selected"""++ @abstractmethod+ async def request_text(+ self,+ url: AbsoluteHttpURL,+ /,+ method: Literal["GET"] = "GET",+ headers: Mapping[str, str] | None = None,+ ) -> str: ...++ async def request_m3u8(+ self,+ url: AbsoluteHttpURL,+ /,+ headers: Mapping[str, str] | None = None,+ *,+ only: Iterable[str] = (),+ exclude: Iterable[str] = ("vp09",),+ ) -> tuple[m3u8.Rendition, m3u8.RenditionDetails | None]:+ m3u8_obj = await self._request_m3u8(url, headers)+ if m3u8_obj.is_variant:+ rendition = m3u8.select_best_rendition(m3u8_obj, only=only, exclude=exclude)+ return await self._resolve_rendition(rendition, headers)+ m3u8_obj.media_type = "video"+ return m3u8.Rendition(m3u8_obj), None++ async def _resolve_rendition(+ self,+ rendition: m3u8.RenditionDetails,+ /,+ headers: Mapping[str, str] | None = None,+ ):+ video, *audio_and_subs = await asyncio.gather(+ *(+ self._request_m3u8(url, headers, name)+ for name, url in zip(+ ("video", "audio", "subtitle"),+ rendition.urls,+ strict=True,+ )+ if url+ )+ )++ return m3u8.Rendition(video, *audio_and_subs), rendition++ async def _request_m3u8(+ self,+ url: AbsoluteHttpURL,+ /,+ headers: Mapping[str, str] | None = None,+ media_type: Literal["video", "audio", "subtitle"] | None = None,+ ) -> m3u8.M3U8:+ content = await self.request_text(url, headers=headers)+ return m3u8.M3U8(content, url.parent, media_type)++[email protected](slots=True)+class SimpleHLSParser(HLSParser):+ """A simple parser that does not depend on the manager.++ DO NOT USE. This is only for testing"""++ _session: aiohttp.ClientSession++ async def request_text(+ self,+ url: AbsoluteHttpURL,+ /,+ method: Literal["GET"] = "GET",+ headers: Mapping[str, str] | None = None,+ ) -> str:+ async with self._session.get(url, headers=headers) as resp:+ return await resp.text()diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 72af539e1..e22d11f79 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -29,9 +29,11 @@ from typing import ()from aiolimiter import AsyncLimiter+from typing_extensions import deprecatedfrom cyberdrop_dl import constantsfrom cyberdrop_dl.clients import HTTPClient, HTTPClientProxy+from cyberdrop_dl.crawlers._hls import HLSParserfrom cyberdrop_dl.data_structures.mediaprops import ISO639Subtitle, Resolutionfrom cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL, MediaItem, ScrapeItemfrom cyberdrop_dl.downloader.downloader import Downloader@@ -54,7 +56,7 @@ from cyberdrop_dl.utils.utilities import ()if TYPE_CHECKING:- from collections.abc import AsyncGenerator, Callable, Coroutine, Generator, Iterable+ from collections.abc import AsyncGenerator, Callable, Coroutine, Generator, Iterable, Mappingfrom http.cookies import BaseCookiefrom types import ModuleType@@ -81,15 +83,15 @@ VALID_RESOLUTION_NAMES = "4K", "8K", "HQ", "Unknown"@dataclasses.dataclass(slots=True, frozen=True)-class PlaceHolderConfig:- include_file_id: bool = True- include_video_codec: bool = True- include_audio_codec: bool = True- include_resolution: bool = True- include_hash: bool = True+class _PlaceHolderConfigInclude:+ file_id: bool = True+ video_codec: bool = True+ audio_codec: bool = True+ resolution: bool = True+ hash: bool = True-_placeholder_config = PlaceHolderConfig()+_include = _PlaceHolderConfigInclude()_DB_PATH_BUILDERS: dict[str, Callable[[AbsoluteHttpURL], str]] = {"url": lambda url: str(url),@@ -144,7 +146,7 @@ class Registry:cls._import_from(sub_module)-class Crawler(HTTPClientProxy, ABC):+class Crawler(HTTPClientProxy, HLSParser, ABC):OLD_DOMAINS: ClassVar[tuple[str, ...]] = ()SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = ()SUPPORTED_PATHS: ClassVar[SupportedPaths] = {}@@ -216,7 +218,6 @@ class Crawler(HTTPClientProxy, ABC):cls,is_abc: bool = False,is_generic: bool = False,- generic_name: str = "",db_path: Literal["url", "name", "path", "path_qs", "path_qs_frag", "path_frag"] | None = None,**kwargs,) -> None:@@ -238,9 +239,8 @@ class Crawler(HTTPClientProxy, ABC):cls.__db_path__ = staticmethod(_DB_PATH_BUILDERS[db_path])if cls.IS_GENERIC:- cls.GENERIC_NAME: str = generic_name or cls.NAMEcls.SCRAPE_MAPPER_KEYS = ()- cls.INFO: CrawlerInfo = CrawlerInfo.generic(cls.GENERIC_NAME, cls.SUPPORTED_PATHS)+ cls.INFO: CrawlerInfo = CrawlerInfo.generic(cls.NAME, cls.SUPPORTED_PATHS)Registry.generic.add(cls)return@@ -415,7 +415,7 @@ class Crawler(HTTPClientProxy, ABC):*,custom_filename: str | None = None,debrid_link: AbsoluteHttpURL | None = None,- m3u8: m3u8.RenditionGroup | None = None,+ m3u8: m3u8.Rendition | None = None,metadata: object = None,referer: AbsoluteHttpURL | None = None,) -> None:@@ -444,7 +444,7 @@ class Crawler(HTTPClientProxy, ABC):await self.handle_media_item(media_item, m3u8)@final- async def _download(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None) -> None:+ async def _download(self, media_item: MediaItem, m3u8: m3u8.Rendition | None) -> None:try:if m3u8:await self.downloader.download_hls(media_item, m3u8)@@ -473,7 +473,7 @@ class Crawler(HTTPClientProxy, ABC):self.manager.progress_manager.download_progress.add_previously_completed()return check_complete- async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:+ async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:await self.manager.states.RUNNING.wait()check_complete = await self.check_complete(media_item.url, media_item.referer)if check_complete:@@ -807,41 +807,41 @@ class Crawler(HTTPClientProxy, ABC):scrape_item.url = redirectself.create_task(self.run(scrape_item))+ @deprecated("Use self.request_m3u8 instead")async def get_m3u8_from_playlist_url(self,m3u8_playlist_url: AbsoluteHttpURL,/,- headers: dict[str, str] | None = None,+ headers: Mapping[str, str] | None = None,*,only: Iterable[str] = (),exclude: Iterable[str] = ("vp09",),- ) -> tuple[m3u8.RenditionGroup, m3u8.RenditionGroupDetails]:+ ) -> tuple[m3u8.Rendition, m3u8.RenditionDetails]:"""Get m3u8 rendition group from a playlist m3u8 (variant m3u8), selecting the best format"""- m3u8_playlist = await self._get_m3u8(m3u8_playlist_url, headers)- rendition_group_info = m3u8.get_best_group_from_playlist(m3u8_playlist, only=only, exclude=exclude)- renditions_urls = rendition_group_info.urls- video = await self._get_m3u8(renditions_urls.video, headers, "video")- audio = await self._get_m3u8(renditions_urls.audio, headers, "audio") if renditions_urls.audio else None- subtitle = (- await self._get_m3u8(renditions_urls.subtitle, headers, "subtitles") if renditions_urls.subtitle else None- )- return m3u8.RenditionGroup(video, audio, subtitle), rendition_group_info+ playlist, info = await self.request_m3u8(m3u8_playlist_url, headers, only=only, exclude=exclude)+ if info is None:+ raise ScrapeError(422, "Not a variant m3u8", origin=m3u8_playlist_url)+ return playlist, info+ @deprecated("Use self.request_m3u8 instead")async def get_m3u8_from_index_url(- self, url: AbsoluteHttpURL, /, headers: dict[str, str] | None = None- ) -> m3u8.RenditionGroup:+ self, url: AbsoluteHttpURL, /, headers: Mapping[str, str] | None = None+ ) -> m3u8.Rendition:"""Get m3u8 rendition group from an index that only has 1 rendition, a video (non variant m3u8)"""- return m3u8.RenditionGroup(await self._get_m3u8(url, headers, "video"))+ playlist, info = await self.request_m3u8(url, headers)+ if info is not None:+ raise ScrapeError(422, "This is a variant m3u8", origin=url)+ return playlist+ @deprecated("Use self._request_m3u8 instead")async def _get_m3u8(self,url: AbsoluteHttpURL,/,- headers: dict[str, str] | None = None,- media_type: Literal["video", "audio", "subtitles"] | None = None,+ headers: Mapping[str, str] | None = None,+ media_type: Literal["video", "audio", "subtitle"] | None = None,) -> m3u8.M3U8:- content = await self.request_text(url, headers=headers)- return m3u8.M3U8(content, url.parent, media_type)+ return await self._request_m3u8(url, headers, media_type)def create_custom_filename(self,@@ -859,30 +859,25 @@ class Crawler(HTTPClientProxy, ABC):calling_args = {name: value for name, value in locals().items() if value is not None and name not in ("self",)}# remove OS separators (if any)stem = sanitize_filename(Path(name).as_posix().replace("/", "-")).strip().removesuffix(ext).strip()- extra_info: list[str] = []-- if _placeholder_config.include_file_id and file_id:- extra_info.append(file_id)- if _placeholder_config.include_video_codec and video_codec:- extra_info.append(video_codec)- if _placeholder_config.include_audio_codec and audio_codec:- extra_info.append(audio_codec)-- if (- _placeholder_config.include_resolution- and resolution- and resolution not in [Resolution.highest(), Resolution.unknown()]- ):- if not isinstance(resolution, Resolution):- resolution = Resolution.parse(resolution)- extra_info.append(resolution.name)- if _placeholder_config.include_hash and hash_string:- assert any(hash_string.startswith(x) for x in HASH_PREFIXES), f"Invalid: {hash_string = }"- extra_info.append(hash_string)+ def extra_info() -> Generator[str]:+ if _include.file_id and file_id:+ yield file_id+ if _include.video_codec and video_codec:+ yield video_codec+ if _include.audio_codec and audio_codec:+ yield audio_codec++ if _include.resolution and resolution and resolution not in (Resolution.highest(), Resolution.unknown()):+ res = resolution if type(resolution) is Resolution else Resolution.parse(resolution)+ yield res.name++ if _include.hash and hash_string:+ assert any(hash_string.startswith(x) for x in HASH_PREFIXES), f"Invalid: {hash_string = }"+ yield hash_string- filename, extra_info_had_invalid_chars = _make_custom_filename(stem, ext, extra_info, only_truncate_stem)- if extra_info_had_invalid_chars:+ filename, had_invalid_chars = _make_custom_filename(stem, ext, list(extra_info()), only_truncate_stem)+ if had_invalid_chars:msg = (f"Custom filename creation for {self.FOLDER_DOMAIN} seems to be broken. "f"Important information was removed while creating a filename. "diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.pyindex 4cab3ec00..27e9030e8 100644--- a/cyberdrop_dl/crawlers/rumble.py+++ b/cyberdrop_dl/crawlers/rumble.py@@ -45,7 +45,7 @@ class Format(NamedTuple):size: inttype: FormatType # On formats where everything else is the same, choose mp4 over webmurl: AbsoluteHttpURL- m3u8: m3u8.RenditionGroup | None = None+ m3u8: m3u8.Rendition | None = None@dataclasses.dataclass(slots=True, frozen=True)diff --git a/cyberdrop_dl/crawlers/tiktok.py b/cyberdrop_dl/crawlers/tiktok.pyindex a3c48f82a..a720bc39f 100644--- a/cyberdrop_dl/crawlers/tiktok.py+++ b/cyberdrop_dl/crawlers/tiktok.py@@ -263,7 +263,7 @@ class TikTokCrawler(Crawler):)scrape_item.add_children()- async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:+ async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:if media_item.ext == ".mp3":media_item.download_folder = media_item.download_folder / "Audios"diff --git a/cyberdrop_dl/crawlers/twitter_images.py b/cyberdrop_dl/crawlers/twitter_images.pyindex 87ef71596..012898966 100644--- a/cyberdrop_dl/crawlers/twitter_images.py+++ b/cyberdrop_dl/crawlers/twitter_images.py@@ -41,7 +41,7 @@ class TwimgCrawler(Crawler):filename, ext = self.get_filename_and_ext(filename)await self.handle_file(link, scrape_item, filename, ext)- async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.RenditionGroup | None = None) -> None:+ async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:_, *media_item.fallbacks = list(_make_download_urls(media_item.url))if media_item.referer == media_item.url and media_item.parents:media_item.referer = media_item.parents[0]diff --git a/cyberdrop_dl/crawlers/voe_sx.py b/cyberdrop_dl/crawlers/voe_sx.pyindex 62a6f0fb2..02b14e9ea 100644--- a/cyberdrop_dl/crawlers/voe_sx.py+++ b/cyberdrop_dl/crawlers/voe_sx.py@@ -133,9 +133,7 @@ class VoeSxCrawler(Crawler):VoeSxCrawler._handle_video(self, scrape_item, video, m3u8)- def _handle_video(- self: Crawler, scrape_item: ScrapeItem, video: VoeVideo, m3u8: m3u8.RenditionGroup | None- ) -> None:+ def _handle_video(self: Crawler, scrape_item: ScrapeItem, video: VoeVideo, m3u8: m3u8.Rendition | None) -> None:custom_filename = self.create_custom_filename(video.title,video.url.suffix,diff --git a/cyberdrop_dl/data_structures/mediaprops.py b/cyberdrop_dl/data_structures/mediaprops.pyindex 5fde9562c..0a4b26673 100644--- a/cyberdrop_dl/data_structures/mediaprops.py+++ b/cyberdrop_dl/data_structures/mediaprops.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport refrom fractions import Fraction-from typing import TYPE_CHECKING, Final, NamedTuple+from typing import TYPE_CHECKING, Final, NamedTuple, finalfrom cyberdrop_dl.exceptions import ScrapeError@@ -49,6 +49,7 @@ class Codecs(NamedTuple):return Codecs(video_codec, audio_codec)+@finalclass Resolution(NamedTuple):width: intheight: intdiff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 06638b226..2bd7839a1 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -64,7 +64,7 @@ if TYPE_CHECKING:from cyberdrop_dl.clients.download_client import DownloadClientfrom cyberdrop_dl.managers.manager import Manager- from cyberdrop_dl.utils.m3u8 import M3U8, RenditionGroup+ from cyberdrop_dl.utils.m3u8 import M3U8, RenditionP = ParamSpec("P")R = TypeVar("R")@@ -189,7 +189,7 @@ class Downloader:return await self.start_download(media_item)@error_handling_wrapper- async def download_hls(self, media_item: MediaItem, m3u8_group: RenditionGroup) -> None:+ async def download_hls(self, media_item: MediaItem, m3u8_group: Rendition) -> None:if media_item.url.path in self.processed_items and not self._ignore_history:return@@ -202,7 +202,7 @@ class Downloader:async with self._download_context(media_item):await self._start_hls_download(media_item, m3u8_group)- async def _start_hls_download(self, media_item: MediaItem, m3u8_group: RenditionGroup) -> None:+ async def _start_hls_download(self, media_item: MediaItem, m3u8_group: Rendition) -> None:media_item.path = media_item.download_folder / media_item.filename# TODO: register database duration from m3u8 info# TODO: compute approx size for UI from the m3u8 info@@ -227,7 +227,7 @@ class Downloader:await self.finalize_download(media_item, downloaded=True)async def _download_rendition_group(- self, media_item: MediaItem, m3u8_group: RenditionGroup+ self, media_item: MediaItem, m3u8_group: Rendition) -> tuple[Path, Path | None, Path | None]:temp_dir = media_item.path.with_suffix(constants.TempExt.HLS)diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.pyindex 4d361a3d4..08ff846a8 100644--- a/cyberdrop_dl/scraper/scrape_mapper.py+++ b/cyberdrop_dl/scraper/scrape_mapper.py@@ -396,7 +396,7 @@ def register_crawler(from_user: bool | Literal["raise"] = False,) -> None:if crawler.IS_GENERIC and include_generics:- keys = (crawler.GENERIC_NAME,)+ keys = (crawler.NAME,)else:keys = crawler.SCRAPE_MAPPER_KEYS@@ -407,7 +407,7 @@ def register_crawler(other = matchif other:msg = (- f"Unable to assign {crawler.PRIMARY_URL} to generic crawler {crawler.GENERIC_NAME}. "+ f"Unable to assign {crawler.PRIMARY_URL} to generic crawler {crawler.NAME}. "f"URL conflicts with URL format of builtin crawler {other.NAME}. ""URL will be ignored")@@ -416,7 +416,7 @@ def register_crawler(logger.error(msg)continueelse:- logger.info(f"Successfully mapped {crawler.PRIMARY_URL} to generic crawler {crawler.GENERIC_NAME}")+ logger.info(f"Successfully mapped {crawler.PRIMARY_URL} to crawler {crawler.NAME}")elif other:msg = f"{domain} from {crawler.NAME} already registered by {other}"diff --git a/cyberdrop_dl/utils/m3u8.py b/cyberdrop_dl/utils/m3u8.pyindex df4f2a10e..181b546eb 100644--- a/cyberdrop_dl/utils/m3u8.py+++ b/cyberdrop_dl/utils/m3u8.py@@ -66,7 +66,7 @@ class MediaURLs(NamedTuple):subtitle: AbsoluteHttpURL | None-class RenditionGroup(NamedTuple):+class Rendition(NamedTuple):video: M3U8audio: M3U8 | None = Nonesubtitle: M3U8 | None = None@@ -96,7 +96,7 @@ class StreamInfo:@dataclass(frozen=True, slots=True, order=True)-class RenditionGroupDetails:+class RenditionDetails:resolution: Resolutioncodecs: Codecsstream_info: StreamInfo@@ -104,7 +104,7 @@ class RenditionGroupDetails:urls: MediaURLs@staticmethod- def new(playlist: Playlist) -> RenditionGroupDetails:+ def new(playlist: Playlist) -> RenditionDetails:assert playlist.uridef get_url(m3u8_obj: Playlist | Media) -> AbsoluteHttpURL:@@ -131,7 +131,7 @@ class RenditionGroupDetails:media_urls = MediaURLs(video_url, audio_url, subtitle_url)stream_info = StreamInfo(**vars(playlist.stream_info))- return RenditionGroupDetails(resolution, codecs, stream_info, media, media_urls)+ return RenditionDetails(resolution, codecs, stream_info, media, media_urls)class M3U8(_M3U8):@@ -139,11 +139,11 @@ class M3U8(_M3U8):self,content: str,base_uri: AbsoluteHttpURL | None = None,- media_type: Literal["video", "audio", "subtitles"] | None = None,+ media_type: Literal["video", "audio", "subtitle"] | None = None,) -> None:if base_uri and base_uri.suffix.casefold() == ".m3u8":base_uri = base_uri.parent- self.media_type: Literal["video", "audio", "subtitles"] | None = media_type+ self.media_type: Literal["video", "audio", "subtitle"] | None = media_typesuper().__init__(content, base_uri=str(base_uri) if base_uri else None)def __repr__(self) -> str:@@ -163,11 +163,11 @@ class VariantM3U8Parser:def __init__(self, m3u8: M3U8) -> None:assert m3u8.is_variantself._m3u8 = m3u8- self.groups = sorted((RenditionGroupDetails.new(playlist) for playlist in m3u8.playlists), reverse=True)+ self.groups = sorted((RenditionDetails.new(playlist) for playlist in m3u8.playlists), reverse=True)def get_rendition_groups(self, only: Iterable[str] = (), *, exclude: Iterable[str] = ()- ) -> Generator[RenditionGroupDetails]:+ ) -> Generator[RenditionDetails]:"""Yields M3U8 options, best to worst"""assert not (only and exclude), "only one of `only` or `exclude` can be supplied, not both"if isinstance(exclude, str):@@ -183,13 +183,13 @@ class VariantM3U8Parser:continueyield group- def get_best_group(self, only: Iterable[str] = (), *, exclude: Iterable[str] = ()) -> RenditionGroupDetails:+ def get_best_group(self, only: Iterable[str] = (), *, exclude: Iterable[str] = ()) -> RenditionDetails:return next(self.get_rendition_groups(only=only, exclude=exclude))-def get_best_group_from_playlist(+def select_best_rendition(m3u8_playlist: M3U8, only: Iterable[str] = (), *, exclude: Iterable[str] = ()-) -> RenditionGroupDetails:+) -> RenditionDetails:return VariantM3U8Parser(m3u8_playlist).get_best_group(only=only, exclude=exclude)