Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add BunnyCDN support
NTFSvolume9c211dde6c97
diff --git a/cyberdrop_dl/crawlers/bunny_cdn.py b/cyberdrop_dl/crawlers/bunny_cdn.pynew file mode 100644index 000000000..afedb91ba--- /dev/null+++ b/cyberdrop_dl/crawlers/bunny_cdn.py@@ -0,0 +1,134 @@+from __future__ import annotations++import dataclasses+import json+from typing import TYPE_CHECKING, ClassVar, override++from cyberdrop_dl import aio+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths+from cyberdrop_dl.downloader.bunny_cdn import BunnyCDNDownloader+from cyberdrop_dl.exceptions import DownloadError+from cyberdrop_dl.mediaprops import Subtitle+from cyberdrop_dl.url_objects import AbsoluteHttpURL+from cyberdrop_dl.utils import css, extr_text+from cyberdrop_dl.utils._url import remove_query_params+from cyberdrop_dl.utils.errors import error_handling_wrapper++if TYPE_CHECKING:+ from collections.abc import Generator++ import bs4++ from cyberdrop_dl.url_objects import ScrapeItem++[email protected]_path_builder("path_qs")+class BunnyCDNCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Video": (+ "/embed/<lib_id>/<video_uuid>",+ "/play/<lib_id>/<video_uuid>",+ ),+ }+ DOMAIN: ClassVar[str] = "bunnycdn"+ FOLDER_DOMAIN: ClassVar[str] = "BunnyCDN"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://iframe.mediadelivery.net")+ OLD_DOMAINS: ClassVar[tuple[str, ...]] = "video.bunnycdn.com", "player.mediadelivery.net"++ downloader: BunnyCDNDownloader++ def __post_init__(self) -> None:+ self.downloader = BunnyCDNDownloader(self.manager) # pyright: ignore[reportIncompatibleVariableOverride]++ @classmethod+ @override+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ url = remove_query_params(super().transform_url(url), keep=("token", "secret"))+ match url.parts[1:]:+ case ["play", _, _]:+ return url.with_path(url.path.replace("/play/", "/embed/"), keep_query=True)+ case _:+ return url++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["embed", _, video_uuid]:+ await self.video(scrape_item, video_uuid)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:+ if await self.check_complete_from_referer(scrape_item.url):+ return++ headers = {"Referer": str(scrape_item.get_referer() or self.PRIMARY_URL)}+ html = await self.request_text(scrape_item.url, headers=headers)+ video = await _extract_video(html)+ src, ext, m3u8 = video.src, ".mp4", None+ if video.original:+ try:+ ext = await self.sniff_ext(video.original, headers=headers)+ except DownloadError:+ pass+ else:+ src = video.original++ if src.suffix == ".m3u8":+ m3u8, _ = await self.request_m3u8(src, headers=headers)++ filename = self.create_custom_filename(name := video.title or video_id, ext, file_id=video_id)+ await self.handle_file(+ src,+ scrape_item,+ name,+ ext,+ custom_filename=filename,+ m3u8=m3u8,+ headers=headers,+ thumbnail=video.thumb,+ )+ self.handle_subs(scrape_item, filename, video.subs)++[email protected](slots=True, frozen=True, kw_only=True)+class Video:+ title: str | None+ thumb: AbsoluteHttpURL+ src: AbsoluteHttpURL+ original: AbsoluteHttpURL | None+ subs: tuple[Subtitle, ...]+++def _extract_subs(video: bs4.Tag) -> Generator[Subtitle]:+ for track in css.iselect(video, "track[src]"):+ kind = css.attr_or_none(track, "kind")+ if not kind or kind in {"subtitles", "captions"}:+ yield Subtitle(+ url=css.attr(track, "src"),+ name=css.attr_or_none(track, "label"),+ lang_code=css.attr_or_none(track, "srclang", "lang", "label") or "unk",+ )++[email protected]_thread+def _extract_video(html: str) -> Video:+ video = css.select(css.soup(html, parse_only="video"), "#main-video")+ parse_url = BunnyCDNCrawler.parse_url+ try:+ original = extr_text(html, "originalUrl =", ";").strip("'")+ except ValueError:+ original = None++ try:+ title = json.loads(css.attr(video, "data-plyr-config"))["title"]+ except (ValueError, LookupError, css.SelectorError):+ title = None++ return Video(+ title=title,+ thumb=parse_url(css.attr(video, "data-poster")),+ src=parse_url(css.select(video, "source", "src")),+ original=parse_url(original) if original else None,+ subs=tuple(_extract_subs(video)),+ )diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex 0297cddc2..0d5073c90 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -608,11 +608,11 @@ class Crawler(HTTPMixin, HLSMixin, ABC):frag="thumbnail",)- async def sniff_ext(self, thumbnail: AbsoluteHttpURL) -> str:+ async def sniff_ext(self, thumbnail: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> str:try:_, ext = self.get_filename_and_ext(thumbnail.name)except NoExtensionError:- async with self.request(thumbnail, self._THUMB_HTTP_METHOD) as resp:+ async with self.request(thumbnail, self._THUMB_HTTP_METHOD, headers=headers or {}) as resp:_, ext = self.get_filename_and_ext(thumbnail.name, mime_type=resp.content_type)return extdiff --git a/cyberdrop_dl/downloader/bunny_cdn.py b/cyberdrop_dl/downloader/bunny_cdn.pynew file mode 100644index 000000000..9320f22f1--- /dev/null+++ b/cyberdrop_dl/downloader/bunny_cdn.py@@ -0,0 +1,67 @@+from __future__ import annotations++import contextlib+import dataclasses+import hashlib+import logging+import random+from contextvars import ContextVar+from typing import TYPE_CHECKING, override++from cyberdrop_dl import aio+from cyberdrop_dl.downloader.http import Downloader++if TYPE_CHECKING:+ from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem+ from cyberdrop_dl.utils.m3u8 import Rendition++logger = logging.getLogger(__name__)++_playback_time = ContextVar[int]("_playback_time", default=0)+_playback_url: ContextVar[AbsoluteHttpURL] = ContextVar("_playback_url")++[email protected](slots=True, frozen=True, kw_only=True)+class PingData:+ url: AbsoluteHttpURL+ headers: dict[str, str]+ secret: str+ context_id: str+ resolution: int = 1080+ period: int = 2+++def _sign_url(url: AbsoluteHttpURL) -> AbsoluteHttpURL:+ return url.update_query(hash=hashlib.md5("_".join(url.query.values()).encode(), usedforsecurity=False).hexdigest())+++class BunnyCDNDownloader(Downloader):+ async def _ping_playback(self, ping: PingData) -> None:+ current_time = _playback_time.get() + ping.period+ _playback_time.set(current_time)+ ping_url = _sign_url(+ ping.url.update_query(+ time=current_time + round(random.random(), 6),+ paused="false",+ resolution=ping.resolution,+ )+ )++ try:+ async with self.client.http_client.raw_request(ping_url, headers=ping.headers):+ ...+ except Exception:+ logger.exception("BunnyCDN ping for %s failed", _playback_url.get())++ @override+ async def _hls(self, media_item: MediaItem, rendition: Rendition) -> None:+ try:+ ping: PingData = media_item.extra_info[media_item.domain]["ping_data"]+ except LookupError:+ background_ping = contextlib.nullcontext()+ else:+ background_ping = aio.backgroud_task(lambda: self._ping_playback(ping), period=ping.period)+ _playback_url.set(media_item.real_url)++ async with background_ping:+ await super()._hls(media_item, rendition)diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.pyindex 64afa0358..4767f8ded 100644--- a/cyberdrop_dl/utils/css.py+++ b/cyberdrop_dl/utils/css.py@@ -74,8 +74,14 @@ def select_tag_text(html: str, tag_name: str) -> str:return select_text(soup(html, parse_only=tag_name), tag_name)-def attr_or_none(tag: Tag, attribute: str) -> str | None:+def attr_or_none(tag: Tag, *attributes: str) -> str | None:"""Same as `tag.get(attribute)` but asserts the result is a single str"""+ for attribute in attributes:+ if result := _attr_or_none(tag, attribute):+ return result+++def _attr_or_none(tag: Tag, attribute: str) -> str | None:attribute_ = attributeif attribute_ == "srcset":if (srcset := tag.get(attribute_)) and type(srcset) is str:@@ -90,12 +96,11 @@ def attr_or_none(tag: Tag, attribute: str) -> str | None:return value-def attr(tag: Tag, attribute: str) -> str:+def attr(tag: Tag, *attributes: str) -> str:"""Same as `tag.get(attribute)` but asserts the result is not `None` and is a single string"""- result = attr_or_none(tag, attribute)- if result is None:- raise SelectorError(f"{attribute = } not found")- return result+ if result := attr_or_none(tag, *attributes):+ return result+ raise SelectorError(f"{attributes = } not found")def text(tag: Tag, *, strip: bool = True) -> str:diff --git a/tests/crawlers/test_cases/bunnycdn.py b/tests/crawlers/test_cases/bunnycdn.pynew file mode 100644index 000000000..692e4d0bd--- /dev/null+++ b/tests/crawlers/test_cases/bunnycdn.py@@ -0,0 +1,42 @@+DOMAIN = "bunnycdn"+TEST_CASES = [+ {+ "url": "https://iframe.mediadelivery.net/embed/136145/32e34c4b-0d72-437c-9abb-05e67657da34",+ "results": [+ {+ "url": "https://vz-b14560c0-654.b-cdn.net/32e34c4b-0d72-437c-9abb-05e67657da34/original",+ "filename": "Sanela ist Teil der #arbeitsmarktkraft [32e34c4b-0d72-437c-9abb-05e67657da34].mp4",+ "debrid_url": None,+ "original_filename": "Sanela ist Teil der #arbeitsmarktkraft",+ "referer": "https://iframe.mediadelivery.net/embed/136145/32e34c4b-0d72-437c-9abb-05e67657da34",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:Loose Files (BunnyCDN)",+ "thumbnail": "https://vz-b14560c0-654.b-cdn.net/32e34c4b-0d72-437c-9abb-05e67657da34/thumbnail_9172dc16.jpg",+ },+ {+ "url": "https://vz-b14560c0-654.b-cdn.net/32e34c4b-0d72-437c-9abb-05e67657da34/captions/de-auto.vtt?ver=1",+ "filename": "Sanela ist Teil der #arbeitsmarktkraft [32e34c4b-0d72-437c-9abb-05e67657da34].de-auto.vtt",+ "debrid_url": None,+ "original_filename": "DE",+ "referer": "https://iframe.mediadelivery.net/embed/136145/32e34c4b-0d72-437c-9abb-05e67657da34#Sanela%20ist%20Teil%20der%20%23arbeitsmarktkraft%20%5B32e34c4b-0d72-437c-9abb-05e67657da34%5D.de-auto.vtt",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:Loose Files (BunnyCDN)",+ "thumbnail": None,+ },+ {+ "url": "https://vz-b14560c0-654.b-cdn.net/32e34c4b-0d72-437c-9abb-05e67657da34/captions/en-auto.vtt?ver=1",+ "filename": "Sanela ist Teil der #arbeitsmarktkraft [32e34c4b-0d72-437c-9abb-05e67657da34].en-auto.vtt",+ "debrid_url": None,+ "original_filename": "EN",+ "referer": "https://iframe.mediadelivery.net/embed/136145/32e34c4b-0d72-437c-9abb-05e67657da34#Sanela%20ist%20Teil%20der%20%23arbeitsmarktkraft%20%5B32e34c4b-0d72-437c-9abb-05e67657da34%5D.en-auto.vtt",+ "album_id": None,+ "uploaded_at": None,+ "download_folder": "re:Loose Files (BunnyCDN)",+ "thumbnail": None,+ },+ ],+ "count": 3,+ },+]