Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add BunnyCDN support

NTFSvolume9c211dde6c97
diff --git a/cyberdrop_dl/crawlers/bunny_cdn.py b/cyberdrop_dl/crawlers/bunny_cdn.py
new file mode 100644
index 000000000..afedb91ba
--- /dev/null
+++ b/cyberdrop_dl/crawlers/bunny_cdn.py
@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+import dataclasses
+import json
+from typing import TYPE_CHECKING, ClassVar, override
+
+from cyberdrop_dl import aio
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
+from cyberdrop_dl.downloader.bunny_cdn import BunnyCDNDownloader
+from cyberdrop_dl.exceptions import DownloadError
+from cyberdrop_dl.mediaprops import Subtitle
+from cyberdrop_dl.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.utils import css, extr_text
+from cyberdrop_dl.utils._url import remove_query_params
+from cyberdrop_dl.utils.errors import error_handling_wrapper
+
+if TYPE_CHECKING:
+ from collections.abc import Generator
+
+ import bs4
+
+ from cyberdrop_dl.url_objects import ScrapeItem
+
+
[email protected]_path_builder("path_qs")
+class BunnyCDNCrawler(Crawler):
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Video": (
+ "/embed/<lib_id>/<video_uuid>",
+ "/play/<lib_id>/<video_uuid>",
+ ),
+ }
+ DOMAIN: ClassVar[str] = "bunnycdn"
+ FOLDER_DOMAIN: ClassVar[str] = "BunnyCDN"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://iframe.mediadelivery.net")
+ OLD_DOMAINS: ClassVar[tuple[str, ...]] = "video.bunnycdn.com", "player.mediadelivery.net"
+
+ downloader: BunnyCDNDownloader
+
+ def __post_init__(self) -> None:
+ self.downloader = BunnyCDNDownloader(self.manager) # pyright: ignore[reportIncompatibleVariableOverride]
+
+ @classmethod
+ @override
+ def transform_url(cls, url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ url = remove_query_params(super().transform_url(url), keep=("token", "secret"))
+ match url.parts[1:]:
+ case ["play", _, _]:
+ return url.with_path(url.path.replace("/play/", "/embed/"), keep_query=True)
+ case _:
+ return url
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["embed", _, video_uuid]:
+ await self.video(scrape_item, video_uuid)
+ case _:
+ raise ValueError
+
+ @error_handling_wrapper
+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
+ if await self.check_complete_from_referer(scrape_item.url):
+ return
+
+ headers = {"Referer": str(scrape_item.get_referer() or self.PRIMARY_URL)}
+ html = await self.request_text(scrape_item.url, headers=headers)
+ video = await _extract_video(html)
+ src, ext, m3u8 = video.src, ".mp4", None
+ if video.original:
+ try:
+ ext = await self.sniff_ext(video.original, headers=headers)
+ except DownloadError:
+ pass
+ else:
+ src = video.original
+
+ if src.suffix == ".m3u8":
+ m3u8, _ = await self.request_m3u8(src, headers=headers)
+
+ filename = self.create_custom_filename(name := video.title or video_id, ext, file_id=video_id)
+ await self.handle_file(
+ src,
+ scrape_item,
+ name,
+ ext,
+ custom_filename=filename,
+ m3u8=m3u8,
+ headers=headers,
+ thumbnail=video.thumb,
+ )
+ self.handle_subs(scrape_item, filename, video.subs)
+
+
[email protected](slots=True, frozen=True, kw_only=True)
+class Video:
+ title: str | None
+ thumb: AbsoluteHttpURL
+ src: AbsoluteHttpURL
+ original: AbsoluteHttpURL | None
+ subs: tuple[Subtitle, ...]
+
+
+def _extract_subs(video: bs4.Tag) -> Generator[Subtitle]:
+ for track in css.iselect(video, "track[src]"):
+ kind = css.attr_or_none(track, "kind")
+ if not kind or kind in {"subtitles", "captions"}:
+ yield Subtitle(
+ url=css.attr(track, "src"),
+ name=css.attr_or_none(track, "label"),
+ lang_code=css.attr_or_none(track, "srclang", "lang", "label") or "unk",
+ )
+
+
+def _extract_video(html: str) -> Video:
+ video = css.select(css.soup(html, parse_only="video"), "#main-video")
+ parse_url = BunnyCDNCrawler.parse_url
+ try:
+ original = extr_text(html, "originalUrl =", ";").strip("'")
+ except ValueError:
+ original = None
+
+ try:
+ title = json.loads(css.attr(video, "data-plyr-config"))["title"]
+ except (ValueError, LookupError, css.SelectorError):
+ title = None
+
+ return Video(
+ title=title,
+ thumb=parse_url(css.attr(video, "data-poster")),
+ src=parse_url(css.select(video, "source", "src")),
+ original=parse_url(original) if original else None,
+ subs=tuple(_extract_subs(video)),
+ )
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index 0297cddc2..0d5073c90 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -608,11 +608,11 @@ class Crawler(HTTPMixin, HLSMixin, ABC):
frag="thumbnail",
)
- async def sniff_ext(self, thumbnail: AbsoluteHttpURL) -> str:
+ async def sniff_ext(self, thumbnail: AbsoluteHttpURL, headers: dict[str, str] | None = None) -> str:
try:
_, ext = self.get_filename_and_ext(thumbnail.name)
except NoExtensionError:
- async with self.request(thumbnail, self._THUMB_HTTP_METHOD) as resp:
+ async with self.request(thumbnail, self._THUMB_HTTP_METHOD, headers=headers or {}) as resp:
_, ext = self.get_filename_and_ext(thumbnail.name, mime_type=resp.content_type)
return ext
diff --git a/cyberdrop_dl/downloader/bunny_cdn.py b/cyberdrop_dl/downloader/bunny_cdn.py
new file mode 100644
index 000000000..9320f22f1
--- /dev/null
+++ b/cyberdrop_dl/downloader/bunny_cdn.py
@@ -0,0 +1,67 @@
+from __future__ import annotations
+
+import contextlib
+import dataclasses
+import hashlib
+import logging
+import random
+from contextvars import ContextVar
+from typing import TYPE_CHECKING, override
+
+from cyberdrop_dl import aio
+from cyberdrop_dl.downloader.http import Downloader
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem
+ from cyberdrop_dl.utils.m3u8 import Rendition
+
+logger = logging.getLogger(__name__)
+
+_playback_time = ContextVar[int]("_playback_time", default=0)
+_playback_url: ContextVar[AbsoluteHttpURL] = ContextVar("_playback_url")
+
+
[email protected](slots=True, frozen=True, kw_only=True)
+class PingData:
+ url: AbsoluteHttpURL
+ headers: dict[str, str]
+ secret: str
+ context_id: str
+ resolution: int = 1080
+ period: int = 2
+
+
+def _sign_url(url: AbsoluteHttpURL) -> AbsoluteHttpURL:
+ return url.update_query(hash=hashlib.md5("_".join(url.query.values()).encode(), usedforsecurity=False).hexdigest())
+
+
+class BunnyCDNDownloader(Downloader):
+ async def _ping_playback(self, ping: PingData) -> None:
+ current_time = _playback_time.get() + ping.period
+ _playback_time.set(current_time)
+ ping_url = _sign_url(
+ ping.url.update_query(
+ time=current_time + round(random.random(), 6),
+ paused="false",
+ resolution=ping.resolution,
+ )
+ )
+
+ try:
+ async with self.client.http_client.raw_request(ping_url, headers=ping.headers):
+ ...
+ except Exception:
+ logger.exception("BunnyCDN ping for %s failed", _playback_url.get())
+
+ @override
+ async def _hls(self, media_item: MediaItem, rendition: Rendition) -> None:
+ try:
+ ping: PingData = media_item.extra_info[media_item.domain]["ping_data"]
+ except LookupError:
+ background_ping = contextlib.nullcontext()
+ else:
+ background_ping = aio.backgroud_task(lambda: self._ping_playback(ping), period=ping.period)
+ _playback_url.set(media_item.real_url)
+
+ async with background_ping:
+ await super()._hls(media_item, rendition)
diff --git a/cyberdrop_dl/utils/css.py b/cyberdrop_dl/utils/css.py
index 64afa0358..4767f8ded 100644
--- a/cyberdrop_dl/utils/css.py
+++ b/cyberdrop_dl/utils/css.py
@@ -74,8 +74,14 @@ def select_tag_text(html: str, tag_name: str) -> str:
return select_text(soup(html, parse_only=tag_name), tag_name)
-def attr_or_none(tag: Tag, attribute: str) -> str | None:
+def attr_or_none(tag: Tag, *attributes: str) -> str | None:
"""Same as `tag.get(attribute)` but asserts the result is a single str"""
+ for attribute in attributes:
+ if result := _attr_or_none(tag, attribute):
+ return result
+
+
+def _attr_or_none(tag: Tag, attribute: str) -> str | None:
attribute_ = attribute
if attribute_ == "srcset":
if (srcset := tag.get(attribute_)) and type(srcset) is str:
@@ -90,12 +96,11 @@ def attr_or_none(tag: Tag, attribute: str) -> str | None:
return value
-def attr(tag: Tag, attribute: str) -> str:
+def attr(tag: Tag, *attributes: str) -> str:
"""Same as `tag.get(attribute)` but asserts the result is not `None` and is a single string"""
- result = attr_or_none(tag, attribute)
- if result is None:
- raise SelectorError(f"{attribute = } not found")
- return result
+ if result := attr_or_none(tag, *attributes):
+ return result
+ raise SelectorError(f"{attributes = } not found")
def text(tag: Tag, *, strip: bool = True) -> str:
diff --git a/tests/crawlers/test_cases/bunnycdn.py b/tests/crawlers/test_cases/bunnycdn.py
new file mode 100644
index 000000000..692e4d0bd
--- /dev/null
+++ b/tests/crawlers/test_cases/bunnycdn.py
@@ -0,0 +1,42 @@
+DOMAIN = "bunnycdn"
+TEST_CASES = [
+ {
+ "url": "https://iframe.mediadelivery.net/embed/136145/32e34c4b-0d72-437c-9abb-05e67657da34",
+ "results": [
+ {
+ "url": "https://vz-b14560c0-654.b-cdn.net/32e34c4b-0d72-437c-9abb-05e67657da34/original",
+ "filename": "Sanela ist Teil der #arbeitsmarktkraft [32e34c4b-0d72-437c-9abb-05e67657da34].mp4",
+ "debrid_url": None,
+ "original_filename": "Sanela ist Teil der #arbeitsmarktkraft",
+ "referer": "https://iframe.mediadelivery.net/embed/136145/32e34c4b-0d72-437c-9abb-05e67657da34",
+ "album_id": None,
+ "uploaded_at": None,
+ "download_folder": "re:Loose Files (BunnyCDN)",
+ "thumbnail": "https://vz-b14560c0-654.b-cdn.net/32e34c4b-0d72-437c-9abb-05e67657da34/thumbnail_9172dc16.jpg",
+ },
+ {
+ "url": "https://vz-b14560c0-654.b-cdn.net/32e34c4b-0d72-437c-9abb-05e67657da34/captions/de-auto.vtt?ver=1",
+ "filename": "Sanela ist Teil der #arbeitsmarktkraft [32e34c4b-0d72-437c-9abb-05e67657da34].de-auto.vtt",
+ "debrid_url": None,
+ "original_filename": "DE",
+ "referer": "https://iframe.mediadelivery.net/embed/136145/32e34c4b-0d72-437c-9abb-05e67657da34#Sanela%20ist%20Teil%20der%20%23arbeitsmarktkraft%20%5B32e34c4b-0d72-437c-9abb-05e67657da34%5D.de-auto.vtt",
+ "album_id": None,
+ "uploaded_at": None,
+ "download_folder": "re:Loose Files (BunnyCDN)",
+ "thumbnail": None,
+ },
+ {
+ "url": "https://vz-b14560c0-654.b-cdn.net/32e34c4b-0d72-437c-9abb-05e67657da34/captions/en-auto.vtt?ver=1",
+ "filename": "Sanela ist Teil der #arbeitsmarktkraft [32e34c4b-0d72-437c-9abb-05e67657da34].en-auto.vtt",
+ "debrid_url": None,
+ "original_filename": "EN",
+ "referer": "https://iframe.mediadelivery.net/embed/136145/32e34c4b-0d72-437c-9abb-05e67657da34#Sanela%20ist%20Teil%20der%20%23arbeitsmarktkraft%20%5B32e34c4b-0d72-437c-9abb-05e67657da34%5D.en-auto.vtt",
+ "album_id": None,
+ "uploaded_at": None,
+ "download_folder": "re:Loose Files (BunnyCDN)",
+ "thumbnail": None,
+ },
+ ],
+ "count": 3,
+ },
+]