Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add cloudflare stream support (#1255)
NTFSvolumea4ea8832aa86
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex 383b09500..f6c5f2aa3 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -14,6 +14,7 @@ from .bunkrr_albums_io import BunkrAlbumsIOCrawlerfrom .buzzheavier import BuzzHeavierCrawlerfrom .camwhores_dot_tv import CamwhoresTVCrawlerfrom .catbox import CatboxCrawler+from .cloudflare_stream import CloudflareStreamCrawlerfrom .coomer import CoomerCrawlerfrom .crawler import Crawlerfrom .cyberdrop import CyberdropCrawlerdiff --git a/cyberdrop_dl/crawlers/cloudflare_stream.py b/cyberdrop_dl/crawlers/cloudflare_stream.pynew file mode 100644index 000000000..521c03dc1--- /dev/null+++ b/cyberdrop_dl/crawlers/cloudflare_stream.py@@ -0,0 +1,85 @@+"""+https://www.cloudflare.com/developer-platform/products/cloudflare-stream/+https://developers.cloudflare.com/stream/+"""++from __future__ import annotations++import uuid+from typing import TYPE_CHECKING, ClassVar++from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.utils.json import JSONWebToken, is_jwt+from cyberdrop_dl.utils.utilities import error_handling_wrapper++if TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem++_DEFAULT_VIDEO_CDN = AbsoluteHttpURL("https://watch.cloudflarestream.com")+++class CloudflareStreamCrawler(Crawler):+ SUPPORTED_DOMAINS: SupportedDomains = "videodelivery.net", "cloudflarestream.com"+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {+ "Public Video": (+ "/embed/___.js?video=<video_uid>",+ "/<video_uid>/watch",+ "/<video_uid>/iframe",+ "/<video_uid>",+ ),+ "Restricted Access Video": (+ "/embed/___.js?video=<jwt_access_token>",+ "/<jwt_access_token>/watch",+ "/<jwt_access_token>/iframe",+ "/<jwt_access_token>",+ ),+ }++ DOMAIN: ClassVar[str] = "cloudflarestream"+ FOLDER_DOMAIN: ClassVar[str] = "CloudflareStream"+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://cloudflarestream.com")++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["embed", _] if video_id := scrape_item.url.query.get("video"):+ return await self.video(scrape_item, video_id)+ case [video_id, *_]:+ return await self.video(scrape_item, video_id)+ case _:+ raise ValueError++ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:+ if is_jwt(video_id):+ # https://developers.cloudflare.com/stream/viewing-videos/securing-your-stream/+ token = video_id+ jwt = JSONWebToken.decode(token)+ video_id = jwt.payload["sub"]+ if jwt.is_expired():+ self.raise_e(scrape_item, ScrapeError(401, "Access token to the video has expired"))+ return+ else:+ token = None++ _ = uuid.UUID(hex=video_id) # raise ValueError if video_id is not a valid uuid+ scrape_item.url = _DEFAULT_VIDEO_CDN / video_id+ return await self._video(scrape_item, video_id, token)++ @error_handling_wrapper+ async def _video(self, scrape_item: ScrapeItem, video_id: str, token: str | None) -> None:+ if await self.check_complete_from_referer(scrape_item):+ return++ m3u8_url = self.PRIMARY_URL / (token or video_id) / "manifest/video.m3u8"+ m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)+ filename, ext = self.get_filename_and_ext(video_id + ".mp4")+ custom_filename = self.create_custom_filename(+ video_id,+ ext,+ file_id=video_id,+ resolution=info.resolution,+ video_codec=info.codecs.video,+ audio_codec=info.codecs.audio,+ )+ await self.handle_file(scrape_item.url, scrape_item, filename, ext, m3u8=m3u8, custom_filename=custom_filename)diff --git a/cyberdrop_dl/utils/json.py b/cyberdrop_dl/utils/json.pyindex f71150ab6..0d676240a 100644--- a/cyberdrop_dl/utils/json.py+++ b/cyberdrop_dl/utils/json.py@@ -1,6 +1,7 @@from __future__ import annotationsimport asyncio+import base64import dataclassesimport datetimeimport enum@@ -9,8 +10,9 @@ import jsonimport json.decoderimport json.scannerimport re+import timefrom collections.abc import Iterable-from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, ParamSpec, Protocol, TypeGuard, TypeVar+from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, ParamSpec, Protocol, Self, TypeGuard, TypeVarif TYPE_CHECKING:from collections.abc import Callable, Iterable@@ -171,3 +173,44 @@ def load_js_obj(string: str, /) -> Any:string = string.replace(old, new)string = re.sub(r"\s\b(?!http)(\w+)\s?:", r' "\1" : ', string) # wrap keys without quotes with double quotesreturn _JS_DECODER.decode(string)++[email protected](frozen=True, slots=True)+class JSONWebToken:+ # https://www.rfc-editor.org/rfc/rfc7519.html+ alg: str+ headers: dict[str, str] = dataclasses.field(repr=False)+ payload: dict[str, Any] = dataclasses.field(repr=False)+ signature: str+ _encoded_token: str++ @classmethod+ def decode(cls, jwt: str, /) -> Self:+ b64_headers, b64_payload, b64_signature = jwt.split(".")+ headers = cls._decode(b64_headers)+ return cls(headers["alg"], headers, cls._decode(b64_payload), b64_signature, jwt)++ @classmethod+ def _decode(cls, value: str, /) -> dict[str, Any]:+ return loads(base64.urlsafe_b64decode(f"{value}==="))++ def is_expired(self, threshold: int = 0) -> bool:+ """Checks if the token has expired or is about to expire.++ threshold is the time in seconds before the token's expiration to consider it as expired.+ """+ expires: int | None = self.payload.get("exp")+ if expires:+ return (expires - time.time()) < threshold+ return False++ def __str__(self) -> str:+ return self._encoded_token+++def is_jwt(string: str) -> bool:+ return string.startswith("eyJ") and string.count(".") == 2+++def jwt_decode(jwt: str) -> dict[str, Any]:+ return JSONWebToken.decode(jwt).payloaddiff --git a/tests/crawlers/test_cases/cloudflare_stream.py b/tests/crawlers/test_cases/cloudflare_stream.pynew file mode 100644index 000000000..cc71f23d3--- /dev/null+++ b/tests/crawlers/test_cases/cloudflare_stream.py@@ -0,0 +1,30 @@+DOMAIN = "cloudflarestream"+TEST_CASES = [+ (+ # This video has subtitles+ "https://customer-eq7kiuol0tk9chox.cloudflarestream.com/embed/sdk-iframe-integration.fla9.latest.js?video=e7bd2dd67e0f8860b4ae81e33a966049",+ [+ {+ "url": "https://watch.cloudflarestream.com/e7bd2dd67e0f8860b4ae81e33a966049",+ "filename": "e7bd2dd67e0f8860b4ae81e33a966049 [e7bd2dd67e0f8860b4ae81e33a966049][avc1][mp4a][720p].mp4",+ "original_filename": "e7bd2dd67e0f8860b4ae81e33a966049.mp4",+ "referer": "https://watch.cloudflarestream.com/e7bd2dd67e0f8860b4ae81e33a966049",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+ (+ "https://customer-u1hok16au7q8ozt4.cloudflarestream.com/1c0006dc4f4299fa33d0dfd161db1c35/watch",+ [+ {+ "url": "https://watch.cloudflarestream.com/1c0006dc4f4299fa33d0dfd161db1c35",+ "filename": "1c0006dc4f4299fa33d0dfd161db1c35 [1c0006dc4f4299fa33d0dfd161db1c35][avc1][mp4a][1080p].mp4",+ "original_filename": "1c0006dc4f4299fa33d0dfd161db1c35.mp4",+ "referer": "https://watch.cloudflarestream.com/1c0006dc4f4299fa33d0dfd161db1c35",+ "album_id": None,+ "datetime": None,+ }+ ],+ ),+]