Skip to content

Bulk asynchronous downloader for multiple file hosts

feat: add cloudflare stream support (#1255)

NTFSvolumea4ea8832aa86
diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.py
index 383b09500..f6c5f2aa3 100644
--- a/cyberdrop_dl/crawlers/__init__.py
+++ b/cyberdrop_dl/crawlers/__init__.py
@@ -14,6 +14,7 @@ from .bunkrr_albums_io import BunkrAlbumsIOCrawler
from .buzzheavier import BuzzHeavierCrawler
from .camwhores_dot_tv import CamwhoresTVCrawler
from .catbox import CatboxCrawler
+from .cloudflare_stream import CloudflareStreamCrawler
from .coomer import CoomerCrawler
from .crawler import Crawler
from .cyberdrop import CyberdropCrawler
diff --git a/cyberdrop_dl/crawlers/cloudflare_stream.py b/cyberdrop_dl/crawlers/cloudflare_stream.py
new file mode 100644
index 000000000..521c03dc1
--- /dev/null
+++ b/cyberdrop_dl/crawlers/cloudflare_stream.py
@@ -0,0 +1,85 @@
+"""
+https://www.cloudflare.com/developer-platform/products/cloudflare-stream/
+https://developers.cloudflare.com/stream/
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import TYPE_CHECKING, ClassVar
+
+from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
+from cyberdrop_dl.exceptions import ScrapeError
+from cyberdrop_dl.utils.json import JSONWebToken, is_jwt
+from cyberdrop_dl.utils.utilities import error_handling_wrapper
+
+if TYPE_CHECKING:
+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem
+
+_DEFAULT_VIDEO_CDN = AbsoluteHttpURL("https://watch.cloudflarestream.com")
+
+
+class CloudflareStreamCrawler(Crawler):
+ SUPPORTED_DOMAINS: SupportedDomains = "videodelivery.net", "cloudflarestream.com"
+ SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
+ "Public Video": (
+ "/embed/___.js?video=<video_uid>",
+ "/<video_uid>/watch",
+ "/<video_uid>/iframe",
+ "/<video_uid>",
+ ),
+ "Restricted Access Video": (
+ "/embed/___.js?video=<jwt_access_token>",
+ "/<jwt_access_token>/watch",
+ "/<jwt_access_token>/iframe",
+ "/<jwt_access_token>",
+ ),
+ }
+
+ DOMAIN: ClassVar[str] = "cloudflarestream"
+ FOLDER_DOMAIN: ClassVar[str] = "CloudflareStream"
+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = AbsoluteHttpURL("https://cloudflarestream.com")
+
+ async def fetch(self, scrape_item: ScrapeItem) -> None:
+ match scrape_item.url.parts[1:]:
+ case ["embed", _] if video_id := scrape_item.url.query.get("video"):
+ return await self.video(scrape_item, video_id)
+ case [video_id, *_]:
+ return await self.video(scrape_item, video_id)
+ case _:
+ raise ValueError
+
+ async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
+ if is_jwt(video_id):
+ # https://developers.cloudflare.com/stream/viewing-videos/securing-your-stream/
+ token = video_id
+ jwt = JSONWebToken.decode(token)
+ video_id = jwt.payload["sub"]
+ if jwt.is_expired():
+ self.raise_e(scrape_item, ScrapeError(401, "Access token to the video has expired"))
+ return
+ else:
+ token = None
+
+ _ = uuid.UUID(hex=video_id) # raise ValueError if video_id is not a valid uuid
+ scrape_item.url = _DEFAULT_VIDEO_CDN / video_id
+ return await self._video(scrape_item, video_id, token)
+
+ @error_handling_wrapper
+ async def _video(self, scrape_item: ScrapeItem, video_id: str, token: str | None) -> None:
+ if await self.check_complete_from_referer(scrape_item):
+ return
+
+ m3u8_url = self.PRIMARY_URL / (token or video_id) / "manifest/video.m3u8"
+ m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)
+ filename, ext = self.get_filename_and_ext(video_id + ".mp4")
+ custom_filename = self.create_custom_filename(
+ video_id,
+ ext,
+ file_id=video_id,
+ resolution=info.resolution,
+ video_codec=info.codecs.video,
+ audio_codec=info.codecs.audio,
+ )
+ await self.handle_file(scrape_item.url, scrape_item, filename, ext, m3u8=m3u8, custom_filename=custom_filename)
diff --git a/cyberdrop_dl/utils/json.py b/cyberdrop_dl/utils/json.py
index f71150ab6..0d676240a 100644
--- a/cyberdrop_dl/utils/json.py
+++ b/cyberdrop_dl/utils/json.py
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
+import base64
import dataclasses
import datetime
import enum
@@ -9,8 +10,9 @@ import json
import json.decoder
import json.scanner
import re
+import time
from collections.abc import Iterable
-from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, ParamSpec, Protocol, TypeGuard, TypeVar
+from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, ParamSpec, Protocol, Self, TypeGuard, TypeVar
if TYPE_CHECKING:
from collections.abc import Callable, Iterable
@@ -171,3 +173,44 @@ def load_js_obj(string: str, /) -> Any:
string = string.replace(old, new)
string = re.sub(r"\s\b(?!http)(\w+)\s?:", r' "\1" : ', string) # wrap keys without quotes with double quotes
return _JS_DECODER.decode(string)
+
+
[email protected](frozen=True, slots=True)
+class JSONWebToken:
+ # https://www.rfc-editor.org/rfc/rfc7519.html
+ alg: str
+ headers: dict[str, str] = dataclasses.field(repr=False)
+ payload: dict[str, Any] = dataclasses.field(repr=False)
+ signature: str
+ _encoded_token: str
+
+ @classmethod
+ def decode(cls, jwt: str, /) -> Self:
+ b64_headers, b64_payload, b64_signature = jwt.split(".")
+ headers = cls._decode(b64_headers)
+ return cls(headers["alg"], headers, cls._decode(b64_payload), b64_signature, jwt)
+
+ @classmethod
+ def _decode(cls, value: str, /) -> dict[str, Any]:
+ return loads(base64.urlsafe_b64decode(f"{value}==="))
+
+ def is_expired(self, threshold: int = 0) -> bool:
+ """Checks if the token has expired or is about to expire.
+
+ threshold is the time in seconds before the token's expiration to consider it as expired.
+ """
+ expires: int | None = self.payload.get("exp")
+ if expires:
+ return (expires - time.time()) < threshold
+ return False
+
+ def __str__(self) -> str:
+ return self._encoded_token
+
+
+def is_jwt(string: str) -> bool:
+ return string.startswith("eyJ") and string.count(".") == 2
+
+
+def jwt_decode(jwt: str) -> dict[str, Any]:
+ return JSONWebToken.decode(jwt).payload
diff --git a/tests/crawlers/test_cases/cloudflare_stream.py b/tests/crawlers/test_cases/cloudflare_stream.py
new file mode 100644
index 000000000..cc71f23d3
--- /dev/null
+++ b/tests/crawlers/test_cases/cloudflare_stream.py
@@ -0,0 +1,30 @@
+DOMAIN = "cloudflarestream"
+TEST_CASES = [
+ (
+ # This video has subtitles
+ "https://customer-eq7kiuol0tk9chox.cloudflarestream.com/embed/sdk-iframe-integration.fla9.latest.js?video=e7bd2dd67e0f8860b4ae81e33a966049",
+ [
+ {
+ "url": "https://watch.cloudflarestream.com/e7bd2dd67e0f8860b4ae81e33a966049",
+ "filename": "e7bd2dd67e0f8860b4ae81e33a966049 [e7bd2dd67e0f8860b4ae81e33a966049][avc1][mp4a][720p].mp4",
+ "original_filename": "e7bd2dd67e0f8860b4ae81e33a966049.mp4",
+ "referer": "https://watch.cloudflarestream.com/e7bd2dd67e0f8860b4ae81e33a966049",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+ (
+ "https://customer-u1hok16au7q8ozt4.cloudflarestream.com/1c0006dc4f4299fa33d0dfd161db1c35/watch",
+ [
+ {
+ "url": "https://watch.cloudflarestream.com/1c0006dc4f4299fa33d0dfd161db1c35",
+ "filename": "1c0006dc4f4299fa33d0dfd161db1c35 [1c0006dc4f4299fa33d0dfd161db1c35][avc1][mp4a][1080p].mp4",
+ "original_filename": "1c0006dc4f4299fa33d0dfd161db1c35.mp4",
+ "referer": "https://watch.cloudflarestream.com/1c0006dc4f4299fa33d0dfd161db1c35",
+ "album_id": None,
+ "datetime": None,
+ }
+ ],
+ ),
+]