Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: remove UUID check (cloudflare stream) (#162)

NTFSvolume8a297cc0457b
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.py
index 86694d099..2cbe7f426 100644
--- a/cyberdrop_dl/crawlers/ashemaletube.py
+++ b/cyberdrop_dl/crawlers/ashemaletube.py
@@ -194,6 +194,6 @@ class AShemaleTubeCrawler(Crawler):
sources_data = json.loads(sources)
url = self.parse_url(sources_data["hlsAuto"])
- rendition, playlist_info = await self.get_m3u8_from_playlist_url(url)
+ rendition, playlist_info = await self.request_m3u8_playlist(url)
return playlist_info.resolution, url, rendition
diff --git a/cyberdrop_dl/crawlers/cloudflare_stream.py b/cyberdrop_dl/crawlers/cloudflare_stream.py
index d57294137..3fad4bbda 100644
--- a/cyberdrop_dl/crawlers/cloudflare_stream.py
+++ b/cyberdrop_dl/crawlers/cloudflare_stream.py
@@ -5,14 +5,13 @@ https://developers.cloudflare.com/stream/
from __future__ import annotations
-import uuid
from typing import TYPE_CHECKING, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths
from cyberdrop_dl.exceptions import ScrapeError
from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import error_handling_wrapper
-from cyberdrop_dl.utils.json import JSONWebToken, is_jwt
+from cyberdrop_dl.utils.json import JSONWebToken
if TYPE_CHECKING:
from cyberdrop_dl.url_objects import ScrapeItem
@@ -21,7 +20,7 @@ _DEFAULT_VIDEO_CDN = AbsoluteHttpURL("https://watch.cloudflarestream.com")
class CloudflareStreamCrawler(Crawler):
- SUPPORTED_DOMAINS: SupportedDomains = "videodelivery.net", "cloudflarestream.com"
+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "videodelivery.net", "cloudflarestream.com"
SUPPORTED_PATHS: ClassVar[SupportedPaths] = {
"Public Video": (
"/embed/___.js?video=<video_uid>",
@@ -51,18 +50,18 @@ class CloudflareStreamCrawler(Crawler):
raise ValueError
async def video(self, scrape_item: ScrapeItem, video_id: str) -> None:
- if is_jwt(video_id):
- # https://developers.cloudflare.com/stream/viewing-videos/securing-your-stream/
- token = video_id
- jwt = JSONWebToken.decode(token)
+ # https://developers.cloudflare.com/stream/viewing-videos/securing-your-stream/
+ try:
+ jwt = JSONWebToken.decode(video_id)
+ except ValueError:
+ token = None
+ else:
+ token = jwt.encoded
video_id = jwt.payload["sub"]
if jwt.is_expired():
self.raise_exc(scrape_item, ScrapeError(401, "Access token to the video has expired"))
return
- else:
- token = None
- _ = uuid.UUID(hex=video_id) # raise ValueError if video_id is not a valid uuid
scrape_item.url = _DEFAULT_VIDEO_CDN / video_id
return await self._video(scrape_item, video_id, token)
@@ -72,7 +71,7 @@ class CloudflareStreamCrawler(Crawler):
return
m3u8_url = self.PRIMARY_URL / (token or video_id) / "manifest/video.m3u8"
- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)
+ m3u8, info = await self.request_m3u8_playlist(m3u8_url)
filename, ext = self.get_filename_and_ext(video_id + ".mp4")
custom_filename = self.create_custom_filename(
video_id,
diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.py
index c4814d149..a56af99c5 100644
--- a/cyberdrop_dl/crawlers/crawler.py
+++ b/cyberdrop_dl/crawlers/crawler.py
@@ -790,8 +790,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):
scrape_item.url = redirect
self.create_task(self.run(scrape_item))
- @deprecated("Use self.request_m3u8 instead")
- async def get_m3u8_from_playlist_url(
+ async def request_m3u8_playlist(
self,
m3u8_playlist_url: AbsoluteHttpURL,
/,
diff --git a/cyberdrop_dl/crawlers/megacloud.py b/cyberdrop_dl/crawlers/megacloud.py
index 7192a9944..888066944 100644
--- a/cyberdrop_dl/crawlers/megacloud.py
+++ b/cyberdrop_dl/crawlers/megacloud.py
@@ -62,7 +62,7 @@ class MegaCloudCrawler(Crawler):
async def _handle_video(self: Crawler, scrape_item: ScrapeItem, video: MegaCloudVideo) -> None:
m3u8_url = video.sources[0]
- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url, headers=_HEADERS)
+ m3u8, info = await self.request_m3u8_playlist(m3u8_url, headers=_HEADERS)
filename, ext = self.get_filename_and_ext(video.id + ".mp4")
video_name = self.create_custom_filename(
video.title or video.id, ext, file_id=video.id, resolution=info.resolution
diff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.py
index a1ca872b0..0cd59f836 100644
--- a/cyberdrop_dl/crawlers/missav.py
+++ b/cyberdrop_dl/crawlers/missav.py
@@ -71,7 +71,7 @@ class MissAVCrawler(Crawler):
uuid = _extract_uuid(soup)
m3u8_url = _M3U8_SERVER / uuid / "playlist.m3u8"
- m3u8, info = await self.get_m3u8_from_playlist_url(
+ m3u8, info = await self.request_m3u8_playlist(
m3u8_url,
headers={"Referer": "https://missav.ws/"},
)
diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.py
index a503440e7..7d3a937dd 100644
--- a/cyberdrop_dl/crawlers/pornhub.py
+++ b/cyberdrop_dl/crawlers/pornhub.py
@@ -272,7 +272,7 @@ class PornHubCrawler(Crawler):
debrid_link = self.parse_url(best_format.url)
if use_hls or best_format is None:
- m3u8, _ = await self.get_m3u8_from_playlist_url(self.parse_url(best_hls.url))
+ m3u8, _ = await self.request_m3u8_playlist(self.parse_url(best_hls.url))
best_format = best_hls
scrape_item.url = page_url
diff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.py
index e70ebf9b1..85175dc12 100644
--- a/cyberdrop_dl/crawlers/rumble.py
+++ b/cyberdrop_dl/crawlers/rumble.py
@@ -198,7 +198,7 @@ class RumbleCrawler(Crawler):
other_formats: list[Format] = [fmt for fmt in formats if fmt.is_single_file or hls_formats.append(fmt)]
async def resolve_m3u8(format: Format) -> Format:
- m3u8, info = await self.get_m3u8_from_playlist_url(format.url)
+ m3u8, info = await self.request_m3u8_playlist(format.url)
return format._replace(
resolution=info.resolution,
m3u8=m3u8,
diff --git a/cyberdrop_dl/crawlers/twitch.py b/cyberdrop_dl/crawlers/twitch.py
index 0f44570c3..b27cb115f 100644
--- a/cyberdrop_dl/crawlers/twitch.py
+++ b/cyberdrop_dl/crawlers/twitch.py
@@ -115,7 +115,7 @@ class TwitchCrawler(Crawler):
token=access_token["value"],
)
- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)
+ m3u8, info = await self.request_m3u8_playlist(m3u8_url)
fps = info.stream_info.frame_rate
filename = self.create_custom_filename(
title,
diff --git a/cyberdrop_dl/crawlers/vidara.py b/cyberdrop_dl/crawlers/vidara.py
index 1ceb66935..e13ce53fb 100644
--- a/cyberdrop_dl/crawlers/vidara.py
+++ b/cyberdrop_dl/crawlers/vidara.py
@@ -39,7 +39,7 @@ class VidaraCrawler(Crawler, db_path="path_qs_frag"):
return
m3u8_url, thumbnail = await self._request_stream(video_id)
- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)
+ m3u8, info = await self.request_m3u8_playlist(m3u8_url)
name, ext = self.get_filename_and_ext(video_id + ".mp4")
custom_filename = self.create_custom_filename(name, ext, resolution=info.resolution)
diff --git a/cyberdrop_dl/crawlers/vsco.py b/cyberdrop_dl/crawlers/vsco.py
index 3a3a97d0a..cecd4c137 100644
--- a/cyberdrop_dl/crawlers/vsco.py
+++ b/cyberdrop_dl/crawlers/vsco.py
@@ -100,7 +100,7 @@ class VSCOCrawler(Crawler):
return
ext = ".mp4"
- m3u8, info = await self.get_m3u8_from_playlist_url(url)
+ m3u8, info = await self.request_m3u8_playlist(url)
res = info.resolution
name, ext = self.get_filename_and_ext(video["id"] + ext)
diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.py
index a18780fc4..6801f1dc7 100644
--- a/cyberdrop_dl/crawlers/xvideos.py
+++ b/cyberdrop_dl/crawlers/xvideos.py
@@ -121,7 +121,7 @@ class XVideosCrawler(Crawler):
scrape_item.uploaded_at = self.parse_iso_date(css.json_ld(soup)["uploadDate"])
script = css.select_text(soup, Selectors.HLS_VIDEO_JS)
m3u8_url = self.parse_url(get_text_between(script, "setVideoHLS('", "')"))
- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)
+ m3u8, info = await self.request_m3u8_playlist(m3u8_url)
custom_filename = self.create_custom_filename(title, ".mp4", file_id=encoded_id, resolution=info.resolution)
# Remove url slug to prevent duplicates in database. It's language specific and not required.
url = scrape_item.url.with_name("_")
diff --git a/cyberdrop_dl/utils/json.py b/cyberdrop_dl/utils/json.py
index d9c6913ed..5fb93eba1 100644
--- a/cyberdrop_dl/utils/json.py
+++ b/cyberdrop_dl/utils/json.py
@@ -181,6 +181,8 @@ class JSONWebToken:
@classmethod
def decode(cls, jwt: str, /) -> Self:
+ if not is_jwt(jwt):
+ raise ValueError("Not a valid JSON Web Token", jwt)
b64_headers, b64_payload, b64_signature = jwt.split(".")
headers = cls._decode(b64_headers)
return cls(headers["alg"], headers, cls._decode(b64_payload), b64_signature, jwt)