Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: remove UUID check (cloudflare stream) (#162)
NTFSvolume8a297cc0457b
diff --git a/cyberdrop_dl/crawlers/ashemaletube.py b/cyberdrop_dl/crawlers/ashemaletube.pyindex 86694d099..2cbe7f426 100644--- a/cyberdrop_dl/crawlers/ashemaletube.py+++ b/cyberdrop_dl/crawlers/ashemaletube.py@@ -194,6 +194,6 @@ class AShemaleTubeCrawler(Crawler):sources_data = json.loads(sources)url = self.parse_url(sources_data["hlsAuto"])- rendition, playlist_info = await self.get_m3u8_from_playlist_url(url)+ rendition, playlist_info = await self.request_m3u8_playlist(url)return playlist_info.resolution, url, renditiondiff --git a/cyberdrop_dl/crawlers/cloudflare_stream.py b/cyberdrop_dl/crawlers/cloudflare_stream.pyindex d57294137..3fad4bbda 100644--- a/cyberdrop_dl/crawlers/cloudflare_stream.py+++ b/cyberdrop_dl/crawlers/cloudflare_stream.py@@ -5,14 +5,13 @@ https://developers.cloudflare.com/stream/from __future__ import annotations-import uuidfrom typing import TYPE_CHECKING, ClassVarfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPathsfrom cyberdrop_dl.exceptions import ScrapeErrorfrom cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import error_handling_wrapper-from cyberdrop_dl.utils.json import JSONWebToken, is_jwt+from cyberdrop_dl.utils.json import JSONWebTokenif TYPE_CHECKING:from cyberdrop_dl.url_objects import ScrapeItem@@ -21,7 +20,7 @@ _DEFAULT_VIDEO_CDN = AbsoluteHttpURL("https://watch.cloudflarestream.com")class CloudflareStreamCrawler(Crawler):- SUPPORTED_DOMAINS: SupportedDomains = "videodelivery.net", "cloudflarestream.com"+ SUPPORTED_DOMAINS: ClassVar[SupportedDomains] = "videodelivery.net", "cloudflarestream.com"SUPPORTED_PATHS: ClassVar[SupportedPaths] = {"Public Video": ("/embed/___.js?video=<video_uid>",@@ -51,18 +50,18 @@ class CloudflareStreamCrawler(Crawler):raise ValueErrorasync def video(self, scrape_item: ScrapeItem, video_id: str) -> None:- if is_jwt(video_id):- # https://developers.cloudflare.com/stream/viewing-videos/securing-your-stream/- token = video_id- jwt = JSONWebToken.decode(token)+ # https://developers.cloudflare.com/stream/viewing-videos/securing-your-stream/+ try:+ jwt = JSONWebToken.decode(video_id)+ except ValueError:+ token = None+ else:+ token = jwt.encodedvideo_id = jwt.payload["sub"]if jwt.is_expired():self.raise_exc(scrape_item, ScrapeError(401, "Access token to the video has expired"))return- else:- token = None- _ = uuid.UUID(hex=video_id) # raise ValueError if video_id is not a valid uuidscrape_item.url = _DEFAULT_VIDEO_CDN / video_idreturn await self._video(scrape_item, video_id, token)@@ -72,7 +71,7 @@ class CloudflareStreamCrawler(Crawler):returnm3u8_url = self.PRIMARY_URL / (token or video_id) / "manifest/video.m3u8"- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)+ m3u8, info = await self.request_m3u8_playlist(m3u8_url)filename, ext = self.get_filename_and_ext(video_id + ".mp4")custom_filename = self.create_custom_filename(video_id,diff --git a/cyberdrop_dl/crawlers/crawler.py b/cyberdrop_dl/crawlers/crawler.pyindex c4814d149..a56af99c5 100644--- a/cyberdrop_dl/crawlers/crawler.py+++ b/cyberdrop_dl/crawlers/crawler.py@@ -790,8 +790,7 @@ class Crawler(HTTPClientProxy, HLSParser, ABC):scrape_item.url = redirectself.create_task(self.run(scrape_item))- @deprecated("Use self.request_m3u8 instead")- async def get_m3u8_from_playlist_url(+ async def request_m3u8_playlist(self,m3u8_playlist_url: AbsoluteHttpURL,/,diff --git a/cyberdrop_dl/crawlers/megacloud.py b/cyberdrop_dl/crawlers/megacloud.pyindex 7192a9944..888066944 100644--- a/cyberdrop_dl/crawlers/megacloud.py+++ b/cyberdrop_dl/crawlers/megacloud.py@@ -62,7 +62,7 @@ class MegaCloudCrawler(Crawler):async def _handle_video(self: Crawler, scrape_item: ScrapeItem, video: MegaCloudVideo) -> None:m3u8_url = video.sources[0]- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url, headers=_HEADERS)+ m3u8, info = await self.request_m3u8_playlist(m3u8_url, headers=_HEADERS)filename, ext = self.get_filename_and_ext(video.id + ".mp4")video_name = self.create_custom_filename(video.title or video.id, ext, file_id=video.id, resolution=info.resolutiondiff --git a/cyberdrop_dl/crawlers/missav.py b/cyberdrop_dl/crawlers/missav.pyindex a1ca872b0..0cd59f836 100644--- a/cyberdrop_dl/crawlers/missav.py+++ b/cyberdrop_dl/crawlers/missav.py@@ -71,7 +71,7 @@ class MissAVCrawler(Crawler):uuid = _extract_uuid(soup)m3u8_url = _M3U8_SERVER / uuid / "playlist.m3u8"- m3u8, info = await self.get_m3u8_from_playlist_url(+ m3u8, info = await self.request_m3u8_playlist(m3u8_url,headers={"Referer": "https://missav.ws/"},)diff --git a/cyberdrop_dl/crawlers/pornhub.py b/cyberdrop_dl/crawlers/pornhub.pyindex a503440e7..7d3a937dd 100644--- a/cyberdrop_dl/crawlers/pornhub.py+++ b/cyberdrop_dl/crawlers/pornhub.py@@ -272,7 +272,7 @@ class PornHubCrawler(Crawler):debrid_link = self.parse_url(best_format.url)if use_hls or best_format is None:- m3u8, _ = await self.get_m3u8_from_playlist_url(self.parse_url(best_hls.url))+ m3u8, _ = await self.request_m3u8_playlist(self.parse_url(best_hls.url))best_format = best_hlsscrape_item.url = page_urldiff --git a/cyberdrop_dl/crawlers/rumble.py b/cyberdrop_dl/crawlers/rumble.pyindex e70ebf9b1..85175dc12 100644--- a/cyberdrop_dl/crawlers/rumble.py+++ b/cyberdrop_dl/crawlers/rumble.py@@ -198,7 +198,7 @@ class RumbleCrawler(Crawler):other_formats: list[Format] = [fmt for fmt in formats if fmt.is_single_file or hls_formats.append(fmt)]async def resolve_m3u8(format: Format) -> Format:- m3u8, info = await self.get_m3u8_from_playlist_url(format.url)+ m3u8, info = await self.request_m3u8_playlist(format.url)return format._replace(resolution=info.resolution,m3u8=m3u8,diff --git a/cyberdrop_dl/crawlers/twitch.py b/cyberdrop_dl/crawlers/twitch.pyindex 0f44570c3..b27cb115f 100644--- a/cyberdrop_dl/crawlers/twitch.py+++ b/cyberdrop_dl/crawlers/twitch.py@@ -115,7 +115,7 @@ class TwitchCrawler(Crawler):token=access_token["value"],)- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)+ m3u8, info = await self.request_m3u8_playlist(m3u8_url)fps = info.stream_info.frame_ratefilename = self.create_custom_filename(title,diff --git a/cyberdrop_dl/crawlers/vidara.py b/cyberdrop_dl/crawlers/vidara.pyindex 1ceb66935..e13ce53fb 100644--- a/cyberdrop_dl/crawlers/vidara.py+++ b/cyberdrop_dl/crawlers/vidara.py@@ -39,7 +39,7 @@ class VidaraCrawler(Crawler, db_path="path_qs_frag"):returnm3u8_url, thumbnail = await self._request_stream(video_id)- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)+ m3u8, info = await self.request_m3u8_playlist(m3u8_url)name, ext = self.get_filename_and_ext(video_id + ".mp4")custom_filename = self.create_custom_filename(name, ext, resolution=info.resolution)diff --git a/cyberdrop_dl/crawlers/vsco.py b/cyberdrop_dl/crawlers/vsco.pyindex 3a3a97d0a..cecd4c137 100644--- a/cyberdrop_dl/crawlers/vsco.py+++ b/cyberdrop_dl/crawlers/vsco.py@@ -100,7 +100,7 @@ class VSCOCrawler(Crawler):returnext = ".mp4"- m3u8, info = await self.get_m3u8_from_playlist_url(url)+ m3u8, info = await self.request_m3u8_playlist(url)res = info.resolutionname, ext = self.get_filename_and_ext(video["id"] + ext)diff --git a/cyberdrop_dl/crawlers/xvideos.py b/cyberdrop_dl/crawlers/xvideos.pyindex a18780fc4..6801f1dc7 100644--- a/cyberdrop_dl/crawlers/xvideos.py+++ b/cyberdrop_dl/crawlers/xvideos.py@@ -121,7 +121,7 @@ class XVideosCrawler(Crawler):scrape_item.uploaded_at = self.parse_iso_date(css.json_ld(soup)["uploadDate"])script = css.select_text(soup, Selectors.HLS_VIDEO_JS)m3u8_url = self.parse_url(get_text_between(script, "setVideoHLS('", "')"))- m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url)+ m3u8, info = await self.request_m3u8_playlist(m3u8_url)custom_filename = self.create_custom_filename(title, ".mp4", file_id=encoded_id, resolution=info.resolution)# Remove url slug to prevent duplicates in database. It's language specific and not required.url = scrape_item.url.with_name("_")diff --git a/cyberdrop_dl/utils/json.py b/cyberdrop_dl/utils/json.pyindex d9c6913ed..5fb93eba1 100644--- a/cyberdrop_dl/utils/json.py+++ b/cyberdrop_dl/utils/json.py@@ -181,6 +181,8 @@ class JSONWebToken:@classmethoddef decode(cls, jwt: str, /) -> Self:+ if not is_jwt(jwt):+ raise ValueError("Not a valid JSON Web Token", jwt)b64_headers, b64_payload, b64_signature = jwt.split(".")headers = cls._decode(b64_headers)return cls(headers["alg"], headers, cls._decode(b64_payload), b64_signature, jwt)