Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
feat: add megacloud support (#1289)
NTFSvolume7f3556561ff7
diff --git a/cyberdrop_dl/clients/download_client.py b/cyberdrop_dl/clients/download_client.pyindex 505836f91..cb6022733 100644--- a/cyberdrop_dl/clients/download_client.py+++ b/cyberdrop_dl/clients/download_client.py@@ -71,6 +71,8 @@ class DownloadClient:"Referer": "https://m.ok.ru/","Origin": "https://m.ok.ru",}+ elif domain == "megacloud":+ download_headers["Referer"] = "https://megacloud.blog/"return download_headersasync def _download(self, domain: str, media_item: MediaItem) -> bool:diff --git a/cyberdrop_dl/crawlers/__init__.py b/cyberdrop_dl/crawlers/__init__.pyindex da9bbef33..d44a716f0 100644--- a/cyberdrop_dl/crawlers/__init__.py+++ b/cyberdrop_dl/crawlers/__init__.py@@ -57,6 +57,7 @@ from .leakedzone import LeakedZoneCrawlerfrom .luscious import LusciousCrawlerfrom .mediafire import MediaFireCrawlerfrom .mega_nz import MegaNzCrawler+from .megacloud import MegaCloudCrawlerfrom .missav import MissAVCrawlerfrom .mixdrop import MixDropCrawlerfrom .motherless import MotherlessCrawlerdiff --git a/cyberdrop_dl/crawlers/megacloud.py b/cyberdrop_dl/crawlers/megacloud.pynew file mode 100644index 000000000..09951cdfb--- /dev/null+++ b/cyberdrop_dl/crawlers/megacloud.py@@ -0,0 +1,166 @@+from __future__ import annotations++import dataclasses+import re+from collections import Counter+from typing import TYPE_CHECKING, Any, ClassVar++from cyberdrop_dl.crawlers.crawler import Crawler+from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL+from cyberdrop_dl.exceptions import ScrapeError+from cyberdrop_dl.utils.utilities import error_handling_wrapper++if TYPE_CHECKING:+ from cyberdrop_dl.data_structures.url_objects import ScrapeItem+++_PRIMARY_URL = AbsoluteHttpURL("https://megacloud.blog")+_V3_SOURCES_URL = _PRIMARY_URL / "embed-2/v3/e-1/getSources"+_HEADERS = {"Origin": str(_PRIMARY_URL), "Referer": str(_PRIMARY_URL) + "/"}++[email protected](frozen=True, slots=True)+class MegaCloudSubtitle:+ label: str+ lang_code: str+ suffix: str+ url: AbsoluteHttpURL++[email protected](slots=True)+class MegaCloudVideo:+ id: str+ sources: tuple[AbsoluteHttpURL, ...]+ subtitles: tuple[MegaCloudSubtitle, ...]+ title: str = ""+++_find_v3_client_key = re.compile(r"([a-zA-Z0-9]{48})").search+++class MegaCloudCrawler(Crawler):+ SUPPORTED_PATHS: ClassVar[dict[str, str]] = {+ "Embed v3": "/embed-2/v3",+ }+ PRIMARY_URL: ClassVar[AbsoluteHttpURL] = _PRIMARY_URL+ DOMAIN: ClassVar[str] = "megacloud"++ async def fetch(self, scrape_item: ScrapeItem) -> None:+ match scrape_item.url.parts[1:]:+ case ["embed-2", "v3", _, _]:+ return await self.embed_v3(scrape_item)+ case _:+ raise ValueError++ @error_handling_wrapper+ async def embed_v3(self, scrape_item: ScrapeItem) -> None:+ canonical_url = scrape_item.url.with_query(None)+ if await self.check_complete_from_referer(canonical_url):+ return++ video = await self._request_video_source(scrape_item.url)+ scrape_item.url = canonical_url+ await self._handle_video(scrape_item, video)++ async def _handle_video(self: Crawler, scrape_item: ScrapeItem, video: MegaCloudVideo) -> None:+ m3u8_url = video.sources[0]+ m3u8, info = await self.get_m3u8_from_playlist_url(m3u8_url, headers=_HEADERS)+ filename, ext = self.get_filename_and_ext(video.id + ".mp4")+ custom_filename = self.create_custom_filename(+ video.title or video.id, ext, file_id=video.id, resolution=info.resolution+ )+ self.create_task(+ self.handle_file(+ scrape_item.url,+ scrape_item,+ filename,+ ext,+ m3u8=m3u8,+ custom_filename=custom_filename,+ )+ )+ video_stem = custom_filename.removesuffix(ext)+ for sub in video.subtitles:+ sub_name, ext = self.get_filename_and_ext(f"{video_stem}.{sub.suffix}")+ self.create_task(+ self.handle_file(+ sub.url,+ scrape_item,+ sub.label,+ ext,+ custom_filename=sub_name,+ )+ )++ async def _get_client_key(self: Crawler, embed_url: AbsoluteHttpURL) -> str:+ content = await self.request_text(embed_url, headers=_HEADERS)+ if match := _find_v3_client_key(content):+ return match.group()++ raise ScrapeError(422, "Unable to extract client key")++ async def _request_video_source(self: Crawler, embed_url: AbsoluteHttpURL) -> MegaCloudVideo:+ video_id = embed_url.name or embed_url.parent.name+ # Explicitly use the unbound _get_client_key so we can call this method from other crawlers without inheritance+ client_key = await MegaCloudCrawler._get_client_key(self, embed_url)+ src_url = _V3_SOURCES_URL.with_query(id=video_id, _k=client_key)+ resp: dict[str, Any] = await self.request_json(src_url, headers=_HEADERS)++ if resp["encrypted"]:+ # TODO: Add logic to handle encrypted videos+ raise ScrapeError(403, "Video is encrypted")++ def parse_subs():+ counter = Counter()+ for track in resp["tracks"]:+ if track["kind"] != "captions":+ continue++ url = self.parse_url(track["file"])+ label: str = track["label"]+ lang_code = _parse_lang_code(url.name, label)+ counter[lang_code] += 1+ if (count := counter[lang_code]) > 1:+ suffix = f"{lang_code}.{count}{url.suffix}"+ else:+ suffix = f"{lang_code}{url.suffix}"++ yield MegaCloudSubtitle(label, lang_code, suffix, url)++ return MegaCloudVideo(+ id=video_id,+ sources=tuple(self.parse_url(x["file"]) for x in resp["sources"]),+ subtitles=tuple(parse_subs()),+ )+++_ISO639_MAP = {+ "arabic": "ara",+ "english": "eng",+ "french": "fre",+ "german": "ger",+ "italian": "ita",+ "portuguese": "por",+ "russian": "rus",+ "spanish": "spa",+ "chinese": "chi",+ "korean": "kor",+ "thai": "tha",+ "indonesian": "ind",+}+++def _parse_lang_code(name: str, label: str) -> str:+ code = name.rsplit("-")[0]+ if len(code) in (2, 3):+ return code++ label = label.casefold()+ if code := _ISO639_MAP.get(label):+ return code++ for lang, code in _ISO639_MAP.items():+ if label.startswith(lang):+ return code++ return "unk"diff --git a/tests/crawlers/test_cases/megacloud.py b/tests/crawlers/test_cases/megacloud.pynew file mode 100644index 000000000..c62bff2c5--- /dev/null+++ b/tests/crawlers/test_cases/megacloud.py@@ -0,0 +1,52 @@+DOMAIN = "megacloud"+TEST_CASES = [+ (+ "https://megacloud.blog/embed-2/v3/e-1/TZb4gRkOQ642?k=1&autoPlay=1&oa=0&asi=1",+ [+ {+ "url": "https://megacloud.blog/embed-2/v3/e-1/TZb4gRkOQ642",+ "debrid_url": "ANY",+ "filename": "TZb4gRkOQ642 [TZb4gRkOQ642][1080p].mp4",+ "referer": "https://megacloud.blog/embed-2/v3/e-1/TZb4gRkOQ642",+ "album_id": None,+ "datetime": None,+ },+ {+ "url": "https://mgstatics.xyz/subtitle/1d75d2cf94d6ab987c8e6fa0af5ae44d/1d75d2cf94d6ab987c8e6fa0af5ae44d.vtt",+ "filename": "TZb4gRkOQ642 [TZb4gRkOQ642][1080p].eng.vtt",+ },+ {+ "url": "https://mgstatics.xyz/subtitle/090c09c12a07d5a96f14a2c1841e0548/ara-6.vtt",+ "filename": "TZb4gRkOQ642 [TZb4gRkOQ642][1080p].ara.vtt",+ },+ {+ "url": "https://mgstatics.xyz/subtitle/090c09c12a07d5a96f14a2c1841e0548/fre-7.vtt",+ "filename": "TZb4gRkOQ642 [TZb4gRkOQ642][1080p].fre.vtt",+ },+ {+ "url": "https://mgstatics.xyz/subtitle/090c09c12a07d5a96f14a2c1841e0548/ger-8.vtt",+ "filename": "TZb4gRkOQ642 [TZb4gRkOQ642][1080p].ger.vtt",+ },+ {+ "url": "https://mgstatics.xyz/subtitle/090c09c12a07d5a96f14a2c1841e0548/ita-9.vtt",+ "filename": "TZb4gRkOQ642 [TZb4gRkOQ642][1080p].ita.vtt",+ },+ {+ "url": "https://mgstatics.xyz/subtitle/090c09c12a07d5a96f14a2c1841e0548/por-3.vtt",+ "filename": "TZb4gRkOQ642 [TZb4gRkOQ642][1080p].por.vtt",+ },+ {+ "url": "https://mgstatics.xyz/subtitle/090c09c12a07d5a96f14a2c1841e0548/rus-10.vtt",+ "filename": "TZb4gRkOQ642 [TZb4gRkOQ642][1080p].rus.vtt",+ },+ {+ "url": "https://mgstatics.xyz/subtitle/090c09c12a07d5a96f14a2c1841e0548/spa-4.vtt",+ "filename": "TZb4gRkOQ642 [TZb4gRkOQ642][1080p].spa.2.vtt",+ },+ {+ "url": "https://mgstatics.xyz/subtitle/090c09c12a07d5a96f14a2c1841e0548/spa-5.vtt",+ "filename": "TZb4gRkOQ642 [TZb4gRkOQ642][1080p].spa.vtt",+ },+ ],+ ),+]