Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: transfer decryption key on media_item (Mega) (#1824)
NTFSvolume5f029ef7407e
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex 6526e2c9b..31d589afe 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -13,14 +13,15 @@ from mega.api import MegaAPIfrom mega.core import MegaCorefrom mega.crypto import b64_to_a32from mega.data_structures import Crypto+from typing_extensions import overridefrom cyberdrop_dl.constants import CDL_USER_AGENTfrom cyberdrop_dl.crawlers.crawler import Crawler, SupportedDomains, SupportedPaths, auto_task_idfrom cyberdrop_dl.downloader.mega_nz import MegaDownloaderfrom cyberdrop_dl.exceptions import LoginError, ScrapeErrorfrom cyberdrop_dl.progress.scraping import show_msg-from cyberdrop_dl.url_objects import AbsoluteHttpURL-from cyberdrop_dl.utils import error_handling_wrapper+from cyberdrop_dl.url_objects import AbsoluteHttpURL, MediaItem+from cyberdrop_dl.utils import error_handling_wrapper, m3u8if TYPE_CHECKING:from mega.filesystem import FileSystem@@ -61,10 +62,13 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):return self.manager.config.auth.meganz.password or Noneasync def __async_post_init__(self) -> None:+ self._decryption_keys: dict[AbsoluteHttpURL, tuple[Crypto, int]] = {}api = MegaAPI(self.manager.http_client._session)api.user_agent = CDL_USER_AGENTself.core = MegaCore(api)+ speed_limiter = self.downloader.client.speed_limiterself.downloader = MegaDownloader(self.manager, self.DOMAIN) # pyright: ignore[reportIncompatibleVariableOverride]+ self.downloader.client.speed_limiter = speed_limiterawait self.login(self.PRIMARY_URL)async def fetch(self, scrape_item: ScrapeItem) -> None:@@ -85,7 +89,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):scrape_item.url = canonical_urlfull_key = b64_to_a32(public_key)- await self._process_file(scrape_item, handle, Crypto.decompose(full_key))+ await self._process_file(scrape_item, handle, Crypto.decompose(full_key)) # pyright: ignore[reportArgumentType]@error_handling_wrapperasync def _process_file(@@ -101,7 +105,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):raise ScrapeError(410, "File not accessible anymore")name = self.core.decrypt_attrs(resp._at, crypto.key, handle).name- self.downloader.register(scrape_item.url, crypto, resp.size)+ self._decryption_keys[scrape_item.url] = crypto, resp.sizefile_url = self.parse_url(resp.url)filename, ext = self.get_filename_and_ext(name)await self.handle_file(scrape_item.url, scrape_item, filename, ext, debrid_link=file_url)@@ -146,6 +150,11 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):self.create_task(self._process_file_task(child_item, file.id, file._crypto, folder_id=folder_id))scrape_item.add_children()+ @override+ async def handle_media_item(self, media_item: MediaItem, m3u8: m3u8.Rendition | None = None) -> None:+ media_item.extra_info.setdefault(self.DOMAIN, {})["key"] = self._decryption_keys.pop(media_item.url)+ await super().handle_media_item(media_item, m3u8)+@error_handling_wrapperasync def login(self, *_) -> None:# This takes a really long time (dozens of seconds)diff --git a/cyberdrop_dl/downloader/mega_nz.py b/cyberdrop_dl/downloader/mega_nz.pyindex 6b57d32b6..3439c771e 100644--- a/cyberdrop_dl/downloader/mega_nz.py+++ b/cyberdrop_dl/downloader/mega_nz.py@@ -11,9 +11,6 @@ from cyberdrop_dl.clients.downloads import DownloadClient, make_speed_checkerfrom cyberdrop_dl.downloader.http import Downloaderif TYPE_CHECKING:- from mega.data_structures import Crypto- from yarl import URL-from cyberdrop_dl.clients.response import AbstractResponsefrom cyberdrop_dl.manager import Managerfrom cyberdrop_dl.progress import ProgressHook@@ -24,7 +21,6 @@ if TYPE_CHECKING:class MegaDownloadClient(DownloadClient): # pyright: ignore[reportGeneralTypeIssues]def __init__(self, manager: Manager) -> None:super().__init__(manager)- self._decrypt_mapping: dict[URL, tuple[Crypto, int]] = {}self._supports_ranges = False@override@@ -36,7 +32,7 @@ class MegaDownloadClient(DownloadClient): # pyright: ignore[reportGeneralTypeIsawait check_free_space()await self._pre_download_check(media_item)- crypto, file_size = self._decrypt_mapping.pop(media_item.url)+ crypto, file_size = media_item.extra_info[media_item.domain]["key"]chunk_decryptor = MegaChunker(crypto.key, crypto.iv, crypto.meta_mac)aiohttp_resp = resp.aiohttp_resp@@ -76,8 +72,5 @@ class MegaDownloader(Downloader):return self._client@property- def max_attempts(self):+ def max_attempts(self) -> int:return 1-- def register(self, url: URL, crypto: Crypto, file_size: int) -> None:- self.client._decrypt_mapping[url] = crypto, file_size