Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
refactor: update mega and transfer it (#1858)
NTFSvolume18ed178972f1
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.pyindex ea62b347d..b0a4f1916 100644--- a/cyberdrop_dl/crawlers/mega_nz.py+++ b/cyberdrop_dl/crawlers/mega_nz.py@@ -96,10 +96,10 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):scrape_item.url = canonical_urlfull_key = b64_to_a32(public_key)- await self._process_file(scrape_item, handle, Crypto.decompose(full_key)) # pyright: ignore[reportArgumentType]+ await self._file(scrape_item, handle, Crypto.decompose(full_key)) # pyright: ignore[reportArgumentType]@error_handling_wrapper- async def _process_file(+ async def _file(self,scrape_item: ScrapeItem,handle: str,@@ -117,7 +117,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):filename, ext = self.get_filename_and_ext(name)await self.handle_file(scrape_item.url, scrape_item, filename, ext, debrid_link=file_url)- _process_file_task = auto_task_id(_process_file)+ _file_task = auto_task_id(_file)@error_handling_wrapperasync def folder(@@ -138,9 +138,9 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):scrape_item.setup_as_album(title, album_id=handle)canonical_url = (self.PRIMARY_URL / "folder" / handle).with_fragment(public_key)scrape_item.url = canonical_url- await self._process_fs(scrape_item, fs, selected_node)+ await self._filesystem(scrape_item, fs, selected_node)- async def _process_fs(self, scrape_item: ScrapeItem, filesystem: FileSystem, selected_node_id: str | None) -> None:+ async def _filesystem(self, scrape_item: ScrapeItem, filesystem: FileSystem, selected_node_id: str | None) -> None:folder_id, public_key = scrape_item.url.name, scrape_item.url.fragmentfor file in filesystem.files_from(selected_node_id):@@ -154,7 +154,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):for part in path.parent.parts[1:]:child_item.add_to_parent_title(part)- self.create_task(self._process_file_task(child_item, file.id, file._crypto, folder_id=folder_id))+ self.create_task(self._file_task(child_item, file.id, file._crypto, folder_id=folder_id))scrape_item.add_children()@overridediff --git a/cyberdrop_dl/crawlers/transfer_it.py b/cyberdrop_dl/crawlers/transfer_it.pyindex 64b1695dc..d20ab7a0a 100644--- a/cyberdrop_dl/crawlers/transfer_it.py+++ b/cyberdrop_dl/crawlers/transfer_it.py@@ -9,7 +9,7 @@ from cyberdrop_dl.url_objects import AbsoluteHttpURLfrom cyberdrop_dl.utils import error_handling_wrapperif TYPE_CHECKING:- from mega import Node+ from mega.data_structures import Nodefrom mega.filesystem import FileSystemfrom cyberdrop_dl.url_objects import ScrapeItem@@ -39,9 +39,9 @@ class TransferItCrawler(Crawler, db_path="path_qs_frag"):root = next(iter(fs))title = self.create_title(root.attributes.name, transfer_id)scrape_item.setup_as_album(title, album_id=transfer_id)- self._process_filesystem(scrape_item, fs, transfer_id)+ self._filesystem(scrape_item, fs, transfer_id)- def _process_filesystem(self, scrape_item: ScrapeItem, fs: FileSystem, transfer_id: str) -> None:+ def _filesystem(self, scrape_item: ScrapeItem, fs: FileSystem, transfer_id: str) -> None:password = scrape_item.url.query.get("pw") or scrape_item.passwordfor file in fs.files: