Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: update mega and transfer it (#1858)

NTFSvolume18ed178972f1
diff --git a/cyberdrop_dl/crawlers/mega_nz.py b/cyberdrop_dl/crawlers/mega_nz.py
index ea62b347d..b0a4f1916 100644
--- a/cyberdrop_dl/crawlers/mega_nz.py
+++ b/cyberdrop_dl/crawlers/mega_nz.py
@@ -96,10 +96,10 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
scrape_item.url = canonical_url
full_key = b64_to_a32(public_key)
- await self._process_file(scrape_item, handle, Crypto.decompose(full_key)) # pyright: ignore[reportArgumentType]
+ await self._file(scrape_item, handle, Crypto.decompose(full_key)) # pyright: ignore[reportArgumentType]
@error_handling_wrapper
- async def _process_file(
+ async def _file(
self,
scrape_item: ScrapeItem,
handle: str,
@@ -117,7 +117,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
filename, ext = self.get_filename_and_ext(name)
await self.handle_file(scrape_item.url, scrape_item, filename, ext, debrid_link=file_url)
- _process_file_task = auto_task_id(_process_file)
+ _file_task = auto_task_id(_file)
@error_handling_wrapper
async def folder(
@@ -138,9 +138,9 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
scrape_item.setup_as_album(title, album_id=handle)
canonical_url = (self.PRIMARY_URL / "folder" / handle).with_fragment(public_key)
scrape_item.url = canonical_url
- await self._process_fs(scrape_item, fs, selected_node)
+ await self._filesystem(scrape_item, fs, selected_node)
- async def _process_fs(self, scrape_item: ScrapeItem, filesystem: FileSystem, selected_node_id: str | None) -> None:
+ async def _filesystem(self, scrape_item: ScrapeItem, filesystem: FileSystem, selected_node_id: str | None) -> None:
folder_id, public_key = scrape_item.url.name, scrape_item.url.fragment
for file in filesystem.files_from(selected_node_id):
@@ -154,7 +154,7 @@ class MegaNzCrawler(Crawler, db_path="path_qs_frag"):
for part in path.parent.parts[1:]:
child_item.add_to_parent_title(part)
- self.create_task(self._process_file_task(child_item, file.id, file._crypto, folder_id=folder_id))
+ self.create_task(self._file_task(child_item, file.id, file._crypto, folder_id=folder_id))
scrape_item.add_children()
@override
diff --git a/cyberdrop_dl/crawlers/transfer_it.py b/cyberdrop_dl/crawlers/transfer_it.py
index 64b1695dc..d20ab7a0a 100644
--- a/cyberdrop_dl/crawlers/transfer_it.py
+++ b/cyberdrop_dl/crawlers/transfer_it.py
@@ -9,7 +9,7 @@ from cyberdrop_dl.url_objects import AbsoluteHttpURL
from cyberdrop_dl.utils import error_handling_wrapper
if TYPE_CHECKING:
- from mega import Node
+ from mega.data_structures import Node
from mega.filesystem import FileSystem
from cyberdrop_dl.url_objects import ScrapeItem
@@ -39,9 +39,9 @@ class TransferItCrawler(Crawler, db_path="path_qs_frag"):
root = next(iter(fs))
title = self.create_title(root.attributes.name, transfer_id)
scrape_item.setup_as_album(title, album_id=transfer_id)
- self._process_filesystem(scrape_item, fs, transfer_id)
+ self._filesystem(scrape_item, fs, transfer_id)
- def _process_filesystem(self, scrape_item: ScrapeItem, fs: FileSystem, transfer_id: str) -> None:
+ def _filesystem(self, scrape_item: ScrapeItem, fs: FileSystem, transfer_id: str) -> None:
password = scrape_item.url.query.get("pw") or scrape_item.password
for file in fs.files: