Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: folders downloads always being skipped as duplicates (mega.nz) (#1241)

NTFSvolumee2d08573de58
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.py
index 5a0d62f98..d03e21410 100644
--- a/cyberdrop_dl/downloader/downloader.py
+++ b/cyberdrop_dl/downloader/downloader.py
@@ -25,6 +25,7 @@ from cyberdrop_dl.exceptions import (
TooManyCrawlerErrors,
)
from cyberdrop_dl.utils import ffmpeg
+from cyberdrop_dl.utils.database.tables.history_table import get_db_path
from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_none, parse_url
@@ -107,7 +108,7 @@ class Downloader:
self.client: DownloadClient = field(init=False)
self.log_prefix = "Download attempt (unsupported domain)" if domain in GENERIC_CRAWLERS else "Download"
- self.processed_items: set = set()
+ self.processed_items: set[str] = set()
self.waiting_items = 0
self._additional_headers = {}
@@ -151,7 +152,7 @@ class Downloader:
async with self._semaphore:
await self.manager.states.RUNNING.wait()
self.waiting_items -= 1
- self.processed_items.add(media_item.url.path)
+ self.processed_items.add(get_db_path(media_item.url, self.domain))
self.update_queued_files(increase_total=False)
async with self.manager.client_manager.download_session_limit:
return await self.start_download(media_item)
diff --git a/cyberdrop_dl/utils/database/tables/history_table.py b/cyberdrop_dl/utils/database/tables/history_table.py
index bc3441613..e9a0a0a36 100644
--- a/cyberdrop_dl/utils/database/tables/history_table.py
+++ b/cyberdrop_dl/utils/database/tables/history_table.py
@@ -25,17 +25,22 @@ DB_UPDATES = (
)
-def get_db_path(url: URL, referer: str = "") -> str:
+def get_db_path(url: URL, domain: str = "") -> str:
"""Gets the URL path to be put into the DB and checked from the DB."""
- url_path = url.path
- if referer and "e-hentai" in referer:
- url_path = url_path.split("keystamp")[0][:-1]
+ # TODO: domain SHOULD be mandatory, not optional. Make it mandatory and update any place that uses it
- if referer and "mediafire" in referer:
- url_path = url.name
+ if domain:
+ if "e-hentai" in domain:
+ return url.path.split("keystamp")[0][:-1]
- return url_path
+ if "mediafire" in domain:
+ return url.name
+
+ if "mega.nz" in domain:
+ return url.path_qs if not (frag := url.fragment) else f"{url.path_qs}#{frag}"
+
+ return url.path
class HistoryTable: