Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: folders downloads always being skipped as duplicates (mega.nz) (#1241)
NTFSvolumee2d08573de58
diff --git a/cyberdrop_dl/downloader/downloader.py b/cyberdrop_dl/downloader/downloader.pyindex 5a0d62f98..d03e21410 100644--- a/cyberdrop_dl/downloader/downloader.py+++ b/cyberdrop_dl/downloader/downloader.py@@ -25,6 +25,7 @@ from cyberdrop_dl.exceptions import (TooManyCrawlerErrors,)from cyberdrop_dl.utils import ffmpeg+from cyberdrop_dl.utils.database.tables.history_table import get_db_pathfrom cyberdrop_dl.utils.logger import logfrom cyberdrop_dl.utils.utilities import error_handling_wrapper, get_size_or_none, parse_url@@ -107,7 +108,7 @@ class Downloader:self.client: DownloadClient = field(init=False)self.log_prefix = "Download attempt (unsupported domain)" if domain in GENERIC_CRAWLERS else "Download"- self.processed_items: set = set()+ self.processed_items: set[str] = set()self.waiting_items = 0self._additional_headers = {}@@ -151,7 +152,7 @@ class Downloader:async with self._semaphore:await self.manager.states.RUNNING.wait()self.waiting_items -= 1- self.processed_items.add(media_item.url.path)+ self.processed_items.add(get_db_path(media_item.url, self.domain))self.update_queued_files(increase_total=False)async with self.manager.client_manager.download_session_limit:return await self.start_download(media_item)diff --git a/cyberdrop_dl/utils/database/tables/history_table.py b/cyberdrop_dl/utils/database/tables/history_table.pyindex bc3441613..e9a0a0a36 100644--- a/cyberdrop_dl/utils/database/tables/history_table.py+++ b/cyberdrop_dl/utils/database/tables/history_table.py@@ -25,17 +25,22 @@ DB_UPDATES = ()-def get_db_path(url: URL, referer: str = "") -> str:+def get_db_path(url: URL, domain: str = "") -> str:"""Gets the URL path to be put into the DB and checked from the DB."""- url_path = url.path- if referer and "e-hentai" in referer:- url_path = url_path.split("keystamp")[0][:-1]+ # TODO: domain SHOULD be mandatory, not optional. Make it mandatory and update any place that uses it- if referer and "mediafire" in referer:- url_path = url.name+ if domain:+ if "e-hentai" in domain:+ return url.path.split("keystamp")[0][:-1]- return url_path+ if "mediafire" in domain:+ return url.name++ if "mega.nz" in domain:+ return url.path_qs if not (frag := url.fragment) else f"{url.path_qs}#{frag}"++ return url.pathclass HistoryTable: