Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: messages being logged twice (#1168)
NTFSvolumec7ad880af540
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.pyindex 7c720d5fe..a393accaf 100644--- a/cyberdrop_dl/crawlers/_forum.py+++ b/cyberdrop_dl/crawlers/_forum.py@@ -141,7 +141,6 @@ class ThreadProtocol(Protocol):def url(self) -> AbsoluteHttpURL: ...-# quotes_2 : embed fauxBlockLinkclass MessageBoardCrawler(Crawler, is_abc=True):"""Base crawler for every MessageBoard.@@ -229,6 +228,9 @@ class MessageBoardCrawler(Crawler, is_abc=True):async def fetch_thread(self, scrape_item: ScrapeItem) -> None:thread_part_index = len(self.PRIMARY_URL.parts)+ # https://github.com/jbsparrow/CyberDropDownloader/issues/1165#issuecomment-3086739753+ if self.PRIMARY_URL.parts[-1] == "":+ thread_part_index -= 1match scrape_item.url.parts[thread_part_index:]:case [thread_part, thread_name_and_id, *_] if thread_part in self.THREAD_PART_NAMES:self.check_thread_recursion(scrape_item)@@ -502,7 +504,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):max_children_error = eif seen:- self.log(f"post #{post.id} {stats = }")+ self.log(f"[{self.FOLDER_DOMAIN}] post #{post.id} {stats = }")if duplicates:msg = f"Found duplicate links in post {scrape_item.url}. Selectors are too generic: {duplicates}"self.log(msg, bug=True)diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.pyindex a8032d50c..094e6d02e 100644--- a/cyberdrop_dl/utils/logger.py+++ b/cyberdrop_dl/utils/logger.py@@ -222,10 +222,9 @@ def log() -> None:"""Simple logging function."""msg = prefix + process_log_msg(message)- logger.log(level, msg, **kwargs)log_debug(msg, level, **kwargs)if bug:- msg = msg.rstrip() + f". Please file a bug report at {NEW_ISSUE_URL}"+ msg = msg.rstrip() + f". Please open a bug report at {NEW_ISSUE_URL}"logger.log(level, msg, **kwargs)diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.pyindex 43c89d779..5a1c349ba 100644--- a/cyberdrop_dl/utils/sorting.py+++ b/cyberdrop_dl/utils/sorting.py@@ -83,7 +83,7 @@ class Sorter:async def run(self) -> None:"""Sorts the files in the download directory into their respective folders."""- if not asyncio.to_thread(self.download_folder.is_dir):+ if not await asyncio.to_thread(self.download_folder.is_dir):log_with_color(f"Download directory ({self.download_folder}) does not exist", "red", 40)return