Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: messages being logged twice (#1168)

NTFSvolumec7ad880af540
diff --git a/cyberdrop_dl/crawlers/_forum.py b/cyberdrop_dl/crawlers/_forum.py
index 7c720d5fe..a393accaf 100644
--- a/cyberdrop_dl/crawlers/_forum.py
+++ b/cyberdrop_dl/crawlers/_forum.py
@@ -141,7 +141,6 @@ class ThreadProtocol(Protocol):
def url(self) -> AbsoluteHttpURL: ...
-# quotes_2 : embed fauxBlockLink
class MessageBoardCrawler(Crawler, is_abc=True):
"""Base crawler for every MessageBoard.
@@ -229,6 +228,9 @@ class MessageBoardCrawler(Crawler, is_abc=True):
async def fetch_thread(self, scrape_item: ScrapeItem) -> None:
thread_part_index = len(self.PRIMARY_URL.parts)
+ # https://github.com/jbsparrow/CyberDropDownloader/issues/1165#issuecomment-3086739753
+ if self.PRIMARY_URL.parts[-1] == "":
+ thread_part_index -= 1
match scrape_item.url.parts[thread_part_index:]:
case [thread_part, thread_name_and_id, *_] if thread_part in self.THREAD_PART_NAMES:
self.check_thread_recursion(scrape_item)
@@ -502,7 +504,7 @@ class HTMLMessageBoardCrawler(MessageBoardCrawler, is_abc=True):
max_children_error = e
if seen:
- self.log(f"post #{post.id} {stats = }")
+ self.log(f"[{self.FOLDER_DOMAIN}] post #{post.id} {stats = }")
if duplicates:
msg = f"Found duplicate links in post {scrape_item.url}. Selectors are too generic: {duplicates}"
self.log(msg, bug=True)
diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.py
index a8032d50c..094e6d02e 100644
--- a/cyberdrop_dl/utils/logger.py
+++ b/cyberdrop_dl/utils/logger.py
@@ -222,10 +222,9 @@ def log(
) -> None:
"""Simple logging function."""
msg = prefix + process_log_msg(message)
- logger.log(level, msg, **kwargs)
log_debug(msg, level, **kwargs)
if bug:
- msg = msg.rstrip() + f". Please file a bug report at {NEW_ISSUE_URL}"
+ msg = msg.rstrip() + f". Please open a bug report at {NEW_ISSUE_URL}"
logger.log(level, msg, **kwargs)
diff --git a/cyberdrop_dl/utils/sorting.py b/cyberdrop_dl/utils/sorting.py
index 43c89d779..5a1c349ba 100644
--- a/cyberdrop_dl/utils/sorting.py
+++ b/cyberdrop_dl/utils/sorting.py
@@ -83,7 +83,7 @@ class Sorter:
async def run(self) -> None:
"""Sorts the files in the download directory into their respective folders."""
- if not asyncio.to_thread(self.download_folder.is_dir):
+ if not await asyncio.to_thread(self.download_folder.is_dir):
log_with_color(f"Download directory ({self.download_folder}) does not exist", "red", 40)
return