Skip to content

Bulk asynchronous downloader for multiple file hosts

refactor: move to std logging (part 2) (#108)

NTFSvolume3cd16e728dfe
diff --git a/cyberdrop_dl/crawlers/luscious.py b/cyberdrop_dl/crawlers/luscious.py
index e3014a485..6dd457037 100644
--- a/cyberdrop_dl/crawlers/luscious.py
+++ b/cyberdrop_dl/crawlers/luscious.py
@@ -7,7 +7,6 @@ from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, SupportedPaths
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
-from cyberdrop_dl.utils.logger import log_debug
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -62,7 +61,7 @@ class LusciousCrawler(Crawler):
filters = [{"name": i, "value": v} for i, v in query.items() if i not in ("page", "display", "q")]
data["variables"] = {"input": {"display": sorting, "filters": filters, "page": page}}
- log_debug(data)
+ self.log.debug(data)
return json.dumps(data)
@error_handling_wrapper
@@ -118,5 +117,5 @@ class LusciousCrawler(Crawler):
data=query,
headers={"Content-Type": "application/json"},
)
- log_debug(json_resp)
+ self.log.debug(json_resp)
return json_resp
diff --git a/cyberdrop_dl/crawlers/realdebrid.py b/cyberdrop_dl/crawlers/realdebrid.py
index 8fe5b59be..d1aefb2be 100644
--- a/cyberdrop_dl/crawlers/realdebrid.py
+++ b/cyberdrop_dl/crawlers/realdebrid.py
@@ -9,7 +9,6 @@ from typing import TYPE_CHECKING, Any, ClassVar
from cyberdrop_dl.crawlers.crawler import Crawler, RateLimit
from cyberdrop_dl.data_structures.url_objects import AbsoluteHttpURL
from cyberdrop_dl.exceptions import ScrapeError
-from cyberdrop_dl.utils.logger import log
from cyberdrop_dl.utils.utilities import error_handling_wrapper
if TYPE_CHECKING:
@@ -118,7 +117,7 @@ class RealDebridCrawler(Crawler):
title = self.create_title(f"files [{url.host}]")
scrape_item.setup_as_album(title)
debrid_link = await self.api.unrestrict(url, scrape_item.password)
- log(f"[{self.FOLDER_DOMAIN}]:\n Original URL: {url}\n Debrid URL: {debrid_link}", 10)
+ self.log.info(f"\n Original URL: {url}\n Debrid URL: {debrid_link}")
filename, ext = self.get_filename_and_ext(debrid_link.name)
await self.handle_file(
url,
diff --git a/cyberdrop_dl/director.py b/cyberdrop_dl/director.py
index 062b816ed..d15232a19 100644
--- a/cyberdrop_dl/director.py
+++ b/cyberdrop_dl/director.py
@@ -2,31 +2,31 @@ from __future__ import annotations
import contextlib
import logging
-from datetime import datetime
from enum import IntEnum
-from functools import wraps
-from pathlib import Path
from typing import TYPE_CHECKING, ParamSpec, TypeVar
-from cyberdrop_dl import aio, constants, env, storage
-from cyberdrop_dl.dependencies import browser_cookie3
+from rich.traceback import install as install_rich_tracebacks
+
+from cyberdrop_dl import aio, storage
from cyberdrop_dl.managers.manager import Manager
from cyberdrop_dl.scraper.scrape_mapper import ScrapeMapper
from cyberdrop_dl.ui import program_ui
from cyberdrop_dl.updates import check_latest_pypi
from cyberdrop_dl.utils.apprise import send_apprise_notifications
-from cyberdrop_dl.utils.logger import LogHandler, QueuedLogger, log_spacer, log_with_color
+from cyberdrop_dl.utils.logger import log_spacer, setup_logging
from cyberdrop_dl.utils.sorting import Sorter
from cyberdrop_dl.utils.utilities import check_partials_and_empty_folders
from cyberdrop_dl.utils.webhook import send_webhook_message
if TYPE_CHECKING:
- from collections.abc import Callable, Coroutine, Sequence
+ from collections.abc import Sequence
logger = logging.getLogger(__name__)
P = ParamSpec("P")
R = TypeVar("R")
+_ = install_rich_tracebacks()
+
class ExitCode(IntEnum):
OK = 0
@@ -36,62 +36,32 @@ class ExitCode(IntEnum):
_C = ExitCode
-def _ui_error_handling_wrapper(
- func: Callable[P, Coroutine[None, None, R]],
-) -> Callable[P, Coroutine[None, None, R | None]]:
- """Wrapper handles errors from the main UI."""
-
- @wraps(func)
- async def wrapper(*args, **kwargs) -> R | None:
- try:
- return await func(*args, **kwargs)
- except* Exception as e:
- exceptions = [e]
- if isinstance(e, ExceptionGroup):
- exceptions = e.exceptions
- if not isinstance(exceptions[0], browser_cookie3.BrowserCookieError):
- msg = "An error occurred, please report this to the developer with your logs file:"
- log_with_color(msg, "bold red", 50, show_in_stats=False)
- for exc in exceptions:
- log_with_color(f" {exc}", "bold red", 50, show_in_stats=False, exc_info=exc)
-
- return wrapper
-
-
-@_ui_error_handling_wrapper
async def _run_manager(manager: Manager) -> None:
"""Runs the program and handles the UI."""
manager.config.resolve_paths()
manager.logs.delete_old_logs()
- debug_log_file_path = _setup_debug_logger(manager)
start_time = manager.start_time
- _setup_main_logger(manager)
- logger.info(f"Using Debug Log: {debug_log_file_path}")
- logger.info("Starting Async Processes...")
- await manager.async_startup()
-
- log_spacer(10)
- async with manager.database:
- logger.info("Starting CDL...\n")
- await _scheduler(manager)
+ with setup_logging(manager.config.logs.main_log):
+ await manager.async_startup()
- manager.progress_manager.print_stats(start_time)
+ log_spacer()
+ async with manager.database:
+ logger.info("Starting CDL...\n")
- log_spacer(20)
- logger.info("Checking for Updates...")
- check_latest_pypi()
- log_spacer(20)
- logger.info("Closing Program...")
- log_with_color("Finished downloading. Enjoy :)", "green", 20, show_in_stats=False)
+ await _runtime(manager)
+ await _post_runtime(manager)
- await send_webhook_message(manager)
- await send_apprise_notifications(manager)
+ manager.progress_manager.print_stats(start_time)
+ log_spacer()
+ check_latest_pypi()
+ log_spacer()
+ logger.info("Closing program...")
+ logger.info("Finished downloading. Enjoy :)", extra={"color": "green"})
-async def _scheduler(manager: Manager) -> None:
- for func in (_runtime, _post_runtime):
- await func(manager)
+ await send_webhook_message(manager)
+ await send_apprise_notifications(manager)
async def _runtime(manager: Manager) -> None:
@@ -104,9 +74,8 @@ async def _runtime(manager: Manager) -> None:
async def _post_runtime(manager: Manager) -> None:
"""Actions to complete after main runtime, and before ui shutdown."""
- log_spacer(20, log_to_console=False)
- msg = f"Running Post-Download Processes For Config: {manager.config_manager.loaded_config}"
- log_with_color(msg, "green", 20)
+ log_spacer()
+ logger.info("Running Post-Download Processes", extra={"color": "green"})
await manager.hash_manager.hash_client.cleanup_dupes_after_download()
@@ -117,58 +86,9 @@ async def _post_runtime(manager: Manager) -> None:
check_partials_and_empty_folders(manager)
if manager.config_manager.settings_data.runtime_options.update_last_forum_post:
- log_spacer(20)
await manager.logs.update_last_forum_post(manager.config.files.input_file)
-def _setup_debug_logger(manager: Manager) -> Path | None:
- if not env.DEBUG_VAR:
- return
-
- log_level = logging.DEBUG
- settings_data = manager.config_manager.settings_data
- settings_data.runtime_options.log_level = log_level
- debug_log_file_path = Path(__file__).parents[1] / "cyberdrop_dl_debug.log"
- if env.DEBUG_LOG_FOLDER:
- debug_log_folder = Path(env.DEBUG_LOG_FOLDER)
- if not debug_log_folder.is_dir():
- msg = "Value of env var 'CDL_DEBUG_LOG_FOLDER' is invalid."
- msg += f" Folder '{debug_log_folder}' does not exists"
- raise FileNotFoundError(None, msg, env.DEBUG_LOG_FOLDER)
- date = datetime.now().strftime("%Y%m%d_%H%M%S")
- debug_log_file_path = debug_log_folder / f"cyberdrop_dl_debug_{date}.log"
-
- file_io = debug_log_file_path.open("w", encoding="utf8")
-
- file_handler = LogHandler(level=log_level, file=file_io, width=500, debug=True)
- queued_logger = QueuedLogger(manager, file_handler, "debug")
-
- logging.getLogger("cyberdrop_dl").addHandler(queued_logger.handler)
-
- # aiosqlite_log = logging.getLogger("aiosqlite")
- # aiosqlite_log.setLevel(log_level)
- # aiosqlite_log.addHandler(file_handler_debug)
-
- return debug_log_file_path.resolve()
-
-
-def _setup_main_logger(manager: Manager) -> None:
- logger = logging.getLogger("cyberdrop_dl")
- file_io = manager.config.logs.main_log.open("w", encoding="utf8")
- settings_data = manager.config_manager.settings_data
- log_level = settings_data.runtime_options.log_level
- logger.setLevel(log_level)
-
- if not manager.parsed_args.cli_only_args.fullscreen_ui:
- constants.CONSOLE_LEVEL = settings_data.runtime_options.console_log_level
- constants.console_handler = LogHandler(level=constants.CONSOLE_LEVEL)
-
- logger.addHandler(constants.console_handler)
- file_handler = LogHandler(level=log_level, file=file_io, width=500)
- queued_logger = QueuedLogger(manager, file_handler)
- logger.addHandler(queued_logger.handler)
-
-
class Director:
"""Creates a manager and runs it"""
diff --git a/cyberdrop_dl/main.py b/cyberdrop_dl/main.py
index d448196a5..8884500cb 100644
--- a/cyberdrop_dl/main.py
+++ b/cyberdrop_dl/main.py
@@ -14,9 +14,7 @@ def main(args: Sequence[str] | None = None) -> None:
def run(args: Sequence[str] | None = None) -> str | int | None:
- from cyberdrop_dl.utils.logger import catch_exceptions
- @catch_exceptions
def run_() -> int:
return _create_director(args).run()
diff --git a/cyberdrop_dl/managers/logs.py b/cyberdrop_dl/managers/logs.py
index 668e8e034..711ba5e55 100644
--- a/cyberdrop_dl/managers/logs.py
+++ b/cyberdrop_dl/managers/logs.py
@@ -9,6 +9,7 @@ from typing import TYPE_CHECKING, Any, Self
from cyberdrop_dl.exceptions import get_origin
from cyberdrop_dl.utils import json
+from cyberdrop_dl.utils.logger import log_spacer
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator
@@ -133,6 +134,7 @@ class LogManager:
def _update_last_forum_post(input_file: Path, last_post_log: Path) -> None:
+ log_spacer()
logger.info("Updating Last Forum Posts...\n")
current_urls, current_base_urls, new_urls, new_base_urls = [], [], [], []
diff --git a/cyberdrop_dl/managers/manager.py b/cyberdrop_dl/managers/manager.py
index 81a310a1e..13ba14053 100644
--- a/cyberdrop_dl/managers/manager.py
+++ b/cyberdrop_dl/managers/manager.py
@@ -2,7 +2,6 @@ from __future__ import annotations
import asyncio
import dataclasses
-import json
import logging
from dataclasses import Field, field
from pathlib import Path
@@ -167,35 +166,32 @@ class Manager:
def args_logging(self) -> None:
"""Logs the runtime arguments."""
- auth_provided = {}
- for site, auth_entries in self.config_manager.authentication_data.model_dump().items():
- auth_provided[site] = all(auth_entries.values())
+ auth = {
+ site: all(credentials.values())
+ for site, credentials in self.config_manager.authentication_data.model_dump().items()
+ }
config_settings = self.config_manager.settings_data.model_copy()
config_settings.runtime_options.deep_scrape = self.config_manager.deep_scrape
- config_settings = config_settings.model_dump_json(indent=4)
- global_settings = self.config_manager.global_settings_data.model_dump_json(indent=4)
- cli_only_args = self.parsed_args.cli_only_args.model_dump_json(indent=4)
- system_info = get_system_information()
-
- args_info = (
- "Starting Cyberdrop-DL Process",
- f"Running Version: {__version__}",
- f"System Info: {system_info}",
- f"Using Config: {self.config_manager.loaded_config}",
- f"Using Config File: {self.config_manager.settings}",
- f"Using Input File: {self.config.files.input_file}",
- f"Using Download Folder: {self.config.files.download_folder}",
- f"Using Database File: {self.appdata.db_file}",
- f"Using CLI only options: {cli_only_args}",
- f"Using Authentication: \n{json.dumps(auth_provided, indent=4, sort_keys=True)}",
- f"Using Settings: \n{config_settings}",
- f"Using Global Settings: \n{global_settings}",
- f"Using ffmpeg version: {ffmpeg.get_ffmpeg_version()}",
- f"Using ffprobe version: {ffmpeg.get_ffprobe_version()}",
- )
- logger.info("\n".join(args_info))
+
+ logger.info(f"Running cyberdrop-dl v{__version__}")
+
+ args_info = {
+ "System": get_system_information(),
+ "Config": self.config_manager.loaded_config,
+ "Config File": self.config_manager.settings,
+ "Input File": self.config.files.input_file,
+ "Download Folder": self.config.files.download_folder,
+ "Database File": self.appdata.db_file,
+ "CLI only options": self.parsed_args.cli_only_args.model_dump(mode="json"),
+ "Auth": auth,
+ "Settings": config_settings.model_dump(mode="json"),
+ "Global Settings": self.config_manager.global_settings_data.model_dump(mode="json"),
+ "ffmpeg version": ffmpeg.get_ffmpeg_version(),
+ "ffprobe version": ffmpeg.get_ffprobe_version(),
+ }
+ logger.info(args_info)
async def close(self) -> None:
diff --git a/cyberdrop_dl/managers/progress_manager.py b/cyberdrop_dl/managers/progress_manager.py
index bfb399743..3d2f7c798 100644
--- a/cyberdrop_dl/managers/progress_manager.py
+++ b/cyberdrop_dl/managers/progress_manager.py
@@ -4,7 +4,6 @@ import logging
import time
from contextlib import asynccontextmanager
from datetime import timedelta
-from functools import partial
from typing import TYPE_CHECKING
from pydantic import ByteSize
@@ -22,10 +21,10 @@ from cyberdrop_dl.ui.progress.hash_progress import HashProgress
from cyberdrop_dl.ui.progress.scraping_progress import ScrapingProgress
from cyberdrop_dl.ui.progress.sort_progress import SortProgress
from cyberdrop_dl.ui.progress.statistic_progress import DownloadStatsProgress, ScrapeStatsProgress
-from cyberdrop_dl.utils.logger import log_spacer, log_with_color
+from cyberdrop_dl.utils.logger import log_spacer
if TYPE_CHECKING:
- from collections.abc import AsyncGenerator
+ from collections.abc import AsyncGenerator, Sequence
from pathlib import Path
from rich.panel import Panel
@@ -36,10 +35,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-log_cyan = partial(log_with_color, style="cyan", level=20)
-log_yellow = partial(log_with_color, style="yellow", level=20)
-log_green = partial(log_with_color, style="green", level=20)
-log_red = partial(log_with_color, style="red", level=20)
spinner = SpinnerColumn(style="green", spinner_name="dots")
@@ -126,67 +121,73 @@ class ProgressManager:
runtime = timedelta(seconds=int(end_time - start_time))
total_data_written = ByteSize(self.file_progress.total_data_written).human_readable(decimal=True)
- log_spacer(20)
+ log_spacer()
logger.info("Printing Stats...\n")
config_path = self.manager.appdata.configs / self.manager.config_manager.loaded_config
config_path_text = get_console_hyperlink(config_path, text=self.manager.config_manager.loaded_config)
input_file_text = get_input(self.manager)
log_folder_text = get_console_hyperlink(self.manager.config.logs.log_folder)
- log_concat("Run Stats (config: ", config_path_text, ")", style="cyan")
- log_concat(" Input File: ", input_file_text, style="yellow")
- log_yellow(f" Input URLs: {self.manager.scrape_mapper.count:,}")
- log_yellow(f" Input URL Groups: {self.manager.scrape_mapper.group_count:,}")
- log_concat(" Log Folder: ", log_folder_text, style="yellow")
- log_yellow(f" Total Runtime: {runtime}")
- log_yellow(f" Total Downloaded Data: {total_data_written}")
-
- log_spacer(20, "")
- log_cyan("Download Stats:")
- log_green(f" Downloaded: {self.download_progress.completed_files:,} files")
- log_yellow(f" Skipped (By Config): {self.download_progress.skipped_files:,} files")
- log_yellow(f" Skipped (Previously Downloaded): {self.download_progress.previously_completed_files:,} files")
- log_red(f" Failed: {self.download_stats_progress.failed_files:,} files")
-
- log_spacer(20, "")
- log_cyan("Unsupported URLs Stats:")
- log_yellow(f" Sent to Jdownloader: {self.scrape_stats_progress.sent_to_jdownloader:,}")
- log_yellow(f" Skipped: {self.scrape_stats_progress.unsupported_urls_skipped:,}")
+ logger.info("Run Stats: ", config_path_text)
+ logger.info(" Input File: ", input_file_text)
+ logger.info(f" Input URLs: {self.manager.scrape_mapper.count:,}")
+ logger.info(f" Input URL Groups: {self.manager.scrape_mapper.group_count:,}")
+ logger.info(" Log Folder: ", log_folder_text)
+ logger.info(f" Total Runtime: {runtime}")
+ logger.info(f" Total Downloaded Data: {total_data_written}")
+
+ log_spacer()
+ logger.info("Download Stats:")
+ logger.info(f" Downloaded: {self.download_progress.completed_files:,} files")
+ logger.info(f" Skipped (By Config): {self.download_progress.skipped_files:,} files")
+ logger.info(f" Skipped (Previously Downloaded): {self.download_progress.previously_completed_files:,} files")
+ logger.info(f" Failed: {self.download_stats_progress.failed_files:,} files")
+
+ log_spacer()
+ logger.info("Unsupported URLs Stats:")
+ logger.info(f" Sent to Jdownloader: {self.scrape_stats_progress.sent_to_jdownloader:,}")
+ logger.info(f" Skipped: {self.scrape_stats_progress.unsupported_urls_skipped:,}")
self.print_dedupe_stats()
- log_spacer(20, "")
- log_cyan("Sort Stats:")
- log_green(f" Audios: {self.sort_progress.audio_count:,}")
- log_green(f" Images: {self.sort_progress.image_count:,}")
- log_green(f" Videos: {self.sort_progress.video_count:,}")
- log_green(f" Other Files: {self.sort_progress.other_count:,}")
+ log_spacer()
+ logger.info("Sort Stats:")
+ logger.info(f" Audios: {self.sort_progress.audio_count:,}")
+ logger.info(f" Images: {self.sort_progress.image_count:,}")
+ logger.info(f" Videos: {self.sort_progress.video_count:,}")
+ logger.info(f" Other Files: {self.sort_progress.other_count:,}")
- last_padding = log_failures(self.scrape_stats_progress.return_totals(), "Scrape Failures:")
- log_failures(self.download_stats_progress.return_totals(), "Download Failures:", last_padding)
+ _log_errors(self.scrape_stats_progress.return_totals(), self.download_stats_progress.return_totals())
def print_dedupe_stats(self) -> None:
- log_spacer(20, "")
- log_cyan("Dupe Stats:")
- log_yellow(f" Newly Hashed: {self.hash_progress.hashed_files:,} files")
- log_yellow(f" Previously Hashed: {self.hash_progress.prev_hashed_files:,} files")
- log_yellow(f" Removed (Downloads): {self.hash_progress.removed_files:,} files")
-
-
-def log_failures(failures: list[UiFailureTotal], title: str = "Failures:", last_padding: int = 0) -> int:
- log_spacer(20, "")
- log_cyan(title)
- if not failures:
- log_green(" None")
- return 0
- error_padding = last_padding
- error_codes = [f.error_code for f in failures if f.error_code is not None]
- if error_codes:
- error_padding = max(len(str(max(error_codes))), error_padding)
- for f in failures:
- error = f.error_code if f.error_code is not None else ""
- log_red(f" {error:>{error_padding}}{' ' if error_padding else ''}{f.msg}: {f.total:,}")
- return error_padding
+ log_spacer()
+ logger.info("Dupe Stats:")
+ logger.info(f" Newly Hashed: {self.hash_progress.hashed_files:,} files")
+ logger.info(f" Previously Hashed: {self.hash_progress.prev_hashed_files:,} files")
+ logger.info(f" Removed (Downloads): {self.hash_progress.removed_files:,} files")
+
+
+def _log_errors(scrape_errors: Sequence[UiFailureTotal], download_errors: Sequence[UiFailureTotal]) -> None:
+ error_codes = (error.code for error in (*scrape_errors, *download_errors) if error.code is not None)
+
+ try:
+ padding = len(str(max(error_codes)))
+ except ValueError:
+ padding = 0
+
+ for title, errors in (
+ ("Scrape Failures:", scrape_errors),
+ ("Download Failures:", download_errors),
+ ):
+ log_spacer()
+ logger.info(title, extra={"color": "red"})
+ if not errors:
+ logger.info(f" {'None':>{padding}}")
+ return
+
+ for error in scrape_errors:
+ error_code = error.code if error.code is not None else ""
+ logger.info(f" {error_code:>{padding}}{' ' if padding else ''}{error.msg}: {error.count:,}")
def get_input(manager: Manager) -> Text | str:
@@ -206,22 +207,3 @@ def get_console_hyperlink(file_path: Path, text: str = "") -> Text:
show_text = text or full_path
file_url = URL(full_path.as_posix()).with_scheme("file")
return Text(str(show_text), style=f"link {file_url}")
-
-
-def concat_as_text(*text_or_str, style: str = "") -> Text:
- result = Text()
- for elem in text_or_str:
- if isinstance(elem, Text):
- text = elem
- if style and text.style != style:
- text.stylize(f"{style} {text.style}")
- else:
- text = Text(elem, style=style)
-
- result.append(text)
- return result
-
-
-def log_concat(*text_or_str, style: str = "", **kwargs) -> None:
- text = concat_as_text(*text_or_str, style=style)
- log_with_color(text, style, **kwargs)
diff --git a/cyberdrop_dl/progress/overflow.py b/cyberdrop_dl/progress/overflow.py
index 5be484910..fea91faa5 100644
--- a/cyberdrop_dl/progress/overflow.py
+++ b/cyberdrop_dl/progress/overflow.py
@@ -51,7 +51,7 @@ class OverFlow:
return ""
-class OverflowPanel:
+class OverFlowPanel:
unit: ClassVar[str]
def __init__(self, *columns: ProgressColumn | str, max_rows: int, expand: bool = True) -> None:
diff --git a/cyberdrop_dl/progress/scraping/downloads.py b/cyberdrop_dl/progress/scraping/downloads.py
index 58ff4a5a5..b9659c934 100644
--- a/cyberdrop_dl/progress/scraping/downloads.py
+++ b/cyberdrop_dl/progress/scraping/downloads.py
@@ -27,7 +27,7 @@ from rich.text import Text
from typing_extensions import override
from cyberdrop_dl.progress import DictProgress, ProgressHook, create_test_live
-from cyberdrop_dl.progress.overflow import OverflowPanel
+from cyberdrop_dl.progress.overflow import OverFlowPanel
if TYPE_CHECKING:
from collections.abc import Generator, Iterable
@@ -112,7 +112,7 @@ class AutoDownloadColumn(DownloadColumn):
@final
-class DownloadsPanel(OverflowPanel):
+class DownloadsPanel(OverFlowPanel):
unit: ClassVar[str] = "file"
@property
diff --git a/cyberdrop_dl/progress/scraping/panel.py b/cyberdrop_dl/progress/scraping/panel.py
index 9a31a3318..22c0ffffe 100644
--- a/cyberdrop_dl/progress/scraping/panel.py
+++ b/cyberdrop_dl/progress/scraping/panel.py
@@ -15,7 +15,7 @@ from rich.text import Text
from cyberdrop_dl import __version__
from cyberdrop_dl.progress import create_test_live
-from cyberdrop_dl.progress.overflow import OverflowPanel
+from cyberdrop_dl.progress.overflow import OverFlowPanel
if TYPE_CHECKING:
from collections.abc import Generator
@@ -61,7 +61,7 @@ class StatusMessage:
@final
-class ScrapingPanel(OverflowPanel):
+class ScrapingPanel(OverFlowPanel):
unit: ClassVar[str] = "URL"
def __init__(self) -> None:
diff --git a/cyberdrop_dl/scraper/scrape_mapper.py b/cyberdrop_dl/scraper/scrape_mapper.py
index 0cfb24caf..4e880f7a2 100644
--- a/cyberdrop_dl/scraper/scrape_mapper.py
+++ b/cyberdrop_dl/scraper/scrape_mapper.py
@@ -463,7 +463,7 @@ def disable_crawlers_by_config(existing_crawlers: dict[str, type[Crawler]], craw
)
logger.info(f"Crawlers disabled by config: \n{crawlers_info}")
- log_spacer(10)
+ log_spacer()
def match_url_to_crawler(existing_crawlers: dict[str, _T], url: AbsoluteHttpURL) -> _T | None:
diff --git a/cyberdrop_dl/ui/progress/statistic_progress.py b/cyberdrop_dl/ui/progress/statistic_progress.py
index f31965aef..b45e3c4ec 100644
--- a/cyberdrop_dl/ui/progress/statistic_progress.py
+++ b/cyberdrop_dl/ui/progress/statistic_progress.py
@@ -42,7 +42,7 @@ class TaskInfo(NamedTuple):
class UiFailureTotal(NamedTuple):
full_msg: str
total: int
- error_code: int | None
+ code: int | None
msg: str
@classmethod
diff --git a/cyberdrop_dl/utils/apprise.py b/cyberdrop_dl/utils/apprise.py
index 09bde9413..fe6127b9d 100644
--- a/cyberdrop_dl/utils/apprise.py
+++ b/cyberdrop_dl/utils/apprise.py
@@ -95,13 +95,13 @@ def _process_results(
else:
final_result = constants.NotificationResult.FAILED
- log_spacer(10, log_to_console=False, log_to_file=not all(result))
+ log_spacer()
rich.print("Apprise notifications results:", final_result.value)
log = logger.debug if all(result) else logger.info
log(f"Apprise notifications results: {final_result.value}")
log(f"PARSED_APPRISE_URLs: \n{json.dumps(all_urls, indent=4)}\n")
log(f"RESULTS_BY_TAGS: \n{json.dumps(result_dict, indent=4)}")
- log_spacer(10, log_to_console=False, log_to_file=not all(result))
+ log_spacer()
parsed_log_lines = _parse_apprise_logs(apprise_logs)
for line in parsed_log_lines:
log(line.msg)
diff --git a/cyberdrop_dl/utils/logger.py b/cyberdrop_dl/utils/logger.py
index 7886fa0c9..709b0eef4 100644
--- a/cyberdrop_dl/utils/logger.py
+++ b/cyberdrop_dl/utils/logger.py
@@ -1,39 +1,44 @@
from __future__ import annotations
import contextlib
+import itertools
import json
import logging
+import os
import queue
-import sys
-from functools import wraps
+from contextvars import ContextVar
+from datetime import datetime
from logging.handlers import QueueHandler, QueueListener
from pathlib import Path
-from typing import IO, TYPE_CHECKING, ParamSpec
+from typing import TYPE_CHECKING, ParamSpec, cast
from rich._log_render import LogRender
from rich.console import Console, Group
-from rich.containers import Lines, Renderables
from rich.logging import RichHandler
-from rich.measure import Measurement
from rich.padding import Padding
from rich.text import Text, TextType
+from typing_extensions import override
-from cyberdrop_dl import constants, env
-from cyberdrop_dl.dependencies import browser_cookie3
-from cyberdrop_dl.exceptions import InvalidYamlError
+from cyberdrop_dl import env
+
+if TYPE_CHECKING:
+ import threading
+ from collections.abc import Generator
logger = logging.getLogger("cyberdrop_dl")
-logger_debug = logging.getLogger("cyberdrop_dl_debug")
-startup_logger = logging.getLogger("cyberdrop_dl_startup")
_DEFAULT_CONSOLE = Console()
-_USER_NAME = Path.home().resolve().name
+_USER_NAME = Path.home().name
_NEW_ISSUE_URL = "https://github.com/NTFSvolume/cdl/issues/new/choose"
+_DEFAULT_CONSOLE_WIDTH = 240
+_LOCK: threading.RLock = cast("threading.RLock", logging._lock) # pyright: ignore[ reportAttributeAccessIssue]
+_MAIN_LOGGER: ContextVar[LogHandler] = ContextVar("_MAIN_LOGGER")
+
+_capture_logs: ContextVar[bool] = ContextVar("_capture_logs", default=False)
if TYPE_CHECKING:
- from collections.abc import Callable, Generator, Iterable
- from datetime import datetime
+ from collections.abc import Callable, Iterable
from rich.console import ConsoleRenderable
@@ -46,18 +51,28 @@ if TYPE_CHECKING:
class RedactedConsole(Console):
"""Custom console to remove username from logs"""
+ @override
def _render_buffer(self, buffer) -> str:
- output: str = super()._render_buffer(buffer)
- return _redact_message(output)
+ return self._redact_message(super()._render_buffer(buffer))
+
+ @classmethod
+ def _redact_message(cls, message: object) -> str:
+ redacted = str(message)
+ for sep in ("\\", "\\\\", "/"):
+ as_tail = sep + _USER_NAME
+ as_part = _USER_NAME + sep
+ redacted = redacted.replace(as_tail, f"{sep}[REDACTED]").replace(as_part, f"[REDACTED]{sep}")
+ return redacted
class JsonLogRecord(logging.LogRecord):
- def getMessage(self) -> str: # noqa: N802
- """`dicts` will be logged as json, lazily"""
+ """`dicts` will be logged as json, lazily"""
+ @override
+ def getMessage(self) -> str:
msg = str(self._proccess_msg(self.msg))
if self.args:
- args = map(self._proccess_msg, self.args)
+ args = tuple(map(self._proccess_msg, self.args))
try:
return msg.format(*args)
except Exception:
@@ -67,7 +82,8 @@ class JsonLogRecord(logging.LogRecord):
@staticmethod
def _proccess_msg(msg: object) -> object:
- # TODO: Use our custom decoder to support more types
+ if callable(dump := getattr(msg, "model_dump_json", None)):
+ return dump()
if isinstance(msg, dict):
return json.dumps(msg, indent=2, ensure_ascii=False, default=str)
return msg
@@ -77,36 +93,64 @@ logging.setLogRecordFactory(JsonLogRecord)
class LogHandler(RichHandler):
- """Rich Handler with default settings, automatic console creation and custom log render to remove padding in files."""
-
- def __init__(
- self, level: int = 10, file: IO[str] | None = None, width: int | None = None, debug: bool = False
- ) -> None:
- is_file: bool = file is not None
- redacted: bool = is_file and not debug
- console_cls = RedactedConsole if redacted else Console
- if file is None and width is None:
- console = _DEFAULT_CONSOLE
- else:
- console = console_cls(file=file, width=width)
+ """Rich Handler with default settings, custom log render to remove padding in files and `color` extra"""
+ def __init__(self, level: int = logging.DEBUG, console: Console | None = None) -> None:
+ self.is_file: bool = bool(console)
+ self._buffer: list[Text] = []
super().__init__(
level,
console,
- show_time=False,
- show_path=False,
- tracebacks_show_locals=debug,
- locals_max_string=constants.DEFAULT_CONSOLE_WIDTH,
+ show_time=self.is_file,
+ rich_tracebacks=True,
+ tracebacks_show_locals=True,
+ locals_max_string=_DEFAULT_CONSOLE_WIDTH,
tracebacks_extra_lines=2,
locals_max_length=20,
+ show_path=False,
)
- if is_file:
+ if self.is_file:
self._log_render = NoPaddingLogRender(
- show_path=False,
show_level=True,
+ show_path=False,
+ level_width=10,
time_format=lambda dt: Text(f"[{dt.isoformat(sep=' ', timespec='milliseconds')}]", style="log.time"),
)
+ @override
+ def render_message(self, record: logging.LogRecord, message: str) -> ConsoleRenderable:
+ """This is the same as the base class, just added the `color` parsing from the extras"""
+ use_markup = bool(getattr(record, "markup", self.markup))
+ color = getattr(record, "color", "")
+ message_text = Text.from_markup(message, style=color) if use_markup else Text(message, style=color)
+
+ highlighter = getattr(record, "highlighter", self.highlighter)
+ if highlighter:
+ message_text = highlighter(message_text)
+
+ if self.keywords is None:
+ self.keywords = self.KEYWORDS
+
+ if self.keywords:
+ _ = message_text.highlight_words(self.keywords, "logging.keyword")
+
+ if self.is_file and _capture_logs.get():
+ self._buffer.append(message_text)
+
+ return message_text
+
+ def export_text(self) -> Text:
+ assert self.lock is not None
+ with self.lock:
+ lines = self._buffer[:]
+ self._buffer.clear()
+
+ eof = Text("\n")
+ text = Text()
+ for line in itertools.chain.from_iterable((line, eof) for line in lines):
+ _ = text.append_text(line)
+ return text
+
class BareQueueHandler(QueueHandler):
"""Sends the log record to the queue as is.
@@ -117,7 +161,7 @@ class BareQueueHandler(QueueHandler):
This made tracebacks render improperly because when the rich handler picks the log record from the queue, it has no traceback.
The original traceback was being formatted as normal text and included as part of the message.
- Having not pickleable objects is only an issue in multi-processing operations (multiprocessing.Queue)
+ We never log from other processes so we do not need that safety check
"""
def prepare(self, record: logging.LogRecord) -> logging.LogRecord:
@@ -144,11 +188,31 @@ class QueuedLogger:
self.log_handler.close()
+def _threaded_logger(log_handler: LogHandler) -> Generator[BareQueueHandler]:
+ """Context-manager to process logs from this handler in another thread.
+
+ It starts a QueueListener and yields the QueueHandler."""
+ q: queue.Queue[logging.LogRecord] = queue.Queue()
+ q_handler: BareQueueHandler = BareQueueHandler(q)
+ listener: QueueListener = QueueListener(q, log_handler, respect_handler_level=True)
+ listener.start()
+ try:
+ yield q_handler
+ finally:
+ try:
+ q_handler.close()
+ finally:
+ listener.stop()
+ for handler in listener.handlers[:]:
+ handler.close()
+
+
class NoPaddingLogRender(LogRender):
- cdl_padding: int = 0
+ _cdl_padding: int = 0
EXCLUDE_PATH_LOGGING_FROM: tuple[str, ...] = "logger.py", "base.py", "session.py", "cache_control.py"
- def __call__( # type: ignore[reportIncompatibleMethodOverride]
+ def __call__( # type: ignore[reportIncompatibleMethodOverride] # pyright: ignore[reportIncompatibleMethodOverride]
self,
console: Console,
renderables: Iterable[ConsoleRenderable],
@@ -169,170 +233,112 @@ class NoPaddingLogRender(LogRender):
else Text(log_time.strftime(time_format), style="log.time")
)
if log_time_display == self._last_time and self.omit_repeated_times:
- output.append(" " * len(log_time_display), style="log.time")
+ _ = output.append(" " * len(log_time_display), style="log.time")
output.pad_right(1)
else:
- output.append(log_time_display)
+ _ = output.append_text(log_time_display)
output.pad_right(1)
self._last_time = log_time_display
+
if self.show_level:
- output.append(level)
+ _ = output.append(level)
output.pad_right(1)
- if not self.cdl_padding:
- self.cdl_padding = _get_renderable_length(output)
+ if not self._cdl_padding:
+ self._cdl_padding = console.measure(output).maximum
if self.show_path and path and not any(path.startswith(p) for p in self.EXCLUDE_PATH_LOGGING_FROM):
path_text = Text(style="log.path")
- path_text.append(path, style=f"link file://{link_path}" if link_path else "")
+ _ = path_text.append(path, style=f"link file://{link_path}" if link_path else "")
if line_no:
- path_text.append(":")
- path_text.append(
+ _ = path_text.append(":")
+ _ = path_text.append(
f"{line_no}",
style=f"link file://{link_path}#{line_no}" if link_path else "",
)
- output.append(path_text)
+ _ = output.append_text(path_text)
output.pad_right(1)
padded_lines: list[ConsoleRenderable] = []
- for renderable in Renderables(renderables): # type: ignore
+ for renderable in renderables:
if isinstance(renderable, Text):
- renderable = _indent_text(renderable, console, self.cdl_padding)
+ renderable = _indent_text(renderable, console, self._cdl_padding)
renderable.stylize("log.message")
- output.append(renderable)
+ _ = output.append_text(renderable)
continue
- padded_lines.append(Padding(renderable, (0, 0, 0, self.cdl_padding), expand=False))
-
- return Group(output, *padded_lines)
+ padded_lines.append(Padding(renderable, (0, 0, 0, self._cdl_padding), expand=False))
-def _get_renderable_length(renderable) -> int:
- measurement = Measurement.get(_DEFAULT_CONSOLE, _DEFAULT_CONSOLE.options, renderable)
- return measurement.maximum
+ return Group(output, *padded_lines)
-def _indent_text(text: Text, console: Console, indent: int = 30) -> Text:
+def _indent_text(text: Text, console: Console, indent: int) -> Text:
"""Indents each line of a Text object except the first one."""
- indent_str = Text("\n" + (" " * indent))
+ padding = Text("\n" + (" " * indent))
new_text = Text()
- new_width = console.width - indent
- lines: Lines = text.wrap(console, width=new_width)
- first_line = lines[0]
- other_lines = lines[1:]
- for line in other_lines:
+ first_line, *rest = text.wrap(console, width=console.width - indent)
+ for line in rest:
line.rstrip()
- new_text.append(indent_str + line)
+ _ = new_text.append_text(padding + line)
first_line.rstrip()
- return first_line.append(new_text)
-
-
-def log(message: object, level: int = 10, bug: bool = False, **kwargs) -> None:
- """Simple logging function."""
- log_debug(message, level, **kwargs)
- if bug:
- args = message, f"Please open a bug report at {_NEW_ISSUE_URL}"
- message = "{}. {}"
- level = 30
- else:
- args = ()
- logger.log(level, message, *args, **kwargs)
-
-
-def log_debug(message: object, level: int = 10, **kwargs) -> None:
- """Simple logging function."""
- if env.DEBUG_VAR:
- logger_debug.log(level, message, **kwargs)
-
+ return first_line.append_text(new_text)
-def log_with_color(message: Text | str, style: str, level: int = 20, show_in_stats: bool = True, **kwargs) -> None:
- """Simple logging function with color."""
- text = message if isinstance(message, Text) else Text(message, style=style)
- log(text.plain, level, **kwargs)
- if constants.CONSOLE_LEVEL >= 50 and "pytest" not in sys.modules:
- _DEFAULT_CONSOLE.print(text)
- if show_in_stats:
- constants.LOG_OUTPUT_TEXT.append_text(text.append("\n"))
-
-def log_spacer(level: int, char: str = "-", *, log_to_console: bool = True, log_to_file: bool = True) -> None:
- spacer = char * min(int(constants.DEFAULT_CONSOLE_WIDTH / 2), 50)
- if log_to_file:
- log(spacer, level)
- if log_to_console and constants.CONSOLE_LEVEL >= 50:
- _DEFAULT_CONSOLE.print("")
- constants.LOG_OUTPUT_TEXT.append("\n", style="black")
-
-
-def _redact_message(message: Exception | Text | str) -> str:
- redacted = str(message)
- separators = ["\\", "\\\\", "/"]
- for sep in separators:
- as_tail = sep + _USER_NAME
- as_part = _USER_NAME + sep
- redacted = redacted.replace(as_tail, f"{sep}[REDACTED]").replace(as_part, f"[REDACTED]{sep}")
- return redacted
+def log_spacer(char: str = "-") -> None:
+ logger.info(char * (_DEFAULT_CONSOLE_WIDTH // 2), stacklevel=2)
@contextlib.contextmanager
-def _setup_startup_logger() -> Generator[None]:
- """Context manager to add a file handler to the startup logger
-
- It will only add it if we have an exception, to prevent creating an empty file"""
- startup_logger.setLevel(10)
- if "pytest" not in sys.modules:
- console_handler = LogHandler(level=10)
- startup_logger.addHandler(console_handler)
- try:
- yield
-
- except Exception:
- try:
- file = Path.cwd() / "startup.log"
- file_handler = LogHandler(
- level=10,
- file=file.open("w", encoding="utf8"),
- width=constants.DEFAULT_CONSOLE_WIDTH,
+def setup_logging(file: Path, /, level: int = logging.DEBUG) -> Generator[None]:
+ logger.setLevel(level)
+ file.parent.mkdir(parents=True, exist_ok=True)
+ with (
+ file.open("w+" if os.name == "nt" else "w", encoding="utf8") as fp,
+ _threaded_logger(LogHandler(level=level)) as console_out,
+ _threaded_logger(
+ main_logger := LogHandler(
+ level=level,
+ console=RedactedConsole(file=fp, width=_DEFAULT_CONSOLE_WIDTH * 2),
)
- startup_logger.addHandler(file_handler)
- except OSError:
- # We could not create the file for some reason
- # Ignore this and just log to the console
- pass
- raise
-
-
-def catch_exceptions(func: Callable[_P, _ExitCode]) -> Callable[_P, _ExitCode]:
- """Decorator to automatically log uncaught exceptions.
-
- Exceptions will be logged to a file in the current working directory
- because the manager setup itself may have failed, therefore we don't know
- what the proper log file path is.
- """
-
- @wraps(func)
- def catch(*args, **kwargs) -> _ExitCode | None:
+ ) as file_out,
+ _setup_debug_logger() as debug_log_file,
+ ):
+ token = _MAIN_LOGGER.set(main_logger)
+ logger.addHandler(console_out)
+ logger.addHandler(file_out)
try:
- with _setup_startup_logger():
- return func(*args, **kwargs)
+ logger.info(f"Debug log file: '{debug_log_file}'")
+ yield
+ finally:
+ _MAIN_LOGGER.reset(token)
- except InvalidYamlError as e:
- startup_logger.error(e.message)
- except browser_cookie3.BrowserCookieError:
- startup_logger.exception("")
+def _setup_debug_logger() -> Generator[Path | None]:
+ if not env.DEBUG_VAR:
+ yield
+ return
- except OSError as e:
- startup_logger.exception(str(e))
+ debug_log_file = Path(__file__).parent.parent.parent / "cyberdrop_dl_debug.log"
- except KeyboardInterrupt:
- startup_logger.info("Exiting...")
- return
+ if env.DEBUG_LOG_FOLDER:
+ debug_log_folder = Path(env.DEBUG_LOG_FOLDER)
- except Exception:
- msg = "An error occurred, please report this to the developer with your logs file:"
- startup_logger.exception(msg)
+ if not debug_log_folder.is_dir():
+ msg = f"Value of env var 'CDL_DEBUG_LOG_FOLDER' is invalid. Folder '{debug_log_folder}' does not exists"
+ raise FileNotFoundError(None, msg, env.DEBUG_LOG_FOLDER)
- return 1
+ now = datetime.now().strftime("%Y%m%d_%H%M%S")
+ debug_log_file = debug_log_folder / f"cyberdrop_dl_debug_{now}.log"
- return catch
+ debug_log_file = debug_log_file.expanduser().resolve().absolute()
+
+ with (
+ debug_log_file.open("w", encoding="utf8") as fp,
+ _threaded_logger(
+ LogHandler(level=logging.DEBUG, console=Console(file=fp, width=_DEFAULT_CONSOLE_WIDTH * 2))
+ ) as debug_handler,
+ ):
+ logger.addHandler(debug_handler)
+ yield debug_log_file
diff --git a/cyberdrop_dl/utils/utilities.py b/cyberdrop_dl/utils/utilities.py
index ee29704f6..77f4c7e4e 100644
--- a/cyberdrop_dl/utils/utilities.py
+++ b/cyberdrop_dl/utils/utilities.py
@@ -11,7 +11,7 @@ import platform
import re
import sys
from collections.abc import Generator
-from functools import partial, wraps
+from functools import wraps
from http import HTTPStatus
from pathlib import Path
from stat import S_ISREG
@@ -45,7 +45,6 @@ from cyberdrop_dl.exceptions import (
get_origin,
)
from cyberdrop_dl.utils import json
-from cyberdrop_dl.utils.logger import log_with_color
if TYPE_CHECKING:
from collections.abc import Callable, Coroutine, Generator, Iterable
@@ -282,22 +281,22 @@ def _partial_files(dir: Path | str) -> Generator[Path]:
def delete_partial_files(manager: Manager) -> None:
"""Deletes partial download files recursively."""
- log_red("Deleting partial downloads...")
+ logger.info("Deleting partial downloads...")
for file in _partial_files(manager.config.files.download_folder):
file.unlink(missing_ok=True)
def check_for_partial_files(manager: Manager) -> None:
"""Checks if there are partial downloads in any subdirectory and logs if found."""
- log_yellow("Checking for partial downloads...")
+ logger.info("Checking for partial downloads...")
has_partial_files = next(_partial_files(manager.config.files.download_folder), None)
if has_partial_files:
- log_yellow("There are partial downloads in the downloads folder")
+ logger.warning("There are partial downloads in the downloads folder")
def delete_empty_folders(manager: Manager) -> None:
"""Deletes empty folders efficiently."""
- log_yellow("Checking for empty folders...")
+ logger.info("Checking for empty folders...")
purge_dir_tree(manager.config.files.download_folder)
sorted_folder = manager.config.sorting.sort_folder
@@ -389,7 +388,7 @@ def get_size_or_none(path: Path) -> int | None:
@functools.cache
-def get_system_information() -> str:
+def get_system_information() -> dict[str, Any]:
def get_common_name() -> str:
system = platform.system()
@@ -418,7 +417,7 @@ def get_system_information() -> str:
"common_name": get_common_name(),
}
_ = system_info.pop("node", None)
- return json.dumps(system_info, indent=4)
+ return system_info
def is_blob_or_svg(link: str) -> bool:
@@ -443,7 +442,3 @@ def unique(iterable: Iterable[_T], *, hashable: bool = True) -> Iterable[_T]:
def xor_decrypt(encrypted_data: bytes, key: bytes) -> str:
data = bytearray(b_input ^ b_key for b_input, b_key in zip(encrypted_data, itertools.cycle(key)))
return data.decode("utf-8", errors="ignore")
-
-
-log_yellow = partial(log_with_color, style="yellow", level=20)
-log_red = partial(log_with_color, style="red", level=20)
diff --git a/cyberdrop_dl/utils/webhook.py b/cyberdrop_dl/utils/webhook.py
index 22d66569e..22d0daecd 100644
--- a/cyberdrop_dl/utils/webhook.py
+++ b/cyberdrop_dl/utils/webhook.py
@@ -90,7 +90,7 @@ async def send_webhook_message(manager: Manager) -> None:
logger.exception("Unable to send webhook notification")
result_to_log = result, str(e)
- log_spacer(10, log_to_console=False)
+ log_spacer()
rich.print("Webhook Notifications Results:", result)
result_to_log = "\n".join(map(str, result_to_log))
logger.info(f"Webhook Notifications Results: {result_to_log}")
diff --git a/tests/test_manager.py b/tests/test_manager.py
index c2127d16a..9beb6fa9d 100644
--- a/tests/test_manager.py
+++ b/tests/test_manager.py
@@ -108,7 +108,7 @@ def test_args_logging_should_censor_webhook(
running_manager.config_manager.settings_data.logs = update_model(logs_model, webhook=webhook)
running_manager.args_logging()
assert logs.messages
- assert "Starting Cyberdrop-DL Process" in logs.text
+ assert "Running cyberdrop-dl " in logs.text
assert webhook not in logs.text
webhook_line = next(msg for msg in logs.text.splitlines() if '"webhook"' in msg)
_, _, webhook_text = webhook_line.partition(":")