Bulk asynchronous downloader for multiple file hosts
Python asynciobunkrcyberdropdownloaderdropboxfilestergofilegoogle-drivemediafiremegamega-nzonedrivepatreonpawchivepinterestpixeldraintransfer-ittwitchtwittervk
fix: restore log levels (#1759)
NTFSvolume65806a9045cb
diff --git a/cyberdrop_dl/__main__.py b/cyberdrop_dl/__main__.pyindex c59e9e33d..87a0af0f6 100644--- a/cyberdrop_dl/__main__.py+++ b/cyberdrop_dl/__main__.py@@ -12,7 +12,7 @@ tracebacks.install_exception_hook()from cyberdrop_dl.cli import CLIargsfrom cyberdrop_dl.config import Config-from cyberdrop_dl.logs import log_spacer, setup_console_logging, setup_file_logging+from cyberdrop_dl.logs import log_spacer, set_console_level, setup_console_logging, setup_file_loggingfrom cyberdrop_dl.managers.manager import AppData, Managerfrom cyberdrop_dl.models.types import HttpURLfrom cyberdrop_dl.progress import REFRESH_RATE, TUI_DISABLED@@ -25,7 +25,10 @@ logger = logging.getLogger("cyberdrop_dl")async def _scrape(manager: Manager) -> None:- with setup_file_logging(manager.config.settings.logs.main_log):+ with setup_file_logging(+ manager.config.settings.logs.main_log,+ level=manager.config.settings.runtime_options.effective_log_level,+ ):await manager.async_startup()REFRESH_RATE.set(manager.config.global_settings.ui_options.refresh_rate)TUI_DISABLED.set(manager.cli_args.ui.is_disabled)@@ -73,6 +76,7 @@ async def _post_runtime(manager: Manager) -> None:def _main(manager: Manager) -> None:+ set_console_level(manager.config.settings.runtime_options.effective_console_log_level)manager.resolve_paths()if not manager.cli_args.download:program_ui.run(manager)diff --git a/cyberdrop_dl/config/settings.py b/cyberdrop_dl/config/settings.pyindex f4db9793f..b5b71c4b4 100755--- a/cyberdrop_dl/config/settings.py+++ b/cyberdrop_dl/config/settings.py@@ -1,9 +1,9 @@# ruff: noqa: RUF012+import loggingimport refrom datetime import date, datetime, timedelta-from logging import DEBUGfrom pathlib import Path-from typing import Annotated+from typing import Annotated, Literalfrom cyclopts import Parameterfrom pydantic import BaseModel, ByteSize, Field, NonNegativeInt, PrivateAttr, field_validator@@ -27,7 +27,7 @@ from cyberdrop_dl.models.types import (NonEmptyStrOrNone,PathOrNone,)-from cyberdrop_dl.models.validators import falsy_as, to_timedelta+from cyberdrop_dl.models.validators import falsy_as, falsy_as_none, to_timedeltafrom cyberdrop_dl.utils import delete_empty_files_and_foldersfrom cyberdrop_dl.utils.strings import validate_format_string@@ -199,20 +199,51 @@ class IgnoreOptions(SettingsGroup):class RuntimeOptions(SettingsGroup):- console_log_level: NonNegativeInt = 100+ log_level: NonNegativeInt | Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "DEBUG"+ "Only log messages of this level or higher to the main log file"+ console_log_level: NonNegativeInt | Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None = None+ "Only log messages of this level or higher to the console. An empty or `None` value will use the same level as `log_level`"deep_scrape: bool = Falsedelete_partial_files: bool = Falseignore_history: bool = Falsejdownloader_autostart: bool = Falsejdownloader_download_dir: PathOrNone = Nonejdownloader_whitelist: ListNonEmptyStr = []- log_level: NonNegativeInt = DEBUG+send_unsupported_to_jdownloader: bool = Falseskip_check_for_empty_folders: bool = Falseskip_check_for_partial_files: bool = Falseslow_download_speed: ByteSizeSerilized = ByteSize(0)update_last_forum_post: bool = True+ @field_validator("log_level", "console_log_level", mode="before")+ @classmethod+ def normalize_log_level(cls, value: object):+ value = falsy_as_none(value)+ if type(value) is str:+ try:+ return logging.getLevelNamesMapping()[value.upper()]+ except KeyError:+ raise ValueError(f"invalid log level name: {value!r}") from None+ return value++ @property+ def effective_log_level(self) -> int:+ if type(self.log_level) is str:+ return logging.getLevelNamesMapping()[self.log_level.upper()]+ assert type(self.log_level) is int+ return self.log_level++ @property+ def effective_console_log_level(self) -> int:+ if type(self.console_log_level) is int and self.console_log_level > logging.CRITICAL:+ self.console_log_level = None+ if not self.console_log_level:+ return self.effective_log_level++ assert type(self.console_log_level) is int+ return self.console_log_level+class Sorting(SettingsGroup):scan_folder: PathOrNone = Nonediff --git a/cyberdrop_dl/logs.py b/cyberdrop_dl/logs.pyindex 0e0ba2e37..607164c48 100644--- a/cyberdrop_dl/logs.py+++ b/cyberdrop_dl/logs.py@@ -7,6 +7,7 @@ import queueimport sysfrom contextvars import ContextVarfrom datetime import datetime+from enum import StrEnumfrom io import StringIOfrom logging.handlers import QueueHandler, QueueListenerfrom pathlib import Path@@ -28,7 +29,7 @@ if TYPE_CHECKING:logger = logging.getLogger("cyberdrop_dl")-for noisy_package in ("aiosqlite",):+for noisy_package in ("aiosqlite", "markdown_it"):logging.getLogger(noisy_package).setLevel(logging.ERROR)_T = TypeVar("_T")@@ -41,6 +42,12 @@ _LOG_TO_CONSOLE: ContextVar[bool] = ContextVar("LOG_TO_CONSOLE", default=True)MAIN_LOG_FILE: ContextVar[Path] = ContextVar("MAIN_LOG_FILE")+class HandlerName(StrEnum):+ CONSOLE = "cdl-console"+ MAIN_LOG = "cdl-main-log-file"+ DEBUG_LOG = "cdl-debug-log-file"++class RedactedConsole(Console):"""Custom console to remove username from logs"""@@ -172,7 +179,9 @@ class BareQueueHandler(QueueHandler):@contextlib.contextmanagerdef _threaded_logger(- log_handler: logging.Handler, *, context_var: ContextVar[QueueListener] | None = None+ log_handler: logging.Handler,+ *,+ context_var: ContextVar[QueueListener] | None = None,) -> Generator[BareQueueHandler]:"""Context-manager to process logs from this handler in another thread"""q: queue.Queue[logging.LogRecord] = queue.Queue()@@ -280,12 +289,24 @@ def log_spacer(char: str = "-") -> None:logger.info(char * 30, stacklevel=2)+def _get_handler(name: str) -> logging.Handler:+ for handler in logging.getLogger().handlers:+ if handler.get_name() == name:+ return handler+ raise LookupError(name)+++def set_console_level(level: int) -> None:+ _get_handler(HandlerName.CONSOLE).setLevel(level)++@contextlib.contextmanager-def setup_console_logging(level: int = logging.INFO) -> Generator[None]:- handler = LogHandler(level, show_time=False)+def setup_console_logging() -> Generator[None]:+ handler = LogHandler(logging.DEBUG, show_time=False)logging.getLogger().setLevel(logging.DEBUG)try:with _threaded_logger(handler, context_var=_CONSOLE_LOG_LISTENER) as q_handler:+ q_handler.set_name(HandlerName.CONSOLE)q_handler.addFilter(lambda _: _LOG_TO_CONSOLE.get())yieldfinally:@@ -295,10 +316,13 @@ def setup_console_logging(level: int = logging.INFO) -> Generator[None]:@contextlib.contextmanager-def setup_file_logging(file: Path, /, level: int = logging.DEBUG) -> Generator[None]:+def setup_file_logging(file: Path, /, *, level: int = logging.DEBUG) -> Generator[None]:file.parent.mkdir(parents=True, exist_ok=True)import mega+ if "pytest" not in sys.modules:+ logging.captureWarnings(True)+with (_setup_debug_logger() as debug_log_file,file.open("w", encoding="utf8") as fp,@@ -307,13 +331,15 @@ def setup_file_logging(file: Path, /, level: int = logging.DEBUG) -> Generator[N_enter_context(mega.LOG_FILE_PROGRESS, False),_threaded_logger(log_handler=LogHandler(- level,+ level=logging.DEBUG,show_time=True,console=RedactedConsole(file=fp, width=_DEFAULT_CONSOLE_WIDTH * 2),),context_var=_MAIN_LOG_LISTENER,- ),+ ) as handler,):+ handler.setLevel(level)+ handler.set_name(HandlerName.MAIN_LOG)logger.info(f"Debug log file: {debug_log_file}")try:yield@@ -353,12 +379,13 @@ def _setup_debug_logger() -> Generator[Path | None]:debug_log_file.open("w", encoding="utf8") as fp,_threaded_logger(LogHandler(- logging.DEBUG,+ level=logging.NOTSET + 1,console=Console(file=fp, width=_DEFAULT_CONSOLE_WIDTH * 2),show_time=True,- )- ),+ ),+ ) as handler,):+ handler.set_name(HandlerName.DEBUG_LOG)yield debug_log_filediff --git a/tests/test_config.py b/tests/test_config.pyindex 52fd538c0..371f8c36a 100644--- a/tests/test_config.py+++ b/tests/test_config.py@@ -127,3 +127,24 @@ class TestMergeDicts:dict2 = {"a": 1}expected = {"a": {"x": 1}}assert merge_dicts(dict1, dict2) == expected+++class TestRuntimeLogsConfig:+ @staticmethod+ def parse(level: object, console_level: object):+ return settings.RuntimeOptions.model_validate({"log_level": level, "console_log_level": console_level})++ def test_default(self) -> None:+ default = settings.RuntimeOptions()+ expected = settings.RuntimeOptions(log_level="debug", console_log_level="debug") # pyright: ignore[reportArgumentType]++ assert default.effective_log_level == 10+ assert default.effective_console_log_level == 10+ assert default.effective_log_level == expected.effective_log_level+ assert default.effective_console_log_level == expected.effective_console_log_level++ def test_falsy_console_log_level(self) -> None:+ for a, b in [(30, None), (40, ""), (20, "none")]:+ config = self.parse(a, b)+ assert config.effective_log_level == a+ assert config.effective_console_log_level == a