Skip to content

Bulk asynchronous downloader for multiple file hosts

fix: restore log levels (#1759)

NTFSvolume65806a9045cb
diff --git a/cyberdrop_dl/__main__.py b/cyberdrop_dl/__main__.py
index c59e9e33d..87a0af0f6 100644
--- a/cyberdrop_dl/__main__.py
+++ b/cyberdrop_dl/__main__.py
@@ -12,7 +12,7 @@ tracebacks.install_exception_hook()
from cyberdrop_dl.cli import CLIargs
from cyberdrop_dl.config import Config
-from cyberdrop_dl.logs import log_spacer, setup_console_logging, setup_file_logging
+from cyberdrop_dl.logs import log_spacer, set_console_level, setup_console_logging, setup_file_logging
from cyberdrop_dl.managers.manager import AppData, Manager
from cyberdrop_dl.models.types import HttpURL
from cyberdrop_dl.progress import REFRESH_RATE, TUI_DISABLED
@@ -25,7 +25,10 @@ logger = logging.getLogger("cyberdrop_dl")
async def _scrape(manager: Manager) -> None:
- with setup_file_logging(manager.config.settings.logs.main_log):
+ with setup_file_logging(
+ manager.config.settings.logs.main_log,
+ level=manager.config.settings.runtime_options.effective_log_level,
+ ):
await manager.async_startup()
REFRESH_RATE.set(manager.config.global_settings.ui_options.refresh_rate)
TUI_DISABLED.set(manager.cli_args.ui.is_disabled)
@@ -73,6 +76,7 @@ async def _post_runtime(manager: Manager) -> None:
def _main(manager: Manager) -> None:
+ set_console_level(manager.config.settings.runtime_options.effective_console_log_level)
manager.resolve_paths()
if not manager.cli_args.download:
program_ui.run(manager)
diff --git a/cyberdrop_dl/config/settings.py b/cyberdrop_dl/config/settings.py
index f4db9793f..b5b71c4b4 100755
--- a/cyberdrop_dl/config/settings.py
+++ b/cyberdrop_dl/config/settings.py
@@ -1,9 +1,9 @@
# ruff: noqa: RUF012
+import logging
import re
from datetime import date, datetime, timedelta
-from logging import DEBUG
from pathlib import Path
-from typing import Annotated
+from typing import Annotated, Literal
from cyclopts import Parameter
from pydantic import BaseModel, ByteSize, Field, NonNegativeInt, PrivateAttr, field_validator
@@ -27,7 +27,7 @@ from cyberdrop_dl.models.types import (
NonEmptyStrOrNone,
PathOrNone,
)
-from cyberdrop_dl.models.validators import falsy_as, to_timedelta
+from cyberdrop_dl.models.validators import falsy_as, falsy_as_none, to_timedelta
from cyberdrop_dl.utils import delete_empty_files_and_folders
from cyberdrop_dl.utils.strings import validate_format_string
@@ -199,20 +199,51 @@ class IgnoreOptions(SettingsGroup):
class RuntimeOptions(SettingsGroup):
- console_log_level: NonNegativeInt = 100
+ log_level: NonNegativeInt | Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "DEBUG"
+ "Only log messages of this level or higher to the main log file"
+ console_log_level: NonNegativeInt | Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None = None
+ "Only log messages of this level or higher to the console. An empty or `None` value will use the same level as `log_level`"
deep_scrape: bool = False
delete_partial_files: bool = False
ignore_history: bool = False
jdownloader_autostart: bool = False
jdownloader_download_dir: PathOrNone = None
jdownloader_whitelist: ListNonEmptyStr = []
- log_level: NonNegativeInt = DEBUG
+
send_unsupported_to_jdownloader: bool = False
skip_check_for_empty_folders: bool = False
skip_check_for_partial_files: bool = False
slow_download_speed: ByteSizeSerilized = ByteSize(0)
update_last_forum_post: bool = True
+ @field_validator("log_level", "console_log_level", mode="before")
+ @classmethod
+ def normalize_log_level(cls, value: object):
+ value = falsy_as_none(value)
+ if type(value) is str:
+ try:
+ return logging.getLevelNamesMapping()[value.upper()]
+ except KeyError:
+ raise ValueError(f"invalid log level name: {value!r}") from None
+ return value
+
+ @property
+ def effective_log_level(self) -> int:
+ if type(self.log_level) is str:
+ return logging.getLevelNamesMapping()[self.log_level.upper()]
+ assert type(self.log_level) is int
+ return self.log_level
+
+ @property
+ def effective_console_log_level(self) -> int:
+ if type(self.console_log_level) is int and self.console_log_level > logging.CRITICAL:
+ self.console_log_level = None
+ if not self.console_log_level:
+ return self.effective_log_level
+
+ assert type(self.console_log_level) is int
+ return self.console_log_level
+
class Sorting(SettingsGroup):
scan_folder: PathOrNone = None
diff --git a/cyberdrop_dl/logs.py b/cyberdrop_dl/logs.py
index 0e0ba2e37..607164c48 100644
--- a/cyberdrop_dl/logs.py
+++ b/cyberdrop_dl/logs.py
@@ -7,6 +7,7 @@ import queue
import sys
from contextvars import ContextVar
from datetime import datetime
+from enum import StrEnum
from io import StringIO
from logging.handlers import QueueHandler, QueueListener
from pathlib import Path
@@ -28,7 +29,7 @@ if TYPE_CHECKING:
logger = logging.getLogger("cyberdrop_dl")
-for noisy_package in ("aiosqlite",):
+for noisy_package in ("aiosqlite", "markdown_it"):
logging.getLogger(noisy_package).setLevel(logging.ERROR)
_T = TypeVar("_T")
@@ -41,6 +42,12 @@ _LOG_TO_CONSOLE: ContextVar[bool] = ContextVar("LOG_TO_CONSOLE", default=True)
MAIN_LOG_FILE: ContextVar[Path] = ContextVar("MAIN_LOG_FILE")
+class HandlerName(StrEnum):
+ CONSOLE = "cdl-console"
+ MAIN_LOG = "cdl-main-log-file"
+ DEBUG_LOG = "cdl-debug-log-file"
+
+
class RedactedConsole(Console):
"""Custom console to remove username from logs"""
@@ -172,7 +179,9 @@ class BareQueueHandler(QueueHandler):
@contextlib.contextmanager
def _threaded_logger(
- log_handler: logging.Handler, *, context_var: ContextVar[QueueListener] | None = None
+ log_handler: logging.Handler,
+ *,
+ context_var: ContextVar[QueueListener] | None = None,
) -> Generator[BareQueueHandler]:
"""Context-manager to process logs from this handler in another thread"""
q: queue.Queue[logging.LogRecord] = queue.Queue()
@@ -280,12 +289,24 @@ def log_spacer(char: str = "-") -> None:
logger.info(char * 30, stacklevel=2)
+def _get_handler(name: str) -> logging.Handler:
+ for handler in logging.getLogger().handlers:
+ if handler.get_name() == name:
+ return handler
+ raise LookupError(name)
+
+
+def set_console_level(level: int) -> None:
+ _get_handler(HandlerName.CONSOLE).setLevel(level)
+
+
@contextlib.contextmanager
-def setup_console_logging(level: int = logging.INFO) -> Generator[None]:
- handler = LogHandler(level, show_time=False)
+def setup_console_logging() -> Generator[None]:
+ handler = LogHandler(logging.DEBUG, show_time=False)
logging.getLogger().setLevel(logging.DEBUG)
try:
with _threaded_logger(handler, context_var=_CONSOLE_LOG_LISTENER) as q_handler:
+ q_handler.set_name(HandlerName.CONSOLE)
q_handler.addFilter(lambda _: _LOG_TO_CONSOLE.get())
yield
finally:
@@ -295,10 +316,13 @@ def setup_console_logging(level: int = logging.INFO) -> Generator[None]:
@contextlib.contextmanager
-def setup_file_logging(file: Path, /, level: int = logging.DEBUG) -> Generator[None]:
+def setup_file_logging(file: Path, /, *, level: int = logging.DEBUG) -> Generator[None]:
file.parent.mkdir(parents=True, exist_ok=True)
import mega
+ if "pytest" not in sys.modules:
+ logging.captureWarnings(True)
+
with (
_setup_debug_logger() as debug_log_file,
file.open("w", encoding="utf8") as fp,
@@ -307,13 +331,15 @@ def setup_file_logging(file: Path, /, level: int = logging.DEBUG) -> Generator[N
_enter_context(mega.LOG_FILE_PROGRESS, False),
_threaded_logger(
log_handler=LogHandler(
- level,
+ level=logging.DEBUG,
show_time=True,
console=RedactedConsole(file=fp, width=_DEFAULT_CONSOLE_WIDTH * 2),
),
context_var=_MAIN_LOG_LISTENER,
- ),
+ ) as handler,
):
+ handler.setLevel(level)
+ handler.set_name(HandlerName.MAIN_LOG)
logger.info(f"Debug log file: {debug_log_file}")
try:
yield
@@ -353,12 +379,13 @@ def _setup_debug_logger() -> Generator[Path | None]:
debug_log_file.open("w", encoding="utf8") as fp,
_threaded_logger(
LogHandler(
- logging.DEBUG,
+ level=logging.NOTSET + 1,
console=Console(file=fp, width=_DEFAULT_CONSOLE_WIDTH * 2),
show_time=True,
- )
- ),
+ ),
+ ) as handler,
):
+ handler.set_name(HandlerName.DEBUG_LOG)
yield debug_log_file
diff --git a/tests/test_config.py b/tests/test_config.py
index 52fd538c0..371f8c36a 100644
--- a/tests/test_config.py
+++ b/tests/test_config.py
@@ -127,3 +127,24 @@ class TestMergeDicts:
dict2 = {"a": 1}
expected = {"a": {"x": 1}}
assert merge_dicts(dict1, dict2) == expected
+
+
+class TestRuntimeLogsConfig:
+ @staticmethod
+ def parse(level: object, console_level: object):
+ return settings.RuntimeOptions.model_validate({"log_level": level, "console_log_level": console_level})
+
+ def test_default(self) -> None:
+ default = settings.RuntimeOptions()
+ expected = settings.RuntimeOptions(log_level="debug", console_log_level="debug") # pyright: ignore[reportArgumentType]
+
+ assert default.effective_log_level == 10
+ assert default.effective_console_log_level == 10
+ assert default.effective_log_level == expected.effective_log_level
+ assert default.effective_console_log_level == expected.effective_console_log_level
+
+ def test_falsy_console_log_level(self) -> None:
+ for a, b in [(30, None), (40, ""), (20, "none")]:
+ config = self.parse(a, b)
+ assert config.effective_log_level == a
+ assert config.effective_console_log_level == a