Skip to content

Bulk asynchronous downloader for multiple file hosts

16.5 KB
Raw Blame
from __future__ import annotations

import contextlib
import json
import logging
import queue
import sys
from collections.abc import MutableMapping
from contextvars import ContextVar
from enum import StrEnum
from io import StringIO
from logging.handlers import QueueHandler, QueueListener
from pathlib import Path
from typing import TYPE_CHECKING, Any, ClassVar, final, override

from rich._log_render import LogRender
from rich.console import Console, Group
from rich.logging import RichHandler
from rich.padding import Padding
from rich.text import Text, TextType

from cyberdrop_dl import env, tracebacks
from cyberdrop_dl.constants import MAIN_LOG_FILE
from cyberdrop_dl.exceptions import CDLConfigRuntimeErrorsGroup
from cyberdrop_dl.utils import dates, enter_context, truncated_preview

if TYPE_CHECKING:
    import datetime as dt
    from collections.abc import Callable, Generator, Iterable

    from rich.console import ConsoleRenderable


logger = logging.getLogger("cyberdrop_dl")
for noisy_package in ("aiosqlite", "markdown_it"):
    logging.getLogger(noisy_package).setLevel(logging.ERROR)

if not env.DEBUG_MODE:
    logging.getLogger("asyncio").setLevel(logging.ERROR)


_USER_NAME = Path.home().name
_DEFAULT_CONSOLE_WIDTH = 240
_MAIN_LOG_LISTENER: ContextVar[QueueListener] = ContextVar("_MAIN_LOG_LISTENER")
_CONSOLE_LOG_LISTENER: ContextVar[QueueListener] = ContextVar("_CONSOLE_LOG_LISTENER")
_LOG_TO_CONSOLE: ContextVar[bool] = ContextVar("LOG_TO_CONSOLE", default=True)
LOG_HTTP_TRAFFIC: ContextVar[bool] = ContextVar("LOG_HTTP_TRAFFIC", default=True)
MAX_ATTACHMENT_SIZE = 20 * 1e6


class HandlerName(StrEnum):
    CONSOLE = "cdl-console"
    MAIN_LOG = "cdl-main-log-file"
    DEBUG_LOG = "cdl-debug-log-file"


class RedactedConsole(Console):
    """Custom console to remove username from logs"""

    @override
    def _render_buffer(self, buffer) -> str:
        return self._redact_message(super()._render_buffer(buffer))

    @classmethod
    def _redact_message(cls, message: object) -> str:
        redacted = str(message)
        for sep in ("\\", "\\\\", "/"):
            as_tail = sep + _USER_NAME
            as_part = _USER_NAME + sep
            redacted = redacted.replace(as_tail, f"{sep}[REDACTED]").replace(as_part, f"[REDACTED]{sep}")
        return redacted


class JsonLogRecord(logging.LogRecord):
    """`dicts` will be logged as json, lazily"""

    @override
    def getMessage(self) -> str:
        msg = str(self._proccess_msg(self.msg))
        args = self.args
        if not args:
            return msg

        if type(self.args) is dict:
            # Override std special case for mappings
            args = [self.args]

        args = tuple(map(self._proccess_msg, args))
        if "%" in msg:
            try:
                return msg % args
            except TypeError as e:
                if not e.args or "not all arguments converted" in e.args[0]:
                    raise

        return msg.format(*args)

    @staticmethod
    def _proccess_msg(obj: object) -> object:
        if callable(dump := getattr(obj, "model_dump_json", None)):
            return dump(indent=2, ensure_ascii=False)
        if callable(dump := getattr(obj, "__json__", None)):
            return json.dumps(dump(), indent=2, ensure_ascii=False, default=str)
        if isinstance(obj, dict):
            return json.dumps(obj, indent=2, ensure_ascii=False, default=str)
        return obj


logging.setLogRecordFactory(JsonLogRecord)


class LoggerAdapter[T: logging.Logger](logging.LoggerAdapter[T]):
    def process[T2: MutableMapping[str, Any]](self, msg: str, kwargs: T2) -> tuple[str, T2]:
        kwargs.setdefault("extra", {}).update(self.extra)
        return msg, kwargs


class CDLFormater(logging.Formatter):
    _CDL_FORMAT: ClassVar[logging.PercentStyle] = logging.PercentStyle("%(message)s")

    def formatMessage(self, record: logging.LogRecord) -> str:  # noqa: N802
        if record.name.startswith("cyberdrop_dl"):
            content = self._CDL_FORMAT.format(record)

        else:
            content = self._style.format(record)

        if getattr(record, "cdl_no_truncate", False):
            return content
        return truncated_preview(content, env.MAX_LOG_MSG_LENGTH)


def _strip_tracebacks(record: logging.LogRecord) -> None:
    if not record.exc_info or record.exc_info == (None, None, None):
        return
    exc_type, exc_value, exc_traceback = record.exc_info
    assert exc_type is not None
    assert exc_value is not None
    tracebacks.strip_frames_from_exc(exc_value)
    exc_traceback = tracebacks.strip_frames(exc_traceback)
    record.exc_info = exc_type, exc_value, exc_traceback


class LogHandler(RichHandler):
    """Rich Handler with default settings, custom log render to remove padding in files and `color` extra"""

    def __init__(
        self,
        level: int = logging.DEBUG,
        console: Console | None = None,
        *,
        show_time: bool,
    ) -> None:
        super().__init__(
            level,
            console,
            show_time=show_time,
            rich_tracebacks=True,
            tracebacks_show_locals=True,
            tracebacks_max_frames=30,
            locals_max_string=_DEFAULT_CONSOLE_WIDTH,
            locals_max_length=20,
            show_path=False,
            show_level=True,
        )
        if show_time:
            self._log_render = NoPaddingLogRender(
                show_time=show_time,
                show_level=True,
                show_path=False,
                level_width=10,
                time_format=lambda dt: Text(f"[{dt.isoformat(sep=' ', timespec='milliseconds')}]", style="log.time"),
            )

        self.setFormatter(CDLFormater("[%(name)s]: %(message)s"))

    @override
    def render_message(self, record: logging.LogRecord, message: str) -> ConsoleRenderable:
        """This is the same as the base class, just added the `color` parsing from the extras"""
        use_markup = bool(getattr(record, "markup", self.markup))
        color = getattr(record, "color", "")
        message_text = Text.from_markup(message, style=color) if use_markup else Text(message, style=color)

        highlighter = getattr(record, "highlighter", self.highlighter)
        if highlighter:
            message_text = highlighter(message_text)

        if self.keywords is None:
            self.keywords = self.KEYWORDS

        if self.keywords:
            _ = message_text.highlight_words(self.keywords, "logging.keyword")

        return message_text

    @override
    def emit(self, record: logging.LogRecord) -> None:
        try:
            _strip_tracebacks(record)
            return super().emit(record)
        except Exception as e:  # noqa: BLE001
            self.handleError(record, e)

    @override
    def handleError(self, record: logging.LogRecord, e: Exception | None = None) -> None:
        logger.critical("LOGGING ERROR. A record has been lost. Unable to log it", exc_info=e)
        return super().handleError(record)


class BareQueueHandler(QueueHandler):
    """Sends the log record to the queue as is.

    The base class formats the record by merging the message and arguments.
    It also removes all other attributes of the record, just in case they have not pickleable objects.

    This made tracebacks render improperly because when the rich handler picks the log record from the queue, it has no traceback.
    The original traceback was being formatted as normal text and included as part of the message.

    We never log from other processes so we do not need that safety check
    """

    def prepare(self, record: logging.LogRecord) -> logging.LogRecord:
        return record


@contextlib.contextmanager
def _threaded_logger(
    log_handler: logging.Handler,
    *,
    context_var: ContextVar[QueueListener] | None = None,
) -> Generator[BareQueueHandler]:
    """Context-manager to process logs from this handler in another thread"""
    q: queue.Queue[logging.LogRecord] = queue.Queue()
    q_handler: BareQueueHandler = BareQueueHandler(q)
    q_listener: QueueListener = QueueListener(q, log_handler, respect_handler_level=True)
    q_listener.start()

    with enter_context(context_var, q_listener) if context_var else contextlib.nullcontext():
        logging.getLogger().addHandler(q_handler)
        try:
            yield q_handler
        finally:
            logging.getLogger().removeHandler(q_handler)
            try:
                q_handler.close()
            finally:
                q_listener.stop()
                for handler in q_listener.handlers[:]:
                    handler.close()


@final
class NoPaddingLogRender(LogRender):
    _cdl_padding: int = 0

    @override
    def __call__(  # type: ignore[reportIncompatibleMethodOverride]  # pyright: ignore[reportIncompatibleMethodOverride]
        self,
        console: Console,
        renderables: Iterable[ConsoleRenderable],
        log_time: dt.datetime | None = None,
        time_format: str | Callable[[dt.datetime], Text] | None = None,
        level: TextType = "",
        path: str | None = None,
        line_no: int | None = None,
        link_path: str | None = None,
    ) -> Group:
        output = Text(no_wrap=True)
        if self.show_time:
            log_time = log_time or console.get_datetime()
            time_format = time_format or self.time_format
            log_time_display = (
                time_format(log_time)
                if callable(time_format)
                else Text(log_time.strftime(time_format), style="log.time")
            )
            if log_time_display == self._last_time and self.omit_repeated_times:
                output.append(" " * len(log_time_display), style="log.time").pad_right(1)
            else:
                output.append_text(log_time_display).pad_right(1)
                self._last_time = log_time_display

        if self.show_level:
            output.append(level).pad_right(1)

        if not self._cdl_padding:
            self._cdl_padding = console.measure(output).maximum

        if self.show_path and path:
            path_text = Text(style="log.path")
            _ = path_text.append(path, style=f"link file://{link_path}" if link_path else "")
            if line_no:
                _ = path_text.append(":")
                _ = path_text.append(
                    f"{line_no}",
                    style=f"link file://{link_path}#{line_no}" if link_path else "",
                )
            output.append_text(path_text).pad_right(1)

        padded_lines: list[ConsoleRenderable] = []

        for renderable in renderables:
            if isinstance(renderable, Text):
                renderable = _indent_text(renderable, console, self._cdl_padding)
                renderable.stylize("log.message")
                _ = output.append_text(renderable)
                continue

            padded_lines.append(Padding(renderable, (0, 0, 0, self._cdl_padding), expand=False))

        return Group(output, *padded_lines)


def _indent_text(text: Text, console: Console, indent: int) -> Text:
    """Indents each line of a Text object except the first one."""
    padding = Text("\n" + (" " * indent))
    new_text = Text()
    first_line, *rest = text.wrap(console, width=console.width - indent)
    for line in rest:
        line.rstrip()
        _ = new_text.append_text(padding + line)
    first_line.rstrip()
    return first_line.append_text(new_text)


def log_spacer(char: str = "-") -> None:
    logger.info(char * 30, stacklevel=2)


def _get_handler(name: str) -> logging.Handler:
    for handler in logging.getLogger().handlers:
        if handler.get_name() == name:
            return handler
    raise LookupError(name)


def set_console_level(level: int) -> None:
    _get_handler(HandlerName.CONSOLE).setLevel(level)


@contextlib.contextmanager
def setup_console_logging() -> Generator[None]:
    handler = LogHandler(logging.DEBUG, show_time=False)
    logging.getLogger().setLevel(logging.DEBUG)
    try:
        with _threaded_logger(handler, context_var=_CONSOLE_LOG_LISTENER) as q_handler:
            q_handler.set_name(HandlerName.CONSOLE)
            q_handler.addFilter(lambda _: _LOG_TO_CONSOLE.get())
            yield
    finally:
        if "pytest" not in sys.modules:
            # Re add it as a normal handler to make sure uncatched exceptions show up
            logging.getLogger().addHandler(handler)


@contextlib.contextmanager
def setup_file_logging(
    file: Path,
    /,
    *,
    level: int = logging.DEBUG,
    log_http_traffic: bool = True,
) -> Generator[None]:
    file.parent.mkdir(parents=True, exist_ok=True)
    import mega

    LOG_HTTP_TRAFFIC.set(log_http_traffic)
    if "pytest" not in sys.modules:
        logging.captureWarnings(capture=True)

    with (
        _setup_debug_logger() as debug_log_file,
        file.open("w", encoding="utf8", errors="backslashreplace") as fp,
        enter_context(MAIN_LOG_FILE, file),
        enter_context(mega.LOG_HTTP_TRAFFIC, log_http_traffic),
        enter_context(mega.LOG_HTTP_TRAFFIC_LEVEL, logging.INFO),
        enter_context(mega.LOG_FILE_PROGRESS, False),
        _threaded_logger(
            log_handler=LogHandler(
                level=logging.DEBUG,
                show_time=True,
                console=RedactedConsole(file=fp, width=_DEFAULT_CONSOLE_WIDTH * 2),
            ),
            context_var=_MAIN_LOG_LISTENER,
        ) as handler,
    ):
        handler.setLevel(level)
        handler.set_name(HandlerName.MAIN_LOG)
        logger.info(f"Debug log file: {debug_log_file}")
        try:
            yield
        except CDLConfigRuntimeErrorsGroup as e:
            with enter_context(_LOG_TO_CONSOLE, False):
                logger.critical("Unrecoverable error", exc_info=e.with_traceback(None))
            raise
        except Exception:
            with enter_context(_LOG_TO_CONSOLE, False):
                logger.critical("Unrecoverable error", exc_info=True)
            raise


@contextlib.contextmanager
def _setup_debug_logger() -> Generator[Path | None]:
    if not env.DEBUG_MODE:
        yield
        return

    debug_log_file = Path(__file__).parent.parent / "cyberdrop_dl_debug.log"

    if env.DEBUG_LOG_FOLDER:
        debug_log_folder = Path(env.DEBUG_LOG_FOLDER).expanduser()

        if not debug_log_folder.exists():
            msg = f"Value of env var 'CDL_DEBUG_LOG_FOLDER' is invalid. Folder '{debug_log_folder}' does not exists"
            raise FileNotFoundError(None, msg, env.DEBUG_LOG_FOLDER)

        if not debug_log_folder.is_dir():
            msg = (
                f"Value of env var 'CDL_DEBUG_LOG_FOLDER' is invalid. Folder '{debug_log_folder}' should a be directory"
            )
            raise NotADirectoryError(None, msg, env.DEBUG_LOG_FOLDER)

        now = dates.now().strftime("%Y%m%d_%H%M%S")
        debug_log_file = debug_log_folder / f"cyberdrop_dl_debug_{now}.log"

    debug_log_file = debug_log_file.resolve().absolute()

    with (
        debug_log_file.open("w", encoding="utf8", errors="backslashreplace") as fp,
        _threaded_logger(
            LogHandler(
                level=logging.NOTSET + 1,
                console=Console(file=fp, width=_DEFAULT_CONSOLE_WIDTH * 2),
                show_time=True,
            ),
        ) as handler,
    ):
        handler.set_name(HandlerName.DEBUG_LOG)
        yield debug_log_file


@contextlib.contextmanager
def capture_logs() -> Generator[StringIO]:
    in_memory_handler = logging.StreamHandler(file := StringIO())
    logging.getLogger().addHandler(in_memory_handler)
    try:
        yield file
    finally:
        logging.getLogger().removeHandler(in_memory_handler)


def export_logs(*, size_limit: float | None = None) -> bytes:
    flush_logs()
    log_file = MAIN_LOG_FILE.get()
    if size_limit and log_file.stat().st_size > size_limit:
        raise RuntimeError(f"Logs file '{log_file}' is too big. Max size expected: {size_limit}")
    return log_file.read_bytes()


def flush_logs() -> None:
    """Wait until every record that is currently queued has been written to disk"""
    listener = _MAIN_LOG_LISTENER.get()
    listener.stop()
    listener.start()


def disable_console_logging():
    try:
        listener = _CONSOLE_LOG_LISTENER.get()
    except LookupError:
        pass
    else:
        listener.stop()
        listener.start()
    return enter_context(_LOG_TO_CONSOLE, False)


@contextlib.contextmanager
def borrow_logger(name: str, level: int = logging.INFO) -> Generator[None]:
    """Context manager to temporarily add our log handlers to a third party logger"""
    _3p_logger = logging.getLogger(name)
    og_level = _3p_logger.level
    _3p_logger.setLevel(level)
    try:
        yield
    finally:
        _3p_logger.setLevel(og_level)