Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

44.2 KB
Raw Blame
"""
Utilities for logging.
"""

# Needed for loguru annotations
from __future__ import annotations

import asyncio
import inspect

# import logging - needed for InterceptHandler compatibility
import logging
import os
import queue
import sys
import contextvars
import threading
from functools import wraps
from typing import Any, Callable

import loguru
from loguru import logger

from ..config.paths import get_logs_directory
from ..database.models import ResearchLog
from ..web.services.socketio_asgi import emit_to_subscribers


# Per-task context for research_id, replacing Flask's g.research_id.
# ContextVar works in both sync and async code and is correctly isolated
# per-task in asyncio (unlike threading.local in async code).
_research_id_var: contextvars.ContextVar[str | None] = contextvars.ContextVar(
    "research_id", default=None
)

_LOG_DIR = get_logs_directory()
# Lazy import: log_utils is imported very early by many bootstrap paths, so
# importing security at module top level risks an import cycle.
from ..security.directory_creation import create_directory  # noqa: E402

create_directory(_LOG_DIR, context="logs directory")


def _register_milestone_level() -> None:
    """Register the custom MILESTONE level.

    Called at import rather than only from ``config_logger()``. Loguru levels
    are process-global, and ``research_service``'s ``progress_callback`` opens
    with ``bound_logger.log("MILESTONE", ...)`` — so in any process that never
    called ``config_logger()`` (i.e. every pytest run; it is invoked only from
    ``web/app.py``'s ``ldr-web`` entry point) that first call raises
    ``ValueError: Level 'MILESTONE' does not exist``.

    The consequence was not a visible failure. The exception surfaced at the
    worker's first progress checkpoint, was re-raised inside the error handler
    and swallowed by a broad ``except``, so the research thread died before
    writing any terminal status and no assertion failed. Whether a given test
    process had the level at all depended on whether an unrelated module that
    registers it (two test modules do so at import time) had been collected
    first — i.e. research-worker tests were order-dependent by construction.
    """
    try:
        logger.level("MILESTONE", no=26, color="<magenta><bold>")
    except ValueError:
        # Already registered — levels are process-global and permanent.
        pass


_register_milestone_level()

# Thread-safe queue for database logs from background threads
_log_queue = queue.Queue(maxsize=1000)

# Thread-local re-entry guard for `database_sink`. If a `logger.*` call
# fires while we're inside the sink (e.g. loguru's own error handler),
# we must not re-enter or the process deadlocks/recurses.
_sink_state = threading.local()
_queue_processor_thread = None
_queue_processor_lock = threading.Lock()
_stop_queue = threading.Event()
"""
Default log directory to use.
"""

# Cap how much of a single log record's message we ship to the browser over
# socket.io. Some diagnostic / agent log lines can carry large payloads
# (model outputs, long tracebacks, extracted text) — many KB per call — which
# is useless in the UI (a single massive blob fills the viewport) and inflates both
# wire traffic and client-side state. Container-log/stderr, file, and DB
# sinks do not apply this cap, so longer text stays available for grep/DB
# queries -- up to the 32 KiB per-string bound the redaction patcher
# (``_sanitize_record``) applies to every sink. The cap bounds the
# *prefix* preserved from the original message; the wire payload is the
# prefix plus a short truncation indicator (~100 bytes), so it can exceed
# this value by that fixed overhead.
FRONTEND_MESSAGE_MAX_LENGTH = 2000

# Cap the size of messages persisted to ResearchLog. The DB column is
# unbounded TEXT, so a long langgraph run can accumulate thousands of
# 10 KB rows — paginated reads (PR #4037) hide the symptom but don't
# stop the storage growth. Same prefix-plus-indicator semantics as
# FRONTEND_MESSAGE_MAX_LENGTH; container-log/stderr/file sinks do not
# apply this cap, so text up to the patcher's 32 KiB per-string bound is
# still available out-of-band.
DATABASE_MESSAGE_MAX_LENGTH = 5000


class InterceptHandler(logging.Handler):
    """
    Intercepts logging messages and forwards them to Loguru's logger.
    """

    def emit(self, record: logging.LogRecord) -> None:
        # Get corresponding Loguru level if it exists.
        try:
            level: str | int = logger.level(record.levelname).name
        except ValueError:
            level = record.levelno

        # Find caller from where originated the logged message.
        frame, depth = inspect.currentframe(), 0
        while frame:
            filename = frame.f_code.co_filename
            is_logging = filename == logging.__file__
            is_frozen = "importlib" in filename and "_bootstrap" in filename
            if depth > 0 and not (is_logging or is_frozen):
                break
            frame = frame.f_back
            depth += 1

        logger.opt(depth=depth, exception=record.exc_info).log(
            level, record.getMessage()
        )


def log_for_research(
    to_wrap: Callable[[str, ...], Any],
) -> Callable[[str, ...], Any]:
    """
    Decorator for a function that's part of the research process. It expects the function to
    take the research ID (UUID) as the first parameter, and configures all log
    messages made during this request to include the research ID.

    Args:
        to_wrap: The function to wrap. Should take the research ID as the first parameter.

    Returns:
        The wrapped function.

    """

    @wraps(to_wrap)
    def wrapped(research_id: str, *args: Any, **kwargs: Any) -> Any:
        cv_handle = _research_id_var.set(research_id)
        try:
            return to_wrap(research_id, *args, **kwargs)
        finally:
            _research_id_var.reset(cv_handle)

    return wrapped


def _get_research_context_fallback() -> dict | None:
    """Read the per-thread research context, if any.

    Used as a fallback when individual log calls don't bind research_id/
    username via ``logger.bind``. The research thread sets this once at
    startup via ``set_search_context``, so every subsequent log call from
    the same thread picks up research_id, username, and user_password
    automatically — without requiring every call site to remember to bind.
    """
    try:
        from .thread_context import get_search_context

        return get_search_context()
    except Exception:
        return None


def _get_request_username() -> str | None:
    """Read the request-scoped username, if any.

    The FastAPI successor to main's ``flask.session.get("username")``
    fallback in the log sinks. Reads the contextvar ``DatabaseMiddleware``
    populates per request (and that background workers push explicitly via
    ``request_user``), so a log call that binds ``research_id`` without a
    ``username`` -- as report_assembly_service does from a real request
    thread -- still resolves an owner instead of being dropped.

    Imported lazily and never allowed to raise: this runs inside a loguru
    sink, where an exception would break logging itself.
    """
    try:
        from .request_context import get_current_username

        return get_current_username()
    except Exception:
        return None


def _get_research_id(record=None) -> str | None:
    """
    Gets the current research ID (UUID), if present.

    Args:
        record: Optional loguru record that might contain bound research_id

    Returns:
        The current research ID (UUID), or None if it does not exist.
    """
    # First check if research_id is bound to the log record
    if record and "extra" in record and "research_id" in record["extra"]:
        return record["extra"]["research_id"]
    # Then check the contextvar (set by @log_for_research on the running
    # task/thread)
    rid = _research_id_var.get()
    if rid:
        return rid
    # Fall back to per-thread research context — research-thread logger
    # calls without an explicit bind still get attributed correctly.
    ctx = _get_research_context_fallback()
    if ctx:
        return ctx.get("research_id")
    return None


# Counters for swallowed exceptions in the logging path. Bare except: pass
# is required here (logging errors must not propagate or recurse), but we
# write a stderr line on each new occurrence so silent failures aren't
# invisible — only active when LDR_APP_DEBUG=true so production stderr
# stays clean.
_silent_exc_counts: dict[str, int] = {}


def _report_silent_exception(
    where: str,
    exc_type_name: str,
    username: str | None = None,
    research_id: str | None = None,
    level: str | None = None,
) -> None:
    """Surface a swallowed logging-path exception to stderr.

    Bypasses ``logger`` to avoid recursing back through ``database_sink``.
    Rate-limited to first occurrence + every 100th repeat for the same
    ``where`` key, so a persistent failure mode doesn't flood the console.

    Note: takes the exception's TYPE NAME as a plain string (not the
    exception object). The caller does ``type(exc).__name__`` and passes
    the result. This is deliberate — CodeQL's taint analyzer treats any
    function frame holding a ``BaseException`` captured from a password-
    bearing call site as tainted, and flags every stderr write inside
    that frame. Receiving only a type-name string severs the flow at
    the boundary.
    """
    if os.environ.get("LDR_APP_DEBUG", "").lower() not in ("1", "true", "yes"):
        return
    n = _silent_exc_counts.get(where, 0) + 1
    _silent_exc_counts[where] = n
    if n != 1 and n % 100 != 0:
        return
    parts = []
    if username is not None:
        parts.append(f"username={username!r}")
    if research_id is not None:
        parts.append(f"research_id={research_id!r}")
    if level is not None:
        parts.append(f"level={level!r}")
    ctx = " ".join(parts)
    # CodeQL's py/clear-text-logging-sensitive-data may flag this stderr
    # write because the function frame is reachable from the logging path.
    # (_write_log_to_database no longer holds any credential at all since
    # #5538 -- it binds to the already-open engine.) The data
    # actually written is only plain strings — `where` (literal),
    # `exc_type_name` (`type(exc).__name__`), and `username/research_id/level`
    # repr'd from the queue entry. No password value ever reaches the
    # formatter; the helper signature deliberately accepts only typed
    # primitives. If CodeQL flags this line, dismiss the alert as a
    # false positive in the Security tab with that justification.
    sys.stderr.write(
        f"[log-utils] {where} swallowed (count={n}): "
        f"{exc_type_name}{(' ' + ctx) if ctx else ''}\n"
    )
    sys.stderr.flush()


def _clear_daemon_thread_credentials() -> None:
    """Drop any DB session / cached credential held by the current thread.

    Invoked at the end of every log-queue drain iteration. The queue
    daemon is a long-lived ``daemon=True`` thread whose credentials the
    dead-thread sweeper (``cleanup_dead_threads``) can never reclaim,
    because the thread stays alive for the whole process. Clearing here
    guarantees the ``log-queue-processor`` thread never retains a
    plaintext credential in ``thread_local_session``'s
    ``_thread_credentials``.

    Best-effort: must never raise, or a single cleanup hiccup would crash
    the daemon and silently stop all subsequent log persistence.
    """
    try:
        from ..database.thread_local_session import cleanup_current_thread

        cleanup_current_thread()
    except Exception:
        pass  # noqa: silent-exception — cleanup must not kill the log daemon


def _process_log_queue():
    """Process logs from the queue in a dedicated background thread.

    Under FastAPI there's no Flask app context to gate on — we just
    drain the queue and write entries directly. Main gated this on
    ``has_app_context()`` and re-queued otherwise; with no such context
    here the write is unconditional, which is safe because
    ``_write_log_to_database`` now refuses to open a closed database.

    Safe to run off the main thread: the write binds a session to the
    user's already-open engine, and the underlying SQLite engines are
    opened with ``check_same_thread=False``. It never reopens a closed
    database, so a post-logout backlog can't resurrect a user's
    connection from this thread (#5538). Errors are swallowed rather
    than crashing the processor.
    """
    while not _stop_queue.is_set():
        try:
            log_entry = _log_queue.get(timeout=0.1)
            if log_entry is None:
                continue
            try:
                _write_log_to_database(log_entry)
            finally:
                # Belt-and-braces credential hygiene (#5538). This daemon is
                # a long-lived daemon=True thread, so the dead-thread
                # credential sweeper (cleanup_dead_threads) never reclaims
                # it. Clear any per-thread DB session / cached credential at
                # the end of every drain iteration so the
                # log-queue-processor thread can never accumulate a
                # plaintext credential in _thread_credentials.
                _clear_daemon_thread_credentials()
        except queue.Empty:
            continue
        except Exception as exc:
            # noqa: silent-exception — must not let logging errors crash the log processor thread.
            # Wrap the report itself: if stderr is closed (broken pipe etc.)
            # an IOError from inside an except handler propagates and would
            # kill the daemon thread, silently breaking all subsequent log
            # persistence for the rest of the process lifetime.
            try:
                _report_silent_exception(
                    "process_log_queue", type(exc).__name__
                )
            except Exception:
                pass  # noqa: silent-exception — broken stderr must not kill the daemon


def _write_log_to_database(log_entry: dict) -> None:
    """Persist one queued log entry into the user's already-open database.

    The queue drain must NEVER reopen a closed database. A log entry that
    was queued while a user was active can be drained by the background
    daemon *after* that user has logged out; reopening their encrypted DB
    here would silently re-add the connection (``is_user_connected`` would
    flip back to True) and re-cache their credentials — resurrecting
    session state with no user action and no attacker involved.

    So we gate strictly on an already-open connection
    (``db_manager.is_user_connected``) and bind to the EXISTING engine via
    ``db_manager.get_session`` — which never opens a new database and needs
    no password. Entries for a user with no live connection (e.g. a
    post-logout backlog) are dropped; the full message is still available
    in the stderr/file sinks.

    In-flight logging is preserved: while a user is active — including a
    running research job, which keeps their DB open — the connection stays
    live, so their logs continue to be written normally.
    """
    from ..database.encrypted_db import db_manager

    username = log_entry.get("username")
    if not username:
        return

    # Gate on an already-open connection. Do NOT lazily reopen the DB:
    # after logout the connection is gone, and reopening it here would
    # resurrect the user's connected state from a stale queued backlog.
    if not db_manager.is_user_connected(username):
        return

    db_session = None
    try:
        # Binds a session to the ALREADY-open engine. Returns None (never
        # reopens) if the connection was closed in the small race between
        # the is_user_connected check above and this call; a rekey sidecar
        # fault in that window raises instead and is reported below like
        # any other failed write (the entry is dropped). We can't use
        # get_user_db_session here: it would resolve+cache a password and
        # lazily reopen the DB, which is exactly the post-logout
        # resurrection this fix removes. The session is explicitly closed
        # in the finally block below, so the QueuePool connection is
        # returned and no FD leaks.
        db_session = db_manager.get_session(username)  # noqa: raw-session
        if db_session is not None:
            db_log = ResearchLog(
                timestamp=log_entry["timestamp"],
                message=log_entry["message"],
                module=log_entry["module"],
                function=log_entry["function"],
                line_no=log_entry["line_no"],
                level=log_entry["level"],
                research_id=log_entry["research_id"],
            )
            db_session.add(db_log)
            db_session.commit()
    except Exception as exc:
        # noqa: silent-exception — DB errors in the logging path must not propagate or recurse.
        # Wrap the report itself so a broken-stderr IOError can't escape and
        # be re-caught by an outer logging-aware handler somewhere upstream.
        try:
            _report_silent_exception(
                "write_log_to_database",
                type(exc).__name__,
                username=log_entry.get("username"),
                research_id=log_entry.get("research_id"),
                level=log_entry.get("level"),
            )
        except Exception:
            pass  # noqa: silent-exception — broken stderr must not bubble out of logging path
    finally:
        # The session is freshly created per drain, bound to the shared
        # per-user engine — close it so its pooled connection is returned
        # (close() also rolls back any pending transaction from a failed
        # commit). Never let a close error escape the logging path.
        if db_session is not None:
            try:
                db_session.close()
            except Exception:
                pass  # noqa: silent-exception — session close failure must not propagate


def database_sink(message: loguru.Message) -> None:
    """
    Sink that saves messages to the database.
    Queues logs from background threads for later processing.

    Args:
        message: The log message to save.

    """
    # Thread-local re-entry guard. Any code path inside this sink (or
    # `_write_log_to_database`) that happens to call a logger — including
    # loguru's own sink-error handler when `catch=True` is set — would
    # re-invoke this function and recurse. The `try/except Exception: pass`
    # in `_write_log_to_database` catches most cases, but a nested sink
    # error handler can still land here. Fail closed for the nested call.
    if getattr(_sink_state, "in_sink", False):
        return
    _sink_state.in_sink = True
    try:
        record = message.record
        research_id = _get_research_id(record)

        # Resolve the username the log belongs to. The queue daemon thread
        # can't read the research thread's ContextVar storage, so we capture
        # it here in the emitting thread.
        #
        # We deliberately do NOT capture the user's password. The queue entry
        # must never carry a plaintext credential: the background drain would
        # otherwise use it to reopen the user's encrypted DB after logout,
        # reinstating their connected state (#5538). The drain writes only
        # when the DB is already open -- see _write_log_to_database.
        #
        # Source priority:
        #   1. logger.bind(...) extras on the record itself
        #   2. per-thread research context (set once when the research thread
        #      starts, so every subsequent log call inherits it without
        #      requiring an explicit bind)
        username = record.get("extra", {}).get("username")
        ctx = _get_research_context_fallback()
        if ctx and not username:
            username = ctx.get("username")
        if research_id is not None and not username:
            # Third source, ported from main: it resolved the request user
            # via `flask.session.get("username")` when a research_id was
            # bound but no username was. That fallback is not dead code --
            # report_assembly_service binds research_id alone from a real
            # request thread, and without this the entry resolves to
            # username=None and is silently dropped by the `if not username`
            # guard downstream. get_current_username() is the direct
            # successor (the contextvar DatabaseMiddleware populates).
            #
            # The `research_id is not None` condition is main's, and it is
            # load-bearing for data retention, not just noise control.
            # DatabaseMiddleware populates that contextvar for EVERY
            # authenticated request, so without the gate any INFO record
            # emitted during any request resolves an owner, clears the skip
            # guard below, and is persisted to that user's app_logs with
            # research_id = NULL. Those rows are unreachable by the only
            # mechanism that ever removes app_logs -- the ON DELETE CASCADE
            # from research_history -- so neither "delete this research" nor
            # "clear all history" can touch them, and there is no retention
            # job. They would accumulate permanently with no way to delete
            # them. main's comment put it as: "ResearchLog is research-scoped
            # by design -- auth and other system DEBUG logs (research_id=None)
            # don't belong there."
            username = _get_request_username()

        # Skip persistence for system logs that have no research context.
        # These can't be written to any per-user encrypted DB and would just
        # churn through the queue + daemon for no useful end state.
        if research_id is None and username is None:
            return

        # Create log entry dict
        log_entry = {
            "timestamp": record["time"],
            # Prefix the message with exception type/value when the record
            # carries one (logger.exception / logger.opt(exception=...)).
            # Keeps the full traceback off the encrypted-DB row (diagnose=False
            # on this sink; password-redaction policy #4182) while still
            # leaving the database log useful for triage.
            "message": _truncate_for_database(
                _exception_context(record) + record["message"]
            ),
            "module": record["name"],
            "function": record["function"],
            "line_no": int(record["line"]),
            "level": record["level"].name,
            "research_id": research_id,
            "username": username,
        }

        # Queue unless we are on a thread where a synchronous SQLCipher write
        # is actually safe — i.e. the main thread with NO event loop running
        # on it (startup, CLI). Everything else goes to the background writer.
        #
        # The event-loop check is load-bearing and its absence was a migration
        # regression. Flask's version of this guard read
        #     if not has_app_context() or current_thread().name != "MainThread"
        # and the port dropped the first clause (correctly — there is no Flask)
        # while keeping a thread-name test whose MEANING INVERTED. Under
        # Werkzeug, request and Socket.IO handlers ran on "Thread-N", so the
        # synchronous branch was effectively startup-only. Under uvicorn the
        # event loop IS "MainThread" (web/app.py calls uvicorn.run() from
        # main()), so every log line emitted on the loop took the synchronous
        # branch: _write_log_to_database -> get_user_db_session -> potentially
        # open_user_database, i.e. SQLCipher PBKDF2 key derivation, engine
        # build and a commit, all on the event loop. With workers=1 that
        # stalls the entire process.
        #
        # Thread name is the wrong question; "is a loop running here" is the
        # right one, and it stays correct however the server is launched.
        # Note no test could observe this: TestClient drives the app from an
        # "asyncio-portal-*" thread, never "MainThread".
        try:
            asyncio.get_running_loop()
            on_event_loop = True
        except RuntimeError:
            on_event_loop = False

        if on_event_loop or threading.current_thread().name != "MainThread":
            try:
                _log_queue.put_nowait(log_entry)
            except queue.Full:
                pass  # Drop log if queue is full to avoid blocking
        else:
            _write_log_to_database(log_entry)
    finally:
        _sink_state.in_sink = False


def _truncate_for_database(message: str) -> str:
    """Bound the persisted size of a log message.

    ``DATABASE_MESSAGE_MAX_LENGTH`` is the preserved-prefix length;
    the stored string is at most that plus a ~100-char suffix that
    reports the original length so debug context is not lost. The
    container-log/stderr/file sinks keep up to the patcher's 32 KiB
    per-string bound.
    """
    if len(message) <= DATABASE_MESSAGE_MAX_LENGTH:
        return message
    return message[:DATABASE_MESSAGE_MAX_LENGTH] + _truncation_suffix(message)


def _truncation_suffix(message: str) -> str:
    """Indicator appended by the DB and frontend caps.

    The reported length counts what the patcher's 32 KiB bound already
    dropped (its omission marker records the count), so it is the length
    before any truncation; the server logs hold at most the first
    ``_LOG_REDACTION_MAX_CHARS`` characters.
    """
    # Deferred import to avoid potential circular imports during module initialization
    from ..security.log_sanitizer import (
        _LOG_REDACTION_MAX_CHARS,
        log_length_before_truncation,
    )

    return (
        f"… (truncated; server logs keep up to {_LOG_REDACTION_MAX_CHARS} "
        f"chars; original length: {log_length_before_truncation(message)} "
        f"chars)"
    )


def _exception_context(record) -> str:
    """Render a short exception prefix from the log record, if any.

    Loguru stores the active exception in ``record["exception"]`` as a
    ``RecordException`` namedtuple (or None) populated by ``.exception()``
    or by ``logger.opt(exception=...)``. Without this, the database sink
    persists messages like ``"LangGraph agent error"`` with the actual
    exception type and message only on stderr — investigators querying
    ``app_logs`` see an empty ERROR row. The exception is always
    separately available in container-log/stderr/file sinks; this just
    gives the DB row enough context to triage. The full traceback is
    deliberately NOT included because ``diagnose=False`` on the database
    sink (see ``config_logger``) and the password-redaction policy
    (#4182) forbid it for encrypted-DB persistence.
    """
    exc = record.get("exception") if record else None
    if not exc:
        return ""
    # RecordException is a namedtuple of (type, value, traceback).
    # ``value`` is the exception instance (or None if un-picklable).
    exc_type = getattr(exc, "type", None)
    exc_value = getattr(exc, "value", None)
    if exc_type is None and isinstance(exc, tuple) and len(exc) >= 1:
        exc_type = exc[0]
        if exc_value is None and len(exc) >= 2:
            exc_value = exc[1]
    if exc_type is None:
        return ""
    # Deferred import to avoid potential circular imports during module initialization
    from ..security.log_sanitizer import (
        _redact_additional_shapes,
        sanitize_error_message,
        type_name_for_log,
    )

    # ``type`` is the original class unless the patcher had to replace it
    # (``security.log_sanitizer._record_type_is_safe``). Its name is read
    # through ``type``'s own descriptor (never a metaclass's ``__name__``)
    # and redacted, as in the patcher's rebuilt
    # ``RuntimeError("<TypeName>: ...")`` text.
    type_name = type_name_for_log(exc_type)
    if exc_value is None:
        return f"[{type_name}] "
    truncated = False
    try:
        value_text = str(exc_value)
        # The patcher renders an application exception as a rebuilt
        # ``RuntimeError("<TypeName>: ...")`` (``RuntimeError("<TypeName>")``
        # when its text is empty) while ``type`` stays the original class;
        # drop the repeated name so the row does not read
        # ``[TypeName: TypeName: ...]`` or ``[TypeName: TypeName]``.
        repeated = "{}: ".format(type_name)
        if type(exc_value) is not exc_type:
            if value_text.startswith(repeated):
                value_text = value_text[len(repeated) :]
            elif value_text == type_name:
                value_text = ""
        if len(value_text) > 4096:
            value_text = value_text[:4096]
        # The credential shapes the log-record path adds on top of
        # sanitize_error_message (``Api-Key:`` headers, slash-bearing DSN
        # passwords, ``;``/``,`` credential parameters, quoted credential
        # keys) run on the 4096-bounded text BEFORE the 240-character cap:
        # a value the cap cuts below a pattern's minimum length (or a DSN
        # cut before its '@') would no longer be recognised, and its head
        # would print. This prefix goes into the DB row with no other
        # redaction pass after it.
        value_text = _redact_additional_shapes(
            sanitize_error_message(value_text)
        )
        if len(value_text) > 240:
            # And once more on the capped text (before the ellipsis, which
            # an unterminated quoted value would otherwise take): these
            # patterns only ever replace part of their input with a
            # marker, so a second pass can hide more, never less.
            value_text = _redact_additional_shapes(value_text[:237])
            truncated = True
    except Exception:
        value_text = "<unprintable exception>"
    if not value_text:
        return f"[{type_name}] "
    # The name and the text are each clean, but a credential pattern can
    # span the two (a class named ``InvalidAuthorization`` and a bare token
    # read as an ``Authorization:`` header): sanitize the joined line.
    line = _redact_additional_shapes(
        sanitize_error_message(f"{type_name}: {value_text}")
    )
    cap = len(type_name) + 2 + 240
    if len(line) > cap - int(truncated):
        line = _redact_additional_shapes(line[: cap - 1])
        truncated = True
    if truncated:
        line += "…"
    return f"[{line}] "


def _truncate_for_frontend(message: str) -> str:
    """Bound the wire size of an outbound log message.

    ``FRONTEND_MESSAGE_MAX_LENGTH`` caps the *preserved prefix* of the
    original message. When truncation kicks in, a short indicator is
    appended that names the original length and points the user at the
    server-side logs, which keep up to 32 KiB of it, so the returned string is
    ``FRONTEND_MESSAGE_MAX_LENGTH`` plus the fixed indicator overhead
    (~100 bytes). Verbose diagnostic / agent logs (model outputs, long
    tracebacks, extracted text) are useless in the UI when displayed in
    full and inflate socket payloads + client-side memory;
    container-log/stderr, file, and DB sinks do not apply this cap.
    """
    if len(message) <= FRONTEND_MESSAGE_MAX_LENGTH:
        return message
    return message[:FRONTEND_MESSAGE_MAX_LENGTH] + _truncation_suffix(message)


def frontend_progress_sink(message: loguru.Message) -> None:
    """
    Sink that sends messages to the frontend.

    Args:
        message: The log message to send.

    """
    record = message.record
    research_id = _get_research_id(record)
    if research_id is None:
        # If we don't have a research ID, don't send anything.
        # Can't use logger here as it causes deadlock
        return

    # Defence in depth (R4-09): never forward policy-audit log lines
    # to WebSocket subscribers. They carry engine names + reason codes
    # which could leak the active scope to a cross-origin observer under
    # CORS=*. Today policy_audit logs don't bind research_id so the
    # research_id guard above already skips them; this filter is the
    # explicit guarantee in case a future call site binds both.
    if record.get("extra", {}).get("policy_audit"):
        return

    # Whose research this is. Subscriptions are keyed by (owner, research_id)
    # because a benchmark id is only unique within one user's database, so an
    # emit without the owner reaches nobody. Resolved the same way
    # ``database_sink`` resolves it: the bound ``username`` extra first, then
    # the per-thread research context set when the research thread started.
    username = record.get("extra", {}).get("username")
    if not username:
        ctx = _get_research_context_fallback()
        if ctx:
            username = ctx.get("username")
    if not username:
        # Same third source as database_sink; see the note there.
        username = _get_request_username()
    if not username:
        # Nothing to scope the emit to. Dropping is the fail-closed choice:
        # the alternative (emit to every subscriber of this id) is exactly
        # the cross-user leak the keying exists to prevent.
        return

    frontend_log = {
        "log_entry": {
            "message": _truncate_for_frontend(record["message"]),
            "type": record["level"].name,  # Keep original case
            "time": record["time"].isoformat(),
        },
    }
    emit_to_subscribers(
        "research_progress",
        research_id,
        frontend_log,
        owner=username,
        enable_logging=False,
    )


def flush_log_queue():
    """Drain all pending logs from the queue to the database.

    Called from the FastAPI lifespan shutdown (see ``fastapi_app.py``)
    to flush whatever the background daemon hadn't written before the
    DB was closed. The request path doesn't call this — the
    ``start_log_queue_processor`` daemon handles steady-state drainage
    so requests never block on DB writes.
    """
    flushed = 0
    while not _log_queue.empty():
        try:
            log_entry = _log_queue.get_nowait()
            _write_log_to_database(log_entry)
            flushed += 1
        except queue.Empty:
            break
        except Exception:
            pass  # noqa: silent-exception — DB errors during log flush must not propagate

    if flushed > 0:
        logger.debug(f"Flushed {flushed} queued log entries to database")


def start_log_queue_processor(app=None) -> threading.Thread:
    """Start the background daemon that drains the log queue into the DB.

    Idempotent: calling twice is a no-op (returns the existing thread).
    Under FastAPI there's no Flask app context to push, so the daemon
    just runs ``_process_log_queue`` directly. The ``app`` parameter is
    accepted for backward compat with any legacy Flask call sites that
    still pass it; it is otherwise ignored.

    Returns:
        The daemon thread (running).
    """
    global _queue_processor_thread
    with _queue_processor_lock:
        if (
            _queue_processor_thread is not None
            and _queue_processor_thread.is_alive()
        ):
            return _queue_processor_thread

        _stop_queue.clear()
        _queue_processor_thread = threading.Thread(
            target=_process_log_queue,
            name="log-queue-processor",
            daemon=True,
        )
        _queue_processor_thread.start()
        thread = _queue_processor_thread
    logger.info("Log queue processor daemon started")
    return thread


def stop_log_queue_processor(timeout: float = 2.0) -> None:
    """Signal the log queue processor to stop and wait briefly for it."""
    global _queue_processor_thread
    _stop_queue.set()
    with _queue_processor_lock:
        thread = _queue_processor_thread
    if thread is not None:
        thread.join(timeout=timeout)
        # Only clear the reference if the thread actually exited. If join
        # timed out the daemon is still running, and clearing the ref would
        # let a subsequent start_log_queue_processor() spawn a second
        # daemon that drains the same queue concurrently. Re-check identity
        # under the lock so we don't accidentally null out a fresh thread
        # that another start spawned in the meantime.
        if not thread.is_alive():
            with _queue_processor_lock:
                if _queue_processor_thread is thread:
                    _queue_processor_thread = None


def _sanitize_record(record) -> None:
    """Global loguru patcher: scrub every record before it reaches a sink.

    Runs once per record, ahead of ALL sinks (stderr, the encrypted per-user
    DB, the browser progress stream, and the persistent, unencrypted
    ``<LDR_DATA_DIR>/logs/*.log`` file), so it is the single chokepoint that
    protects every sink regardless of per-call-site logging discipline:

    * Control / format characters are stripped from the message (prevents log
      forging via CR/LF, RLO, zero-width, etc.).
    * Credential *shapes* and bare ``field=value`` secrets are redacted from
      the message — the sink-level backstop for any call site that logged a
      credential without going through ``scrub_error`` first.
    * Structured ``extra`` / ``logger.bind(...)`` values are redacted by
      sensitive key name AND by embedded credential shape, catching the
      recurring family bug of a secret carried as a bound structured value
      (a token/URL nested under a key the name-based checks miss).
    * The *rendered* exception graph (value, notes, chained and grouped
      exceptions) is scrubbed too, as a rebuilt copy, so a credential
      embedded in an exception's string does not render verbatim to the
      default-format stderr / file sinks — the message scrub alone never
      reaches it.

    All of this is ``security.log_sanitizer.redact_log_record``, which the
    MCP subprocess installs as its own patcher too (``mcp/server.py``'s
    ``configure_mcp_logging``); it fails closed per field. It is imported
    lazily (never at this module's import time): importing ``..security``
    runs ``install_audit_hook()``, and ``log_utils`` must stay importable
    before the security layer is wired.
    """
    from ..security.log_sanitizer import redact_log_record

    redact_log_record(record)


def install_log_redaction() -> None:
    """Install the shared credential-redaction patcher without adding sinks.

    Entry points that configure their own sinks, such as the benchmark CLI,
    use this before enabling their package logger. Importing this module
    alone does not change the host process's Loguru configuration.
    """
    logger.configure(patcher=_sanitize_record)


def config_logger(name: str, debug: bool = False) -> None:
    """
    Configures the default logger.

    Args:
        name: The name to use for the log file.
        debug: Whether to enable unsafe debug logging.

    """
    install_log_redaction()

    logger.enable("local_deep_research")
    logger.remove()

    # Log to console (stderr), database, and frontend (Socket.IO).
    # All three sinks track the stderr level — sending DEBUG logs to the
    # DB and to every connected Socket.IO client in production would flood
    # storage and the network with high-frequency internal events.
    # `diagnose=False` on the DB/frontend sinks: loguru's `diagnose=True`
    # dumps local variables on exceptions, which can contain request
    # bodies, session data, or DB rows — we don't want those leaving the
    # stderr sink.
    sink_level = "DEBUG" if debug else "INFO"

    # loguru's diagnose=True renders repr() of every local variable in every
    # traceback frame on exceptions. Under LDR_APP_DEBUG that would dump
    # credentials living in frame locals (api_key, SQLCipher password,
    # Authorization headers) into the stderr sink. Gate diagnose behind a
    # separate explicit opt-in so enabling LDR_APP_DEBUG for general debug
    # output does not also enable localvar dumps. Default OFF even when debug
    # is on, and never enabled on the DB/frontend sinks.
    #
    # security/secure_logging.py gates provider/engine exception tracebacks
    # on the same two flags (its is_diagnose_mode()). Deliberate divergence:
    # the wrapper reads LDR_APP_DEBUG from the environment only, while the
    # ``debug`` argument here may also come from the app.debug DB setting /
    # legacy server_config.json — so DB-enabled debug affects this sink's
    # diagnose but never the wrapper's traceback gate. Kept inline (not
    # env_truthy) because log_utils must not import security at module
    # level: security/__init__.py runs install_audit_hook() on import.
    diagnose = debug and os.environ.get(
        "LDR_LOGURU_DIAGNOSE", ""
    ).strip().lower() in ("1", "true", "yes")

    # ``diagnose`` renders the repr() of every frame-local — which can
    # include the SQLCipher master password and other credentials — into
    # the rendered exception block. Allow it ONLY on the ephemeral, local
    # stderr sink the operator explicitly opted into. The database sink
    # PERSISTS into the user's own encrypted DB and the frontend sink
    # SHIPS to the browser, so they must NEVER render frame locals, even
    # under LDR_LOGURU_DIAGNOSE. This single chokepoint protects every
    # credential-bearing exception handler app-wide against the
    # frame-locals leak, independent of per-site logging discipline
    # (#4182).
    #
    # enqueue=True on stderr: loguru hands the formatted record to a
    # multiprocessing.SimpleQueue and a dedicated background thread does the
    # actual stderr write, decoupling a log call from stderr I/O for up to
    # one queue pipe buffer (~64KB on Linux) rather than never blocking.
    # Without it, when stderr back-pressures (e.g. a slow/full `docker logs`
    # pipe in CI) the lock-holder blocks mid-write and ALL logging threads
    # pile up behind the lock, freezing the request pipeline for ~60s
    # (#4431). Captured forensically: 3/5 server threads parked in loguru's
    # _protected_lock under load. This is a bounded grace window, not
    # immunity: SimpleQueue.put() still runs inside Handler.emit's
    # _protected_lock, and its OS-pipe send_bytes() blocks once that buffer
    # fills — e.g. if the background writer thread itself stalls on a
    # wedged stderr long enough to drain nothing, reproducing the same
    # pile-up (loguru/_handler.py Handler.emit / _queued_writer,
    # multiprocessing/queues.py SimpleQueue.put). The database/progress
    # sinks keep their own emitting-thread context capture and are left
    # synchronous.
    logger.add(sys.stderr, level=sink_level, diagnose=diagnose, enqueue=True)
    logger.add(database_sink, level=sink_level, diagnose=False)
    logger.add(frontend_progress_sink, level=sink_level, diagnose=False)

    if debug:
        logger.warning(
            "DEBUG logging is enabled (LDR_APP_DEBUG=true). "
            "Logs may contain sensitive data (queries, answers, API responses). "
            "Do NOT use in production."
        )

    if diagnose:
        logger.warning(
            "LDR_LOGURU_DIAGNOSE is enabled: exception tracebacks will include "
            "local variable values, which may contain credentials (API keys, "
            "passwords, tokens). Do NOT use in production."
        )

    # Optionally log to file if enabled (disabled by default for security)
    # Check environment variable first, then database setting
    enable_file_logging = (
        os.environ.get("LDR_ENABLE_FILE_LOGGING", "").lower() == "true"
    )

    # File logging is controlled only by environment variable for simplicity
    # Database settings are not available at logger initialization time

    if enable_file_logging:
        log_file = _LOG_DIR / f"{name}.log"
        logger.add(
            log_file,
            level="DEBUG",
            rotation="10 MB",
            retention="7 days",
            compression="zip",
            # diagnose=False: the file sink is persistent and unencrypted,
            # so — like the DB and frontend sinks — it must never render
            # frame-local credentials, even under LDR_LOGURU_DIAGNOSE.
            # Frame-local dumps go only to the ephemeral stderr sink, which
            # is what the policy comment above describes (#4182).
            diagnose=False,
        )
        logger.warning(
            f"File logging enabled - logs will be written to {log_file}. "
            "WARNING: Log files are unencrypted and may contain sensitive data!"
        )

    # Add a special log level for milestones. Also registered at import
    # (see _register_milestone_level) so processes that never call this
    # function still have it; kept here for the re-entrant case.
    _register_milestone_level()