~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""Tests for log_utils module."""
import logging
import queue
import threading
from datetime import datetime
from unittest.mock import Mock, patch, MagicMock
import pytest
class TestInterceptHandler:
"""Tests for InterceptHandler class."""
def test_emit_forwards_to_loguru(self):
"""Should forward log records to loguru."""
from local_deep_research.utilities.log_utils import InterceptHandler
handler = InterceptHandler()
# Create a mock log record
record = logging.LogRecord(
name="test",
level=logging.INFO,
pathname="test.py",
lineno=1,
msg="Test message",
args=(),
exc_info=None,
)
with patch(
"local_deep_research.utilities.log_utils.logger"
) as mock_logger:
mock_logger.level.return_value = Mock(name="INFO")
mock_opt = Mock()
mock_logger.opt.return_value = mock_opt
handler.emit(record)
mock_logger.opt.assert_called()
mock_opt.log.assert_called()
def test_handles_unknown_level(self):
"""Should handle unknown log levels by using levelno."""
from local_deep_research.utilities.log_utils import InterceptHandler
handler = InterceptHandler()
record = logging.LogRecord(
name="test",
level=35, # Non-standard level
pathname="test.py",
lineno=1,
msg="Test",
args=(),
exc_info=None,
)
record.levelname = "CUSTOM"
with patch(
"local_deep_research.utilities.log_utils.logger"
) as mock_logger:
mock_logger.level.side_effect = ValueError("Unknown level")
mock_opt = Mock()
mock_logger.opt.return_value = mock_opt
handler.emit(record)
mock_opt.log.assert_called()
class TestLogForResearch:
"""Tests for log_for_research decorator.
The decorator now uses a contextvar (not Flask `g`) — Flask is gone
after the FastAPI migration. Tests verify the contextvar lifecycle
(set on entry, reset on exit) instead.
"""
def test_sets_research_id_in_contextvar(self):
"""Should set research_id in the contextvar during execution."""
from local_deep_research.utilities import log_utils
captured = {}
@log_utils.log_for_research
def test_func(research_id):
captured["value"] = log_utils._research_id_var.get()
return "done"
test_func("test-uuid-123")
assert captured["value"] == "test-uuid-123"
def test_resets_research_id_after_function(self):
"""Should reset the contextvar after the wrapped function returns."""
from local_deep_research.utilities import log_utils
@log_utils.log_for_research
def test_func(research_id):
return "result"
# Outside any wrapped call, the var carries its default sentinel.
before = log_utils._research_id_var.get()
test_func("uuid")
after = log_utils._research_id_var.get()
assert after == before
def test_preserves_function_metadata(self):
"""Should preserve function name and docstring."""
from local_deep_research.utilities.log_utils import log_for_research
@log_for_research
def documented_func(research_id):
"""My documentation."""
pass
assert documented_func.__name__ == "documented_func"
assert documented_func.__doc__ == "My documentation."
def test_passes_args_and_kwargs(self):
"""Should pass arguments correctly."""
from local_deep_research.utilities.log_utils import log_for_research
@log_for_research
def test_func(research_id, arg1, kwarg1=None):
return (research_id, arg1, kwarg1)
result = test_func("uuid", "value1", kwarg1="value2")
assert result == ("uuid", "value1", "value2")
@pytest.mark.skip(
reason="Tests pre-migration database_sink that mocked Flask `has_app_context`. "
"Post-FastAPI-migration, database_sink routes by thread name + contextvars; "
"live behavior is covered by "
"tests/utilities/test_log_utils_deep_coverage.py::TestDatabaseSink and "
"tests/web/test_log_sink_never_writes_on_event_loop.py. "
"Rewrite scoped to a follow-up."
)
class TestDatabaseSink:
"""Tests for database_sink function."""
def test_creates_log_entry_dict(self):
"""Should create log entry dictionary from message."""
from local_deep_research.utilities.log_utils import database_sink
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Test log message",
"name": "test_module",
"function": "test_function",
"line": 42,
"level": Mock(name="INFO"),
# research_id is required — ResearchLog rows are research-scoped,
# logs with no research context are filtered out at the queue
# boundary (they'd never reach the panel anyway).
"extra": {"research_id": "rid-1"},
}
# Post-FastAPI-migration the sink queues whenever the emitting
# thread is not MainThread (no Flask app-context check anymore).
with patch(
"local_deep_research.utilities.log_utils.threading.current_thread"
) as mock_thread:
mock_thread.return_value.name = "log-test-worker"
with patch(
"local_deep_research.utilities.log_utils._log_queue"
) as mock_queue:
database_sink(mock_message)
# Should queue the log since we're not in app context
mock_queue.put_nowait.assert_called_once()
def test_queues_log_from_non_main_thread(self):
"""Should queue log when not in main thread."""
from local_deep_research.utilities.log_utils import database_sink
import local_deep_research.utilities.log_utils as module
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Test message",
"name": "module",
"function": "func",
"line": 1,
"level": Mock(name="DEBUG"),
"extra": {"research_id": "test-uuid"},
}
# Mock has_app_context to return True but thread name is not MainThread
mock_thread = Mock()
mock_thread.name = "WorkerThread"
with patch.object(module, "has_app_context", return_value=True):
with patch.object(
module, "_get_research_id", return_value="test-uuid"
):
with patch.object(
threading, "current_thread", return_value=mock_thread
):
with patch.object(module, "_log_queue") as mock_queue:
database_sink(mock_message)
# Should queue since not MainThread
mock_queue.put_nowait.assert_called_once()
def test_handles_full_queue_gracefully(self):
"""Should not raise when queue is full."""
from local_deep_research.utilities.log_utils import database_sink
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Test",
"name": "mod",
"function": "f",
"line": 1,
"level": Mock(name="INFO"),
"extra": {},
}
with patch(
"local_deep_research.utilities.log_utils.has_app_context",
return_value=False,
):
with patch(
"local_deep_research.utilities.log_utils._log_queue"
) as mock_queue:
mock_queue.put_nowait.side_effect = queue.Full()
# Should not raise
database_sink(mock_message)
def test_writes_to_database_in_main_thread(self):
"""Should write to the DB when in main thread with app context and
the user's DB is already open — binding to the existing connection,
never reopening it (no password is used)."""
from local_deep_research.utilities.log_utils import database_sink
from local_deep_research.database.encrypted_db import db_manager
import local_deep_research.utilities.log_utils as module
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Test message",
"name": "test_module",
"function": "test_func",
"line": 42,
"level": Mock(name="INFO"),
"extra": {"username": "testuser", "research_id": "test-uuid"},
}
mock_thread = Mock()
mock_thread.name = "MainThread"
mock_session = MagicMock()
with patch.object(module, "has_app_context", return_value=True):
with patch.object(
threading, "current_thread", return_value=mock_thread
):
with patch.object(
db_manager, "is_user_connected", return_value=True
) as mock_conn:
with patch.object(
db_manager, "get_session", return_value=mock_session
) as mock_get_session:
with patch.object(
db_manager, "open_user_database"
) as mock_open:
database_sink(mock_message)
# Gated on the already-open connection, bound to it.
mock_conn.assert_called_once_with("testuser")
mock_get_session.assert_called_once_with("testuser")
# The drain must never reopen the DB.
mock_open.assert_not_called()
mock_session.add.assert_called_once()
mock_session.commit.assert_called_once()
mock_session.close.assert_called_once()
def test_handles_database_error_gracefully(self):
"""Should not raise on database errors when writing."""
from local_deep_research.utilities.log_utils import database_sink
import local_deep_research.utilities.log_utils as module
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Test",
"name": "mod",
"function": "f",
"line": 1,
"level": Mock(name="INFO"),
"extra": {"research_id": "test-uuid"},
}
mock_thread = Mock()
mock_thread.name = "MainThread"
mock_g = Mock()
mock_g.get.return_value = None
with patch.object(module, "has_app_context", return_value=True):
with patch.object(module, "g", mock_g):
with patch.object(
threading, "current_thread", return_value=mock_thread
):
with patch(
"local_deep_research.database.session_context.get_user_db_session",
side_effect=Exception("DB error"),
):
# Should not raise
database_sink(mock_message)
def test_extracts_research_id_from_record_extra(self):
"""Should extract research_id from record extra."""
from local_deep_research.utilities.log_utils import database_sink
import local_deep_research.utilities.log_utils as module
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Test",
"name": "mod",
"function": "f",
"line": 1,
"level": Mock(name="INFO"),
"extra": {"research_id": "record-uuid"},
}
with patch.object(module, "has_app_context", return_value=False):
with patch.object(module, "_log_queue") as mock_queue:
database_sink(mock_message)
# Verify the queued log entry contains the research_id
call_args = mock_queue.put_nowait.call_args[0][0]
assert call_args["research_id"] == "record-uuid"
def test_extracts_research_id_from_flask_g(self):
"""Should extract research_id from Flask g when not in record."""
from local_deep_research.utilities.log_utils import database_sink
import local_deep_research.utilities.log_utils as module
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Test",
"name": "mod",
"function": "f",
"line": 1,
"level": Mock(name="INFO"),
"extra": {},
}
mock_g = Mock()
mock_g.get.return_value = "flask-uuid"
with patch.object(module, "has_app_context", return_value=True):
with patch.object(module, "g", mock_g):
with patch.object(module, "_log_queue") as mock_queue:
# Use non-main thread to queue instead of write
mock_thread = Mock()
mock_thread.name = "WorkerThread"
with patch.object(
threading, "current_thread", return_value=mock_thread
):
database_sink(mock_message)
# Verify the queued log entry contains the research_id
call_args = mock_queue.put_nowait.call_args[0][0]
assert call_args["research_id"] == "flask-uuid"
def test_record_research_id_takes_priority_over_flask(self):
"""Record research_id should take priority over Flask g."""
from local_deep_research.utilities.log_utils import database_sink
import local_deep_research.utilities.log_utils as module
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Test",
"name": "mod",
"function": "f",
"line": 1,
"level": Mock(name="INFO"),
"extra": {"research_id": "record-uuid"},
}
mock_g = Mock()
mock_g.get.return_value = "flask-uuid"
with patch.object(module, "has_app_context", return_value=True):
with patch.object(module, "g", mock_g):
with patch.object(module, "_log_queue") as mock_queue:
# Use non-main thread to queue instead of write
mock_thread = Mock()
mock_thread.name = "WorkerThread"
with patch.object(
threading, "current_thread", return_value=mock_thread
):
database_sink(mock_message)
# Record research_id should win
call_args = mock_queue.put_nowait.call_args[0][0]
assert call_args["research_id"] == "record-uuid"
def test_captures_username_but_never_password_into_queue_entry(self):
"""The username is still captured from the per-thread research
context so the daemon can attribute the row, but the queue entry
must NEVER carry a plaintext password. Carrying it would let the
background drain reopen the user's encrypted DB after logout and
reinstate their connected state (post-logout resurrection)."""
from local_deep_research.utilities.log_utils import database_sink
import local_deep_research.utilities.log_utils as module
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Test",
"name": "mod",
"function": "f",
"line": 1,
"level": Mock(name="INFO"),
"extra": {"research_id": "rid-1"},
}
fake_ctx = {
"research_id": "rid-1",
"username": "alice",
"user_password": "pw-from-thread-ctx", # gitleaks:allow
}
with patch.object(
module, "_get_research_context_fallback", return_value=fake_ctx
):
with patch.object(module, "has_app_context", return_value=False):
with patch.object(module, "_log_queue") as mock_queue:
database_sink(mock_message)
call_args = mock_queue.put_nowait.call_args[0][0]
assert call_args["username"] == "alice"
# No plaintext credential anywhere in the queue entry.
assert "user_password" not in call_args
assert "pw-from-thread-ctx" not in call_args.values()
def test_skips_persistence_when_no_research_or_username(self):
"""Logs without a research_id AND without a username must be dropped
at the sink, not queued. ResearchLog is research-scoped — system
DEBUG logs (auth, settings, etc.) attached via flask_session would
churn through the queue and never resolve to a valid row."""
from local_deep_research.utilities.log_utils import database_sink
import local_deep_research.utilities.log_utils as module
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "system debug log",
"name": "mod",
"function": "f",
"line": 1,
"level": Mock(name="DEBUG"),
"extra": {},
}
# No research context anywhere — neither thread context nor Flask.
with patch.object(
module, "_get_research_context_fallback", return_value=None
):
with patch.object(module, "has_app_context", return_value=False):
with patch.object(module, "_log_queue") as mock_queue:
database_sink(mock_message)
mock_queue.put_nowait.assert_not_called()
class TestTruncateForDatabase:
"""Tests for _truncate_for_database helper and database_sink integration.
Mirrors the cap discipline of _truncate_for_frontend: a long message is
cut at DATABASE_MESSAGE_MAX_LENGTH chars and a short indicator is
appended that reports the original length. Stops langgraph fetch logs
(which inline 10 KB page bodies) from being persisted in full into
ResearchLog rows.
"""
def test_short_message_passes_through_unchanged(self):
from local_deep_research.utilities.log_utils import (
DATABASE_MESSAGE_MAX_LENGTH,
_truncate_for_database,
)
short = "small payload"
assert len(short) < DATABASE_MESSAGE_MAX_LENGTH
assert _truncate_for_database(short) == short
def test_message_at_exact_cap_is_not_truncated(self):
"""The cap is inclusive — a message of exactly the cap length must
pass through unchanged."""
from local_deep_research.utilities.log_utils import (
DATABASE_MESSAGE_MAX_LENGTH,
_truncate_for_database,
)
exact = "Y" * DATABASE_MESSAGE_MAX_LENGTH
out = _truncate_for_database(exact)
assert out == exact
assert "truncated" not in out
def test_long_message_truncated_with_indicator(self):
from local_deep_research.utilities.log_utils import (
DATABASE_MESSAGE_MAX_LENGTH,
_truncate_for_database,
)
big = "X" * (DATABASE_MESSAGE_MAX_LENGTH + 5000)
out = _truncate_for_database(big)
assert out.startswith("X" * DATABASE_MESSAGE_MAX_LENGTH)
assert "truncated" in out
assert str(len(big)) in out # original length is surfaced
# Indicator overhead is bounded (~100 chars), well under the cap.
assert len(out) < DATABASE_MESSAGE_MAX_LENGTH + 200
def test_forged_omission_marker_does_not_break_the_caps(self):
"""A message ending in a forged marker with a huge count is capped.
Parsing a 4,301-digit count raised ``ValueError`` (Python's int
digit limit), so both caps raised and their sinks lost the record.
"""
from local_deep_research.utilities.log_utils import (
DATABASE_MESSAGE_MAX_LENGTH,
FRONTEND_MESSAGE_MAX_LENGTH,
_truncate_for_database,
_truncate_for_frontend,
)
message = (
"q " * 3_000
+ " [... "
+ "9" * 4_301
+ " characters omitted from log output]"
)
suffix = f"original length: {len(message)} chars)"
out = _truncate_for_database(message)
assert out.startswith(message[:DATABASE_MESSAGE_MAX_LENGTH])
assert out.endswith(suffix)
out = _truncate_for_frontend(message)
assert out.startswith(message[:FRONTEND_MESSAGE_MAX_LENGTH])
assert out.endswith(suffix)
def test_database_sink_queues_truncated_message(self):
"""database_sink must apply the cap BEFORE queueing so the 10 KB
blob never sits in _log_queue (bounded to 1000 entries; without the
cap, that's a 10 MB worst-case transient)."""
from local_deep_research.utilities.log_utils import (
DATABASE_MESSAGE_MAX_LENGTH,
database_sink,
)
big = "Z" * (DATABASE_MESSAGE_MAX_LENGTH + 1000)
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": big,
"name": "test_module",
"function": "test_function",
"line": 1,
"level": Mock(name="INFO"),
"extra": {"research_id": "rid-truncate"},
}
# Post-FastAPI-migration database_sink routes by thread name:
# background threads queue, MainThread writes directly. Force the
# queue path by faking a non-main thread name (pytest runs on
# MainThread).
fake_thread = Mock()
fake_thread.name = "ResearchWorker"
with patch(
"local_deep_research.utilities.log_utils.threading.current_thread",
return_value=fake_thread,
):
with patch(
"local_deep_research.utilities.log_utils._log_queue"
) as mock_queue:
database_sink(mock_message)
mock_queue.put_nowait.assert_called_once()
queued = mock_queue.put_nowait.call_args[0][0]
# The dict put on the queue carries the truncated string, not the
# original 10KB+ blob.
assert len(queued["message"]) < len(big)
assert queued["message"].startswith("Z" * DATABASE_MESSAGE_MAX_LENGTH)
assert "truncated" in queued["message"]
def test_database_sink_persists_exception_context_on_error_records(self):
"""When ``logger.exception`` is used, the persisted
``app_logs.message`` previously contained only the bare message
string ("LangGraph agent error") with the exception type and value
stripped out. The DB sink now prefixes a short ``[Type: value]``
context when the record carries an active exception, so an
investigator querying ``app_logs`` can distinguish the same line
emitted under different failure modes without the full traceback
(which the diagnose=False policy on the encrypted-DB sink forbids
per #4182).
"""
from loguru._logger import RecordException
from local_deep_research.utilities.log_utils import database_sink
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "LangGraph agent error",
"name": "strategies.langgraph_agent_strategy",
"function": "analyze_topic",
"line": 1598,
"level": Mock(name="ERROR"),
"extra": {"research_id": "rid-ctx"},
# RecordException is loguru's internal namedtuple
# (type, value, traceback). The DB sink reads it via
# record.get("exception") and renders a short prefix.
"exception": RecordException(
ValueError, ValueError("Unknown search engine 'foo'"), None
),
}
# Post-FastAPI-migration the sink queues whenever the emitting
# thread is not MainThread (no Flask app-context check anymore).
with patch(
"local_deep_research.utilities.log_utils.threading.current_thread"
) as mock_thread:
mock_thread.return_value.name = "log-test-worker"
with patch(
"local_deep_research.utilities.log_utils._log_queue"
) as mock_queue:
database_sink(mock_message)
mock_queue.put_nowait.assert_called_once()
queued = mock_queue.put_nowait.call_args[0][0]
assert queued["message"].startswith(
"[ValueError: Unknown search engine 'foo'] "
)
# The original bare message is preserved as the suffix so downstream
# filters and search by message text still work.
assert "LangGraph agent error" in queued["message"]
# No traceback frame is ever rendered to the encrypted-DB row.
assert "Traceback" not in queued["message"]
def test_database_sink_unchanged_when_record_has_no_exception(self):
"""Records without an active exception (e.g. INFO/DEBUG logs)
pass through unchanged — the prefix is only added when loguru
attached a RecordException to the record."""
from local_deep_research.utilities.log_utils import database_sink
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Search completed",
"name": "mod",
"function": "fn",
"line": 1,
"level": Mock(name="INFO"),
"extra": {"research_id": "rid-noexc"},
"exception": None,
}
# Post-FastAPI-migration the sink queues whenever the emitting
# thread is not MainThread (no Flask app-context check anymore).
with patch(
"local_deep_research.utilities.log_utils.threading.current_thread"
) as mock_thread:
mock_thread.return_value.name = "log-test-worker"
with patch(
"local_deep_research.utilities.log_utils._log_queue"
) as mock_queue:
database_sink(mock_message)
queued = mock_queue.put_nowait.call_args[0][0]
assert queued["message"] == "Search completed"
assert "[" not in queued["message"]
def test_database_sink_truncates_long_exception_values(self):
"""An exception whose ``str()`` is hundreds of chars long should
be bounded — the same prefix-then-truncation discipline that
applies to plain message bodies also applies to the exception
context, so a 5 KB ``repr()`` doesn't blow past the per-row
``DATABASE_MESSAGE_MAX_LENGTH`` budget."""
from loguru._logger import RecordException
from local_deep_research.utilities.log_utils import (
DATABASE_MESSAGE_MAX_LENGTH,
database_sink,
)
long_value = "x" * 600
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "boom",
"name": "mod",
"function": "fn",
"line": 1,
"level": Mock(name="ERROR"),
"extra": {"research_id": "rid-bigexc"},
"exception": RecordException(
RuntimeError, RuntimeError(long_value), None
),
}
# Post-FastAPI-migration the sink queues whenever the emitting
# thread is not MainThread (no Flask app-context check anymore).
with patch(
"local_deep_research.utilities.log_utils.threading.current_thread"
) as mock_thread:
mock_thread.return_value.name = "log-test-worker"
with patch(
"local_deep_research.utilities.log_utils._log_queue"
) as mock_queue:
database_sink(mock_message)
queued = mock_queue.put_nowait.call_args[0][0]
# 240 char cap on the value preview (per _exception_context's own
# documented contract), with ellipsis once exceeded. The full
# 600-char value must NOT be present.
assert long_value not in queued["message"]
assert "…" in queued["message"]
assert len(queued["message"]) < DATABASE_MESSAGE_MAX_LENGTH + 200
def test_database_sink_handles_tuple_exceptions(self):
"""Standard tuple sys.exc_info() stored in record['exception'] should be supported."""
from local_deep_research.utilities.log_utils import database_sink
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "tuple exc",
"name": "mod",
"function": "fn",
"line": 1,
"level": Mock(name="ERROR"),
"extra": {"research_id": "rid-tuple"},
"exception": (TypeError, TypeError("invalid type"), None),
}
# Post-FastAPI-migration the sink queues whenever the emitting
# thread is not MainThread (no Flask app-context check anymore).
with patch(
"local_deep_research.utilities.log_utils.threading.current_thread"
) as mock_thread:
mock_thread.return_value.name = "log-test-worker"
with patch(
"local_deep_research.utilities.log_utils._log_queue"
) as mock_queue:
database_sink(mock_message)
queued = mock_queue.put_nowait.call_args[0][0]
assert queued["message"].startswith("[TypeError: invalid type] ")
def test_database_sink_handles_unprintable_exceptions(self):
"""If str(exc_value) raises an exception, _exception_context should not crash."""
from local_deep_research.utilities.log_utils import database_sink
class FaultyException(Exception):
def __str__(self):
raise RuntimeError("str failed")
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "faulty str",
"name": "mod",
"function": "fn",
"line": 1,
"level": Mock(name="ERROR"),
"extra": {"research_id": "rid-faulty"},
"exception": (FaultyException, FaultyException(), None),
}
# Post-FastAPI-migration the sink queues whenever the emitting
# thread is not MainThread (no Flask app-context check anymore).
with patch(
"local_deep_research.utilities.log_utils.threading.current_thread"
) as mock_thread:
mock_thread.return_value.name = "log-test-worker"
with patch(
"local_deep_research.utilities.log_utils._log_queue"
) as mock_queue:
database_sink(mock_message)
queued = mock_queue.put_nowait.call_args[0][0]
assert "[FaultyException: <unprintable exception>]" in queued["message"]
def test_database_sink_sanitizes_credentials_in_exception_context(self):
"""Exception messages containing credential shapes (e.g. DSN passwords or API keys)
must be scrubbed via sanitize_error_message in _exception_context before being written to app_logs."""
from loguru._logger import RecordException
from local_deep_research.utilities.log_utils import database_sink
secret_dsn = "postgresql://user:supersecretpass@localhost:5432/db"
mock_message = Mock()
mock_message.record = {
"time": datetime.now(),
"message": "Connection error",
"name": "mod",
"function": "fn",
"line": 1,
"level": Mock(name="ERROR"),
"extra": {"research_id": "rid-sanitized"},
"exception": RecordException(
RuntimeError,
RuntimeError(f"Failed to connect to {secret_dsn}"),
None,
),
}
# Post-FastAPI-migration the sink queues whenever the emitting
# thread is not MainThread (no Flask app-context check anymore).
with patch(
"local_deep_research.utilities.log_utils.threading.current_thread"
) as mock_thread:
mock_thread.return_value.name = "log-test-worker"
with patch(
"local_deep_research.utilities.log_utils._log_queue"
) as mock_queue:
database_sink(mock_message)
queued = mock_queue.put_nowait.call_args[0][0]
assert "supersecretpass" not in queued["message"]
assert queued["message"].startswith(
"[RuntimeError: Failed to connect to postgresql://[REDACTED]"
)
def test_exception_context_precaps_large_exception_messages(self):
"""_exception_context should pre-cap massive exception message strings before running sanitize_error_message."""
from loguru._logger import RecordException
from local_deep_research.utilities.log_utils import _exception_context
large_err_msg = "A" * 10000 + "Bearer secret-token-12345"
record = {
"exception": RecordException(
RuntimeError, RuntimeError(large_err_msg), None
)
}
result = _exception_context(record)
# Verify it was capped to 240 chars (237 + '…')
assert len(result) <= 260
assert result.startswith("[RuntimeError: AAAAAA")
assert result.endswith("…] ")
def test_exception_context_scrubs_token_inside_retained_region(self):
"""Companion to the pre-cap test: place a Bearer token early in the
message (well inside both the 4096 pre-cap and the 240 render cap)
so it survives truncation and must be removed by
``sanitize_error_message`` rather than merely dropped by the cap.
The pre-cap test alone can pass even if scrubbing regressed, because
it hides the token past both caps. This variant exercises the
sanitization step itself on the retained prefix.
"""
from loguru._logger import RecordException
from local_deep_research.utilities.log_utils import _exception_context
# ~pos 40 → inside the 237-char rendered prefix and the 4096 pre-cap.
err_msg = "auth failed: " + "x" * 30 + " Bearer secret-token-12345"
record = {
"exception": RecordException(
RuntimeError, RuntimeError(err_msg), None
)
}
result = _exception_context(record)
assert "secret-token-12345" not in result
assert "Bearer [REDACTED]" in result
@pytest.mark.skip(
reason="Tests pre-migration frontend_progress_sink that mocked Flask "
"`has_app_context` + `g`. Post-FastAPI-migration, the sink uses "
"contextvars + the Socket.IO emitter. Coverage moved to "
"tests/utilities/test_log_utils_deep_coverage.py::TestFrontendProgressSink, "
"tests/web/services/test_socket_service_emit_contracts.py and "
"tests/web/services/test_socketio_asgi_contracts.py."
)
class TestFrontendProgressSink:
"""Tests for frontend_progress_sink function."""
def test_skips_when_no_research_id(self):
"""Should skip when no research_id is available."""
from local_deep_research.utilities.log_utils import (
frontend_progress_sink,
)
mock_message = Mock()
mock_message.record = {
"message": "Test",
"level": Mock(name="INFO"),
"time": Mock(isoformat=Mock(return_value="2024-01-01T00:00:00")),
"extra": {},
}
with patch(
"local_deep_research.utilities.log_utils._get_research_id",
return_value=None,
):
with patch(
"local_deep_research.utilities.log_utils.SocketIOService"
) as mock_socket:
frontend_progress_sink(mock_message)
# Should not emit anything
mock_socket.return_value.emit_to_subscribers.assert_not_called()
def test_emits_to_subscribers_with_research_id(self):
"""Should emit to subscribers when research_id is present."""
from local_deep_research.utilities.log_utils import (
frontend_progress_sink,
)
mock_message = Mock()
mock_message.record = {
"message": "Progress update",
"level": Mock(name="INFO"),
"time": Mock(isoformat=Mock(return_value="2024-01-01T12:00:00")),
"extra": {"research_id": "test-uuid"},
}
with patch(
"local_deep_research.utilities.log_utils._get_research_id",
return_value="test-uuid",
):
with patch(
"local_deep_research.utilities.log_utils.SocketIOService"
) as mock_socket:
frontend_progress_sink(mock_message)
mock_socket.return_value.emit_to_subscribers.assert_called_once()
call_args = (
mock_socket.return_value.emit_to_subscribers.call_args
)
assert call_args[0][0] == "progress"
assert call_args[0][1] == "test-uuid"
def test_short_messages_pass_through_unchanged(self):
"""Messages under the cap must reach the wire byte-for-byte."""
from local_deep_research.utilities.log_utils import (
FRONTEND_MESSAGE_MAX_LENGTH,
frontend_progress_sink,
)
short_msg = "small payload"
assert len(short_msg) < FRONTEND_MESSAGE_MAX_LENGTH
mock_message = Mock()
mock_message.record = {
"message": short_msg,
"level": Mock(name="INFO"),
"time": Mock(isoformat=Mock(return_value="2024-01-01T00:00:00")),
"extra": {"research_id": "rid"},
}
with (
patch(
"local_deep_research.utilities.log_utils._get_research_id",
return_value="rid",
),
patch(
"local_deep_research.utilities.log_utils.SocketIOService"
) as mock_socket,
):
frontend_progress_sink(mock_message)
payload = mock_socket.return_value.emit_to_subscribers.call_args[0][2]
assert payload["log_entry"]["message"] == short_msg
def test_long_messages_truncated_with_indicator(self):
"""Messages exceeding the cap must be truncated and carry a clear
indicator that points the user at the server logs (which keep up
to 32 KiB of the text). Other log levels / sinks are unaffected by
this sink."""
from local_deep_research.utilities.log_utils import (
FRONTEND_MESSAGE_MAX_LENGTH,
frontend_progress_sink,
)
big_msg = "X" * (FRONTEND_MESSAGE_MAX_LENGTH + 5000)
mock_message = Mock()
mock_message.record = {
"message": big_msg,
"level": Mock(name="INFO"),
"time": Mock(isoformat=Mock(return_value="2024-01-01T00:00:00")),
"extra": {"research_id": "rid"},
}
with (
patch(
"local_deep_research.utilities.log_utils._get_research_id",
return_value="rid",
),
patch(
"local_deep_research.utilities.log_utils.SocketIOService"
) as mock_socket,
):
frontend_progress_sink(mock_message)
payload = mock_socket.return_value.emit_to_subscribers.call_args[0][2]
out = payload["log_entry"]["message"]
# Truncated to the cap plus indicator
assert out.startswith("X" * FRONTEND_MESSAGE_MAX_LENGTH)
assert "truncated" in out
assert str(len(big_msg)) in out # original length surfaced
# Sanity: cap + indicator is still much smaller than the original
assert len(out) < len(big_msg)
def test_message_at_exact_cap_is_not_truncated(self):
"""A message of exactly ``FRONTEND_MESSAGE_MAX_LENGTH`` chars
must pass through unchanged — the cap is inclusive."""
from local_deep_research.utilities.log_utils import (
FRONTEND_MESSAGE_MAX_LENGTH,
frontend_progress_sink,
)
exact = "Y" * FRONTEND_MESSAGE_MAX_LENGTH
mock_message = Mock()
mock_message.record = {
"message": exact,
"level": Mock(name="INFO"),
"time": Mock(isoformat=Mock(return_value="2024-01-01T00:00:00")),
"extra": {"research_id": "rid"},
}
with (
patch(
"local_deep_research.utilities.log_utils._get_research_id",
return_value="rid",
),
patch(
"local_deep_research.utilities.log_utils.SocketIOService"
) as mock_socket,
):
frontend_progress_sink(mock_message)
payload = mock_socket.return_value.emit_to_subscribers.call_args[0][2]
assert payload["log_entry"]["message"] == exact
assert "truncated" not in payload["log_entry"]["message"]
class TestFlushLogQueue:
"""Tests for flush_log_queue function."""
def test_flushes_all_queued_logs(self):
"""Should flush all logs from queue."""
from local_deep_research.utilities.log_utils import flush_log_queue
import local_deep_research.utilities.log_utils as module
from local_deep_research.database.encrypted_db import db_manager
log_entries = [
{
"timestamp": datetime.now(),
"message": "Log 1",
"module": "mod",
"function": "f",
"line_no": 1,
"level": "INFO",
"research_id": "rid-1",
"username": "alice",
},
{
"timestamp": datetime.now(),
"message": "Log 2",
"module": "mod",
"function": "f",
"line_no": 2,
"level": "INFO",
"research_id": "rid-1",
"username": "alice",
},
]
mock_session = MagicMock()
with patch.object(module, "_log_queue") as mock_queue:
mock_queue.empty.side_effect = [False, False, True]
mock_queue.get_nowait.side_effect = log_entries + [queue.Empty()]
with patch.object(
db_manager, "is_user_connected", return_value=True
):
with patch.object(
db_manager, "get_session", return_value=mock_session
):
flush_log_queue()
# Should have written 2 logs
assert mock_session.add.call_count == 2
assert mock_session.commit.call_count == 2
def test_handles_empty_queue(self):
"""Should handle empty queue gracefully."""
from local_deep_research.utilities.log_utils import flush_log_queue
with patch(
"local_deep_research.utilities.log_utils._log_queue"
) as mock_queue:
mock_queue.empty.return_value = True
# Should not raise
flush_log_queue()
class TestConfigLogger:
"""Tests for config_logger function."""
def test_configures_logger(self):
"""Should configure logger with sinks."""
from local_deep_research.utilities.log_utils import config_logger
with patch(
"local_deep_research.utilities.log_utils.logger"
) as mock_logger:
config_logger("test_app")
mock_logger.enable.assert_called_with("local_deep_research")
mock_logger.remove.assert_called_once()
# Should add multiple sinks
assert mock_logger.add.call_count >= 3
def test_adds_file_logging_when_enabled(self):
"""Should add file logging when environment variable is set."""
from local_deep_research.utilities.log_utils import config_logger
with patch.dict("os.environ", {"LDR_ENABLE_FILE_LOGGING": "true"}):
with patch(
"local_deep_research.utilities.log_utils.logger"
) as mock_logger:
config_logger("test_app")
# Should add 4 sinks (stderr, database, frontend, file)
assert mock_logger.add.call_count >= 4
def test_creates_milestone_level(self):
"""Should create MILESTONE log level."""
from local_deep_research.utilities.log_utils import config_logger
with patch(
"local_deep_research.utilities.log_utils.logger"
) as mock_logger:
config_logger("test_app")
mock_logger.level.assert_called()
def test_handles_existing_milestone_level(self):
"""Should handle case where MILESTONE level already exists."""
from local_deep_research.utilities.log_utils import config_logger
with patch(
"local_deep_research.utilities.log_utils.logger"
) as mock_logger:
mock_logger.level.side_effect = ValueError("Level already exists")
# Should not raise
config_logger("test_app")
class _FakeDBManager:
"""Minimal stand-in for the encrypted ``db_manager`` singleton.
Mirrors the real resurrection hazard: ``get_session`` yields a session
only for an already-open connection (like the real one, which returns
None when the user isn't in ``connections``), while
``open_user_database`` is the buggy path that RE-ADDS a connection.
A correct drain must never call ``open_user_database`` for a
logged-out user, so ``is_user_connected`` must stay False.
"""
def __init__(self, connected):
self._connections = set(connected)
self.open_called = 0
self.get_session_calls = []
self.last_session = None
def is_user_connected(self, username):
return username in self._connections
def get_session(self, username):
self.get_session_calls.append(username)
if username not in self._connections:
# Real get_session returns None for a closed DB — never reopens.
return None
self.last_session = MagicMock()
return self.last_session
def open_user_database(self, username, password=None):
# The resurrection path the drain must NOT take: reopening re-adds
# the connection so is_user_connected would flip back to True.
self.open_called += 1
self._connections.add(username)
return MagicMock()
class TestWriteLogToDatabase:
"""Tests for _write_log_to_database function."""
@staticmethod
def _entry(username="alice", research_id="test-uuid", level="INFO"):
return {
"timestamp": datetime.now(),
"message": "Test message",
"module": "test_module",
"function": "test_func",
"line_no": 42,
"level": level,
"research_id": research_id,
"username": username,
}
def test_writes_research_log_when_user_connected(self):
"""A connected user's entry is written by binding to the EXISTING
open connection — no reopen, no password. This is also the
research-in-progress case: a running research job keeps the user's
DB open, so in-flight logs must still be persisted."""
from local_deep_research.utilities.log_utils import (
_write_log_to_database,
)
fake = _FakeDBManager(connected={"alice"})
with patch(
"local_deep_research.database.encrypted_db.db_manager", fake
):
_write_log_to_database(self._entry(username="alice"))
assert fake.get_session_calls == ["alice"]
assert fake.open_called == 0 # never reopened
session = fake.last_session
session.add.assert_called_once()
session.commit.assert_called_once()
session.close.assert_called_once()
def test_drops_entry_and_never_reopens_when_user_not_connected(self):
"""A backlog entry drained AFTER logout (connection closed) must be
dropped: the DB must not be reopened and ``is_user_connected`` must
stay False. Asserts no post-logout resurrection."""
from local_deep_research.utilities.log_utils import (
_write_log_to_database,
)
# User already logged out — no live connection.
fake = _FakeDBManager(connected=set())
with patch(
"local_deep_research.database.encrypted_db.db_manager", fake
):
_write_log_to_database(self._entry(username="loggedout"))
# No resurrection: DB never reopened, connection still closed,
# and no session was created for the write.
assert fake.open_called == 0
assert fake.is_user_connected("loggedout") is False
assert fake.get_session_calls == []
assert fake.last_session is None
def test_drops_entry_with_no_username(self):
"""An entry with no username can't map to any per-user DB — drop it
without touching the connection registry."""
from local_deep_research.utilities.log_utils import (
_write_log_to_database,
)
fake = _FakeDBManager(connected={"alice"})
with patch(
"local_deep_research.database.encrypted_db.db_manager", fake
):
_write_log_to_database(self._entry(username=None))
assert fake.get_session_calls == []
assert fake.open_called == 0
def test_does_not_cache_credential_on_drain_thread(self):
"""Draining a connected user's entry must NOT leave a credential
cached in the thread-local session manager for this thread — the
drain binds to the existing engine instead of going through the
password-caching get_metrics_session path."""
from local_deep_research.utilities.log_utils import (
_write_log_to_database,
)
from local_deep_research.database.thread_local_session import (
thread_session_manager,
)
tid = threading.get_ident()
thread_session_manager._thread_credentials.pop(tid, None)
fake = _FakeDBManager(connected={"alice"})
with patch(
"local_deep_research.database.encrypted_db.db_manager", fake
):
_write_log_to_database(self._entry(username="alice"))
assert tid not in thread_session_manager._thread_credentials
def test_handles_database_error_gracefully(self):
"""Should not raise if the write itself fails for a connected user."""
from local_deep_research.utilities.log_utils import (
_write_log_to_database,
)
from local_deep_research.database.encrypted_db import db_manager
with patch.object(db_manager, "is_user_connected", return_value=True):
with patch.object(
db_manager,
"get_session",
side_effect=Exception("DB error"),
):
# Should not raise
_write_log_to_database(self._entry(username="alice"))
class TestClearDaemonThreadCredentials:
"""Tests for the belt-and-suspenders per-drain credential cleanup."""
def test_clears_cached_credential_for_current_thread(self):
"""After a drain iteration the daemon thread must retain no
credential in ``_thread_credentials`` — the long-lived daemon is
never reclaimed by the dead-thread sweeper, so it clears its own
entry every iteration."""
from local_deep_research.utilities.log_utils import (
_clear_daemon_thread_credentials,
)
from local_deep_research.database.thread_local_session import (
thread_session_manager,
)
tid = threading.get_ident()
thread_session_manager._thread_credentials[tid] = (
"alice",
"secret-pw", # gitleaks:allow
)
try:
_clear_daemon_thread_credentials()
assert tid not in thread_session_manager._thread_credentials
finally:
thread_session_manager._thread_credentials.pop(tid, None)
def test_never_raises(self):
"""Cleanup failures must be swallowed — a hiccup must not crash the
log daemon and stop all subsequent log persistence."""
from local_deep_research.utilities import log_utils
with patch(
"local_deep_research.database.thread_local_session.cleanup_current_thread",
side_effect=Exception("boom"),
):
# Must not raise.
log_utils._clear_daemon_thread_credentials()
class TestLogQueueProcessorDaemon:
"""Tests for start_log_queue_processor / stop_log_queue_processor."""
def _make_fake_app(self):
"""Return a Flask-app-like stub that supports ``with app.app_context()``."""
fake_app = MagicMock()
ctx = MagicMock()
ctx.__enter__ = Mock(return_value=None)
ctx.__exit__ = Mock(return_value=False)
fake_app.app_context.return_value = ctx
return fake_app
def test_spawns_daemon_thread(self):
# audit: PUNCHLIST reviewed 2026-05 — KEEP (OK).
from local_deep_research.utilities import log_utils
app = self._make_fake_app()
try:
thread = log_utils.start_log_queue_processor(app)
assert isinstance(thread, threading.Thread)
assert thread.daemon is True
assert thread.is_alive()
assert thread.name == "log-queue-processor"
finally:
log_utils.stop_log_queue_processor(timeout=2.0)
def test_is_idempotent(self):
from local_deep_research.utilities import log_utils
app = self._make_fake_app()
try:
first = log_utils.start_log_queue_processor(app)
second = log_utils.start_log_queue_processor(app)
assert first is second
finally:
log_utils.stop_log_queue_processor(timeout=2.0)
def test_stop_joins_thread(self):
# audit: PUNCHLIST reviewed 2026-05 — KEEP (OK).
from local_deep_research.utilities import log_utils
app = self._make_fake_app()
thread = log_utils.start_log_queue_processor(app)
log_utils.stop_log_queue_processor(timeout=2.0)
assert not thread.is_alive()
def test_stop_does_not_clear_ref_when_join_times_out(self):
"""If the daemon doesn't exit before join() times out, the
module-level reference must stay populated — otherwise a
subsequent start would happily spawn a second daemon competing
for the same queue."""
from local_deep_research.utilities import log_utils
# Stand-in thread that ignores the stop signal so join() always
# times out. Daemon=True so it doesn't keep the test process alive.
live_thread = threading.Thread(
target=lambda: threading.Event().wait(), daemon=True
)
live_thread.start()
log_utils._queue_processor_thread = live_thread
try:
log_utils.stop_log_queue_processor(timeout=0.05)
assert log_utils._queue_processor_thread is live_thread, (
"thread reference cleared despite join timing out — would "
"let start_log_queue_processor spawn a duplicate daemon"
)
finally:
log_utils._queue_processor_thread = None