~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Branch/defensive-path tests for connection_cleanup that exercise fallback
and error-handling paths not reached by test_connection_cleanup.py.
Targets uncovered lines in src/local_deep_research/web/auth/connection_cleanup.py:
- 56-68: _count_open_fds fallback via resource.getrlimit + os.fstat loop
- 107->113: scheduler.is_running == False branch
- 157-167: periodic pool reset block (_last_dispose_time gate, delegation to
db.reset_connection_pools() with exception swallowed, reset count log)
- 179-182: engine.pool.checkedout() raising inside FD monitor
- 190-195: high-FD (>800) warning and outer FD-monitor exception handler
"""
import datetime
from unittest.mock import MagicMock, patch
import pytest
from local_deep_research.web.auth import connection_cleanup as cc_module
from local_deep_research.web.auth.connection_cleanup import (
cleanup_idle_connections,
)
from local_deep_research.web.auth.session_manager import SessionManager
@pytest.fixture
def sm():
with patch(
"local_deep_research.web.auth.session_manager.get_security_default",
return_value=1,
):
mgr = SessionManager()
mgr.session_timeout = datetime.timedelta(seconds=1)
mgr.remember_me_timeout = datetime.timedelta(seconds=2)
return mgr
@pytest.fixture
def db():
mock = MagicMock()
mock.get_connected_usernames.return_value = set()
mock.connections = {}
# _connections_lock must be a context manager
mock._connections_lock = MagicMock()
mock._connections_lock.__enter__ = MagicMock(return_value=None)
mock._connections_lock.__exit__ = MagicMock(return_value=None)
return mock
@pytest.fixture(autouse=True)
def reset_dispose_timer():
"""Ensure _last_dispose_time doesn't leak across tests in this file.
Set it far below any realistic time.monotonic() return so the
elapsed-interval gate (>= _DISPOSE_INTERVAL_SECONDS, 1800s) is always
satisfied by default. On freshly-booted systems time.monotonic() can
still be below 1800s, so initializing to 0.0 isn't sufficient.
"""
original = cc_module._last_dispose_time
cc_module._last_dispose_time = -1e9
yield
cc_module._last_dispose_time = original
class TestCountOpenFdsFallback:
"""When /proc/self/fd is unavailable, _count_open_fds falls back to
resource.getrlimit + os.fstat loop."""
def test_falls_back_when_proc_fd_not_a_directory(self):
"""is_dir() returning False triggers the resource.getrlimit path."""
from local_deep_research.web.auth.connection_cleanup import (
_count_open_fds,
)
fake_limit = (5, 1024)
valid_fds = {0, 1, 2, 4}
def fake_fstat(fd):
if fd in valid_fds:
return MagicMock()
raise OSError("bad fd")
with (
patch(
"local_deep_research.web.auth.connection_cleanup.Path"
) as MockPath,
patch(
"local_deep_research.web.auth.connection_cleanup.os.fstat",
side_effect=fake_fstat,
),
patch("resource.getrlimit", return_value=fake_limit),
):
mock_proc = MagicMock()
mock_proc.is_dir.return_value = False
MockPath.return_value = mock_proc
result = _count_open_fds()
assert result == len(valid_fds)
def test_falls_back_when_proc_fd_iterdir_raises(self):
"""is_dir() True but iterdir() raising OSError falls through to fstat."""
from local_deep_research.web.auth.connection_cleanup import (
_count_open_fds,
)
fake_limit = (3, 1024)
def fake_fstat(fd):
if fd in (0, 1):
return MagicMock()
raise OSError("bad fd")
with (
patch(
"local_deep_research.web.auth.connection_cleanup.Path"
) as MockPath,
patch(
"local_deep_research.web.auth.connection_cleanup.os.fstat",
side_effect=fake_fstat,
),
patch("resource.getrlimit", return_value=fake_limit),
):
mock_proc = MagicMock()
mock_proc.is_dir.return_value = True
mock_proc.iterdir.side_effect = OSError("permission denied")
MockPath.return_value = mock_proc
result = _count_open_fds()
assert result == 2
class TestSchedulerNotRunningBranch:
"""When scheduler.is_running is False, unregister_user is not called."""
@patch(
"local_deep_research.scheduler.background.get_background_job_scheduler",
)
@patch(
"local_deep_research.web.auth.connection_cleanup.get_usernames_with_active_research",
return_value=set(),
)
def test_unregister_skipped_when_scheduler_not_running(
self, _mock_research, mock_get_sched, sm, db
):
mock_scheduler = MagicMock()
mock_scheduler.is_running = False
mock_get_sched.return_value = mock_scheduler
db.get_connected_usernames.return_value = {"alice"}
cleanup_idle_connections(sm, db)
mock_scheduler.unregister_user.assert_not_called()
db.close_user_database.assert_called_once_with("alice")
class TestPeriodicPoolDispose:
"""When the dispose interval has elapsed, all engines are disposed and
individual dispose() exceptions are swallowed."""
@patch(
"local_deep_research.scheduler.background.get_background_job_scheduler",
)
@patch(
"local_deep_research.web.auth.connection_cleanup.get_usernames_with_active_research",
return_value=set(),
)
def test_disposes_all_engines_and_swallows_errors(
self, _mock_research, _mock_sched, sm, db
):
# _last_dispose_time = 0 (from autouse fixture), so the interval
# elapsed gate is satisfied and the dispose block runs. The sweep
# delegates to the fenced DatabaseManager.reset_connection_pools()
# (rekey-sidecar-aware checkpoint+dispose per engine; per-engine
# failures are swallowed and logged there, type name only).
db.reset_connection_pools.return_value = 2
cleanup_idle_connections(sm, db)
# Delegated exactly once, and the count feeds the info log.
db.reset_connection_pools.assert_called_once_with()
# The timer was updated so a rapid second call would NOT re-dispose.
assert cc_module._last_dispose_time > 0
@patch(
"local_deep_research.scheduler.background.get_background_job_scheduler",
)
@patch(
"local_deep_research.web.auth.connection_cleanup.get_usernames_with_active_research",
return_value=set(),
)
def test_dispose_failures_surface_as_warnings(
self, _mock_research, _mock_sched, sm, db
):
"""The pool-dispose workaround for ADR-0004's WAL/SHM handle leak
depends on this sweep succeeding. A whole-sweep failure must
surface at WARNING so an operator sees it, but only the exception
TYPE NAME (no value) — the value can carry sensitive locals — and
must not take the periodic scheduler down. Per-engine failure
logging (user + type name) lives inside
DatabaseManager.reset_connection_pools and is covered by its own
suite.
"""
db.reset_connection_pools.side_effect = RuntimeError(
"potentially-sensitive details"
)
with patch.object(cc_module.logger, "warning") as mock_warn:
# Must not raise: the scheduler calls this periodically.
cleanup_idle_connections(sm, db)
warning_calls = [str(call) for call in mock_warn.call_args_list]
assert any("RuntimeError" in call for call in warning_calls)
# The exception's message text must not appear in any warning —
# only the type name is logged.
assert not any(
"potentially-sensitive details" in call for call in warning_calls
)
@patch(
"local_deep_research.scheduler.background.get_background_job_scheduler",
)
@patch(
"local_deep_research.web.auth.connection_cleanup.get_usernames_with_active_research",
return_value=set(),
)
def test_skip_dispose_when_interval_not_elapsed(
self, _mock_research, _mock_sched, sm, db
):
"""If _last_dispose_time is recent, the pool-reset block is skipped.
The sweep no longer disposes engines itself; it delegates to
``db.reset_connection_pools()`` (sidecar + lifecycle lock, WAL
checkpoint, dispose), so that call is what the interval gate must
suppress. Asserting ``engine.dispose`` would pass even with the
gate removed.
"""
import time
cc_module._last_dispose_time = time.monotonic() # just now
before = cc_module._last_dispose_time
engine = MagicMock()
db.connections = {"alice": engine}
cleanup_idle_connections(sm, db)
db.reset_connection_pools.assert_not_called()
engine.dispose.assert_not_called()
# The timer was not advanced either.
assert cc_module._last_dispose_time == before
class TestFdMonitoringErrorPaths:
"""Exception handling inside and around the FD monitor block."""
@patch(
"local_deep_research.web.auth.connection_cleanup._count_open_fds",
return_value=801,
)
@patch(
"local_deep_research.scheduler.background.get_background_job_scheduler",
)
@patch(
"local_deep_research.web.auth.connection_cleanup.get_usernames_with_active_research",
return_value=set(),
)
def test_high_fd_warning_and_checkedout_exception(
self, _mock_research, _mock_sched, _mock_fd, sm, db
):
"""FD count > 800 emits a warning; engine.pool.checkedout() raising
is swallowed silently."""
flaky_engine = MagicMock()
flaky_engine.pool.checkedout.side_effect = RuntimeError("pool gone")
db.connections = {"alice": flaky_engine}
with patch.object(cc_module.logger, "warning") as mock_warn:
cleanup_idle_connections(sm, db)
# Verify checkedout was attempted (the inner try path ran).
flaky_engine.pool.checkedout.assert_called_once()
# High-FD warning fired with the count.
assert any(
"801" in str(call) or "High FD" in str(call)
for call in mock_warn.call_args_list
)
@patch(
"local_deep_research.web.auth.connection_cleanup._count_open_fds",
side_effect=RuntimeError("getrlimit unavailable"),
)
@patch(
"local_deep_research.scheduler.background.get_background_job_scheduler",
)
@patch(
"local_deep_research.web.auth.connection_cleanup.get_usernames_with_active_research",
return_value=set(),
)
def test_outer_fd_monitor_exception_swallowed(
self, _mock_research, _mock_sched, _mock_fd, sm, db
):
"""If _count_open_fds itself raises, cleanup still completes."""
db.get_connected_usernames.return_value = set()
# Should not propagate.
cleanup_idle_connections(sm, db)