~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
fix(database): reclaim unused sidecar gates (#7705)
Chanhyo Jung1abc64c1da13
diff --git a/changelog.d/7687.bugfix.md b/changelog.d/7687.bugfix.mdnew file mode 100644index 0000000000..f8a398d892--- /dev/null+++ b/changelog.d/7687.bugfix.md@@ -0,0 +1 @@+Reclaim unused in-process sidecar gates while preserving the shared gate for active holders and waiters.diff --git a/src/local_deep_research/database/rekey_lock.py b/src/local_deep_research/database/rekey_lock.pyindex 08f07fc8da..8c1c3cad19 100644--- a/src/local_deep_research/database/rekey_lock.py+++ b/src/local_deep_research/database/rekey_lock.py@@ -8,6 +8,7 @@ import statimport tempfileimport threadingimport time+import weakreffrom collections.abc import Generatorfrom contextlib import contextmanagerfrom dataclasses import dataclass@@ -247,7 +248,10 @@ _CONTENTION_RETRY_INTERVAL = 0.01# processes' probes) so genuinely foreign holders still fail with the# typed contention error. Request-path callers bound the gate wait and# the flock retries together, by one deadline (``timeout``).-_sidecar_gates: dict[Path, threading.Lock] = {}+# Waiters and holders keep a strong local reference through gate release.+_sidecar_gates: weakref.WeakValueDictionary[Path, threading.Lock] = (+ weakref.WeakValueDictionary()+)_sidecar_gates_guard = threading.Lock()diff --git a/tests/database/test_rekey_gate_registry.py b/tests/database/test_rekey_gate_registry.pynew file mode 100644index 0000000000..9a10265ac6--- /dev/null+++ b/tests/database/test_rekey_gate_registry.py@@ -0,0 +1,222 @@+"""Lifetime and identity regressions for real in-process sidecar gates."""++import gc+import threading+import weakref++import pytest++from local_deep_research.database import rekey_lock as sut+++def _assert_reclaimed(path, reference):+ gc.collect()+ assert reference() is None, "owned gate is still strongly retained"+ with sut._sidecar_gates_guard:+ assert path not in sut._sidecar_gates, "owned registry key remains"+++def test_independent_paths_and_reclamation(tmp_path):+ first_path = tmp_path / "first.lock"+ second_path = tmp_path / "second.lock"+ first = sut._get_sidecar_gate(first_path)+ second = sut._get_sidecar_gate(second_path)+ assert first is sut._get_sidecar_gate(first_path)+ assert second is sut._get_sidecar_gate(second_path)+ assert first is not second+ first_ref, second_ref = weakref.ref(first), weakref.ref(second)+ del first+ _assert_reclaimed(first_path, first_ref)+ assert second_ref() is second+ del second+ _assert_reclaimed(second_path, second_ref)++[email protected]("failure", ["none", "body", "acquisition", "timeout"])+def test_wrapper_cleanup(tmp_path, monkeypatch, failure):+ path = tmp_path / "cleanup.lock"+ request = sut._SidecarLockRequest(+ db_path=tmp_path / "user.db",+ lock_path=path,+ mode=sut.AuthorizationLeaseMode.EXCLUSIVE,+ blocking=False,+ )+ references = []+ real_getter = sut._get_sidecar_gate++ def observe(lock_path):+ gate = real_getter(lock_path)+ references.append(weakref.ref(gate))+ return gate++ real_inner = sut._acquire_sidecar_lock_locked++ def inner(_request):+ if failure == "acquisition":+ raise ValueError("acquisition")+ yield from real_inner(_request)++ monkeypatch.setattr(sut, "_get_sidecar_gate", observe)+ monkeypatch.setattr(sut, "_acquire_sidecar_lock_locked", inner)+ held = real_getter(path)+ if failure == "timeout":+ held.acquire()+ try:+ if failure == "none":+ with sut._acquire_sidecar_lock(request):+ assert references[-1]() is not None+ else:+ expected = (+ sut.RekeyLockContentionError+ if failure == "timeout"+ else ValueError+ )+ with pytest.raises(expected):+ with sut._acquire_sidecar_lock(request):+ raise ValueError("body")+ assert held.locked() is (failure == "timeout")+ finally:+ if held.locked():+ held.release()+ del held+ _assert_reclaimed(path, references[-1])+++def test_waiter_keeps_gate_across_holder_exit_and_new_arrival(+ tmp_path, monkeypatch+):+ path = tmp_path / "contended.lock"+ request = sut._SidecarLockRequest(+ db_path=tmp_path / "user.db",+ lock_path=path,+ mode=sut.AuthorizationLeaseMode.EXCLUSIVE,+ blocking=False,+ )+ looked_up = threading.Event()+ resume = threading.Event()+ errors = []+ references = {}+ active = 0+ real_getter = sut._get_sidecar_gate++ def observe(lock_path):+ gate = real_getter(lock_path)+ name = threading.current_thread().name+ references[name] = weakref.ref(gate)+ if name == "registry-waiter":+ looked_up.set()+ assert resume.wait(10), "waiter was not released"+ return gate++ def inner(_request):+ nonlocal active+ active += 1+ try:+ assert active == 1, "wrapper critical sections overlap"+ yield+ finally:+ active -= 1++ def waiter():+ try:+ with pytest.raises(sut.RekeyLockContentionError):+ with sut._acquire_sidecar_lock(request):+ pytest.fail("waiter entered while newcomer held the gate")+ except BaseException as error:+ # Do not keep traceback frames (and their gate references) alive.+ errors.append(f"{type(error).__name__}: {error}")++ monkeypatch.setattr(sut, "_get_sidecar_gate", observe)+ monkeypatch.setattr(sut, "_acquire_sidecar_lock_locked", inner)+ thread = threading.Thread(target=waiter, name="registry-waiter")+ try:+ with sut._acquire_sidecar_lock(request):+ holder_ref = references[threading.current_thread().name]+ thread.start()+ assert looked_up.wait(10), "waiter did not look up the gate"+ assert references[thread.name]() is holder_ref()+ # Only the paused waiter now owns a strong reference to the old gate.+ gc.collect()+ assert holder_ref() is not None+ with sut._acquire_sidecar_lock(request):+ newcomer_ref = references[threading.current_thread().name]+ resume.set()+ thread.join(10)+ assert not thread.is_alive(), "waiter did not settle"+ assert not errors, errors+ assert newcomer_ref() is holder_ref(), (+ "new arrival split gate identity"+ )+ finally:+ resume.set()+ if thread.ident is not None:+ thread.join(10)+ assert not thread.is_alive(), "test-owned thread survived cleanup"+ assert not errors, errors+ assert active == 0+ _assert_reclaimed(path, holder_ref)+++def test_blocking_waiter_and_independent_path_use_real_sidecars(+ tmp_path, monkeypatch+):+ path = tmp_path / "waiting.lock"+ other_path = tmp_path / "independent.lock"++ def request(lock_path, gate_wait=None):+ return sut._SidecarLockRequest(+ db_path=tmp_path / "user.db",+ lock_path=lock_path,+ mode=sut.AuthorizationLeaseMode.EXCLUSIVE,+ blocking=False,+ gate_wait=gate_wait,+ )++ looked_up = threading.Event()+ entered = threading.Event()+ release = threading.Event()+ references = {}+ errors = []+ real_getter = sut._get_sidecar_gate++ def observe(lock_path):+ gate = real_getter(lock_path)+ references[lock_path] = weakref.ref(gate)+ if threading.current_thread().name == "blocking-registry-waiter":+ looked_up.set()+ return gate++ def waiter():+ try:+ with sut._acquire_sidecar_lock(request(path, gate_wait=10)):+ entered.set()+ assert release.wait(10), "blocking waiter was not released"+ except BaseException as error:+ errors.append(f"{type(error).__name__}: {error}")++ monkeypatch.setattr(sut, "_get_sidecar_gate", observe)+ thread = threading.Thread(target=waiter, name="blocking-registry-waiter")+ try:+ with sut._acquire_sidecar_lock(request(path)):+ holder_ref = references[path]+ thread.start()+ assert looked_up.wait(10), "blocking waiter did not look up gate"+ assert references[path]() is holder_ref()+ # An independent path can acquire while this holder owns its gate.+ with sut._acquire_sidecar_lock(request(other_path)):+ other_ref = references[other_path]+ assert other_ref() is not holder_ref()+ assert entered.wait(10), "blocking waiter did not acquire after release"+ with pytest.raises(sut.RekeyLockContentionError):+ with sut._acquire_sidecar_lock(request(path)):+ pytest.fail(+ "new arrival entered while blocking waiter held gate"+ )+ finally:+ release.set()+ if thread.ident is not None:+ thread.join(10)+ assert not thread.is_alive(), "blocking waiter survived cleanup"+ assert not errors, errors+ _assert_reclaimed(path, holder_ref)+ _assert_reclaimed(other_path, other_ref)