Skip to content

~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.

fix(database): reclaim unused sidecar gates (#7705)

Chanhyo Jung1abc64c1da13
diff --git a/changelog.d/7687.bugfix.md b/changelog.d/7687.bugfix.md
new file mode 100644
index 0000000000..f8a398d892
--- /dev/null
+++ b/changelog.d/7687.bugfix.md
@@ -0,0 +1 @@
+Reclaim unused in-process sidecar gates while preserving the shared gate for active holders and waiters.
diff --git a/src/local_deep_research/database/rekey_lock.py b/src/local_deep_research/database/rekey_lock.py
index 08f07fc8da..8c1c3cad19 100644
--- a/src/local_deep_research/database/rekey_lock.py
+++ b/src/local_deep_research/database/rekey_lock.py
@@ -8,6 +8,7 @@ import stat
import tempfile
import threading
import time
+import weakref
from collections.abc import Generator
from contextlib import contextmanager
from dataclasses import dataclass
@@ -247,7 +248,10 @@ _CONTENTION_RETRY_INTERVAL = 0.01
# processes' probes) so genuinely foreign holders still fail with the
# typed contention error. Request-path callers bound the gate wait and
# the flock retries together, by one deadline (``timeout``).
-_sidecar_gates: dict[Path, threading.Lock] = {}
+# Waiters and holders keep a strong local reference through gate release.
+_sidecar_gates: weakref.WeakValueDictionary[Path, threading.Lock] = (
+ weakref.WeakValueDictionary()
+)
_sidecar_gates_guard = threading.Lock()
diff --git a/tests/database/test_rekey_gate_registry.py b/tests/database/test_rekey_gate_registry.py
new file mode 100644
index 0000000000..9a10265ac6
--- /dev/null
+++ b/tests/database/test_rekey_gate_registry.py
@@ -0,0 +1,222 @@
+"""Lifetime and identity regressions for real in-process sidecar gates."""
+
+import gc
+import threading
+import weakref
+
+import pytest
+
+from local_deep_research.database import rekey_lock as sut
+
+
+def _assert_reclaimed(path, reference):
+ gc.collect()
+ assert reference() is None, "owned gate is still strongly retained"
+ with sut._sidecar_gates_guard:
+ assert path not in sut._sidecar_gates, "owned registry key remains"
+
+
+def test_independent_paths_and_reclamation(tmp_path):
+ first_path = tmp_path / "first.lock"
+ second_path = tmp_path / "second.lock"
+ first = sut._get_sidecar_gate(first_path)
+ second = sut._get_sidecar_gate(second_path)
+ assert first is sut._get_sidecar_gate(first_path)
+ assert second is sut._get_sidecar_gate(second_path)
+ assert first is not second
+ first_ref, second_ref = weakref.ref(first), weakref.ref(second)
+ del first
+ _assert_reclaimed(first_path, first_ref)
+ assert second_ref() is second
+ del second
+ _assert_reclaimed(second_path, second_ref)
+
+
[email protected]("failure", ["none", "body", "acquisition", "timeout"])
+def test_wrapper_cleanup(tmp_path, monkeypatch, failure):
+ path = tmp_path / "cleanup.lock"
+ request = sut._SidecarLockRequest(
+ db_path=tmp_path / "user.db",
+ lock_path=path,
+ mode=sut.AuthorizationLeaseMode.EXCLUSIVE,
+ blocking=False,
+ )
+ references = []
+ real_getter = sut._get_sidecar_gate
+
+ def observe(lock_path):
+ gate = real_getter(lock_path)
+ references.append(weakref.ref(gate))
+ return gate
+
+ real_inner = sut._acquire_sidecar_lock_locked
+
+ def inner(_request):
+ if failure == "acquisition":
+ raise ValueError("acquisition")
+ yield from real_inner(_request)
+
+ monkeypatch.setattr(sut, "_get_sidecar_gate", observe)
+ monkeypatch.setattr(sut, "_acquire_sidecar_lock_locked", inner)
+ held = real_getter(path)
+ if failure == "timeout":
+ held.acquire()
+ try:
+ if failure == "none":
+ with sut._acquire_sidecar_lock(request):
+ assert references[-1]() is not None
+ else:
+ expected = (
+ sut.RekeyLockContentionError
+ if failure == "timeout"
+ else ValueError
+ )
+ with pytest.raises(expected):
+ with sut._acquire_sidecar_lock(request):
+ raise ValueError("body")
+ assert held.locked() is (failure == "timeout")
+ finally:
+ if held.locked():
+ held.release()
+ del held
+ _assert_reclaimed(path, references[-1])
+
+
+def test_waiter_keeps_gate_across_holder_exit_and_new_arrival(
+ tmp_path, monkeypatch
+):
+ path = tmp_path / "contended.lock"
+ request = sut._SidecarLockRequest(
+ db_path=tmp_path / "user.db",
+ lock_path=path,
+ mode=sut.AuthorizationLeaseMode.EXCLUSIVE,
+ blocking=False,
+ )
+ looked_up = threading.Event()
+ resume = threading.Event()
+ errors = []
+ references = {}
+ active = 0
+ real_getter = sut._get_sidecar_gate
+
+ def observe(lock_path):
+ gate = real_getter(lock_path)
+ name = threading.current_thread().name
+ references[name] = weakref.ref(gate)
+ if name == "registry-waiter":
+ looked_up.set()
+ assert resume.wait(10), "waiter was not released"
+ return gate
+
+ def inner(_request):
+ nonlocal active
+ active += 1
+ try:
+ assert active == 1, "wrapper critical sections overlap"
+ yield
+ finally:
+ active -= 1
+
+ def waiter():
+ try:
+ with pytest.raises(sut.RekeyLockContentionError):
+ with sut._acquire_sidecar_lock(request):
+ pytest.fail("waiter entered while newcomer held the gate")
+ except BaseException as error:
+ # Do not keep traceback frames (and their gate references) alive.
+ errors.append(f"{type(error).__name__}: {error}")
+
+ monkeypatch.setattr(sut, "_get_sidecar_gate", observe)
+ monkeypatch.setattr(sut, "_acquire_sidecar_lock_locked", inner)
+ thread = threading.Thread(target=waiter, name="registry-waiter")
+ try:
+ with sut._acquire_sidecar_lock(request):
+ holder_ref = references[threading.current_thread().name]
+ thread.start()
+ assert looked_up.wait(10), "waiter did not look up the gate"
+ assert references[thread.name]() is holder_ref()
+ # Only the paused waiter now owns a strong reference to the old gate.
+ gc.collect()
+ assert holder_ref() is not None
+ with sut._acquire_sidecar_lock(request):
+ newcomer_ref = references[threading.current_thread().name]
+ resume.set()
+ thread.join(10)
+ assert not thread.is_alive(), "waiter did not settle"
+ assert not errors, errors
+ assert newcomer_ref() is holder_ref(), (
+ "new arrival split gate identity"
+ )
+ finally:
+ resume.set()
+ if thread.ident is not None:
+ thread.join(10)
+ assert not thread.is_alive(), "test-owned thread survived cleanup"
+ assert not errors, errors
+ assert active == 0
+ _assert_reclaimed(path, holder_ref)
+
+
+def test_blocking_waiter_and_independent_path_use_real_sidecars(
+ tmp_path, monkeypatch
+):
+ path = tmp_path / "waiting.lock"
+ other_path = tmp_path / "independent.lock"
+
+ def request(lock_path, gate_wait=None):
+ return sut._SidecarLockRequest(
+ db_path=tmp_path / "user.db",
+ lock_path=lock_path,
+ mode=sut.AuthorizationLeaseMode.EXCLUSIVE,
+ blocking=False,
+ gate_wait=gate_wait,
+ )
+
+ looked_up = threading.Event()
+ entered = threading.Event()
+ release = threading.Event()
+ references = {}
+ errors = []
+ real_getter = sut._get_sidecar_gate
+
+ def observe(lock_path):
+ gate = real_getter(lock_path)
+ references[lock_path] = weakref.ref(gate)
+ if threading.current_thread().name == "blocking-registry-waiter":
+ looked_up.set()
+ return gate
+
+ def waiter():
+ try:
+ with sut._acquire_sidecar_lock(request(path, gate_wait=10)):
+ entered.set()
+ assert release.wait(10), "blocking waiter was not released"
+ except BaseException as error:
+ errors.append(f"{type(error).__name__}: {error}")
+
+ monkeypatch.setattr(sut, "_get_sidecar_gate", observe)
+ thread = threading.Thread(target=waiter, name="blocking-registry-waiter")
+ try:
+ with sut._acquire_sidecar_lock(request(path)):
+ holder_ref = references[path]
+ thread.start()
+ assert looked_up.wait(10), "blocking waiter did not look up gate"
+ assert references[path]() is holder_ref()
+ # An independent path can acquire while this holder owns its gate.
+ with sut._acquire_sidecar_lock(request(other_path)):
+ other_ref = references[other_path]
+ assert other_ref() is not holder_ref()
+ assert entered.wait(10), "blocking waiter did not acquire after release"
+ with pytest.raises(sut.RekeyLockContentionError):
+ with sut._acquire_sidecar_lock(request(path)):
+ pytest.fail(
+ "new arrival entered while blocking waiter held gate"
+ )
+ finally:
+ release.set()
+ if thread.ident is not None:
+ thread.join(10)
+ assert not thread.is_alive(), "blocking waiter survived cleanup"
+ assert not errors, errors
+ _assert_reclaimed(path, holder_ref)
+ _assert_reclaimed(other_path, other_ref)