Personal AI, On Personal Devices
Python
fix(memory): cross-process locking for LocalFactStore (#797)
havolline13e7eac70a69d
diff --git a/src/openjarvis/memory/store.py b/src/openjarvis/memory/store.pyindex 2841043a..fbad20e5 100644--- a/src/openjarvis/memory/store.py+++ b/src/openjarvis/memory/store.py@@ -9,24 +9,107 @@ caps the total number of facts, and is safe to call from multiple threads.from __future__ import annotations+import errnoimport jsonimport os+import sys+import tempfileimport threadingimport timefrom abc import ABC, abstractmethod+from contextlib import contextmanagerfrom dataclasses import asdict, dataclassfrom pathlib import Path-from typing import Any, Iterable, List+from typing import Any, Iterable, Iterator, Listfrom openjarvis.core.paths import get_config_dirfrom openjarvis.core.registry import FactStoreRegistry+if sys.platform != "win32":+ import fcntl+++_WINDOWS_LOCK_RETRY_SECONDS = 0.01+_WINDOWS_LOCK_RETRY_ERRNOS = {errno.EACCES, errno.EAGAIN, errno.EDEADLK}+_WINDOWS_LOCK_RETRY_WINERRORS = {33} # ERROR_LOCK_VIOLATION+++def _lock_windows_blocking(+ file_obj: Any,+ *,+ locking_api: Any | None = None,+ retry_interval: float = _WINDOWS_LOCK_RETRY_SECONDS,+) -> None:+ """Acquire the first-byte Windows lock, retrying until it is available.++ ``msvcrt.LK_LOCK`` only retries a fixed number of times before raising.+ Use the non-blocking operation in our own retry loop so contention never+ turns an otherwise valid store operation into a spurious failure.+ """+ if locking_api is None:+ import msvcrt as locking_api++ while True:+ file_obj.seek(0)+ try:+ locking_api.locking(file_obj.fileno(), locking_api.LK_NBLCK, 1)+ return+ except OSError as exc:+ if (+ exc.errno not in _WINDOWS_LOCK_RETRY_ERRNOS+ and getattr(exc, "winerror", None) not in _WINDOWS_LOCK_RETRY_WINERRORS+ ):+ raise+ time.sleep(retry_interval)+++def _unlock_windows(file_obj: Any, *, locking_api: Any | None = None) -> None:+ if locking_api is None:+ import msvcrt as locking_api++ file_obj.seek(0)+ locking_api.locking(file_obj.fileno(), locking_api.LK_UNLCK, 1)+def _default_fact_path() -> Path:"""Return the env-aware default JSONL path for automatic memory facts."""return get_config_dir() / "memory_facts.jsonl"+@contextmanager+def _cross_process_lock(lock_path: Path) -> Iterator[None]:+ """Hold an exclusive, blocking OS-level lock on *lock_path*.++ ``threading.Lock`` only serializes within one process, so it cannot+ protect the load -> modify -> flush cycle when two separate+ ``LocalFactStore`` instances (e.g. in different processes) touch the+ same file (#755). This blocks until the lock is free rather than+ failing fast, since callers want to wait their turn, not error out.+ """+ lock_path.parent.mkdir(parents=True, exist_ok=True)+ f = open(lock_path, "a+b")+ try:+ if sys.platform == "win32":+ # msvcrt locks a byte range; the file needs at least one byte+ # for the region at offset 0 to be lockable.+ f.seek(0, os.SEEK_END)+ if f.tell() == 0:+ f.write(b"\0")+ f.flush()+ _lock_windows_blocking(f)+ else:+ fcntl.flock(f.fileno(), fcntl.LOCK_EX)+ try:+ yield+ finally:+ if sys.platform == "win32":+ _unlock_windows(f)+ else:+ fcntl.flock(f.fileno(), fcntl.LOCK_UN)+ finally:+ f.close()++@dataclass(slots=True)class Fact:"""A single durable memory entry."""@@ -120,12 +203,30 @@ class LocalFactStore(FactStore):def _flush(self) -> None:"""Atomically rewrite the JSONL file from the in-memory list."""self._path.parent.mkdir(parents=True, exist_ok=True)- tmp = self._path.with_suffix(self._path.suffix + ".tmp")payload = "".join(json.dumps(asdict(f), ensure_ascii=False) + "\n" for f in self._facts)- tmp.write_text(payload, encoding="utf-8")- os.replace(tmp, self._path)+ # A fixed temp filename lets two concurrent writers interleave into+ # the same temp file before either os.replace() runs (#755); a+ # unique name per flush avoids that collision entirely.+ fd, tmp_name = tempfile.mkstemp(+ dir=str(self._path.parent),+ prefix=self._path.name + ".",+ suffix=".tmp",+ )+ try:+ with os.fdopen(fd, "w", encoding="utf-8") as tmp_f:+ tmp_f.write(payload)+ os.replace(tmp_name, self._path)+ except BaseException:+ try:+ os.unlink(tmp_name)+ except OSError:+ pass+ raise++ def _lock_path(self) -> Path:+ return self._path.with_suffix(self._path.suffix + ".lock")def _sync_from_disk_locked(self) -> None:"""Refresh in-memory facts from disk while holding ``self._lock``."""@@ -137,7 +238,7 @@ class LocalFactStore(FactStore):text = (text or "").strip()if not text:return False- with self._lock:+ with self._lock, _cross_process_lock(self._lock_path()):self._sync_from_disk_locked()lowered = text.lower()if any(f.text.lower() == lowered for f in self._facts):@@ -155,7 +256,7 @@ class LocalFactStore(FactStore):return list(self._facts)def clear(self) -> int:- with self._lock:+ with self._lock, _cross_process_lock(self._lock_path()):self._sync_from_disk_locked()removed = len(self._facts)self._facts = []diff --git a/tests/memory/test_fact_store.py b/tests/memory/test_fact_store.pyindex 149af13e..fc054732 100644--- a/tests/memory/test_fact_store.py+++ b/tests/memory/test_fact_store.py@@ -3,17 +3,47 @@from __future__ import annotationsimport json+import multiprocessing+import threading+import timeimport pytestfrom openjarvis.core.registry import FactStoreRegistryfrom openjarvis.memory.store import (LocalFactStore,+ _cross_process_lock,+ _lock_windows_blocking,create_fact_store,load_configured_facts,)+def _add_facts_in_process(+ path: str,+ worker_id: int,+ count: int,+ start: multiprocessing.synchronize.Event,+) -> None:+ start.wait()+ store = LocalFactStore(path)+ for index in range(count):+ store.add(f"process-{worker_id}-{index}")+++def _mutate_facts_in_process(+ path: str,+ operation: str,+ ready: multiprocessing.synchronize.Event,+) -> None:+ store = LocalFactStore(path)+ ready.set()+ if operation == "clear":+ store.clear()+ else:+ store.add("new fact")++def test_add_and_list(tmp_path):store = LocalFactStore(tmp_path / "facts.jsonl")assert store.add("User prefers concise answers") is True@@ -93,6 +123,150 @@ def test_external_clear_does_not_resurrect_stale_facts(tmp_path):assert [f.text for f in LocalFactStore(path).list()] == ["new fact"]+def test_concurrent_instances_do_not_lose_writes(tmp_path):+ """Regression for #755: independent store instances (simulating two+ processes) racing on the same file must not lose writes.++ ``threading.Lock`` only serializes within one process/instance, so it+ does nothing to protect the read (load) -> modify -> write (flush)+ cycle across separate ``LocalFactStore`` instances, and a fixed temp+ filename lets concurrent flushes interleave into the same temp file+ before either ``os.replace()`` runs. Hammer the same file from several+ instances at once and confirm every write survives.+ """+ path = tmp_path / "facts.jsonl"+ n_workers = 8+ n_per_worker = 15++ def worker(worker_id: int) -> None:+ store = LocalFactStore(path)+ for j in range(n_per_worker):+ store.add(f"fact-{worker_id}-{j}")++ threads = [threading.Thread(target=worker, args=(i,)) for i in range(n_workers)]+ for t in threads:+ t.start()+ for t in threads:+ t.join(timeout=30)+ assert not t.is_alive(), "worker thread hung"++ final = LocalFactStore(path)+ assert final.count() == n_workers * n_per_worker+++def test_multiprocess_instances_do_not_lose_writes(tmp_path):+ """Separate spawned interpreters must serialize their update cycles."""+ path = tmp_path / "multiprocess-facts.jsonl"+ context = multiprocessing.get_context("spawn")+ start = context.Event()+ n_workers = 4+ n_per_worker = 10+ processes = [+ context.Process(+ target=_add_facts_in_process,+ args=(str(path), worker_id, n_per_worker, start),+ )+ for worker_id in range(n_workers)+ ]++ for process in processes:+ process.start()+ start.set()+ for process in processes:+ process.join(timeout=20)+ assert process.exitcode == 0++ assert LocalFactStore(path).count() == n_workers * n_per_worker+++def test_multiprocess_clear_and_add_are_serialized(tmp_path):+ """A clear racing an add must not corrupt or resurrect pre-clear facts."""+ path = tmp_path / "clear-race.jsonl"+ LocalFactStore(path).add("old fact")+ context = multiprocessing.get_context("spawn")+ clear_ready = context.Event()+ add_ready = context.Event()+ clear_process = context.Process(+ target=_mutate_facts_in_process,+ args=(str(path), "clear", clear_ready),+ )+ add_process = context.Process(+ target=_mutate_facts_in_process,+ args=(str(path), "add", add_ready),+ )++ # Hold the actual OS lock until both processes are contending for it.+ # Releasing it lets the operations run in either valid serialized order.+ lock_path = path.with_suffix(path.suffix + ".lock")+ with _cross_process_lock(lock_path):+ clear_process.start()+ add_process.start()+ assert clear_ready.wait(timeout=10)+ assert add_ready.wait(timeout=10)+ time.sleep(0.1)+ assert clear_process.is_alive()+ assert add_process.is_alive()++ clear_process.join(timeout=20)+ add_process.join(timeout=20)+ assert clear_process.exitcode == 0+ assert add_process.exitcode == 0++ final_texts = [fact.text for fact in LocalFactStore(path).list()]+ assert "old fact" not in final_texts+ assert final_texts in ([], ["new fact"])+++def test_windows_lock_retries_transient_contention(tmp_path):+ """Simulate the Windows CRT's lock violation before eventual success."""++ class ContendedWindowsLock:+ LK_NBLCK = 1++ def __init__(self) -> None:+ self.calls = 0++ def locking(self, _fd: int, mode: int, nbytes: int) -> None:+ assert mode == self.LK_NBLCK+ assert nbytes == 1+ self.calls += 1+ if self.calls < 4:+ raise OSError(13, "permission denied")++ locking_api = ContendedWindowsLock()+ lock_path = tmp_path / "simulated-windows.lock"+ with lock_path.open("w+b") as lock_file:+ lock_file.write(b"\0")+ lock_file.flush()+ _lock_windows_blocking(+ lock_file,+ locking_api=locking_api,+ retry_interval=0,+ )++ assert locking_api.calls == 4+++def test_windows_lock_does_not_hide_unexpected_errors(tmp_path):+ class BrokenWindowsLock:+ LK_NBLCK = 1++ @staticmethod+ def locking(_fd: int, _mode: int, _nbytes: int) -> None:+ raise OSError(22, "invalid argument")++ lock_path = tmp_path / "broken-windows.lock"+ with lock_path.open("w+b") as lock_file:+ lock_file.write(b"\0")+ lock_file.flush()+ with pytest.raises(OSError, match="invalid argument"):+ _lock_windows_blocking(+ lock_file,+ locking_api=BrokenWindowsLock(),+ retry_interval=0,+ )++def test_load_skips_malformed_lines(tmp_path):path = tmp_path / "facts.jsonl"path.write_text(