Skip to content

Personal AI, On Personal Devices

Python

fix(memory): cross-process locking for LocalFactStore (#797)

havolline13e7eac70a69d
diff --git a/src/openjarvis/memory/store.py b/src/openjarvis/memory/store.py
index 2841043a..fbad20e5 100644
--- a/src/openjarvis/memory/store.py
+++ b/src/openjarvis/memory/store.py
@@ -9,24 +9,107 @@ caps the total number of facts, and is safe to call from multiple threads.
from __future__ import annotations
+import errno
import json
import os
+import sys
+import tempfile
import threading
import time
from abc import ABC, abstractmethod
+from contextlib import contextmanager
from dataclasses import asdict, dataclass
from pathlib import Path
-from typing import Any, Iterable, List
+from typing import Any, Iterable, Iterator, List
from openjarvis.core.paths import get_config_dir
from openjarvis.core.registry import FactStoreRegistry
+if sys.platform != "win32":
+ import fcntl
+
+
+_WINDOWS_LOCK_RETRY_SECONDS = 0.01
+_WINDOWS_LOCK_RETRY_ERRNOS = {errno.EACCES, errno.EAGAIN, errno.EDEADLK}
+_WINDOWS_LOCK_RETRY_WINERRORS = {33} # ERROR_LOCK_VIOLATION
+
+
+def _lock_windows_blocking(
+ file_obj: Any,
+ *,
+ locking_api: Any | None = None,
+ retry_interval: float = _WINDOWS_LOCK_RETRY_SECONDS,
+) -> None:
+ """Acquire the first-byte Windows lock, retrying until it is available.
+
+ ``msvcrt.LK_LOCK`` only retries a fixed number of times before raising.
+ Use the non-blocking operation in our own retry loop so contention never
+ turns an otherwise valid store operation into a spurious failure.
+ """
+ if locking_api is None:
+ import msvcrt as locking_api
+
+ while True:
+ file_obj.seek(0)
+ try:
+ locking_api.locking(file_obj.fileno(), locking_api.LK_NBLCK, 1)
+ return
+ except OSError as exc:
+ if (
+ exc.errno not in _WINDOWS_LOCK_RETRY_ERRNOS
+ and getattr(exc, "winerror", None) not in _WINDOWS_LOCK_RETRY_WINERRORS
+ ):
+ raise
+ time.sleep(retry_interval)
+
+
+def _unlock_windows(file_obj: Any, *, locking_api: Any | None = None) -> None:
+ if locking_api is None:
+ import msvcrt as locking_api
+
+ file_obj.seek(0)
+ locking_api.locking(file_obj.fileno(), locking_api.LK_UNLCK, 1)
+
def _default_fact_path() -> Path:
"""Return the env-aware default JSONL path for automatic memory facts."""
return get_config_dir() / "memory_facts.jsonl"
+@contextmanager
+def _cross_process_lock(lock_path: Path) -> Iterator[None]:
+ """Hold an exclusive, blocking OS-level lock on *lock_path*.
+
+ ``threading.Lock`` only serializes within one process, so it cannot
+ protect the load -> modify -> flush cycle when two separate
+ ``LocalFactStore`` instances (e.g. in different processes) touch the
+ same file (#755). This blocks until the lock is free rather than
+ failing fast, since callers want to wait their turn, not error out.
+ """
+ lock_path.parent.mkdir(parents=True, exist_ok=True)
+ f = open(lock_path, "a+b")
+ try:
+ if sys.platform == "win32":
+ # msvcrt locks a byte range; the file needs at least one byte
+ # for the region at offset 0 to be lockable.
+ f.seek(0, os.SEEK_END)
+ if f.tell() == 0:
+ f.write(b"\0")
+ f.flush()
+ _lock_windows_blocking(f)
+ else:
+ fcntl.flock(f.fileno(), fcntl.LOCK_EX)
+ try:
+ yield
+ finally:
+ if sys.platform == "win32":
+ _unlock_windows(f)
+ else:
+ fcntl.flock(f.fileno(), fcntl.LOCK_UN)
+ finally:
+ f.close()
+
+
@dataclass(slots=True)
class Fact:
"""A single durable memory entry."""
@@ -120,12 +203,30 @@ class LocalFactStore(FactStore):
def _flush(self) -> None:
"""Atomically rewrite the JSONL file from the in-memory list."""
self._path.parent.mkdir(parents=True, exist_ok=True)
- tmp = self._path.with_suffix(self._path.suffix + ".tmp")
payload = "".join(
json.dumps(asdict(f), ensure_ascii=False) + "\n" for f in self._facts
)
- tmp.write_text(payload, encoding="utf-8")
- os.replace(tmp, self._path)
+ # A fixed temp filename lets two concurrent writers interleave into
+ # the same temp file before either os.replace() runs (#755); a
+ # unique name per flush avoids that collision entirely.
+ fd, tmp_name = tempfile.mkstemp(
+ dir=str(self._path.parent),
+ prefix=self._path.name + ".",
+ suffix=".tmp",
+ )
+ try:
+ with os.fdopen(fd, "w", encoding="utf-8") as tmp_f:
+ tmp_f.write(payload)
+ os.replace(tmp_name, self._path)
+ except BaseException:
+ try:
+ os.unlink(tmp_name)
+ except OSError:
+ pass
+ raise
+
+ def _lock_path(self) -> Path:
+ return self._path.with_suffix(self._path.suffix + ".lock")
def _sync_from_disk_locked(self) -> None:
"""Refresh in-memory facts from disk while holding ``self._lock``."""
@@ -137,7 +238,7 @@ class LocalFactStore(FactStore):
text = (text or "").strip()
if not text:
return False
- with self._lock:
+ with self._lock, _cross_process_lock(self._lock_path()):
self._sync_from_disk_locked()
lowered = text.lower()
if any(f.text.lower() == lowered for f in self._facts):
@@ -155,7 +256,7 @@ class LocalFactStore(FactStore):
return list(self._facts)
def clear(self) -> int:
- with self._lock:
+ with self._lock, _cross_process_lock(self._lock_path()):
self._sync_from_disk_locked()
removed = len(self._facts)
self._facts = []
diff --git a/tests/memory/test_fact_store.py b/tests/memory/test_fact_store.py
index 149af13e..fc054732 100644
--- a/tests/memory/test_fact_store.py
+++ b/tests/memory/test_fact_store.py
@@ -3,17 +3,47 @@
from __future__ import annotations
import json
+import multiprocessing
+import threading
+import time
import pytest
from openjarvis.core.registry import FactStoreRegistry
from openjarvis.memory.store import (
LocalFactStore,
+ _cross_process_lock,
+ _lock_windows_blocking,
create_fact_store,
load_configured_facts,
)
+def _add_facts_in_process(
+ path: str,
+ worker_id: int,
+ count: int,
+ start: multiprocessing.synchronize.Event,
+) -> None:
+ start.wait()
+ store = LocalFactStore(path)
+ for index in range(count):
+ store.add(f"process-{worker_id}-{index}")
+
+
+def _mutate_facts_in_process(
+ path: str,
+ operation: str,
+ ready: multiprocessing.synchronize.Event,
+) -> None:
+ store = LocalFactStore(path)
+ ready.set()
+ if operation == "clear":
+ store.clear()
+ else:
+ store.add("new fact")
+
+
def test_add_and_list(tmp_path):
store = LocalFactStore(tmp_path / "facts.jsonl")
assert store.add("User prefers concise answers") is True
@@ -93,6 +123,150 @@ def test_external_clear_does_not_resurrect_stale_facts(tmp_path):
assert [f.text for f in LocalFactStore(path).list()] == ["new fact"]
+def test_concurrent_instances_do_not_lose_writes(tmp_path):
+ """Regression for #755: independent store instances (simulating two
+ processes) racing on the same file must not lose writes.
+
+ ``threading.Lock`` only serializes within one process/instance, so it
+ does nothing to protect the read (load) -> modify -> write (flush)
+ cycle across separate ``LocalFactStore`` instances, and a fixed temp
+ filename lets concurrent flushes interleave into the same temp file
+ before either ``os.replace()`` runs. Hammer the same file from several
+ instances at once and confirm every write survives.
+ """
+ path = tmp_path / "facts.jsonl"
+ n_workers = 8
+ n_per_worker = 15
+
+ def worker(worker_id: int) -> None:
+ store = LocalFactStore(path)
+ for j in range(n_per_worker):
+ store.add(f"fact-{worker_id}-{j}")
+
+ threads = [threading.Thread(target=worker, args=(i,)) for i in range(n_workers)]
+ for t in threads:
+ t.start()
+ for t in threads:
+ t.join(timeout=30)
+ assert not t.is_alive(), "worker thread hung"
+
+ final = LocalFactStore(path)
+ assert final.count() == n_workers * n_per_worker
+
+
+def test_multiprocess_instances_do_not_lose_writes(tmp_path):
+ """Separate spawned interpreters must serialize their update cycles."""
+ path = tmp_path / "multiprocess-facts.jsonl"
+ context = multiprocessing.get_context("spawn")
+ start = context.Event()
+ n_workers = 4
+ n_per_worker = 10
+ processes = [
+ context.Process(
+ target=_add_facts_in_process,
+ args=(str(path), worker_id, n_per_worker, start),
+ )
+ for worker_id in range(n_workers)
+ ]
+
+ for process in processes:
+ process.start()
+ start.set()
+ for process in processes:
+ process.join(timeout=20)
+ assert process.exitcode == 0
+
+ assert LocalFactStore(path).count() == n_workers * n_per_worker
+
+
+def test_multiprocess_clear_and_add_are_serialized(tmp_path):
+ """A clear racing an add must not corrupt or resurrect pre-clear facts."""
+ path = tmp_path / "clear-race.jsonl"
+ LocalFactStore(path).add("old fact")
+ context = multiprocessing.get_context("spawn")
+ clear_ready = context.Event()
+ add_ready = context.Event()
+ clear_process = context.Process(
+ target=_mutate_facts_in_process,
+ args=(str(path), "clear", clear_ready),
+ )
+ add_process = context.Process(
+ target=_mutate_facts_in_process,
+ args=(str(path), "add", add_ready),
+ )
+
+ # Hold the actual OS lock until both processes are contending for it.
+ # Releasing it lets the operations run in either valid serialized order.
+ lock_path = path.with_suffix(path.suffix + ".lock")
+ with _cross_process_lock(lock_path):
+ clear_process.start()
+ add_process.start()
+ assert clear_ready.wait(timeout=10)
+ assert add_ready.wait(timeout=10)
+ time.sleep(0.1)
+ assert clear_process.is_alive()
+ assert add_process.is_alive()
+
+ clear_process.join(timeout=20)
+ add_process.join(timeout=20)
+ assert clear_process.exitcode == 0
+ assert add_process.exitcode == 0
+
+ final_texts = [fact.text for fact in LocalFactStore(path).list()]
+ assert "old fact" not in final_texts
+ assert final_texts in ([], ["new fact"])
+
+
+def test_windows_lock_retries_transient_contention(tmp_path):
+ """Simulate the Windows CRT's lock violation before eventual success."""
+
+ class ContendedWindowsLock:
+ LK_NBLCK = 1
+
+ def __init__(self) -> None:
+ self.calls = 0
+
+ def locking(self, _fd: int, mode: int, nbytes: int) -> None:
+ assert mode == self.LK_NBLCK
+ assert nbytes == 1
+ self.calls += 1
+ if self.calls < 4:
+ raise OSError(13, "permission denied")
+
+ locking_api = ContendedWindowsLock()
+ lock_path = tmp_path / "simulated-windows.lock"
+ with lock_path.open("w+b") as lock_file:
+ lock_file.write(b"\0")
+ lock_file.flush()
+ _lock_windows_blocking(
+ lock_file,
+ locking_api=locking_api,
+ retry_interval=0,
+ )
+
+ assert locking_api.calls == 4
+
+
+def test_windows_lock_does_not_hide_unexpected_errors(tmp_path):
+ class BrokenWindowsLock:
+ LK_NBLCK = 1
+
+ @staticmethod
+ def locking(_fd: int, _mode: int, _nbytes: int) -> None:
+ raise OSError(22, "invalid argument")
+
+ lock_path = tmp_path / "broken-windows.lock"
+ with lock_path.open("w+b") as lock_file:
+ lock_file.write(b"\0")
+ lock_file.flush()
+ with pytest.raises(OSError, match="invalid argument"):
+ _lock_windows_blocking(
+ lock_file,
+ locking_api=BrokenWindowsLock(),
+ retry_interval=0,
+ )
+
+
def test_load_skips_malformed_lines(tmp_path):
path = tmp_path / "facts.jsonl"
path.write_text(