Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
"""Stable mirroring for Hermes native MEMORY.md / USER.md entries.
Hermes' built-in memory tool selects entries by text and reports their full
previous content after a committed write. OpenViking needs an exact ``viking://``
file URI to update or delete a memory safely. This module keeps the identity map
in a small profile-scoped registry and serializes mirror operations through one
FIFO worker.
The registry is intentionally narrow: it tracks only memories created through
the built-in-memory mirror. Session-extracted memories and explicit
``viking_remember`` writes are outside its ownership and are never guessed at or
deleted by similarity.
"""
from __future__ import annotations
import hashlib
import json
import logging
import queue
import threading
import time
from pathlib import Path
from typing import Any, Dict, Optional
from agent.memory_provider import spawn_context_thread
from utils import atomic_json_write
logger = logging.getLogger("plugins.memory.openviking")
_REGISTRY_VERSION = 2
_REGISTRY_RELATIVE_PATH = Path("openviking") / "memory_mirror_registry.json"
_SUPPORTED_ACTIONS = frozenset({"add", "replace", "remove"})
_POLL_SECONDS = 0.05
_REGISTRY_LOCKS_GUARD = threading.Lock()
_REGISTRY_LOCKS: Dict[Path, threading.Lock] = {}
class _MappingError(RuntimeError):
"""A destructive mirror operation could not resolve one exact URI."""
def _registry_lock(path: Path) -> threading.Lock:
"""Return one process-wide lock for a profile mirror registry."""
key = path.resolve(strict=False)
with _REGISTRY_LOCKS_GUARD:
lock = _REGISTRY_LOCKS.get(key)
if lock is None:
lock = threading.Lock()
_REGISTRY_LOCKS[key] = lock
return lock
def _connection_fingerprint(client: Any) -> str:
"""Identify the captured OpenViking connection without storing its key."""
identity = [
str(getattr(client, "_endpoint", "") or "").rstrip("/"),
str(getattr(client, "_api_key", "") or ""),
str(getattr(client, "_account", "") or ""),
str(getattr(client, "_user", "") or ""),
str(getattr(client, "_agent", "") or ""),
]
encoded = json.dumps(identity, ensure_ascii=True, separators=(",", ":")).encode()
return hashlib.sha256(encoded).hexdigest()
class NativeMemoryMirror:
"""FIFO, profile-scoped mirror of Hermes native memory into OpenViking."""
def __init__(self, provider: Any):
self._provider = provider
self._queue: queue.Queue[Dict[str, Any]] = queue.Queue()
self._state_lock = threading.Lock()
self._worker: Optional[threading.Thread] = None
self._shutting_down = False
def enqueue(
self,
action: str,
target: str,
content: str,
*,
metadata: Optional[Dict[str, Any]] = None,
subdir: str,
client: Any,
) -> None:
"""Queue one committed built-in-memory mutation in call order."""
if action not in _SUPPORTED_ACTIONS:
return
normalized_content = str(content or "").strip()
if action in {"add", "replace"} and not normalized_content:
return
event = {
"action": action,
"target": str(target or "memory"),
# The native MemoryStore strips content before it commits. Mirror
# the committed value rather than the raw tool argument.
"content": normalized_content,
"metadata": dict(metadata or {}),
"subdir": str(subdir),
"client": client,
"connection": _connection_fingerprint(client),
}
with self._state_lock:
if self._shutting_down:
logger.warning(
"OpenViking memory mirror skipped %s during provider shutdown",
action,
)
return
self._queue.put(event)
if self._worker is None or not self._worker.is_alive():
self._worker = spawn_context_thread(
self._run,
daemon=True,
name="openviking-memory-mirror",
)
self._worker.start()
def shutdown(self, timeout: float = 5.0) -> None:
"""Drain queued mirror operations, then stop the FIFO worker."""
with self._state_lock:
self._shutting_down = True
worker = self._worker
if worker is None:
return
deadline = time.monotonic() + max(0.0, timeout)
while self._queue.unfinished_tasks and time.monotonic() < deadline:
time.sleep(0.01)
remaining = max(0.0, deadline - time.monotonic())
if worker.is_alive() and remaining:
worker.join(timeout=remaining)
if worker.is_alive():
logger.warning("OpenViking memory mirror worker did not drain before shutdown")
def _run(self) -> None:
while True:
with self._state_lock:
stopping = self._shutting_down
if stopping and self._queue.empty():
return
try:
event = self._queue.get(timeout=_POLL_SECONDS)
except queue.Empty:
continue
try:
self._apply(event)
except _MappingError as exc:
logger.warning("OpenViking memory mirror skipped: %s", exc)
except Exception as exc:
logger.warning("OpenViking memory mirror failed: %s", exc)
finally:
self._queue.task_done()
def _registry_path(self) -> Path:
root = str(getattr(self._provider, "_hermes_home", "") or "").strip()
if not root:
from hermes_constants import get_hermes_home
root = str(get_hermes_home())
return Path(root) / _REGISTRY_RELATIVE_PATH
def _load_registry(self, path: Path) -> Dict[str, Any]:
if not path.exists():
return {"version": _REGISTRY_VERSION, "entries": []}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception as exc:
raise RuntimeError(f"cannot read mirror registry {path}: {exc}") from exc
if not isinstance(payload, dict):
raise RuntimeError(f"invalid mirror registry format: expected a JSON object: {path}")
if "version" not in payload:
raise RuntimeError(f"invalid mirror registry format: missing version: {path}")
found_version = payload["version"]
if found_version != _REGISTRY_VERSION:
raise RuntimeError(
"unsupported mirror registry version "
f"{found_version!r}: expected {_REGISTRY_VERSION}: {path}"
)
entries = payload.get("entries")
if not isinstance(entries, list):
raise RuntimeError(f"invalid mirror registry entries: {path}")
for entry in entries:
if not isinstance(entry, dict):
raise RuntimeError(f"invalid mirror registry entry: {path}")
if not all(
isinstance(entry.get(key), str)
for key in ("connection", "target", "uri", "content")
):
raise RuntimeError(f"invalid mirror registry entry fields: {path}")
return payload
@staticmethod
def _save_registry(path: Path, registry: Dict[str, Any]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
atomic_json_write(path, registry, mode=0o600)
@staticmethod
def _resolve_mapping(
registry: Dict[str, Any],
*,
connection: str,
target: str,
previous_content: Any,
action: str,
) -> tuple[int, Dict[str, str]]:
if not isinstance(previous_content, str) or not previous_content:
raise _MappingError(
f"{action} requires authoritative previous_content from Hermes; "
"upgrade Hermes to a version with committed-entry metadata; "
"leaving OpenViking unchanged"
)
matches = [
(index, entry)
for index, entry in enumerate(registry["entries"])
if entry["connection"] == connection
and entry["target"] == target
and previous_content == entry["content"]
]
if not matches:
raise _MappingError(
f"{action} has no stable OpenViking URI mapping for target={target!r}; "
"leaving OpenViking unchanged"
)
if len(matches) != 1:
raise _MappingError(
f"{action} matched {len(matches)} OpenViking URI mappings for "
f"target={target!r}; leaving OpenViking unchanged"
)
return matches[0]
@staticmethod
def _warn_failed_indexing(result: Dict[str, Any], uri: str) -> None:
failed = [
name for name in ("semantic_status", "vector_status") if result.get(name) == "failed"
]
if failed:
logger.warning(
"OpenViking memory file updated at %s, but indexing failed (%s); "
"search results may be stale. The URI mapping was retained.",
uri,
", ".join(failed),
)
def _apply(self, event: Dict[str, Any]) -> None:
path = self._registry_path()
# Provider instances can share one profile and registry. Keep the
# complete remote-mutation + registry-update sequence atomic in this
# process so concurrent read-modify-write cycles cannot lose mappings.
with _registry_lock(path):
self._apply_locked(path, event)
def _apply_locked(self, path: Path, event: Dict[str, Any]) -> None:
action = event["action"]
target = event["target"]
content = event["content"]
metadata = event["metadata"]
connection = event["connection"]
registry = self._load_registry(path)
client = event["client"]
if action == "add":
# The native store treats exact duplicate adds as idempotent. Mirror
# the same way if a duplicate notification ever reaches us.
if any(
entry["connection"] == connection
and entry["target"] == target
and entry["content"] == content
for entry in registry["entries"]
):
return
requested_uri = self._provider._build_memory_uri(
event["subdir"],
client=client,
require_confirmed_user=True,
)
response = client.post(
"/api/v1/content/write",
{"uri": requested_uri, "content": content, "mode": "create"},
)
result = response.get("result", {}) if isinstance(response, dict) else {}
if isinstance(result, dict) and result.get("content_updated") is False:
raise RuntimeError("OpenViking memory file was not updated")
canonical_uri = (
str(result.get("uri") or "").strip() if isinstance(result, dict) else ""
) or requested_uri
registry["entries"].append(
{
"connection": connection,
"target": target,
"uri": canonical_uri,
"content": content,
}
)
self._save_registry(path, registry)
return
index, mapping = self._resolve_mapping(
registry,
connection=connection,
target=target,
previous_content=metadata.get("previous_content"),
action=action,
)
uri = mapping["uri"]
if action == "replace":
response = client.post(
"/api/v1/content/write",
{"uri": uri, "content": content, "mode": "replace", "wait": True},
)
result = response.get("result", {}) if isinstance(response, dict) else {}
if isinstance(result, dict) and result.get("content_updated") is False:
raise RuntimeError("OpenViking memory file was not updated")
registry["entries"][index] = {
"connection": connection,
"target": target,
"uri": uri,
"content": content,
}
self._save_registry(path, registry)
if isinstance(result, dict):
self._warn_failed_indexing(result, uri)
return
client.delete(
"/api/v1/fs",
params={"uri": uri, "recursive": False, "wait": True},
)
registry["entries"].pop(index)
self._save_registry(path, registry)
_MIRROR_ATTR = "_native_memory_mirror"
def enqueue_native_memory_write(
provider: Any,
action: str,
target: str,
content: str,
*,
metadata: Optional[Dict[str, Any]] = None,
subdir: str,
client: Any,
) -> None:
"""Lazily create the provider's FIFO mirror and enqueue one operation."""
with provider._native_memory_mirror_lock:
if provider._shutting_down:
logger.warning(
"OpenViking memory mirror skipped %s during provider shutdown",
action,
)
return
mirror = getattr(provider, _MIRROR_ATTR, None)
if mirror is None:
mirror = NativeMemoryMirror(provider)
setattr(provider, _MIRROR_ATTR, mirror)
mirror.enqueue(
action,
target,
content,
metadata=metadata,
subdir=subdir,
client=client,
)
def shutdown_native_memory_mirror(provider: Any, timeout: float = 5.0) -> None:
"""Drain the provider's native-memory mirror if it was ever used."""
with provider._native_memory_mirror_lock:
mirror = getattr(provider, _MIRROR_ATTR, None)
if mirror is not None:
mirror.shutdown(timeout=timeout)