Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(server): refresh read-replica API key index via store watcher (#3857)
dingben9097fef4783d
diff --git a/openviking/server/api_keys/legacy.py b/openviking/server/api_keys/legacy.pyindex 89eaf0818..a045f6f44 100644--- a/openviking/server/api_keys/legacy.py+++ b/openviking/server/api_keys/legacy.py@@ -2,6 +2,7 @@# SPDX-License-Identifier: AGPL-3.0"""Legacy API Key management (original implementation)."""+import asyncioimport fnmatchimport hashlibimport hmac@@ -81,6 +82,8 @@ class LegacyAPIKeyManager:self._accounts: Dict[str, AccountInfo] = {}# Prefix index: key_prefix -> list[UserKeyEntry]self._prefix_index: Dict[str, list[UserKeyEntry]] = {}+ # Serializes reload() so overlapping refreshes can't interleave.+ self._reload_lock = asyncio.Lock()def _discard_account_state(self, account_id: str) -> None:"""Remove an account and its key index entries from in-memory state."""@@ -117,7 +120,7 @@ class LegacyAPIKeyManager:logger.exception("Failed to persist rollback for account %s", account_id)async def load(self) -> None:- """Load accounts and user keys from VikingFS into memory."""+ """Load keys into memory (writable startup path; migrates plaintext; see reload())."""accounts_data = await self._read_json(ACCOUNTS_PATH)if accounts_data is None:# First run: create default account@@ -125,65 +128,123 @@ class LegacyAPIKeyManager:accounts_data = {"accounts": {"default": {"created_at": now}}}await self._write_json(ACCOUNTS_PATH, accounts_data)+ accounts, prefix_index = await self._build_state(accounts_data, allow_migration=True)+ self._accounts = accounts+ self._prefix_index = prefix_index++ logger.info(+ "LegacyAPIKeyManager loaded: %d accounts, %d user keys",+ len(self._accounts),+ sum(len(info.users) for info in self._accounts.values()),+ )++ async def reload(self) -> None:+ """Read-only refresh: re-read store and atomically swap state (never writes/migrates)."""+ async with self._reload_lock:+ accounts_data = await self._read_json(ACCOUNTS_PATH)+ if accounts_data is None:+ # Store not initialized yet (reader started before writer): keep state.+ return++ accounts, prefix_index = await self._build_state(accounts_data, allow_migration=False)+ # Atomic swap: rebind so readers never observe a half-built index.+ self._accounts = accounts+ self._prefix_index = prefix_index++ logger.debug(+ "LegacyAPIKeyManager reloaded: %d accounts, %d user keys",+ len(self._accounts),+ sum(len(info.users) for info in self._accounts.values()),+ )++ async def _build_state(+ self, accounts_data: dict, *, allow_migration: bool+ ) -> tuple[Dict[str, AccountInfo], Dict[str, list[UserKeyEntry]]]:+ """Build fresh (accounts, prefix_index) state; migrate plaintext only if allow_migration."""+ accounts: Dict[str, AccountInfo] = {}+ prefix_index: Dict[str, list[UserKeyEntry]] = {}+for account_id, info in accounts_data.get("accounts", {}).items():users_path = USERS_PATH_TEMPLATE.format(account_id=account_id)users_data = await self._read_json(users_path)users = users_data.get("users", {}) if users_data else {}- self._accounts[account_id] = AccountInfo(+ accounts[account_id] = AccountInfo(created_at=info.get("created_at", ""),users=users,)for user_id, user_info in users.items():key_or_hash = user_info.get("key", "")- if key_or_hash:- # Check if it's a hashed key- if key_or_hash.startswith("$argon2"):- # Already hashed- stored_key = key_or_hash- is_hashed = True- key_prefix = user_info.get("key_prefix", "")- else:- # Plaintext key- if self._api_key_hashing_enabled:- # If API key hashing enabled, migrate to hashed- stored_key = self._hash_api_key(key_or_hash)- is_hashed = True- key_prefix = self._get_key_prefix(key_or_hash)- # Update storage- user_info["key"] = stored_key- user_info["key_prefix"] = key_prefix- await self._save_users_json(account_id)- logger.info(- "Migrated API key for user %s in account %s", user_id, account_id- )- else:- # If API key hashing not enabled, keep as plaintext- stored_key = key_or_hash- is_hashed = False- # For plaintext keys, compute prefix on the fly for indexing- key_prefix = self._get_key_prefix(key_or_hash)-- entry = UserKeyEntry(- account_id=account_id,- user_id=user_id,- role=Role(user_info.get("role", "user")),- key_or_hash=stored_key,- is_hashed=is_hashed,- )+ if not key_or_hash:+ continue++ if key_or_hash.startswith("$argon2"):+ # Already hashed+ stored_key = key_or_hash+ is_hashed = True+ key_prefix = user_info.get("key_prefix", "")+ elif self._api_key_hashing_enabled and allow_migration:+ # Migrate plaintext to hashed and persist.+ stored_key = self._hash_api_key(key_or_hash)+ is_hashed = True+ key_prefix = self._get_key_prefix(key_or_hash)+ user_info["key"] = stored_key+ user_info["key_prefix"] = key_prefix+ await self._save_users_json_for(account_id, accounts)+ logger.info("Migrated API key for user %s in account %s", user_id, account_id)+ else:+ # Keep plaintext (hashing off or read-only refresh); prefix on the fly.+ stored_key = key_or_hash+ is_hashed = False+ key_prefix = self._get_key_prefix(key_or_hash)++ entry = UserKeyEntry(+ account_id=account_id,+ user_id=user_id,+ role=Role(user_info.get("role", "user")),+ key_or_hash=stored_key,+ is_hashed=is_hashed,+ )++ # Add to prefix index+ if key_prefix:+ if key_prefix not in prefix_index:+ prefix_index[key_prefix] = []+ prefix_index[key_prefix].append(entry)++ return accounts, prefix_index++ async def compute_store_signature(self) -> tuple:+ """Return a cheap (path, size, modTime) signature over accounts.json + all users.json."""+ signature: list[tuple] = []- # Add to prefix index- if key_prefix:- if key_prefix not in self._prefix_index:- self._prefix_index[key_prefix] = []- self._prefix_index[key_prefix].append(entry)+ accounts_data = await self._read_json(ACCOUNTS_PATH)+ signature.append(await self._stat_signature(ACCOUNTS_PATH))- logger.info(- "LegacyAPIKeyManager loaded: %d accounts, %d user keys",- len(self._accounts),- sum(len(info.users) for info in self._accounts.values()),- )+ if accounts_data:+ for account_id in accounts_data.get("accounts", {}):+ users_path = USERS_PATH_TEMPLATE.format(account_id=account_id)+ signature.append(await self._stat_signature(users_path))++ return tuple(signature)++ async def _stat_signature(self, path: str) -> tuple:+ """Return a (path, size, mod_time) tuple for one file; missing/error yields a sentinel."""+ try:+ info = await self._async_agfs.stat(path)+ except AGFSNotFoundError:+ return (path, None, None)+ except Exception:+ logger.debug("Failed to stat %s for key-store signature", path, exc_info=True)+ return (path, None, None)++ if not isinstance(info, dict):+ return (path, None, None)++ size = info.get("size")+ mod_time = info.get("modTime", info.get("mod_time", info.get("mtime")))+ return (path, size, mod_time)def resolve(self, api_key: str) -> ResolvedIdentity:"""Resolve an API key to identity. Sequential matching: root key first, then user key index."""@@ -701,6 +762,10 @@ class LegacyAPIKeyManager:async def _save_users_json(self, account_id: str) -> None:"""Persist a single account's user registry."""+ await self._save_users_json_for(account_id, self._accounts)++ async def _save_users_json_for(self, account_id: str, accounts: Dict[str, AccountInfo]) -> None:+ """Persist one account's user registry from the given accounts map."""path = USERS_PATH_TEMPLATE.format(account_id=account_id)try:lease = await self._async_agfs.pathlock_acquire_exact(path, timeout_secs=10.0)@@ -711,7 +776,7 @@ class LegacyAPIKeyManager:conflict_type="user_registry_busy",) from exctry:- account = self._accounts.get(account_id)+ account = accounts.get(account_id)if account is None:returnawait self._write_json(path, {"users": account.users}, lease_ref=lease)diff --git a/openviking/server/api_keys/new.py b/openviking/server/api_keys/new.pyindex 1aed199be..f8221697f 100644--- a/openviking/server/api_keys/new.py+++ b/openviking/server/api_keys/new.py@@ -113,6 +113,14 @@ class NewAPIKeyManager:sum(len(info.users) for info in self._legacy._accounts.values()),)+ async def reload(self) -> None:+ """Read-only refresh of in-memory key state; delegates to the legacy manager."""+ await self._legacy.reload()++ async def compute_store_signature(self) -> tuple:+ """Return a cheap change signature for the on-disk key store."""+ return await self._legacy.compute_store_signature()+def resolve(self, api_key: str) -> ResolvedIdentity:"""Resolve an API key to identity.diff --git a/openviking/server/app.py b/openviking/server/app.pyindex 548b24f5a..c09b81c32 100644--- a/openviking/server/app.py+++ b/openviking/server/app.py@@ -346,6 +346,12 @@ def create_app(await shutdown_usage_audit(app=app)await shutdown_metrics_async(app=app)task_tracker.stop_cleanup_loop()+ auth_plugin_state = getattr(app.state, "auth_plugin", None)+ if auth_plugin_state is not None:+ try:+ await auth_plugin_state.shutdown()+ except Exception as e: # noqa: BLE001+ logger.warning("Auth plugin shutdown failed: %s", e)if oauth_gc_task is not None:oauth_gc_task.cancel()try:diff --git a/openviking/server/auth/plugin.py b/openviking/server/auth/plugin.pyindex f483336f3..4585393d8 100644--- a/openviking/server/auth/plugin.py+++ b/openviking/server/auth/plugin.py@@ -113,6 +113,10 @@ class AuthPlugin(abc.ABC):"""return True+ async def shutdown(self) -> None:+ """Release resources acquired during ``initialize`` (default no-op; override to cancel)."""+ pass+def can_skip_api_key_for_bot_proxy(self) -> bool:"""Whether the bot proxy may skip API key validation.diff --git a/openviking/server/auth/plugins/api_key.py b/openviking/server/auth/plugins/api_key.pyindex 3acc05c5d..126357aee 100644--- a/openviking/server/auth/plugins/api_key.py+++ b/openviking/server/auth/plugins/api_key.py@@ -4,6 +4,7 @@from __future__ import annotations+import asyncioimport sysfrom typing import Optional@@ -13,6 +14,9 @@ from openviking.server.api_keys import APIKeyManagerfrom openviking.server.auth.plugin import AuthPluginfrom openviking.server.identity import ResolvedIdentity, Rolefrom openviking_cli.exceptions import PermissionDeniedError, UnauthenticatedError+from openviking_cli.utils import get_logger++_watch_logger = get_logger(__name__)_API_KEY_ROOT_ALLOWED_PATHS = {"/api/v1/system/status",@@ -57,6 +61,10 @@ class ApiKeyAuthPlugin(AuthPlugin):auth_mode = "api_key"+ def __init__(self) -> None:+ # Background key-store poll task; None when the watcher is disabled.+ self._watch_task: Optional[asyncio.Task] = None+async def resolve_identity(self,request: Request,@@ -204,6 +212,58 @@ class ApiKeyAuthPlugin(AuthPlugin):await api_key_manager.load()app.state.api_key_manager = api_key_manager+ # Read replicas never rewrite the store, so poll for writer-side changes.+ if getattr(config, "api_key_watch_enabled", False):+ interval = getattr(config, "api_key_watch_interval_seconds", 30.0)+ self._watch_task = asyncio.create_task(self._watch_key_store(api_key_manager, interval))+ _watch_logger.info("API key store watcher started (interval=%.1fs)", interval)++ async def shutdown(self) -> None:+ """Cancel the background key-store watcher, if running."""+ if self._watch_task is None:+ return+ self._watch_task.cancel()+ try:+ await self._watch_task+ except asyncio.CancelledError:+ pass+ except Exception: # noqa: BLE001+ _watch_logger.warning("API key store watcher exited with error", exc_info=True)+ finally:+ self._watch_task = None++ @staticmethod+ async def _watch_key_store(api_key_manager: APIKeyManager, interval: float) -> None:+ """Poll the key store and reload on signature change; errors are swallowed to survive."""+ # Guard against a misconfigured non-positive interval spinning hot.+ interval = interval if interval and interval > 0 else 30.0+ try:+ last_signature = await api_key_manager.compute_store_signature()+ except Exception: # noqa: BLE001+ _watch_logger.warning(+ "Initial key-store signature failed; watcher will retry", exc_info=True+ )+ last_signature = None++ while True:+ await asyncio.sleep(interval)+ try:+ signature = await api_key_manager.compute_store_signature()+ except Exception: # noqa: BLE001+ _watch_logger.debug("Key-store signature check failed", exc_info=True)+ continue++ if signature == last_signature:+ continue++ try:+ await api_key_manager.reload()+ last_signature = signature+ _watch_logger.info("API key store change detected; in-memory index reloaded")+ except Exception: # noqa: BLE001+ # Keep last_signature unchanged so we retry the reload next tick.+ _watch_logger.warning("API key store reload failed; will retry", exc_info=True)+def get_request_context_checks(self,path: str,diff --git a/openviking/server/config.py b/openviking/server/config.pyindex 74deb5257..3eb6dfd70 100644--- a/openviking/server/config.py+++ b/openviking/server/config.py@@ -290,6 +290,11 @@ class ServerConfig(BaseModel):bot_api_url: str = "http://localhost:18790" # Vikingbot OpenAPIChannel URL (default port)encryption_enabled: bool = False # Whether file-level AES encryption is enabledapi_key_hashing_enabled: bool = False # Whether API key Argon2id hashing is enabled (default: false, rely on file encryption)+ # When true, poll the shared key store and reload the in-memory index on change so+ # read replicas pick up writer-side user add/rotate/remove. Default off (single writer).+ api_key_watch_enabled: bool = False+ # Poll interval; each check only stats registry files and reads fully on change.+ api_key_watch_interval_seconds: float = 30.0observability: ObservabilityConfig = Field(default_factory=ObservabilityConfig)usage_reporter: UsageReporterConfig = Field(default_factory=UsageReporterConfig)# Public-facing base URL emitted in MCP-issued upload instructions. Seediff --git a/tests/server/test_api_key_manager.py b/tests/server/test_api_key_manager.pyindex f023545be..bb6b1d178 100644--- a/tests/server/test_api_key_manager.py+++ b/tests/server/test_api_key_manager.py@@ -890,3 +890,154 @@ def test_new_api_key_manager_public_api_parity_with_legacy():f"NewAPIKeyManager is missing public methods present on "f"LegacyAPIKeyManager: {sorted(missing)}")+++# ---- Read-only reload / change-signature tests ----+#+# Read replicas load the key store once at startup and never rewrite it. A user+# registered / rotated / removed on the writer is therefore invisible to a+# reader until it refreshes its in-memory index. reload() + compute_store_+# signature() give the optional background watcher a read-only way to converge.+++async def test_reload_picks_up_user_registered_by_writer(manager_service):+ """A reader's reload() should see a user the writer registered afterwards.++ This reproduces the read-replica auth failure: two managers over the same+ store, a user created via the "writer", initially rejected by the "reader",+ then accepted after reload().+ """+ writer = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await writer.load()+ reader = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await reader.load()++ acct = _uid()+ await writer.create_account(acct, "alice")+ user_key = await writer.register_user(acct, "bob")++ # Reader has not refreshed yet: the new user's key is unknown.+ with pytest.raises(UnauthenticatedError):+ reader.resolve(user_key)++ await reader.reload()++ identity = reader.resolve(user_key)+ assert identity.account_id == acct+ assert identity.user_id == "bob"+++async def test_reload_reflects_removed_user(manager_service):+ """reload() should drop a user the writer removed after the reader loaded."""+ writer = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await writer.load()++ acct = _uid()+ await writer.create_account(acct, "alice")+ user_key = await writer.register_user(acct, "bob")++ reader = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await reader.load()+ # Reader accepts bob at first.+ assert reader.resolve(user_key).user_id == "bob"++ await writer.remove_user(acct, "bob")+ await reader.reload()++ with pytest.raises(UnauthenticatedError):+ reader.resolve(user_key)+++async def test_reload_does_not_write_to_store(manager_service, monkeypatch):+ """reload() must never persist: read replicas mount a read-only volume."""+ writer = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await writer.load()+ acct = _uid()+ await writer.create_account(acct, "alice")++ reader = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await reader.load()++ async def _fail_write_json(*args, **kwargs):+ raise AssertionError("reload() must not write to the key store")++ monkeypatch.setattr(reader._legacy, "_write_json", _fail_write_json)+ monkeypatch.setattr(reader._legacy, "_save_accounts_json", _fail_write_json)+ monkeypatch.setattr(reader._legacy, "_save_users_json", _fail_write_json)++ # Must not raise: no write path should be exercised.+ await reader.reload()+++async def test_reload_with_hashing_enabled_does_not_migrate(manager_service, monkeypatch):+ """reload() with hashing on must index plaintext keys without migrating them.++ load() migrates plaintext -> Argon2id (a write); reload() is read-only and+ must skip that migration while still resolving the plaintext key.+ """+ acct = _uid()+ seed_mgr = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await seed_mgr._write_json(+ ACCOUNTS_PATH, {"accounts": {acct: {"created_at": "2026-04-16T00:00:00+00:00"}}}+ )+ await seed_mgr._write_json(+ f"/local/{acct}/_system/users.json",+ {"users": {"alice": {"role": "admin", "key": "legacy-plaintext-key-alice"}}},+ )++ reader = APIKeyManager(+ root_key=ROOT_KEY,+ viking_fs=manager_service.viking_fs,+ api_key_hashing_enabled=True,+ )++ async def _fail_write_json(*args, **kwargs):+ raise AssertionError("reload() must not migrate plaintext keys")++ monkeypatch.setattr(reader._legacy, "_write_json", _fail_write_json)+ monkeypatch.setattr(reader._legacy, "_save_users_json", _fail_write_json)++ await reader.reload()++ identity = reader.resolve("legacy-plaintext-key-alice")+ assert identity.account_id == acct+ assert identity.user_id == "alice"+++async def test_reload_tolerates_uninitialized_store(manager_service):+ """reload() before the store exists must keep current state, not wipe it."""+ reader = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ # Deliberately skip load(): accounts.json does not exist yet (reader started+ # before the writer bootstrapped the store).+ await reader.reload()+ # Root key still resolves; no crash from a missing store.+ assert reader.resolve(ROOT_KEY).role == Role.ROOT+++async def test_store_signature_changes_on_user_registration(manager_service):+ """The change signature must move when users.json changes (not just accounts)."""+ writer = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await writer.load()+ acct = _uid()+ await writer.create_account(acct, "alice")++ before = await writer.compute_store_signature()+ await writer.register_user(acct, "bob")+ after = await writer.compute_store_signature()++ assert before != after+++async def test_store_signature_stable_without_changes(manager_service):+ """Two consecutive signatures with no store change must be equal.++ This is what lets the watcher skip a full reload on an unchanged store.+ """+ writer = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await writer.load()+ acct = _uid()+ await writer.create_account(acct, "alice")++ first = await writer.compute_store_signature()+ second = await writer.compute_store_signature()+ assert first == seconddiff --git a/tests/server/test_auth.py b/tests/server/test_auth.pyindex 2e53db19b..189ea12da 100644--- a/tests/server/test_auth.py+++ b/tests/server/test_auth.py@@ -3,6 +3,7 @@"""Tests for multi-tenant authentication (openviking/server/auth.py)."""+import asyncioimport uuidimport httpx@@ -1467,3 +1468,95 @@ async def test_trusted_mode_admin_api_http_route_without_identity():assert response.json()["result"]["role"] == "root"assert response.json()["result"]["account_id"] == "trusted"assert response.json()["result"]["user_id"] == "trusted"+++# ---- API key store watcher (read-replica refresh) tests ----+#+# When server.api_key_watch_enabled is set, the api_key plugin starts a+# background task that polls the shared key store and reloads the in-memory+# index on change, so read replicas converge on writer-side user changes.+++class _FakeManager:+ """Minimal APIKeyManager stand-in that records watcher interactions."""++ def __init__(self, signatures):+ # signatures: list of signatures returned on successive calls; the last+ # value is repeated once exhausted.+ self._signatures = list(signatures)+ self.reload_calls = 0+ self.signature_calls = 0++ async def compute_store_signature(self):+ self.signature_calls += 1+ idx = min(self.signature_calls - 1, len(self._signatures) - 1)+ return self._signatures[idx]++ async def reload(self):+ self.reload_calls += 1+++async def test_watcher_disabled_by_default(auth_service):+ """No watch task should start when api_key_watch_enabled is unset."""+ from openviking.server.auth.plugins import ApiKeyAuthPlugin++ app = create_app(config=ServerConfig(root_api_key=ROOT_KEY), service=auth_service)+ plugin = ApiKeyAuthPlugin()+ config = ServerConfig(root_api_key=ROOT_KEY)+ await plugin.initialize(app, auth_service, config)+ try:+ assert plugin._watch_task is None+ finally:+ await plugin.shutdown()+++async def test_watcher_reloads_on_signature_change(auth_service):+ """The watcher reloads only when the store signature changes."""+ from openviking.server.auth.plugins import ApiKeyAuthPlugin++ manager = _FakeManager(signatures=[("sig-a",), ("sig-b",)])+ task = asyncio.create_task(+ ApiKeyAuthPlugin._watch_key_store(manager, interval=0.01)+ )+ try:+ await asyncio.sleep(0.1)+ finally:+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ # First signature is the baseline; the change to sig-b triggers a reload.+ assert manager.reload_calls >= 1+++async def test_watcher_skips_reload_when_unchanged(auth_service):+ """A stable signature must not trigger any reload."""+ from openviking.server.auth.plugins import ApiKeyAuthPlugin++ manager = _FakeManager(signatures=[("sig-stable",)])+ task = asyncio.create_task(+ ApiKeyAuthPlugin._watch_key_store(manager, interval=0.01)+ )+ try:+ await asyncio.sleep(0.08)+ finally:+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task+ assert manager.reload_calls == 0+++async def test_shutdown_cancels_watch_task(auth_service):+ """shutdown() must cancel a running watch task and clear the handle."""+ from openviking.server.auth.plugins import ApiKeyAuthPlugin++ app = create_app(config=ServerConfig(root_api_key=ROOT_KEY), service=auth_service)+ plugin = ApiKeyAuthPlugin()+ config = ServerConfig(+ root_api_key=ROOT_KEY,+ api_key_watch_enabled=True,+ api_key_watch_interval_seconds=0.01,+ )+ await plugin.initialize(app, auth_service, config)+ assert plugin._watch_task is not None+ await plugin.shutdown()+ assert plugin._watch_task is None