Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(admin): avoid full reload for list endpoints (#4935)
Jiahui Zhou182f3d58de4b
diff --git a/openviking/server/api_keys/legacy.py b/openviking/server/api_keys/legacy.pyindex 1a8d55c84..fb5aa5052 100644--- a/openviking/server/api_keys/legacy.py+++ b/openviking/server/api_keys/legacy.py@@ -217,6 +217,80 @@ class LegacyAPIKeyManager:await self._reload_unlocked()return True+ async def refresh_accounts_from_store(self) -> None:+ """Refresh account metadata without reading user or group registries."""+ async with self._mutation_lock, self._reload_lock:+ await self._refresh_accounts_from_store_unlocked()++ async def _refresh_accounts_from_store_unlocked(self) -> None:+ accounts_data = await self._read_json(ACCOUNTS_PATH)+ if accounts_data is None:+ return+ persisted_accounts = accounts_data.get("accounts", {})++ for account_id in set(self._accounts) - set(persisted_accounts):+ self._discard_account_state(account_id)+ for account_id, info in persisted_accounts.items():+ created_at = info.get("created_at", "")+ account = self._accounts.get(account_id)+ if account is not None and account.created_at != created_at:+ self._discard_account_state(account_id)+ account = None+ if account is None:+ self._accounts[account_id] = AccountInfo(+ created_at=created_at,+ groups_loaded=False,+ )+ else:+ account.created_at = created_at++ async def refresh_account_users_from_store(self, account_id: str) -> None:+ """Refresh one account's users without reading unrelated registries."""+ async with self._mutation_lock, self._reload_lock:+ await self._refresh_account_users_from_store_unlocked(account_id)++ async def _refresh_account_users_from_store_unlocked(self, account_id: str) -> None:+ users_path = USERS_PATH_TEMPLATE.format(account_id=account_id)+ users_data = await self._read_json(users_path)+ account = self._accounts.get(account_id)+ if users_data is None and account is None:+ raise NotFoundError(account_id, "account")++ users = users_data.get("users", {}) if users_data else {}+ prefix_entries: list[tuple[str, UserKeyEntry]] = []+ for user_id, user_info in users.items():+ key_or_hash = user_info.get("key", "")+ if not key_or_hash:+ continue+ key_prefix = user_info.get("key_prefix", "") or self._get_key_prefix(+ key_or_hash+ )+ if key_prefix:+ prefix_entries.append(+ (+ key_prefix,+ UserKeyEntry(+ account_id=account_id,+ user_id=user_id,+ role=Role(user_info.get("role", "user")),+ key_or_hash=key_or_hash,+ is_hashed=key_or_hash.startswith("$argon2"),+ ),+ )+ )++ if account is None:+ account = AccountInfo(created_at="", groups_loaded=False)+ self._accounts[account_id] = account++ for user_id, user_info in account.users.items():+ self._remove_key_index_entry(account_id, user_id, user_info)++ account.users = users+ for key_prefix, entry in prefix_entries:+ self._prefix_index.setdefault(key_prefix, []).append(entry)+ self._rebuild_account_group_index(account_id)+async def _update_identity_registry_signatures(self, account_ids: set[str] | None = None) -> None:@@ -617,7 +691,12 @@ class LegacyAPIKeyManager:(accounts_data.get("accounts", {}).get(account_id) or {}).get("created_at")or now)- account = AccountInfo(created_at=created_at, users={}, groups={})+ account = AccountInfo(+ created_at=created_at,+ users={},+ groups={},+ groups_loaded=False,+ )self._accounts[account_id] = accountfor user_id, user_info in persisted_users.items():account.users.setdefault(user_id, dict(user_info))@@ -722,6 +801,7 @@ class LegacyAPIKeyManager:if not isinstance(deletion, dict) or deletion.get("task_id") != task_id:return False+ await self._load_account_groups_if_needed(account_id, account)old_groups = copy.deepcopy(account.groups)groups = copy.deepcopy(account.groups)for group in groups.values():@@ -975,6 +1055,7 @@ class LegacyAPIKeyManager:raise InvalidArgumentError(error)async with self._reload_lock:account = self._require_account(account_id)+ await self._load_account_groups_if_needed(account_id, account)if group_id in account.groups:raise AlreadyExistsError(group_id, "group")groups = copy.deepcopy(account.groups)@@ -996,6 +1077,7 @@ class LegacyAPIKeyManager:async def add_group_member(self, account_id: str, group_id: str, user_id: str) -> bool:async with self._reload_lock:account = self._require_account(account_id)+ await self._load_account_groups_if_needed(account_id, account)if user_id not in account.users:raise NotFoundError(user_id, "user")group = self._require_group(account_id, group_id)@@ -1010,6 +1092,7 @@ class LegacyAPIKeyManager:async def remove_group_member(self, account_id: str, group_id: str, user_id: str) -> bool:async with self._reload_lock:account = self._require_account(account_id)+ await self._load_account_groups_if_needed(account_id, account)group = self._require_group(account_id, group_id)if user_id not in group.get("members", []):return False@@ -1023,6 +1106,7 @@ class LegacyAPIKeyManager:async def delete_group(self, account_id: str, group_id: str) -> None:async with self._reload_lock:account = self._require_account(account_id)+ await self._load_account_groups_if_needed(account_id, account)group = self._require_group(account_id, group_id)if group.get("members"):raise FailedPreconditionError("Group must be empty before deletion")@@ -1074,6 +1158,23 @@ class LegacyAPIKeyManager:raise NotFoundError(group_id, "group")return group+ async def ensure_account_groups_loaded(self, account_id: str) -> None:+ """Load one account's groups when its account metadata was discovered alone."""+ async with self._reload_lock:+ account = self._require_account(account_id)+ await self._load_account_groups_if_needed(account_id, account)++ async def _load_account_groups_if_needed(+ self, account_id: str, account: AccountInfo+ ) -> None:+ if account.groups_loaded:+ return+ groups_path = GROUPS_PATH_TEMPLATE.format(account_id=account_id)+ groups_data = await self._read_json(groups_path)+ account.groups = groups_data.get("groups", {}) if groups_data else {}+ account.groups_loaded = True+ self._rebuild_account_group_index(account_id)+@staticmethoddef _group_result(group_id: str, group: dict) -> dict:return {@@ -1116,6 +1217,7 @@ class LegacyAPIKeyManager:) -> None:await self._write_groups_json(account_id, groups)account.groups = groups+ account.groups_loaded = Trueself._rebuild_account_group_index(account_id)def _remove_key_index_entry(self, account_id: str, user_id: str, user_info: dict) -> None:diff --git a/openviking/server/api_keys/models.py b/openviking/server/api_keys/models.pyindex d8ccfed62..3f1608de5 100644--- a/openviking/server/api_keys/models.py+++ b/openviking/server/api_keys/models.py@@ -39,3 +39,4 @@ class AccountInfo:created_at: strusers: Dict[str, dict] = field(default_factory=dict)groups: Dict[str, dict] = field(default_factory=dict)+ groups_loaded: bool = Truediff --git a/openviking/server/api_keys/new.py b/openviking/server/api_keys/new.pyindex c508e173f..19afc1c02 100644--- a/openviking/server/api_keys/new.py+++ b/openviking/server/api_keys/new.py@@ -131,6 +131,18 @@ class NewAPIKeyManager:"""Refresh account/user registry state for a management read when needed."""return await self._legacy.refresh_identity_registry_if_changed(account_id)+ async def refresh_accounts_from_store(self) -> None:+ """Refresh account metadata without loading user or group registries."""+ await self._legacy.refresh_accounts_from_store()++ async def refresh_account_users_from_store(self, account_id: str) -> None:+ """Refresh one account's users without loading unrelated registries."""+ await self._legacy.refresh_account_users_from_store(account_id)++ async def ensure_account_groups_loaded(self, account_id: str) -> None:+ """Load one account's groups after an account-only refresh."""+ await self._legacy.ensure_account_groups_loaded(account_id)+def resolve(self, api_key: str) -> ResolvedIdentity:"""Resolve an API key to identity.diff --git a/openviking/server/routers/admin.py b/openviking/server/routers/admin.pyindex 5cacbcd0f..ce18170f1 100644--- a/openviking/server/routers/admin.py+++ b/openviking/server/routers/admin.py@@ -161,6 +161,12 @@ def _should_expose_user_key(request: Request) -> bool:return config.get_effective_auth_mode() != "trusted"+def _registry_watcher_running(request: Request) -> bool:+ plugin = getattr(request.app.state, "auth_plugin", None)+ watch_task = getattr(plugin, "_watch_task", None)+ return watch_task is not None and not watch_task.done()++def _check_account_access(ctx: RequestContext, account_id: str) -> None:"""ADMIN can only operate on their own account."""if ctx.role == Role.ADMIN and ctx.account_id != account_id:@@ -173,10 +179,14 @@ async def _check_account_exists(manager = getattr(request.app.state, "api_key_manager", None)if manager is None:return None- await manager.refresh_identity_registry_if_changed(refresh_scope)+ watcher_running = _registry_watcher_running(request)+ if not watcher_running:+ await manager.refresh_accounts_from_store()accounts = manager.get_accounts()if not any(item.get("account_id") == account_id for item in accounts):raise NotFoundError(account_id, "account")+ if refresh_scope is not None and not watcher_running:+ await manager.refresh_account_users_from_store(refresh_scope)return manager@@ -337,7 +347,8 @@ async def list_accounts():"""List accounts in creation order. `name` supports wildcard (* and ?) matching."""manager = _get_api_key_manager(request)- await manager.refresh_identity_registry_if_changed()+ if not _registry_watcher_running(request):+ await manager.refresh_accounts_from_store()accounts = manager.get_accounts(name_filter=name, limit=limit, page=page)return Response(status="ok", result=accounts)@@ -518,7 +529,8 @@ async def list_users("""List users in an account, in creation order. `name` supports wildcard (* and ?) matching."""_check_account_access(ctx, account_id)manager = _get_api_key_manager(request)- await manager.refresh_identity_registry_if_changed(account_id)+ if not _registry_watcher_running(request):+ await manager.refresh_account_users_from_store(account_id)expose_key = _should_expose_user_key(request)users = manager.get_users(account_id,@@ -681,7 +693,9 @@ async def list_groups(ctx: RequestContext = Depends(get_request_context),):_check_account_access(ctx, account_id)- result = _get_api_key_manager(request).get_groups(account_id)+ manager = _get_api_key_manager(request)+ await manager.ensure_account_groups_loaded(account_id)+ result = manager.get_groups(account_id)return Response(status="ok", result=result)@@ -707,7 +721,9 @@ async def list_group_members(ctx: RequestContext = Depends(get_request_context),):_check_account_access(ctx, account_id)- members = _get_api_key_manager(request).get_group_members(account_id, group_id)+ manager = _get_api_key_manager(request)+ await manager.ensure_account_groups_loaded(account_id)+ members = manager.get_group_members(account_id, group_id)return Response(status="ok", result={"group_id": group_id, "members": members})diff --git a/tests/server/test_admin_api.py b/tests/server/test_admin_api.pyindex 4905d9609..9efbe19bf 100644--- a/tests/server/test_admin_api.py+++ b/tests/server/test_admin_api.py@@ -524,6 +524,101 @@ async def test_list_accounts(admin_client: httpx.AsyncClient):assert acct in account_ids+async def test_list_accounts_without_watcher_reads_only_accounts_registry(+ lightweight_admin_client: httpx.AsyncClient,+ lightweight_admin_app: FastAPI,+ monkeypatch: pytest.MonkeyPatch,+):+ """Without a watcher, account listing must not load user registries."""+ manager = lightweight_admin_app.state.api_key_manager+ original_read_json = manager._legacy._read_json+ read_paths: list[str] = []++ async def _record_read(path: str):+ read_paths.append(path)+ return await original_read_json(path)++ monkeypatch.setattr(manager._legacy, "_read_json", _record_read)++ resp = await lightweight_admin_client.get(+ "/api/v1/admin/accounts?limit=5&page=1", headers=root_headers()+ )++ assert resp.status_code == 200, resp.text+ assert read_paths == ["/local/_system/accounts.json"]+++async def test_list_users_without_watcher_reads_only_target_user_registry(+ lightweight_admin_client: httpx.AsyncClient,+ lightweight_admin_app: FastAPI,+ monkeypatch: pytest.MonkeyPatch,+):+ """Without a watcher, user listing must not load unrelated registries."""+ acct = _uid()+ await lightweight_admin_client.post(+ "/api/v1/admin/accounts",+ json={"account_id": acct, "admin_user_id": "alice"},+ headers=root_headers(),+ )+ manager = lightweight_admin_app.state.api_key_manager+ original_read_json = manager._legacy._read_json+ read_paths: list[str] = []++ async def _record_read(path: str):+ read_paths.append(path)+ return await original_read_json(path)++ monkeypatch.setattr(manager._legacy, "_read_json", _record_read)++ resp = await lightweight_admin_client.get(+ f"/api/v1/admin/accounts/{acct}/users", headers=root_headers()+ )++ assert resp.status_code == 200, resp.text+ assert read_paths == [f"/local/{acct}/_system/users.json"]+++async def test_list_users_for_default_account_without_users_file_returns_empty_list(+ lightweight_admin_client: httpx.AsyncClient,+):+ """The initialized default account may legitimately have no users.json."""+ resp = await lightweight_admin_client.get(+ "/api/v1/admin/accounts/default/users", headers=root_headers()+ )++ assert resp.status_code == 200, resp.text+ assert resp.json()["result"] == []+++async def test_list_accounts_with_watcher_uses_memory(+ lightweight_admin_client: httpx.AsyncClient,+ lightweight_admin_app: FastAPI,+ monkeypatch: pytest.MonkeyPatch,+):+ """A running watcher keeps list requests on the in-memory fast path."""+ loop = asyncio.get_running_loop()+ watch_task = loop.create_future()+ lightweight_admin_app.state.auth_plugin._watch_task = watch_task++ async def _unexpected_read(_path: str):+ raise AssertionError("list accounts should not read AGFS while watcher is running")++ manager = lightweight_admin_app.state.api_key_manager+ monkeypatch.setattr(manager._legacy, "_read_json", _unexpected_read)+ try:+ resp = await lightweight_admin_client.get(+ "/api/v1/admin/accounts", headers=root_headers()+ )+ users_resp = await lightweight_admin_client.get(+ "/api/v1/admin/accounts/default/users", headers=root_headers()+ )+ finally:+ watch_task.cancel()++ assert resp.status_code == 200, resp.text+ assert users_resp.status_code == 200, users_resp.text++async def test_identity_settings_refreshes_a_stale_registry_on_demand(admin_client: httpx.AsyncClient,admin_app: FastAPI,@@ -546,7 +641,7 @@ async def test_identity_settings_refreshes_a_stale_registry_on_demand(headers=root_headers(),)assert account_settings.status_code == 200, account_settings.text- assert replica.has_user(acct, "trusted-user") is True+ assert replica.has_user(acct, "trusted-user") is Falseawait writer.ensure_trusted_identities({acct: {"trusted-user-2"}})assert replica.has_user(acct, "trusted-user-2") is False@@ -556,6 +651,48 @@ async def test_identity_settings_refreshes_a_stale_registry_on_demand(headers=root_headers(),)assert user_settings.status_code == 200, user_settings.text+ assert replica.has_user(acct, "trusted-user") is True+ assert replica.has_user(acct, "trusted-user-2") is True+++async def test_identity_settings_without_watcher_use_scoped_registry_reads(+ lightweight_admin_client: httpx.AsyncClient,+ lightweight_admin_app: FastAPI,+ monkeypatch: pytest.MonkeyPatch,+):+ """Settings checks must not fall back to a full identity-registry reload."""+ acct = _uid()+ await lightweight_admin_client.post(+ "/api/v1/admin/accounts",+ json={"account_id": acct, "admin_user_id": "alice"},+ headers=root_headers(),+ )+ manager = lightweight_admin_app.state.api_key_manager+ original_read_json = manager._legacy._read_json+ read_paths: list[str] = []++ async def _record_read(path: str):+ read_paths.append(path)+ return await original_read_json(path)++ monkeypatch.setattr(manager._legacy, "_read_json", _record_read)++ account_resp = await lightweight_admin_client.get(+ f"/api/v1/admin/accounts/{acct}/settings", headers=root_headers()+ )+ assert account_resp.status_code == 200, account_resp.text+ assert read_paths == ["/local/_system/accounts.json"]++ read_paths.clear()+ user_resp = await lightweight_admin_client.get(+ f"/api/v1/admin/accounts/{acct}/users/alice/settings",+ headers=root_headers(),+ )+ assert user_resp.status_code == 200, user_resp.text+ assert read_paths == [+ "/local/_system/accounts.json",+ f"/local/{acct}/_system/users.json",+ ]async def test_delete_account(admin_client: httpx.AsyncClient):diff --git a/tests/server/test_api_key_manager.py b/tests/server/test_api_key_manager.pyindex fcc44b900..478c33c6a 100644--- a/tests/server/test_api_key_manager.py+++ b/tests/server/test_api_key_manager.py@@ -244,6 +244,174 @@ async def test_account_registry_refresh_check_stats_only_accounts_file(assert stat_paths == [ACCOUNTS_PATH]+async def test_refresh_accounts_from_store_does_not_read_user_registries(+ manager: APIKeyManager, monkeypatch: pytest.MonkeyPatch+):+ """An account-list refresh reads only accounts.json and keeps cached users."""+ acct = _uid()+ await manager.create_account(acct, "alice")+ cached_users = manager._legacy._accounts[acct].users+ original_read_json = manager._legacy._read_json+ read_paths: list[str] = []++ async def _record_read(path: str):+ read_paths.append(path)+ return await original_read_json(path)++ monkeypatch.setattr(manager._legacy, "_read_json", _record_read)++ await manager.refresh_accounts_from_store()++ assert read_paths == [ACCOUNTS_PATH]+ assert manager._legacy._accounts[acct].users is cached_users+ account = next(item for item in manager.get_accounts() if item["account_id"] == acct)+ assert account["user_count"] == 1+++async def test_refresh_account_users_from_store_reads_only_target_registry(+ manager: APIKeyManager, monkeypatch: pytest.MonkeyPatch+):+ """A user-list refresh reads and replaces only the requested account's users."""+ target = _uid()+ unrelated = _uid()+ await manager.create_account(target, "alice")+ await manager.create_account(unrelated, "bob")+ unrelated_users = manager._legacy._accounts[unrelated].users+ original_read_json = manager._legacy._read_json+ read_paths: list[str] = []++ async def _record_read(path: str):+ read_paths.append(path)+ return await original_read_json(path)++ monkeypatch.setattr(manager._legacy, "_read_json", _record_read)++ await manager.refresh_account_users_from_store(target)++ assert read_paths == [USERS_PATH_TEMPLATE.format(account_id=target)]+ assert manager._legacy._accounts[unrelated].users is unrelated_users+++async def test_refresh_account_users_from_store_is_atomic_on_invalid_data(+ manager: APIKeyManager, monkeypatch: pytest.MonkeyPatch+):+ """An invalid persisted user must not partially replace authentication state."""+ acct = _uid()+ user_key = await manager.create_account(acct, "alice")+ cached_users = manager._legacy._accounts[acct].users+ users_path = USERS_PATH_TEMPLATE.format(account_id=acct)+ original_read_json = manager._legacy._read_json++ async def _read_invalid_role(path: str):+ if path == users_path:+ return {"users": {"mallory": {"role": "invalid", "key": "bad-key"}}}+ return await original_read_json(path)++ monkeypatch.setattr(manager._legacy, "_read_json", _read_invalid_role)++ with pytest.raises(ValueError):+ await manager.refresh_account_users_from_store(acct)++ assert manager._legacy._accounts[acct].users is cached_users+ identity = manager.resolve(user_key)+ assert identity.account_id == acct+ assert identity.user_id == "alice"+++async def test_scoped_store_refresh_observes_another_manager(+ manager: APIKeyManager, manager_service+):+ """Scoped refreshes expose another manager's account and user changes."""+ replica = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await replica.load()+ acct = _uid()++ await manager.create_account(acct, "alice")+ await replica.refresh_accounts_from_store()+ account = next(item for item in replica.get_accounts() if item["account_id"] == acct)+ assert account["user_count"] == 0++ await replica.refresh_account_users_from_store(acct)+ assert replica.get_users(acct, expose_key=False) == [+ {"user_id": "alice", "role": "admin"}+ ]+ account = next(item for item in replica.get_accounts() if item["account_id"] == acct)+ assert account["user_count"] == 1+++async def test_account_only_refresh_loads_groups_before_group_write(manager_service):+ """A partially discovered account must not overwrite persisted groups."""+ writer = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ reader = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await writer.load()+ await reader.load()+ acct = _uid()++ await writer.create_account(acct, "alice")+ await writer.create_group(acct, "engineering")+ await writer.add_group_member(acct, "engineering", "alice")++ await reader.refresh_accounts_from_store()+ discovered = reader._legacy._accounts[acct]+ assert discovered.groups_loaded is False+ assert discovered.groups == {}++ await reader.create_group(acct, "finance")++ verifier = APIKeyManager(root_key=ROOT_KEY, viking_fs=manager_service.viking_fs)+ await verifier.load()+ assert verifier.get_groups(acct) == [+ {"group_id": "engineering", "member_count": 1},+ {"group_id": "finance", "member_count": 0},+ ]+ assert verifier.get_group_members(acct, "engineering") == ["alice"]+++async def test_account_only_refresh_discards_recreated_account_state(+ manager: APIKeyManager, monkeypatch: pytest.MonkeyPatch+):+ """A recreated account must not retain users or keys from its old incarnation."""+ acct = _uid()+ old_key = await manager.create_account(acct, "alice")+ original_read_json = manager._legacy._read_json++ async def _read_recreated_account(path: str):+ if path == ACCOUNTS_PATH:+ return {"accounts": {acct: {"created_at": "recreated"}}}+ return await original_read_json(path)++ monkeypatch.setattr(manager._legacy, "_read_json", _read_recreated_account)++ await manager.refresh_accounts_from_store()++ account = next(item for item in manager.get_accounts() if item["account_id"] == acct)+ assert account == {+ "account_id": acct,+ "created_at": "recreated",+ "user_count": 0,+ }+ with pytest.raises(UnauthenticatedError):+ manager.resolve(old_key)+++async def test_refresh_known_account_without_users_file_returns_empty_users(+ manager: APIKeyManager, monkeypatch: pytest.MonkeyPatch+):+ """A missing users.json is an empty registry for an account already in memory."""+ original_read_json = manager._legacy._read_json++ async def _missing_default_users(path: str):+ if path == USERS_PATH_TEMPLATE.format(account_id="default"):+ return None+ return await original_read_json(path)++ monkeypatch.setattr(manager._legacy, "_read_json", _missing_default_users)++ await manager.refresh_account_users_from_store("default")++ assert manager.get_users("default", expose_key=False) == []++async def test_trusted_identity_merge_refreshes_only_affected_user_signatures(manager: APIKeyManager, monkeypatch: pytest.MonkeyPatch):