Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(bot): Limit grep\glob\search tool's user scope, grep use one pattern, Optimize client initialization. (#2240)
DuTaof2bc994b4d12
diff --git a/bot/tests/test_openviking_api_key_type.py b/bot/tests/test_openviking_api_key_type.pyindex 0fffcce70..3579e8400 100644--- a/bot/tests/test_openviking_api_key_type.py+++ b/bot/tests/test_openviking_api_key_type.py@@ -2,8 +2,7 @@ from pathlib import Pathfrom types import SimpleNamespaceimport pytest--from vikingbot.agent.tools.ov_file import VikingSearchTool+from vikingbot.agent.tools.ov_file import VikingGrepTool, VikingSearchToolfrom vikingbot.config.schema import SessionKeyfrom vikingbot.hooks.base import HookContextfrom vikingbot.hooks.builtins.openviking_hooks import OpenVikingCompactHook@@ -38,6 +37,15 @@ class _DummyHTTPClient:async def admin_list_accounts(self):return []+ async def find(self, *_args, **_kwargs):+ return []++ async def search(self, *_args, **_kwargs):+ return {"memories": [], "resources": [], "skills": []}++ async def grep(self, *_args, **_kwargs):+ return {"matches": []}+async def close(self):return None@@ -140,6 +148,9 @@ async def test_compact_hook_user_mode_commits_once(monkeypatch):def __init__(self):self.calls = []+ def should_sender_fanout(self):+ return False+async def commit(self, session_id, messages, user_id=None):self.calls.append((session_id, user_id, len(messages)))return {"success": "committed"}@@ -263,6 +274,49 @@ async def test_skill_memory_uri_respects_namespace_policy(monkeypatch):)+def test_openviking_grep_schema_requires_single_string_pattern():+ tool = VikingGrepTool()++ assert tool.parameters["properties"]["pattern"]["type"] == "string"+++async def test_openviking_grep_passes_admin_user_id(monkeypatch):+ tool = VikingGrepTool()+ calls = []++ class _FakeClient:+ admin_user_id = "admin"++ async def grep(self, uri, pattern, case_insensitive=False, user_id=None):+ calls.append((uri, pattern, case_insensitive, user_id))+ return {+ "matches": [+ {+ "uri": "viking://resources/doc.md",+ "line": 3,+ "content": "hello admin scoped grep",+ }+ ]+ }++ async def _fake_get_client(_tool_context):+ return _FakeClient()++ monkeypatch.setattr(tool, "_get_client", _fake_get_client)++ result = await tool.execute(+ SimpleNamespace(workspace_id="workspace"),+ uri="viking://resources/",+ pattern="hello",+ case_insensitive=True,+ )++ assert calls == [("viking://resources/", "hello", True, "admin")]+ assert "Found 1 match for pattern 'hello':" in result+ assert "viking://resources/doc.md" in result++@pytest.mark.asyncioasync def test_openviking_search_uses_policy_scoped_user_namespace(monkeypatch):monkeypatch.setattr(ov_server_module, "load_config", lambda: _make_config("root"))@@ -280,13 +334,17 @@ async def test_openviking_search_uses_policy_scoped_user_namespace(monkeypatch):}]- async def _search(query, target_uri=None, limit=20):+ async def _search(query, target_uri=None, limit=20, user_id=None):calls.append(target_uri)return {"memories": [{"uri": target_uri, "abstract": "a", "score": 0.9, "is_leaf": True}]}+ async def _fake_get_client(_tool_context):+ return client+monkeypatch.setattr(client.client, "admin_list_accounts", _accounts)- monkeypatch.setattr(client.client, "search", _search)- monkeypatch.setattr(tool, "_get_client", lambda _tool_context: client)+ monkeypatch.setattr(client, "search", _search)+ monkeypatch.setattr(tool, "_get_client", _fake_get_client)+ await client._load_namespace_policy()tool_context = SimpleNamespace(workspace_id="workspace", memory_user_ids=["sender-1"])result = await tool.execute(tool_context, query="hello")@@ -303,14 +361,19 @@ async def test_openviking_search_user_key_mode_uses_current_user_namespace(monkecalls = []- async def _search(query, target_uri=None, limit=20):+ async def _search(query, target_uri=None, limit=20, user_id=None):calls.append(target_uri)return {"memories": [{"uri": target_uri, "abstract": "a", "score": 0.9, "is_leaf": True}]}- monkeypatch.setattr(client.client, "search", _search)- monkeypatch.setattr(tool, "_get_client", lambda _tool_context: client)+ async def _fake_get_client(_tool_context):+ return client++ monkeypatch.setattr(client, "search", _search)+ monkeypatch.setattr(tool, "_get_client", _fake_get_client)- tool_context = SimpleNamespace(workspace_id="workspace", memory_user_ids=["sender-1", "sender-2"])+ tool_context = SimpleNamespace(+ workspace_id="workspace", memory_user_ids=["sender-1", "sender-2"]+ )result = await tool.execute(tool_context, query="hello")assert "viking://user/memories/" in resultdiff --git a/bot/vikingbot/agent/tools/ov_file.py b/bot/vikingbot/agent/tools/ov_file.pyindex 8aeae597b..ed09a093e 100644--- a/bot/vikingbot/agent/tools/ov_file.py+++ b/bot/vikingbot/agent/tools/ov_file.py@@ -4,7 +4,7 @@ import asyncioimport jsonfrom abc import ABCfrom pathlib import Path-from typing import Any, Optional, Union+from typing import Any, Optionalimport httpxfrom loguru import logger@@ -89,6 +89,7 @@ class VikingSearchTool(OVFileTool):def description(self) -> str:return ("Using query to search for resources (knowledge, code, files, workflow, etc.) in OpenViking. "+ "Result: Only URIs and summaries are included here. To view the full content, use openviking_multi_read tool.""This operation performs semantic retrieval, not full character matching. Please avoid repeated calls with similar queries as much as possible.""bad-case: after searching with ‘Nate Joanna dog playdate 3:00 pm', another search was performed using 'Nate Joanna dog playdate'.")@@ -249,37 +250,38 @@ class VikingSearchTool(OVFileTool):) -> str:try:client = await self._get_client(tool_context)- search_client = client.admin_user_client or client.client+ admin_user_id = client.admin_user_id- # If no target_uri specified, use memory_user_ids to search specific user memoriesif not target_uri and tool_context.memory_user_ids:- all_results = []- user_ids = tool_context.memory_user_ids- if client._is_user_key_mode():- user_ids = [None]-- for user_id in user_ids:- user_uri = client._memory_target_uri(user_id)- logger.info(f"openviking_search: searching {user_uri} for query: {query}")- results = await search_client.search(query, target_uri=user_uri, limit=20)- if results:- memories = [- item- for item in self._extract_search_items(results)- if item.get("type") == "memory"- ]- all_results.extend(memories)-- if not all_results:- return f"No results found for query: {query}"+ user_ids = tool_context.memory_user_ids if client.should_sender_fanout() else [None]+ grouped_items = {+ "memory": [],+ "resource": [],+ "skill": [],+ }++ for memory_user_id in user_ids:+ results = await client.search(+ query,+ target_uri=client._memory_target_uri(memory_user_id),+ limit=20,+ user_id=admin_user_id,+ )+ filtered_items = self._filter_search_items(results, min_score=min_score)+ for item_type, items in filtered_items.items():+ grouped_items[item_type].extend(items)- grouped_items = self._filter_search_items(all_results, min_score=min_score)total = sum(len(items) for items in grouped_items.values())if total == 0:return f"No results found for query: {query}"return self._format_search_items_json(grouped_items, min_score=min_score)- results = await search_client.search(query, target_uri=target_uri, limit=20)+ results = await client.search(+ query,+ target_uri=target_uri,+ limit=20,+ user_id=admin_user_id,+ )if not results:return f"No results found for query: {query}"@@ -352,7 +354,7 @@ class VikingAddResourceTool(OVFileTool):class VikingGrepTool(OVFileTool):- """Tool to search Viking resources using regex patterns."""+ """Tool to search Viking resources using a regex pattern."""@propertydef name(self) -> str:@@ -361,7 +363,8 @@ class VikingGrepTool(OVFileTool):@propertydef description(self) -> str:return (- "Search Viking resources using regex patterns (like grep). Supports multiple patterns to search concurrently."+ "Search Viking resources using a regex pattern (like grep)."+ "Result: Only URIs and summaries are included here. To view the full content, use openviking_multi_read tool.""Please avoid repeated calls with similar queries as much as possible.")@@ -375,9 +378,8 @@ class VikingGrepTool(OVFileTool):"description": "The whole Viking URI to search within (e.g., viking://resources/)",},"pattern": {- "type": "array",- "items": {"type": "string"},- "description": "Regex pattern or array of regex patterns to search for",+ "type": "string",+ "description": "Regex pattern to search for",},"case_insensitive": {"type": "boolean",@@ -392,71 +394,51 @@ class VikingGrepTool(OVFileTool):self,tool_context: "ToolContext",uri: str,- pattern: Union[str, list[str]],+ pattern: str,case_insensitive: bool = False,**kwargs: Any,) -> str:try:client = await self._get_client(tool_context)- patterns = [pattern] if isinstance(pattern, str) else pattern+ result = await client.grep(+ uri,+ pattern,+ case_insensitive=case_insensitive,+ user_id=client.admin_user_id,+ )+ if isinstance(result, dict):+ matches = result.get("matches", [])+ else:+ matches = getattr(result, "matches", [])- # Limit concurrent requests to avoid overwhelming the server and memory- max_concurrent = 10- semaphore = asyncio.Semaphore(max_concurrent)+ if not matches:+ return f"No matches found for pattern: '{pattern}'"- async def run_grep(p: str) -> tuple[str, list[Any]]:- async with semaphore:- try:- result = await client.grep(uri, p, case_insensitive=case_insensitive)- if isinstance(result, dict):- matches = result.get("matches", [])- else:- matches = getattr(result, "matches", [])- return (p, matches)- except Exception as e:- logger.warning(f"Error searching for pattern '{p}': {e}")- return (p, [])+ merged_results: dict[str, list[tuple[int, str]]] = {}- tasks = [run_grep(p) for p in patterns]- results = await asyncio.gather(*tasks)+ for match in matches:+ if isinstance(match, dict):+ match_uri = match.get("uri", "unknown")+ line = match.get("line", "?")+ content = match.get("content", "")+ else:+ match_uri = getattr(match, "uri", "unknown")+ line = getattr(match, "line", "?")+ content = getattr(match, "content", "")- # Merge results by URI- merged_results: dict[str, list[tuple[int, str, str]]] = {}- total_matches = 0+ if match_uri not in merged_results:+ merged_results[match_uri] = []+ merged_results[match_uri].append((line, content))- for p, matches in results:- if not matches:- continue- total_matches += len(matches)- for match in matches:- if isinstance(match, dict):- match_uri = match.get("uri", "unknown")- line = match.get("line", "?")- content = match.get("content", "")- else:- match_uri = getattr(match, "uri", "unknown")- line = getattr(match, "line", "?")- content = getattr(match, "content", "")-- if match_uri not in merged_results:- merged_results[match_uri] = []- merged_results[match_uri].append((line, content, p))-- if not merged_results:- pattern_str = ", ".join(f"'{p}'" for p in patterns)- return f"No matches found for patterns: {pattern_str}"-- # Format outputresult_lines = [- f"Found {total_matches} match{'es' if total_matches != 1 else ''} across {len(patterns)} pattern{'s' if len(patterns) != 1 else ''}:"+ f"Found {len(matches)} match{'es' if len(matches) != 1 else ''} for pattern '{pattern}':"]- for match_uri, matches in merged_results.items():- # Sort matches by line number- matches.sort(key=lambda x: int(x[0]) if str(x[0]).isdigit() else 0)+ for match_uri, uri_matches in merged_results.items():+ uri_matches.sort(key=lambda x: int(x[0]) if str(x[0]).isdigit() else 0)result_lines.append(f"\n📄 {match_uri}")- for line, content, pattern_name in matches:- result_lines.append(f" Line {line} (pattern: '{pattern_name}'):")+ for line, content in uri_matches:+ result_lines.append(f" Line {line}:")result_lines.append(f" {content}")return "\n".join(result_lines)@@ -473,7 +455,10 @@ class VikingGlobTool(OVFileTool):@propertydef description(self) -> str:- return "Find Viking resources using glob patterns (like **/*.md, *.py)."+ return (+ "Find Viking resources using glob patterns (like **/*.md, *.py)."+ "Result: Only URIs and summaries are included here. To view the full content, use openviking_multi_read tool."+ )@propertydef parameters(self) -> dict[str, Any]:@@ -498,7 +483,7 @@ class VikingGlobTool(OVFileTool):) -> str:try:client = await self._get_client(tool_context)- result = await client.glob(pattern, uri=uri or None)+ result = await client.glob(pattern, uri=uri or None, user_id=client.admin_user_id)if isinstance(result, dict):matches = result.get("matches", [])diff --git a/bot/vikingbot/openviking_mount/ov_server.py b/bot/vikingbot/openviking_mount/ov_server.pyindex 24de185ca..6aa6af21a 100644--- a/bot/vikingbot/openviking_mount/ov_server.py+++ b/bot/vikingbot/openviking_mount/ov_server.py@@ -24,7 +24,6 @@ class VikingClient:raise ValueError(f"Invalid ov_server.api_key_type: {self.api_key_type}")self._apikey_manager = None- self.admin_user_client = Noneself._namespace_policy = {"isolate_user_scope_by_agent": False,"isolate_agent_scope_by_user": False,@@ -69,21 +68,6 @@ class VikingClient:await self.client.initialize()await self._load_namespace_policy()- if self._is_root_key_mode() and self.admin_user_id:- user_exists = await self._check_user_exists(self.admin_user_id)- if not user_exists:- await self._initialize_user(self.admin_user_id, role="admin")- admin_user_api_key = await self._get_or_create_user_apikey(self.admin_user_id)- if admin_user_api_key:- self.admin_user_client = ov.AsyncHTTPClient(- url=self.openviking_config.server_url,- api_key=admin_user_api_key,- agent_id=self.agent_id,- account=self.account_id,- user=self.admin_user_id,- )- await self.admin_user_client.initialize()-@classmethodasync def create(cls, agent_id: Optional[str] = None):"""Factory method to create and initialize a VikingClient instance.@@ -256,11 +240,22 @@ class VikingClient:return resultasync def search(- self, query: str, target_uri: str | list[str] | None = None, limit: int = 10+ self,+ query: str,+ target_uri: str | list[str] | None = None,+ limit: int = 10,+ user_id: Optional[str] = None,) -> Dict[str, Any]:- # session = self.client.session()+ client = self.client+ should_close = False+ if user_id:+ client, should_close = await self._get_user_scoped_client(user_id)- result = await self.client.search(query, target_uri=target_uri, limit=limit)+ try:+ result = await client.search(query, target_uri=target_uri, limit=limit)+ finally:+ if should_close:+ await client.close()# 将 FindResult 对象转换为 JSON mapreturn {@@ -330,28 +325,24 @@ class VikingClient:logger.warning(f"Failed to initialize user {user_id}: {e}")return False- async def _get_or_create_user_apikey(self, user_id: str) -> Optional[str]:+ async def _get_or_create_user_apikey(self, user_id: str, role: str = "user") -> Optional[str]:"""获取或创建用户的 API key。"""if self._is_user_key_mode() or not self._apikey_manager or not user_id:return None- # Step 1: Check local storage firstapi_key = self._apikey_manager.get_apikey(user_id)if api_key:return api_keytry:- # 2a. Remove user if existsuser_exists = await self._check_user_exists(user_id)if user_exists:await self.client.admin_remove_user(self.account_id, user_id)- # 2b. Recreate user - this will save API key in _initialize_user- success = await self._initialize_user(user_id)+ success = await self._initialize_user(user_id, role=role)if not success:logger.warning(f"Failed to recreate user {user_id}")return None- # 2c. Get API key from local storage (it was saved by _initialize_user)api_key = self._apikey_manager.get_apikey(user_id)if api_key:return api_key@@ -362,6 +353,41 @@ class VikingClient:logger.error(f"Error getting or creating API key for user {user_id}: {e}")return None+ async def _get_user_scoped_client(self, user_id: Optional[str]) -> tuple[Any, bool]:+ effective_user_id = self._effective_user_id(user_id)+ if not effective_user_id:+ return self.client, False++ if self._is_root_key_mode():+ if not self._apikey_manager:+ raise RuntimeError("User API key manager is unavailable for user-scoped client")++ role = "admin" if effective_user_id == self.admin_user_id else "user"+ user_exists = await self._check_user_exists(effective_user_id)+ if not user_exists:+ success = await self._initialize_user(effective_user_id, role=role)+ if not success:+ raise RuntimeError(f"Failed to initialize user {effective_user_id}")++ user_api_key = await self._get_or_create_user_apikey(effective_user_id, role=role)+ if not user_api_key:+ raise RuntimeError(f"Failed to get API key for user {effective_user_id}")++ client_kwargs = {+ "url": self.openviking_config.server_url,+ "api_key": user_api_key,+ "agent_id": self.agent_id,+ }+ if effective_user_id == self.admin_user_id:+ client_kwargs["account"] = self.account_id+ client_kwargs["user"] = self.admin_user_id++ client = ov.AsyncHTTPClient(**client_kwargs)+ await client.initialize()+ return client, True++ return self.client, False+async def search_memory(self, query: str, user_ids: str | list[str], agent_user_id: str, limit: int = 10) -> dict[str, list[Any]]:@@ -423,19 +449,40 @@ class VikingClient:case_insensitive: bool = False,node_limit: Optional[int] = 10,exclude_uri: Optional[str] = None,+ user_id: Optional[str] = None,) -> Dict[str, Any]:"""通过模式(正则表达式)搜索内容"""- return await self.client.grep(- uri,- pattern,- case_insensitive=case_insensitive,- node_limit=node_limit,- exclude_uri=exclude_uri,- )+ client = self.client+ should_close = False+ if user_id:+ client, should_close = await self._get_user_scoped_client(user_id)- async def glob(self, pattern: str, uri: Optional[str] = None) -> Dict[str, Any]:+ try:+ return await client.grep(+ uri,+ pattern,+ case_insensitive=case_insensitive,+ node_limit=node_limit,+ exclude_uri=exclude_uri,+ )+ finally:+ if should_close:+ await client.close()++ async def glob(+ self, pattern: str, uri: Optional[str] = None, user_id: Optional[str] = None+ ) -> Dict[str, Any]:"""通过 glob 模式匹配文件"""- return await self.client.glob(pattern, uri=uri)+ client = self.client+ should_close = False+ if user_id:+ client, should_close = await self._get_user_scoped_client(user_id)++ try:+ return await client.glob(pattern, uri=uri)+ finally:+ if should_close:+ await client.close()async def commit(self, session_id: str, messages: list[dict[str, Any]], user_id: str = None):"""提交会话"""@@ -587,4 +634,4 @@ async def account_test():if __name__ == "__main__":asyncio.run(main_test())- # asyncio.run(account_test())\ No newline at end of file+ # asyncio.run(account_test())