Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(storage): optimize ls operation (#2257)
baojun-zhang0b41ed397606
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex 8f6a5ea93..07c6aaeb2 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -16,6 +16,7 @@ import asyncioimport contextvarsimport hashlibimport json+import osimport refrom contextlib import contextmanagerfrom dataclasses import dataclass, field@@ -85,6 +86,35 @@ def _is_directory_not_empty_error(message: str) -> bool:)+def _get_cpu_count() -> int:+ """Return the number of CPUs available to this process.++ Tries process_cpu_count (Python 3.13+, cgroup-aware),+ falls back to sched_getaffinity (Linux),+ then os.cpu_count (may report host CPUs in containers).+ """+ if hasattr(os, "process_cpu_count"):+ return os.process_cpu_count() or 1+ try:+ return len(os.sched_getaffinity(0))+ except (AttributeError, NotImplementedError):+ return os.cpu_count() or 1+++def _get_abstract_worker_count() -> int:+ default = max(4, min(12, min(32, _get_cpu_count() + 4) // 2))+ env_val = os.getenv("OPENVIKING_FILE_OPS_CONCURRENCY")+ if env_val is not None:+ try:+ return max(1, int(env_val))+ except ValueError:+ pass+ return max(1, default)+++_ABSTRACT_WORKER_COUNT = _get_abstract_worker_count()++# ========== Dataclass ==========@@ -1095,27 +1125,51 @@ class VikingFS:abs_limit: int,ctx: Optional[RequestContext] = None,) -> None:- """Batch fetch abstracts for entries.+ """Batch fetch abstracts for entries using a fixed-size worker pool.++ Non-directory entries receive an empty abstract immediately.+ Directory entries are processed concurrently via a worker pool,+ using _read_abstract_for_known_dir to skip redundant stat() calls.Args:entries: List of entries to fetch abstracts forabs_limit: Maximum length for abstract truncation"""- semaphore = asyncio.Semaphore(6)+ dir_jobs = []+ for index, entry in enumerate(entries):+ if not entry.get("isDir", False):+ entry["abstract"] = ""+ continue+ dir_jobs.append((index, entry))++ if not dir_jobs:+ return++ worker_count = min(_ABSTRACT_WORKER_COUNT, len(dir_jobs))++ cursor = 0+ cursor_lock = asyncio.Lock()+ results: Dict[int, str] = {}++ async def worker() -> None:+ nonlocal cursor+ while True:+ async with cursor_lock:+ if cursor >= len(dir_jobs):+ return+ index, entry = dir_jobs[cursor]+ cursor += 1- async def fetch_abstract(index: int, entry: Dict[str, Any]) -> tuple[int, str]:- async with semaphore:- if not entry.get("isDir", False):- return index, ""try:- abstract = await self.abstract(entry["uri"], ctx=ctx)- return index, abstract+ abstract = await self._read_abstract_for_known_dir(entry["uri"], ctx=ctx)except Exception:- return index, "[.abstract.md is not ready]"+ abstract = "[.abstract.md is not ready]"++ results[index] = abstract++ await asyncio.gather(*(worker() for _ in range(worker_count)))- tasks = [fetch_abstract(i, entry) for i, entry in enumerate(entries)]- abstract_results = await asyncio.gather(*tasks)- for index, abstract in abstract_results:+ for index, abstract in results.items():if len(abstract) > abs_limit:abstract = abstract[: abs_limit - 3] + "..."entries[index]["abstract"] = abstract@@ -1251,26 +1305,17 @@ class VikingFS:# ========== VikingFS Specific Capabilities ==========- async def abstract(+ async def _read_abstract_file(self,+ path: str,uri: str,ctx: Optional[RequestContext] = None,) -> str:- """Read directory's L0 summary (.abstract.md)."""- self._ensure_access(uri, ctx)- path = self._uri_to_path(uri, ctx=ctx)- try:- info = await self._async_agfs.stat(path)- except Exception as exc:- mapped = map_exception(exc, resource=uri)- if mapped is not None:- raise mapped from exc- raise- if not info.get("isDir", info.get("is_dir")):- raise FailedPreconditionError(- f"{uri} is not a directory",- details={"resource": uri, "expected": "directory"},- )+ """Read and decrypt/decode .abstract.md from a known directory path.++ Does NOT perform stat or isDir check -- caller is responsible for+ ensuring the path points to a directory.+ """file_path = f"{path}/.abstract.md"try:content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))@@ -1280,7 +1325,6 @@ class VikingFS:if mapped is not None:raise mapped from excraise- # Fallback to default if .abstract.md doesn't existreturn f"# {uri} [Directory abstract is not ready]"if self._encryptor:@@ -1289,6 +1333,42 @@ class VikingFS:return self._decode_bytes(content_bytes)+ async def _read_abstract_for_known_dir(+ self,+ uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> str:+ """Read .abstract.md for a directory that is already known to be a directory.++ Bypasses stat() and isDir check. Caller (i.e. _batch_fetch_abstracts)+ must guarantee that the URI points to a directory.+ """+ self._ensure_access(uri, ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ return await self._read_abstract_file(path, uri, ctx=ctx)++ async def abstract(+ self,+ uri: str,+ ctx: Optional[RequestContext] = None,+ ) -> str:+ """Read directory's L0 summary (.abstract.md)."""+ self._ensure_access(uri, ctx)+ path = self._uri_to_path(uri, ctx=ctx)+ try:+ info = await self._async_agfs.stat(path)+ except Exception as exc:+ mapped = map_exception(exc, resource=uri)+ if mapped is not None:+ raise mapped from exc+ raise+ if not info.get("isDir", info.get("is_dir")):+ raise FailedPreconditionError(+ f"{uri} is not a directory",+ details={"resource": uri, "expected": "directory"},+ )+ return await self._read_abstract_file(path, uri, ctx=ctx)+async def overview(self,uri: str,@@ -2339,7 +2419,6 @@ class VikingFS:all_entries.append(new_entry)elif show_all_hidden:all_entries.append(new_entry)- # call abstract in parallel 6 threadsawait self._batch_fetch_abstracts(all_entries, abs_limit, ctx=ctx)return all_entries