Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(storage): optimize ls operation (#2257)

baojun-zhang0b41ed397606
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 8f6a5ea93..07c6aaeb2 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -16,6 +16,7 @@ import asyncio
import contextvars
import hashlib
import json
+import os
import re
from contextlib import contextmanager
from dataclasses import dataclass, field
@@ -85,6 +86,35 @@ def _is_directory_not_empty_error(message: str) -> bool:
)
+def _get_cpu_count() -> int:
+ """Return the number of CPUs available to this process.
+
+ Tries process_cpu_count (Python 3.13+, cgroup-aware),
+ falls back to sched_getaffinity (Linux),
+ then os.cpu_count (may report host CPUs in containers).
+ """
+ if hasattr(os, "process_cpu_count"):
+ return os.process_cpu_count() or 1
+ try:
+ return len(os.sched_getaffinity(0))
+ except (AttributeError, NotImplementedError):
+ return os.cpu_count() or 1
+
+
+def _get_abstract_worker_count() -> int:
+ default = max(4, min(12, min(32, _get_cpu_count() + 4) // 2))
+ env_val = os.getenv("OPENVIKING_FILE_OPS_CONCURRENCY")
+ if env_val is not None:
+ try:
+ return max(1, int(env_val))
+ except ValueError:
+ pass
+ return max(1, default)
+
+
+_ABSTRACT_WORKER_COUNT = _get_abstract_worker_count()
+
+
# ========== Dataclass ==========
@@ -1095,27 +1125,51 @@ class VikingFS:
abs_limit: int,
ctx: Optional[RequestContext] = None,
) -> None:
- """Batch fetch abstracts for entries.
+ """Batch fetch abstracts for entries using a fixed-size worker pool.
+
+ Non-directory entries receive an empty abstract immediately.
+ Directory entries are processed concurrently via a worker pool,
+ using _read_abstract_for_known_dir to skip redundant stat() calls.
Args:
entries: List of entries to fetch abstracts for
abs_limit: Maximum length for abstract truncation
"""
- semaphore = asyncio.Semaphore(6)
+ dir_jobs = []
+ for index, entry in enumerate(entries):
+ if not entry.get("isDir", False):
+ entry["abstract"] = ""
+ continue
+ dir_jobs.append((index, entry))
+
+ if not dir_jobs:
+ return
+
+ worker_count = min(_ABSTRACT_WORKER_COUNT, len(dir_jobs))
+
+ cursor = 0
+ cursor_lock = asyncio.Lock()
+ results: Dict[int, str] = {}
+
+ async def worker() -> None:
+ nonlocal cursor
+ while True:
+ async with cursor_lock:
+ if cursor >= len(dir_jobs):
+ return
+ index, entry = dir_jobs[cursor]
+ cursor += 1
- async def fetch_abstract(index: int, entry: Dict[str, Any]) -> tuple[int, str]:
- async with semaphore:
- if not entry.get("isDir", False):
- return index, ""
try:
- abstract = await self.abstract(entry["uri"], ctx=ctx)
- return index, abstract
+ abstract = await self._read_abstract_for_known_dir(entry["uri"], ctx=ctx)
except Exception:
- return index, "[.abstract.md is not ready]"
+ abstract = "[.abstract.md is not ready]"
+
+ results[index] = abstract
+
+ await asyncio.gather(*(worker() for _ in range(worker_count)))
- tasks = [fetch_abstract(i, entry) for i, entry in enumerate(entries)]
- abstract_results = await asyncio.gather(*tasks)
- for index, abstract in abstract_results:
+ for index, abstract in results.items():
if len(abstract) > abs_limit:
abstract = abstract[: abs_limit - 3] + "..."
entries[index]["abstract"] = abstract
@@ -1251,26 +1305,17 @@ class VikingFS:
# ========== VikingFS Specific Capabilities ==========
- async def abstract(
+ async def _read_abstract_file(
self,
+ path: str,
uri: str,
ctx: Optional[RequestContext] = None,
) -> str:
- """Read directory's L0 summary (.abstract.md)."""
- self._ensure_access(uri, ctx)
- path = self._uri_to_path(uri, ctx=ctx)
- try:
- info = await self._async_agfs.stat(path)
- except Exception as exc:
- mapped = map_exception(exc, resource=uri)
- if mapped is not None:
- raise mapped from exc
- raise
- if not info.get("isDir", info.get("is_dir")):
- raise FailedPreconditionError(
- f"{uri} is not a directory",
- details={"resource": uri, "expected": "directory"},
- )
+ """Read and decrypt/decode .abstract.md from a known directory path.
+
+ Does NOT perform stat or isDir check -- caller is responsible for
+ ensuring the path points to a directory.
+ """
file_path = f"{path}/.abstract.md"
try:
content_bytes = self._handle_agfs_read(await self._async_agfs.read(file_path))
@@ -1280,7 +1325,6 @@ class VikingFS:
if mapped is not None:
raise mapped from exc
raise
- # Fallback to default if .abstract.md doesn't exist
return f"# {uri} [Directory abstract is not ready]"
if self._encryptor:
@@ -1289,6 +1333,42 @@ class VikingFS:
return self._decode_bytes(content_bytes)
+ async def _read_abstract_for_known_dir(
+ self,
+ uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> str:
+ """Read .abstract.md for a directory that is already known to be a directory.
+
+ Bypasses stat() and isDir check. Caller (i.e. _batch_fetch_abstracts)
+ must guarantee that the URI points to a directory.
+ """
+ self._ensure_access(uri, ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ return await self._read_abstract_file(path, uri, ctx=ctx)
+
+ async def abstract(
+ self,
+ uri: str,
+ ctx: Optional[RequestContext] = None,
+ ) -> str:
+ """Read directory's L0 summary (.abstract.md)."""
+ self._ensure_access(uri, ctx)
+ path = self._uri_to_path(uri, ctx=ctx)
+ try:
+ info = await self._async_agfs.stat(path)
+ except Exception as exc:
+ mapped = map_exception(exc, resource=uri)
+ if mapped is not None:
+ raise mapped from exc
+ raise
+ if not info.get("isDir", info.get("is_dir")):
+ raise FailedPreconditionError(
+ f"{uri} is not a directory",
+ details={"resource": uri, "expected": "directory"},
+ )
+ return await self._read_abstract_file(path, uri, ctx=ctx)
+
async def overview(
self,
uri: str,
@@ -2339,7 +2419,6 @@ class VikingFS:
all_entries.append(new_entry)
elif show_all_hidden:
all_entries.append(new_entry)
- # call abstract in parallel 6 threads
await self._batch_fetch_abstracts(all_entries, abs_limit, ctx=ctx)
return all_entries