Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
perf(session): reduce commit phase1 latency (#4687)
Qin Haojie02e31f2d661a
diff --git a/openviking/service/task_store.py b/openviking/service/task_store.pyindex 1e122c93a..0522961eb 100644--- a/openviking/service/task_store.py+++ b/openviking/service/task_store.py@@ -46,6 +46,7 @@ class PersistentTaskStore:def __init__(self, agfs: Any) -> None:self._agfs = agfs if isinstance(agfs, AsyncAGFSClient) else AsyncAGFSClient(agfs)+ self._ensured_task_dirs: set[tuple[str, str]] = set()async def create(self, task: Any) -> None:await self._write_task(task)@@ -92,7 +93,11 @@ class PersistentTaskStore:async def delete(self, task_id: str, *, account_id: str, user_id: Optional[str] = None) -> None:if not user_id:return- await self._agfs.rm(self._task_path(account_id, user_id, task_id), force=True)+ await self._agfs.rm(+ self._task_path(account_id, user_id, task_id),+ force=True,+ auto_pathlock=False,+ )async def _write_task(self, task: Any) -> None:account_id = getattr(task, "account_id", None)@@ -100,16 +105,43 @@ class PersistentTaskStore:if not account_id or not user_id:raise ValueError("PersistentTaskStore requires account_id and user_id")await self._ensure_task_dir(account_id, user_id)- await self._agfs.write(- self._task_path(account_id, user_id, task.task_id),- json.dumps(_task_to_payload(task), ensure_ascii=False).encode("utf-8"),- )+ path = self._task_path(account_id, user_id, task.task_id)+ payload = json.dumps(_task_to_payload(task), ensure_ascii=False).encode("utf-8")+ try:+ await self._write_task_payload(path, payload)+ except (AGFSNotFoundError, FileNotFoundError):+ self._ensured_task_dirs.discard((account_id, user_id))+ await self._ensure_task_dir(account_id, user_id)+ await self._write_task_payload(path, payload)async def _ensure_task_dir(self, account_id: str, user_id: str) -> None:- await self._mkdir_if_missing(self._account_dir(account_id))- await self._mkdir_if_missing(self._system_dir(account_id))- await self._mkdir_if_missing(self._task_root_dir(account_id))- await self._mkdir_if_missing(self._task_dir(account_id, user_id))+ cache_key = (account_id, user_id)+ if cache_key in self._ensured_task_dirs:+ return++ seen_paths: set[str] = set()+ for path in self._task_dir_chain(account_id, user_id):+ if path in seen_paths:+ continue+ seen_paths.add(path)+ await self._mkdir_if_missing(path)+ self._ensured_task_dirs.add(cache_key)++ def _task_dir_chain(self, account_id: str, user_id: str) -> tuple[str, ...]:+ return (+ self._account_dir(account_id),+ self._system_dir(account_id),+ self._task_root_dir(account_id),+ self._task_dir(account_id, user_id),+ )++ async def _write_task_payload(self, path: str, payload: bytes) -> None:+ # TaskTracker is the owner of task mutations and serializes updates per+ # task. PersistentTaskStore does not implement store-level revision/CAS,+ # so AGFS pathlock cannot make multiple independent writers correct; it+ # only adds one storage lock around every task lifecycle write. Avoid+ # that extra PathLock overhead on this internal task file.+ await self._agfs.write(path, payload, auto_pathlock=False)async def _mkdir_if_missing(self, path: str) -> None:try:diff --git a/openviking/session/session.py b/openviking/session/session.pyindex 2ca9f658c..9707f1aa3 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -2107,42 +2107,50 @@ class Session:)phase1_stage = "phase1_persist"try:- await self._write_phase1_marker(- archive_uri,- queue_message=queue_msg.to_dict(),- original_messages=original_messages,- archived_messages=messages_to_archive,- retained_messages=retained_messages,- keep_recent_count=keep_recent_count,- retention_mode=retention_mode,- keep_recent_turn_count=effective_keep_turns if turn_mode else 0,- retained_message_token_budget=effective_token_budget if turn_mode else 0,- min_raw_tail_steps=effective_min_tail,- agent_evolution_enabled=agent_evolution_enabled,- agent_memory_skip_reason=agent_memory_skip_reason,- )-- # Archive raw remains durable and recoverable before any live- # conversation history is removed from the root JSONL.+ archive_persist_tasks = [+ self._write_phase1_marker(+ archive_uri,+ queue_message=queue_msg.to_dict(),+ original_messages=original_messages,+ archived_messages=messages_to_archive,+ retained_messages=retained_messages,+ keep_recent_count=keep_recent_count,+ retention_mode=retention_mode,+ keep_recent_turn_count=effective_keep_turns if turn_mode else 0,+ retained_message_token_budget=effective_token_budget if turn_mode else 0,+ min_raw_tail_steps=effective_min_tail,+ agent_evolution_enabled=agent_evolution_enabled,+ agent_memory_skip_reason=agent_memory_skip_reason,+ )+ ]if self._viking_fs:lines = [m.to_jsonl() for m in messages_to_archive]- await self._viking_fs.write_file(- uri=f"{archive_uri}/messages.jsonl",- content="\n".join(lines) + "\n",- ctx=self.ctx,- )- if retention_plan is not None:- await self._merge_archive_meta(- archive_uri,- {- "retention_plan": self._retention_plan_meta(- retention_plan,- keep_recent_turn_count=effective_keep_turns,- retained_message_token_budget=effective_token_budget,- min_raw_tail_steps=effective_min_tail,- )- },+ archive_persist_tasks.append(+ self._viking_fs.write_file(+ uri=f"{archive_uri}/messages.jsonl",+ content="\n".join(lines) + "\n",+ ctx=self.ctx,)+ )+ archive_persist_results = await asyncio.gather(+ *archive_persist_tasks,+ return_exceptions=True,+ )+ for result in archive_persist_results:+ if isinstance(result, BaseException):+ raise result+ if retention_plan is not None:+ await self._merge_archive_meta(+ archive_uri,+ {+ "retention_plan": self._retention_plan_meta(+ retention_plan,+ keep_recent_turn_count=effective_keep_turns,+ retained_message_token_budget=effective_token_budget,+ min_raw_tail_steps=effective_min_tail,+ )+ },+ )phase1_stage = "queue_enqueue"await get_queue_manager().enqueue(@@ -5294,45 +5302,53 @@ class Session:lines = [m.to_jsonl() for m in messages]content = "\n".join(lines) + "\n" if lines else ""-- await viking_fs.write_file(- uri=f"{self._session_uri}/messages.jsonl",- content=content,- ctx=self.ctx,- lease_ref=lease_ref,+ abstract_content = render_abstract_overview(+ ContextLevel.ABSTRACT,+ self._session_uri,+ abstract,+ {+ "generated_by": {+ "component": "Session",+ "trigger": "session_update",+ }+ },)- await viking_fs.write_file(- uri=f"{self._session_uri}/.abstract.md",- content=render_abstract_overview(- ContextLevel.ABSTRACT,- self._session_uri,- abstract,- {- "generated_by": {- "component": "Session",- "trigger": "session_update",- }- },- ),- ctx=self.ctx,- lease_ref=lease_ref,+ overview_content = render_abstract_overview(+ ContextLevel.OVERVIEW,+ self._session_uri,+ overview,+ {+ "generated_by": {+ "component": "Session",+ "trigger": "session_update",+ }+ },)- await viking_fs.write_file(- uri=f"{self._session_uri}/.overview.md",- content=render_abstract_overview(- ContextLevel.OVERVIEW,- self._session_uri,- overview,- {- "generated_by": {- "component": "Session",- "trigger": "session_update",- }- },++ root_write_results = await asyncio.gather(+ viking_fs.write_file(+ uri=f"{self._session_uri}/messages.jsonl",+ content=content,+ ctx=self.ctx,+ lease_ref=lease_ref,),- ctx=self.ctx,- lease_ref=lease_ref,+ viking_fs.write_file(+ uri=f"{self._session_uri}/.abstract.md",+ content=abstract_content,+ ctx=self.ctx,+ lease_ref=lease_ref,+ ),+ viking_fs.write_file(+ uri=f"{self._session_uri}/.overview.md",+ content=overview_content,+ ctx=self.ctx,+ lease_ref=lease_ref,+ ),+ return_exceptions=True,)+ for result in root_write_results:+ if isinstance(result, BaseException):+ raise resultdef _generate_abstract(self) -> str:"""Generate one-sentence summary for session."""diff --git a/tests/test_task_tracker.py b/tests/test_task_tracker.pyindex 6deed5469..7579d9f7e 100644--- a/tests/test_task_tracker.py+++ b/tests/test_task_tracker.py@@ -62,12 +62,32 @@ class _FakeAgfs:self.files = {}self.dirs = {"/", "/local"}self.fail_rm = False+ self.mkdir_calls = []+ self.write_calls = []+ self.rm_calls = []def mkdir(self, path: str, mode: str = "755"):+ self.mkdir_calls.append({"path": path, "mode": mode})self.dirs.add(path.rstrip("/") or "/")return {"message": "created", "mode": mode}- def write(self, path: str, data):+ def write(+ self,+ path: str,+ data,+ max_retries: int = 3,+ *,+ fs_ctx=None,+ auto_pathlock: bool = True,+ ):+ self.write_calls.append(+ {+ "path": path,+ "max_retries": max_retries,+ "fs_ctx": fs_ctx,+ "auto_pathlock": auto_pathlock,+ }+ )if isinstance(data, str):data = data.encode("utf-8")self.files[path] = data@@ -102,7 +122,24 @@ class _FakeAgfs:children[name] = {"name": name, "path": f"{prefix}/{name}", "is_dir": False}return list(children.values())- def rm(self, path: str, recursive: bool = False, force: bool = False):+ def rm(+ self,+ path: str,+ recursive: bool = False,+ force: bool = False,+ *,+ fs_ctx=None,+ auto_pathlock: bool = True,+ ):+ self.rm_calls.append(+ {+ "path": path,+ "recursive": recursive,+ "force": force,+ "fs_ctx": fs_ctx,+ "auto_pathlock": auto_pathlock,+ }+ )if self.fail_rm:raise OSError("simulated delete failure")self.files.pop(path, None)@@ -111,6 +148,7 @@ class _FakeAgfs:class _FakeAgfsExistingDir(_FakeAgfs):def mkdir(self, path: str, mode: str = "755"):+ self.mkdir_calls.append({"path": path, "mode": mode})normalized = path.rstrip("/") or "/"if normalized in self.dirs:raise AGFSAlreadyExistsError(f"already exists: {path}")@@ -450,6 +488,7 @@ async def test_evict_keeps_cached_task_when_persistent_delete_fails():assert await tracker.get(t.task_id) is Noneassert await tracker._store.get(t.task_id, **_owner_kwargs()) is None+ assert all(call["auto_pathlock"] is False for call in agfs.rm_calls)async def test_evict_keeps_recent_completed(tracker: TaskTracker):@@ -527,6 +566,7 @@ async def test_persistent_store_writes_task_record_json():raw = agfs.files[f"/local/acme/_system/tasks/alice/{task.task_id}.json"]payload = json.loads(raw.decode("utf-8"))+ assert agfs.write_calls[-1]["auto_pathlock"] is Falseassert payload["task_id"] == task.task_idassert payload["task_type"] == "add_resource"assert payload["account_id"] == "acme"@@ -543,6 +583,7 @@ async def test_persistent_store_writes_task_record_json():agfs.files[f"/local/acme/_system/tasks/alice/{task.task_id}.json"].decode("utf-8"))assert terminal_payload["auth"] == {}+ assert all(call["auto_pathlock"] is False for call in agfs.write_calls)async def test_persistent_store_keeps_tasktracker_tasks_dict():@@ -579,11 +620,20 @@ async def test_persistent_store_ignores_existing_task_dirs():tracker = TaskTracker(store=PersistentTaskStore(agfs))first = await tracker.create("session_commit", resource_id="sess-1", **_owner_kwargs())+ first_mkdir_calls = list(agfs.mkdir_calls)second = await tracker.create("session_commit", resource_id="sess-2", **_owner_kwargs())assert first.task_id != second.task_idassert agfs.files[f"/local/acme/_system/tasks/alice/{first.task_id}.json"]assert agfs.files[f"/local/acme/_system/tasks/alice/{second.task_id}.json"]+ assert [call["path"] for call in first_mkdir_calls] == [+ "/local/acme",+ "/local/acme/_system",+ "/local/acme/_system/tasks",+ "/local/acme/_system/tasks/alice",+ ]+ assert agfs.mkdir_calls == first_mkdir_calls+ assert all(call["auto_pathlock"] is False for call in agfs.write_calls)async def test_create_requires_owner(tracker: TaskTracker):