Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(session): decouple Phase 2 extraction from live session files (#6332)
DuTaob281505a6d60
diff --git a/docs/en/api/17-tasks.md b/docs/en/api/17-tasks.mdindex f1314d103..3146f6dde 100644--- a/docs/en/api/17-tasks.md+++ b/docs/en/api/17-tasks.md@@ -197,6 +197,8 @@ Persisted task files now contain `execution_events`. Rolling back requires a ver`memories_extracted` in the completed task result reports per-category counts for this commit only. Sum its values when you want the total for this commit.+`token_usage` normally reports cumulative session usage after the statistics update. If Phase 2 cannot read valid session metadata after bounded retries for transient storage errors, extraction still completes without changing the session's `.meta.json`. The statistics failure is logged, and `token_usage` retains its full structure with every counter set to `0`; these zeros indicate unavailable statistics, not zero actual usage. That commit's cumulative token and memory counts are not backfilled automatically; its extraction results and telemetry remain available.+---### cancel_task()diff --git a/docs/zh/api/17-tasks.md b/docs/zh/api/17-tasks.mdindex 1db0eaa59..265db49f5 100644--- a/docs/zh/api/17-tasks.md+++ b/docs/zh/api/17-tasks.md@@ -195,6 +195,8 @@ ov task status uuid-xxx}```+完成任务结果中的 `memories_extracted` 是本次提交各类别的提取数量;`token_usage` 通常是统计更新后的 session 累计用量。如果 Phase 2 无法读到有效的 session 元数据(临时存储错误会有限重试),提取仍可完成,保留 session 的 `.meta.json` 原样并记录统计失败日志。此时 `token_usage` 保持完整结构、所有计数返回 `0`,表示统计不可用,并不代表实际消耗为零。本次累计 token 和记忆数量不会自动补计,提取结果与遥测数据仍然保留。+---### cancel_task()diff --git a/openviking/session/session.py b/openviking/session/session.pyindex c1c411424..5ec8c3c31 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -594,6 +594,64 @@ class Session:).to_dict()return found+ async def _read_commit_meta(self) -> SessionMeta:+ """Read a valid root metadata snapshot under the caller's Session lock.++ Commit recovery and statistics must never replace unreadable metadata+ with this instance's defaults or an earlier in-memory snapshot.+ """++ async def read() -> str:+ return await self._viking_fs.read_file(f"{self._session_uri}/.meta.json", ctx=self.ctx)++ def is_retryable(exc: Exception) -> bool:+ if isinstance(exc, (TimeoutError, ConnectionError)):+ return True+ mapped = map_exception(exc)+ return mapped is not None and mapped.code in {"UNAVAILABLE", "DEADLINE_EXCEEDED"}++ content = await retry_async(+ read,+ max_retries=2,+ base_delay=0.1,+ max_delay=0.3,+ jitter=False,+ is_retryable=is_retryable,+ logger=logger,+ operation_name="session commit metadata read",+ )+ data = json.loads(content)+ if not isinstance(data, dict):+ raise ValueError("Session metadata must be an object")+ # Preserve the normal loader's legacy coercions before validating the+ # counters that recovery and statistics will persist.+ meta = SessionMeta.from_dict(data)+ for name in (+ "message_count",+ "total_message_count",+ "commit_count",+ "pending_tokens",+ "keep_recent_count",+ "keep_recent_turn_count",+ "retained_message_token_budget",+ "min_raw_tail_steps",+ ):+ value = getattr(meta, name)+ if name == "total_message_count" and value is None:+ continue+ if type(value) is not int or value < 0:+ raise ValueError(f"Invalid Session metadata counter: {name}")+ for name in ("llm_token_usage", "embedding_token_usage", "memories_extracted"):+ counters = getattr(meta, name)+ if not isinstance(counters, dict) or any(+ type(value) is not int or value < 0 for value in counters.values()+ ):+ raise ValueError(f"Invalid Session metadata counters: {name}")+ for name in ("memory_policy", "auto_commit_policy"):+ if getattr(meta, name) is not None and not isinstance(getattr(meta, name), dict):+ raise ValueError(f"Invalid Session metadata policy: {name}")+ return meta+async def _rebuild_pending_tokens(self) -> None:"""Recompute ``pending_tokens`` from the current message list.@@ -1247,13 +1305,14 @@ class Session:# Root is authoritative and proves the rewrite completed. Reconcile# metadata that may have been interrupted immediately afterwards.try:- meta_content = await self._viking_fs.read_file(- f"{self._session_uri}/.meta.json",- ctx=self.ctx,+ self._meta = await self._read_commit_meta()+ except Exception as exc:+ await self._write_failed_marker(+ archive_uri,+ stage="phase1_recovery",+ error=f"Cannot restore Phase 1 metadata: {exc}",)- self._meta = SessionMeta.from_dict(json.loads(meta_content))- except Exception:- pass+ return Falseself._messages = live_messagesself._remember_retention_policy(keep_recent_count=max(0, int(marker.get("keep_recent_count", 0) or 0)),@@ -1270,6 +1329,10 @@ class Session:self._archives.archive_index_from_uri(archive_uri),)self._meta.last_commit_at = get_current_timestamp()+ if isinstance(queue_message, dict) and queue_message.get("record_auto_commit_success"):+ # Recovery must restore the Phase 1 throttle stamp even if+ # the later Phase 2 statistics update is skipped.+ self._meta.last_auto_commit_at = get_current_timestamp()await self._rebuild_pending_tokens()await self._save_meta()await self._write_phase1_ready_marker(archive_uri)@@ -1667,11 +1730,9 @@ class Session:# this intentional registration window as an orphan.phase1_stage = "publication_lease"archive_path = self._viking_fs._uri_to_path(archive_uri, ctx=self.ctx)- publication_lease = (- await self._viking_fs._async_agfs.pathlock_acquire_exact(- archive_path,- timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS,- )+ publication_lease = await self._viking_fs._async_agfs.pathlock_acquire_exact(+ archive_path,+ timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS,)await self._write_phase1_ready_marker(archive_uri)if record_idle_commit_success:@@ -2511,12 +2572,15 @@ class Session:# Phase 2 complete — update meta with telemetry and commit infosnapshot = telemetry.finish("ok")_publish_telemetry_summary_best_effort(snapshot)- await self._merge_and_save_commit_meta(+ updated_meta = await self._merge_and_save_commit_meta(archive_index=archive_index,memories_extracted=memories_extracted,telemetry_snapshot=snapshot,record_auto_commit_success=record_auto_commit_success,)+ # Keep the response schema stable when cumulative statistics are+ # unavailable. This zero snapshot is never persisted to the session.+ usage_meta = updated_meta if updated_meta is not None else SessionMeta()result_payload = {"session_id": self.session_id,@@ -2539,13 +2603,13 @@ class Session:"agent_evolution_enabled": agent_evolution_enabled,"agent_memory_skip_reason": agent_memory_skip_reason,"token_usage": {- "llm": dict(self._meta.llm_token_usage),- "embedding": dict(self._meta.embedding_token_usage),+ "llm": dict(usage_meta.llm_token_usage),+ "embedding": dict(usage_meta.embedding_token_usage),"total": {- "total_tokens": self._meta.llm_token_usage["total_tokens"]- + self._meta.embedding_token_usage["total_tokens"],- "cached_tokens": self._meta.llm_token_usage["cached_tokens"],- "reasoning_tokens": self._meta.llm_token_usage["reasoning_tokens"],+ "total_tokens": usage_meta.llm_token_usage["total_tokens"]+ + usage_meta.embedding_token_usage["total_tokens"],+ "cached_tokens": usage_meta.llm_token_usage["cached_tokens"],+ "reasoning_tokens": usage_meta.llm_token_usage["reasoning_tokens"],},},}@@ -2974,22 +3038,23 @@ class Session:telemetry_snapshot: Any,*,record_auto_commit_success: bool = False,- ) -> None:- """Merge Phase 2 results without overwriting concurrent root updates."""+ ) -> Optional[SessionMeta]:+ """Return saved metadata, or None when its read fails and statistics are skipped."""session_path = self._viking_fs._uri_to_path(self._session_uri, ctx=self.ctx)lease = await self._viking_fs._async_agfs.pathlock_acquire_exact(session_path, timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS)try:- latest_meta = self._metatry:- meta_content = await self._viking_fs.read_file(- f"{self._session_uri}/.meta.json",- ctx=self.ctx,+ latest_meta = await self._read_commit_meta()+ except Exception as exc:+ reason = f"{type(exc).__name__}: {exc}"+ logger.warning(+ "Session %s extraction completed without updating statistics: %s",+ self.session_id,+ reason,)- latest_meta = SessionMeta.from_dict(json.loads(meta_content))- except Exception:- latest_meta = self._meta+ return Noneif telemetry_snapshot:llm = telemetry_snapshot.summary.get("tokens", {}).get("llm", {})@@ -3012,29 +3077,16 @@ class Session:latest_meta.memories_extracted.get("total", 0) + count)latest_meta.last_commit_at = get_current_timestamp()- latest_meta.message_count = await self._read_live_message_count()if record_auto_commit_success:# Mirror the Phase 1 success stamp so the persisted meta reflects# a clean auto-commit even after Phase 2 reloads the latest meta.latest_meta.last_auto_commit_at = get_current_timestamp()self._meta = latest_metaawait self._save_meta()+ return latest_metafinally:await self._viking_fs._async_agfs.pathlock_release(lease)- async def _read_live_message_count(self) -> int:- """Count current live session messages from persisted storage."""- if not self._viking_fs:- return len(self._messages)- try:- content = await self._viking_fs.read_file(- f"{self._session_uri}/messages.jsonl",- ctx=self.ctx,- )- except Exception:- return len(self._messages)- return len([line for line in content.strip().split("\n") if line.strip()])-async def _read_live_messages_strict(self) -> List[Message]:"""Read the authoritative root JSONL without silently dropping corrupt rows."""if not self._viking_fs:diff --git a/openviking/storage/queuefs/session_commit_processor.py b/openviking/storage/queuefs/session_commit_processor.pyindex 7d42519b7..ae43fec0c 100644--- a/openviking/storage/queuefs/session_commit_processor.py+++ b/openviking/storage/queuefs/session_commit_processor.py@@ -75,7 +75,6 @@ class SessionCommitProcessor(DequeueHandlerBase):user_id=ctx.user.user_id,)return True- await session.load()processed = await session.resume_queued_commit(msg)if not processed:from openviking.storage.queuefs import QueueManager, get_queue_managerdiff --git a/tests/session/conftest.py b/tests/session/conftest.pyindex 88ae682a5..958bf1fb7 100644--- a/tests/session/conftest.py+++ b/tests/session/conftest.py@@ -62,7 +62,6 @@ async def client():breakawait asyncio.sleep(0)- await queued_session.load()try:while not await queued_session.resume_queued_commit(message):await asyncio.sleep(0)diff --git a/tests/session/test_session_retention_integration.py b/tests/session/test_session_retention_integration.pyindex 7bb3bb44d..8a779b50f 100644--- a/tests/session/test_session_retention_integration.py+++ b/tests/session/test_session_retention_integration.py@@ -1627,7 +1627,6 @@ async def test_session_state_lock_serializes_root_meta_without_blocking_archive_phase2 = client(session_id=initial.session_id)appending = client(session_id=initial.session_id)- await phase2.load()await appending.load()phase2_inside_save = asyncio.Event()@@ -1875,6 +1874,14 @@ async def test_interrupted_phase1_recovers_when_root_rewrite_is_durable():session = client(session_id="phase1_reconcile_durable_root_test")await session.ensure_exists()+ session.meta.memory_policy = {"memory_types": ["profile"]}+ legacy_meta = session.meta.to_dict()+ legacy_meta.update(+ pending_tokens="123", keep_recent_count="1", memories_extracted={"total": "4"}+ )+ await session._viking_fs.write_file(+ f"{session.uri}/.meta.json", json.dumps(legacy_meta), ctx=session.ctx+ )original = [_text_message("u1", "user", "archive me"),_text_message("a1", "assistant", "retain me"),@@ -1884,7 +1891,7 @@ async def test_interrupted_phase1_recovers_when_root_rewrite_is_durable(archive_uri = await _write_archive(session, 1, [original[0]])await session._write_phase1_marker(archive_uri,- queue_message={"task_id": "synthetic"},+ queue_message={"task_id": "synthetic", "record_auto_commit_success": True},original_messages=original,archived_messages=[original[0]],retained_messages=[original[1]],@@ -1901,9 +1908,28 @@ async def test_interrupted_phase1_recovers_when_root_rewrite_is_durable(AsyncMock(side_effect=lambda task_id: task_id == "synthetic"),)- assert await session._ensure_phase1_ready(archive_uri)- assert (await session._read_phase1_meta(archive_uri))["status"] == "ready"- assert session.meta.message_count == 1+ recovering = client(session_id=session.session_id)+ original_read = recovering._viking_fs.read_file+ failed_once = False++ async def transient_meta_read(uri, *args, **kwargs):+ nonlocal failed_once+ if uri == f"{session.uri}/.meta.json" and not failed_once:+ failed_once = True+ raise AGFSIoError("temporary metadata read failure")+ return await original_read(uri, *args, **kwargs)++ monkeypatch.setattr(recovering._viking_fs, "read_file", transient_meta_read)+ assert await recovering._ensure_phase1_ready(archive_uri)+ assert (await recovering._read_phase1_meta(archive_uri))["status"] == "ready"+ assert recovering.meta.message_count == 1+ assert recovering.meta.memory_policy == {"memory_types": ["profile"]}+ assert recovering.meta.memories_extracted == {"total": 4}+ assert recovering.meta.last_auto_commit_at+ persisted = json.loads(+ await recovering._viking_fs.read_file(f"{session.uri}/.meta.json", ctx=recovering.ctx)+ )+ assert persisted["last_auto_commit_at"] == recovering.meta.last_auto_commit_atasync def test_interrupted_phase1_before_root_rewrite_becomes_failed(diff --git a/tests/storage/test_session_commit_processor_identity.py b/tests/storage/test_session_commit_processor_identity.pyindex cf60c99ad..78c7ced3c 100644--- a/tests/storage/test_session_commit_processor_identity.py+++ b/tests/storage/test_session_commit_processor_identity.py@@ -31,9 +31,6 @@ class _FakeSession:async def exists(self) -> bool:return True- async def load(self) -> None:- return None-async def resume_queued_commit(self, msg) -> bool:root = get_root_observability_context()self._captured["account_id"] = root.account_id if root else None@@ -74,11 +71,12 @@ async def test_process_binds_committing_identity_to_root_context(monkeypatch):with pytest.raises(OSError, match="store unavailable"):await processor.on_dequeue(message)tracker.create.side_effect = None- session.load = AsyncMock(side_effect=ValueError("invalid session message"))+ original_resume = session.resume_queued_commit+ session.resume_queued_commit = AsyncMock(side_effect=ValueError("invalid archive message"))result = await processor.on_dequeue(message)assert result.outcome is ProcessOutcome.FAILED- assert result.error == "invalid session message"- session.load.side_effect = None+ assert result.error == "invalid archive message"+ session.resume_queued_commit = original_resumeassert (await processor.on_dequeue(message)).outcome is ProcessOutcome.SUCCESSassert captured == {"account_id": "acme", "user_id": "alice"}assert get_root_observability_context() is Nonediff --git a/tests/unit/session/test_session_commit_resume.py b/tests/unit/session/test_session_commit_resume.pyindex 500ed3cd0..716b39b3c 100644--- a/tests/unit/session/test_session_commit_resume.py+++ b/tests/unit/session/test_session_commit_resume.py@@ -11,11 +11,13 @@ from unittest.mock import AsyncMockimport pytestfrom openviking.message import Message, TextPart, ToolPart+from openviking.pyagfs.exceptions import AGFSConnectionErrorfrom openviking.service.task_store import PersistentTaskStorefrom openviking.service.task_tracker import TaskStatus, TaskTracker, set_task_trackerfrom openviking.session.memory_policy import MemoryPolicy-from openviking.session.session import Session+from openviking.session.session import Session, SessionMetafrom openviking.storage.queuefs.session_commit_msg import SessionCommitMsg+from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessorfrom tests.test_task_tracker import _FakeAgfs@@ -67,7 +69,8 @@ class _MemoryVikingFS:async def read_file(self, uri, ctx=None):if uri not in self.files:raise FileNotFoundError(uri)- return self.files[uri]+ content = self.files[uri]+ return content.decode("utf-8") if isinstance(content, bytes) else contentasync def write_file(self, uri, content, ctx=None, lease_ref=None):self.files[uri] = content@@ -78,6 +81,11 @@ class _MemoryVikingFS:async def exists(self, uri, ctx=None):return uri in self.files or any(path.startswith(f"{uri}/") for path in self.files)+ async def stat(self, uri, ctx=None, skip_count=False):+ if not await self.exists(uri, ctx):+ raise FileNotFoundError(uri)+ return {"path": uri}+async def ls(self, uri, ctx=None):prefix = f"{uri}/"names = {@@ -206,47 +214,96 @@ async def test_phase2_cancellation_marks_archive_only_when_requested(@pytest.mark.asyncio-async def test_resume_queued_commit_continues_phase2(monkeypatch):+async def test_resume_queued_commit_preserves_live_state_when_statistics_are_unavailable(+ monkeypatch,+):session_uri = "viking://user/default/sessions/session-1"archive_uri = f"{session_uri}/history/archive_001"- archived = Message(id="archived", role="user", parts=[TextPart("old")])- retained = Message(id="retained", role="assistant", parts=[TextPart("new")])+ meta_uri = f"{session_uri}/.meta.json"+ live_uri = f"{session_uri}/messages.jsonl"+ archived = Message(id="archived", role="user", parts=[TextPart("archive input")])+ latest_meta = SessionMeta(+ session_id="session-1",+ message_count=7,+ total_message_count=50,+ commit_count=2,+ auto_commit_policy={"keep_recent_count": 3},+ )+ latest_meta.llm_token_usage["total_tokens"] = 100+ original_meta = json.dumps(latest_meta.to_dict())files = {- f"{session_uri}/messages.jsonl": f"{retained.to_jsonl()}\n",- f"{session_uri}/.meta.json": json.dumps(- {"session_id": "session-1", "message_count": 1, "commit_count": 1}- ),- f"{archive_uri}/messages.jsonl": f"{archived.to_jsonl()}\n",+ live_uri: "{live file being rewritten",+ meta_uri: original_meta,+ f"{archive_uri}/messages.jsonl": archived.to_jsonl() + "\n",+ f"{archive_uri}/.meta.json": json.dumps({"phase1": {"status": "ready"}}),}- viking_fs = _MemoryVikingFS(files)- session = Session(viking_fs=viking_fs, session_id="session-1", session_uri=session_uri)+ storage = _MemoryVikingFS(files)+ original_read = storage.read_file++ async def read(uri, ctx=None):+ if uri == meta_uri:+ raise AGFSConnectionError("metadata unavailable")+ return await original_read(uri, ctx)++ reads = AsyncMock(side_effect=read)+ monkeypatch.setattr(storage, "read_file", reads)+ extract = AsyncMock(return_value={"contexts": [SimpleNamespace(category="profile")]})+ summarize = AsyncMock(return_value="# Session Summary\n\nArchive summarized.")+ session = Session(+ viking_fs=storage,+ session_id="session-1",+ session_uri=session_uri,+ session_compressor=SimpleNamespace(extract_long_term_memories=extract),+ )+ # Neither persisted statistics nor a stale instance may supply the fallback.+ session.meta.llm_token_usage["total_tokens"] = 999tracker = TaskTracker(_TaskStore())- set_task_tracker(tracker)- monkeypatch.setattr(session, "_run_memory_extraction", AsyncMock())+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)+ monkeypatch.setattr(session, "_generate_archive_summary_async", summarize)+ monkeypatch.setattr(+ "openviking.session.session.get_openviking_config",+ lambda: SimpleNamespace(+ memory=SimpleNamespace(extraction_enabled=True, session_skill_extraction_enabled=False)+ ),+ )message = SessionCommitMsg(task_id="task-1",- session_id="session-1",+ session_id=session.session_id,session_uri=session_uri,archive_uri=archive_uri,- user={"account_id": "default", "user_id": "default"},- memory_policy={"memory_types": []},- auto_commit_policy={"pending_token_threshold": 8000},+ user=session.user.to_dict(),+ memory_policy=MemoryPolicy(memory_types={"profile"}, enable_working_memory=True).to_dict(),)-- try:- await session.resume_queued_commit(message)- finally:- set_task_tracker(None)-- session._run_memory_extraction.assert_awaited_once()- assert session._run_memory_extraction.await_args.kwargs["task_id"] == "task-1"- assert session._run_memory_extraction.await_args.kwargs["agent_evolution_enabled"] is True- assert session._run_memory_extraction.await_args.kwargs["auto_commit_policy"] == {- "pending_token_threshold": 8000+ processor = SessionCommitProcessor(SimpleNamespace(session=lambda *args, **kwargs: session))++ await processor.on_dequeue(message.to_dict())++ task = await tracker.get(message.task_id)+ assert task.status is TaskStatus.COMPLETED+ assert task.result["memories_extracted"] == {"profile": 1}+ assert task.result["token_usage"] == {+ "llm": {+ "prompt_tokens": 0,+ "completion_tokens": 0,+ "total_tokens": 0,+ "cached_tokens": 0,+ "reasoning_tokens": 0,+ },+ "embedding": {"total_tokens": 0},+ "total": {"total_tokens": 0, "cached_tokens": 0, "reasoning_tokens": 0},}- assert [item.id for item in session._run_memory_extraction.await_args.kwargs["messages"]] == [- "archived"- ]+ assert files[meta_uri] == original_meta+ assert f"{archive_uri}/.done" in files+ assert [message.id for message in summarize.await_args.args[0]] == ["archived"]+ assert [message.id for message in extract.await_args.kwargs["messages"]] == ["archived"]+ assert extract.await_args.kwargs["allowed_memory_types"] == {"profile"}++ await processor.on_dequeue(message.to_dict())++ extract.assert_awaited_once()+ summarize.assert_awaited_once()+ assert all(call.args[0] != live_uri for call in reads.await_args_list)+ assert files[meta_uri] == original_meta@pytest.mark.asyncio