Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(session): decouple Phase 2 extraction from live session files (#6332)

DuTaob281505a6d60
diff --git a/docs/en/api/17-tasks.md b/docs/en/api/17-tasks.md
index f1314d103..3146f6dde 100644
--- a/docs/en/api/17-tasks.md
+++ b/docs/en/api/17-tasks.md
@@ -197,6 +197,8 @@ Persisted task files now contain `execution_events`. Rolling back requires a ver
`memories_extracted` in the completed task result reports per-category counts for this commit only. Sum its values when you want the total for this commit.
+`token_usage` normally reports cumulative session usage after the statistics update. If Phase 2 cannot read valid session metadata after bounded retries for transient storage errors, extraction still completes without changing the session's `.meta.json`. The statistics failure is logged, and `token_usage` retains its full structure with every counter set to `0`; these zeros indicate unavailable statistics, not zero actual usage. That commit's cumulative token and memory counts are not backfilled automatically; its extraction results and telemetry remain available.
+
---
### cancel_task()
diff --git a/docs/zh/api/17-tasks.md b/docs/zh/api/17-tasks.md
index 1db0eaa59..265db49f5 100644
--- a/docs/zh/api/17-tasks.md
+++ b/docs/zh/api/17-tasks.md
@@ -195,6 +195,8 @@ ov task status uuid-xxx
}
```
+完成任务结果中的 `memories_extracted` 是本次提交各类别的提取数量;`token_usage` 通常是统计更新后的 session 累计用量。如果 Phase 2 无法读到有效的 session 元数据(临时存储错误会有限重试),提取仍可完成,保留 session 的 `.meta.json` 原样并记录统计失败日志。此时 `token_usage` 保持完整结构、所有计数返回 `0`,表示统计不可用,并不代表实际消耗为零。本次累计 token 和记忆数量不会自动补计,提取结果与遥测数据仍然保留。
+
---
### cancel_task()
diff --git a/openviking/session/session.py b/openviking/session/session.py
index c1c411424..5ec8c3c31 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -594,6 +594,64 @@ class Session:
).to_dict()
return found
+ async def _read_commit_meta(self) -> SessionMeta:
+ """Read a valid root metadata snapshot under the caller's Session lock.
+
+ Commit recovery and statistics must never replace unreadable metadata
+ with this instance's defaults or an earlier in-memory snapshot.
+ """
+
+ async def read() -> str:
+ return await self._viking_fs.read_file(f"{self._session_uri}/.meta.json", ctx=self.ctx)
+
+ def is_retryable(exc: Exception) -> bool:
+ if isinstance(exc, (TimeoutError, ConnectionError)):
+ return True
+ mapped = map_exception(exc)
+ return mapped is not None and mapped.code in {"UNAVAILABLE", "DEADLINE_EXCEEDED"}
+
+ content = await retry_async(
+ read,
+ max_retries=2,
+ base_delay=0.1,
+ max_delay=0.3,
+ jitter=False,
+ is_retryable=is_retryable,
+ logger=logger,
+ operation_name="session commit metadata read",
+ )
+ data = json.loads(content)
+ if not isinstance(data, dict):
+ raise ValueError("Session metadata must be an object")
+ # Preserve the normal loader's legacy coercions before validating the
+ # counters that recovery and statistics will persist.
+ meta = SessionMeta.from_dict(data)
+ for name in (
+ "message_count",
+ "total_message_count",
+ "commit_count",
+ "pending_tokens",
+ "keep_recent_count",
+ "keep_recent_turn_count",
+ "retained_message_token_budget",
+ "min_raw_tail_steps",
+ ):
+ value = getattr(meta, name)
+ if name == "total_message_count" and value is None:
+ continue
+ if type(value) is not int or value < 0:
+ raise ValueError(f"Invalid Session metadata counter: {name}")
+ for name in ("llm_token_usage", "embedding_token_usage", "memories_extracted"):
+ counters = getattr(meta, name)
+ if not isinstance(counters, dict) or any(
+ type(value) is not int or value < 0 for value in counters.values()
+ ):
+ raise ValueError(f"Invalid Session metadata counters: {name}")
+ for name in ("memory_policy", "auto_commit_policy"):
+ if getattr(meta, name) is not None and not isinstance(getattr(meta, name), dict):
+ raise ValueError(f"Invalid Session metadata policy: {name}")
+ return meta
+
async def _rebuild_pending_tokens(self) -> None:
"""Recompute ``pending_tokens`` from the current message list.
@@ -1247,13 +1305,14 @@ class Session:
# Root is authoritative and proves the rewrite completed. Reconcile
# metadata that may have been interrupted immediately afterwards.
try:
- meta_content = await self._viking_fs.read_file(
- f"{self._session_uri}/.meta.json",
- ctx=self.ctx,
+ self._meta = await self._read_commit_meta()
+ except Exception as exc:
+ await self._write_failed_marker(
+ archive_uri,
+ stage="phase1_recovery",
+ error=f"Cannot restore Phase 1 metadata: {exc}",
)
- self._meta = SessionMeta.from_dict(json.loads(meta_content))
- except Exception:
- pass
+ return False
self._messages = live_messages
self._remember_retention_policy(
keep_recent_count=max(0, int(marker.get("keep_recent_count", 0) or 0)),
@@ -1270,6 +1329,10 @@ class Session:
self._archives.archive_index_from_uri(archive_uri),
)
self._meta.last_commit_at = get_current_timestamp()
+ if isinstance(queue_message, dict) and queue_message.get("record_auto_commit_success"):
+ # Recovery must restore the Phase 1 throttle stamp even if
+ # the later Phase 2 statistics update is skipped.
+ self._meta.last_auto_commit_at = get_current_timestamp()
await self._rebuild_pending_tokens()
await self._save_meta()
await self._write_phase1_ready_marker(archive_uri)
@@ -1667,11 +1730,9 @@ class Session:
# this intentional registration window as an orphan.
phase1_stage = "publication_lease"
archive_path = self._viking_fs._uri_to_path(archive_uri, ctx=self.ctx)
- publication_lease = (
- await self._viking_fs._async_agfs.pathlock_acquire_exact(
- archive_path,
- timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS,
- )
+ publication_lease = await self._viking_fs._async_agfs.pathlock_acquire_exact(
+ archive_path,
+ timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS,
)
await self._write_phase1_ready_marker(archive_uri)
if record_idle_commit_success:
@@ -2511,12 +2572,15 @@ class Session:
# Phase 2 complete — update meta with telemetry and commit info
snapshot = telemetry.finish("ok")
_publish_telemetry_summary_best_effort(snapshot)
- await self._merge_and_save_commit_meta(
+ updated_meta = await self._merge_and_save_commit_meta(
archive_index=archive_index,
memories_extracted=memories_extracted,
telemetry_snapshot=snapshot,
record_auto_commit_success=record_auto_commit_success,
)
+ # Keep the response schema stable when cumulative statistics are
+ # unavailable. This zero snapshot is never persisted to the session.
+ usage_meta = updated_meta if updated_meta is not None else SessionMeta()
result_payload = {
"session_id": self.session_id,
@@ -2539,13 +2603,13 @@ class Session:
"agent_evolution_enabled": agent_evolution_enabled,
"agent_memory_skip_reason": agent_memory_skip_reason,
"token_usage": {
- "llm": dict(self._meta.llm_token_usage),
- "embedding": dict(self._meta.embedding_token_usage),
+ "llm": dict(usage_meta.llm_token_usage),
+ "embedding": dict(usage_meta.embedding_token_usage),
"total": {
- "total_tokens": self._meta.llm_token_usage["total_tokens"]
- + self._meta.embedding_token_usage["total_tokens"],
- "cached_tokens": self._meta.llm_token_usage["cached_tokens"],
- "reasoning_tokens": self._meta.llm_token_usage["reasoning_tokens"],
+ "total_tokens": usage_meta.llm_token_usage["total_tokens"]
+ + usage_meta.embedding_token_usage["total_tokens"],
+ "cached_tokens": usage_meta.llm_token_usage["cached_tokens"],
+ "reasoning_tokens": usage_meta.llm_token_usage["reasoning_tokens"],
},
},
}
@@ -2974,22 +3038,23 @@ class Session:
telemetry_snapshot: Any,
*,
record_auto_commit_success: bool = False,
- ) -> None:
- """Merge Phase 2 results without overwriting concurrent root updates."""
+ ) -> Optional[SessionMeta]:
+ """Return saved metadata, or None when its read fails and statistics are skipped."""
session_path = self._viking_fs._uri_to_path(self._session_uri, ctx=self.ctx)
lease = await self._viking_fs._async_agfs.pathlock_acquire_exact(
session_path, timeout_secs=_SESSION_PHASE1_LOCK_TIMEOUT_SECONDS
)
try:
- latest_meta = self._meta
try:
- meta_content = await self._viking_fs.read_file(
- f"{self._session_uri}/.meta.json",
- ctx=self.ctx,
+ latest_meta = await self._read_commit_meta()
+ except Exception as exc:
+ reason = f"{type(exc).__name__}: {exc}"
+ logger.warning(
+ "Session %s extraction completed without updating statistics: %s",
+ self.session_id,
+ reason,
)
- latest_meta = SessionMeta.from_dict(json.loads(meta_content))
- except Exception:
- latest_meta = self._meta
+ return None
if telemetry_snapshot:
llm = telemetry_snapshot.summary.get("tokens", {}).get("llm", {})
@@ -3012,29 +3077,16 @@ class Session:
latest_meta.memories_extracted.get("total", 0) + count
)
latest_meta.last_commit_at = get_current_timestamp()
- latest_meta.message_count = await self._read_live_message_count()
if record_auto_commit_success:
# Mirror the Phase 1 success stamp so the persisted meta reflects
# a clean auto-commit even after Phase 2 reloads the latest meta.
latest_meta.last_auto_commit_at = get_current_timestamp()
self._meta = latest_meta
await self._save_meta()
+ return latest_meta
finally:
await self._viking_fs._async_agfs.pathlock_release(lease)
- async def _read_live_message_count(self) -> int:
- """Count current live session messages from persisted storage."""
- if not self._viking_fs:
- return len(self._messages)
- try:
- content = await self._viking_fs.read_file(
- f"{self._session_uri}/messages.jsonl",
- ctx=self.ctx,
- )
- except Exception:
- return len(self._messages)
- return len([line for line in content.strip().split("\n") if line.strip()])
-
async def _read_live_messages_strict(self) -> List[Message]:
"""Read the authoritative root JSONL without silently dropping corrupt rows."""
if not self._viking_fs:
diff --git a/openviking/storage/queuefs/session_commit_processor.py b/openviking/storage/queuefs/session_commit_processor.py
index 7d42519b7..ae43fec0c 100644
--- a/openviking/storage/queuefs/session_commit_processor.py
+++ b/openviking/storage/queuefs/session_commit_processor.py
@@ -75,7 +75,6 @@ class SessionCommitProcessor(DequeueHandlerBase):
user_id=ctx.user.user_id,
)
return True
- await session.load()
processed = await session.resume_queued_commit(msg)
if not processed:
from openviking.storage.queuefs import QueueManager, get_queue_manager
diff --git a/tests/session/conftest.py b/tests/session/conftest.py
index 88ae682a5..958bf1fb7 100644
--- a/tests/session/conftest.py
+++ b/tests/session/conftest.py
@@ -62,7 +62,6 @@ async def client(
):
break
await asyncio.sleep(0)
- await queued_session.load()
try:
while not await queued_session.resume_queued_commit(message):
await asyncio.sleep(0)
diff --git a/tests/session/test_session_retention_integration.py b/tests/session/test_session_retention_integration.py
index 7bb3bb44d..8a779b50f 100644
--- a/tests/session/test_session_retention_integration.py
+++ b/tests/session/test_session_retention_integration.py
@@ -1627,7 +1627,6 @@ async def test_session_state_lock_serializes_root_meta_without_blocking_archive_
phase2 = client(session_id=initial.session_id)
appending = client(session_id=initial.session_id)
- await phase2.load()
await appending.load()
phase2_inside_save = asyncio.Event()
@@ -1875,6 +1874,14 @@ async def test_interrupted_phase1_recovers_when_root_rewrite_is_durable(
):
session = client(session_id="phase1_reconcile_durable_root_test")
await session.ensure_exists()
+ session.meta.memory_policy = {"memory_types": ["profile"]}
+ legacy_meta = session.meta.to_dict()
+ legacy_meta.update(
+ pending_tokens="123", keep_recent_count="1", memories_extracted={"total": "4"}
+ )
+ await session._viking_fs.write_file(
+ f"{session.uri}/.meta.json", json.dumps(legacy_meta), ctx=session.ctx
+ )
original = [
_text_message("u1", "user", "archive me"),
_text_message("a1", "assistant", "retain me"),
@@ -1884,7 +1891,7 @@ async def test_interrupted_phase1_recovers_when_root_rewrite_is_durable(
archive_uri = await _write_archive(session, 1, [original[0]])
await session._write_phase1_marker(
archive_uri,
- queue_message={"task_id": "synthetic"},
+ queue_message={"task_id": "synthetic", "record_auto_commit_success": True},
original_messages=original,
archived_messages=[original[0]],
retained_messages=[original[1]],
@@ -1901,9 +1908,28 @@ async def test_interrupted_phase1_recovers_when_root_rewrite_is_durable(
AsyncMock(side_effect=lambda task_id: task_id == "synthetic"),
)
- assert await session._ensure_phase1_ready(archive_uri)
- assert (await session._read_phase1_meta(archive_uri))["status"] == "ready"
- assert session.meta.message_count == 1
+ recovering = client(session_id=session.session_id)
+ original_read = recovering._viking_fs.read_file
+ failed_once = False
+
+ async def transient_meta_read(uri, *args, **kwargs):
+ nonlocal failed_once
+ if uri == f"{session.uri}/.meta.json" and not failed_once:
+ failed_once = True
+ raise AGFSIoError("temporary metadata read failure")
+ return await original_read(uri, *args, **kwargs)
+
+ monkeypatch.setattr(recovering._viking_fs, "read_file", transient_meta_read)
+ assert await recovering._ensure_phase1_ready(archive_uri)
+ assert (await recovering._read_phase1_meta(archive_uri))["status"] == "ready"
+ assert recovering.meta.message_count == 1
+ assert recovering.meta.memory_policy == {"memory_types": ["profile"]}
+ assert recovering.meta.memories_extracted == {"total": 4}
+ assert recovering.meta.last_auto_commit_at
+ persisted = json.loads(
+ await recovering._viking_fs.read_file(f"{session.uri}/.meta.json", ctx=recovering.ctx)
+ )
+ assert persisted["last_auto_commit_at"] == recovering.meta.last_auto_commit_at
async def test_interrupted_phase1_before_root_rewrite_becomes_failed(
diff --git a/tests/storage/test_session_commit_processor_identity.py b/tests/storage/test_session_commit_processor_identity.py
index cf60c99ad..78c7ced3c 100644
--- a/tests/storage/test_session_commit_processor_identity.py
+++ b/tests/storage/test_session_commit_processor_identity.py
@@ -31,9 +31,6 @@ class _FakeSession:
async def exists(self) -> bool:
return True
- async def load(self) -> None:
- return None
-
async def resume_queued_commit(self, msg) -> bool:
root = get_root_observability_context()
self._captured["account_id"] = root.account_id if root else None
@@ -74,11 +71,12 @@ async def test_process_binds_committing_identity_to_root_context(monkeypatch):
with pytest.raises(OSError, match="store unavailable"):
await processor.on_dequeue(message)
tracker.create.side_effect = None
- session.load = AsyncMock(side_effect=ValueError("invalid session message"))
+ original_resume = session.resume_queued_commit
+ session.resume_queued_commit = AsyncMock(side_effect=ValueError("invalid archive message"))
result = await processor.on_dequeue(message)
assert result.outcome is ProcessOutcome.FAILED
- assert result.error == "invalid session message"
- session.load.side_effect = None
+ assert result.error == "invalid archive message"
+ session.resume_queued_commit = original_resume
assert (await processor.on_dequeue(message)).outcome is ProcessOutcome.SUCCESS
assert captured == {"account_id": "acme", "user_id": "alice"}
assert get_root_observability_context() is None
diff --git a/tests/unit/session/test_session_commit_resume.py b/tests/unit/session/test_session_commit_resume.py
index 500ed3cd0..716b39b3c 100644
--- a/tests/unit/session/test_session_commit_resume.py
+++ b/tests/unit/session/test_session_commit_resume.py
@@ -11,11 +11,13 @@ from unittest.mock import AsyncMock
import pytest
from openviking.message import Message, TextPart, ToolPart
+from openviking.pyagfs.exceptions import AGFSConnectionError
from openviking.service.task_store import PersistentTaskStore
from openviking.service.task_tracker import TaskStatus, TaskTracker, set_task_tracker
from openviking.session.memory_policy import MemoryPolicy
-from openviking.session.session import Session
+from openviking.session.session import Session, SessionMeta
from openviking.storage.queuefs.session_commit_msg import SessionCommitMsg
+from openviking.storage.queuefs.session_commit_processor import SessionCommitProcessor
from tests.test_task_tracker import _FakeAgfs
@@ -67,7 +69,8 @@ class _MemoryVikingFS:
async def read_file(self, uri, ctx=None):
if uri not in self.files:
raise FileNotFoundError(uri)
- return self.files[uri]
+ content = self.files[uri]
+ return content.decode("utf-8") if isinstance(content, bytes) else content
async def write_file(self, uri, content, ctx=None, lease_ref=None):
self.files[uri] = content
@@ -78,6 +81,11 @@ class _MemoryVikingFS:
async def exists(self, uri, ctx=None):
return uri in self.files or any(path.startswith(f"{uri}/") for path in self.files)
+ async def stat(self, uri, ctx=None, skip_count=False):
+ if not await self.exists(uri, ctx):
+ raise FileNotFoundError(uri)
+ return {"path": uri}
+
async def ls(self, uri, ctx=None):
prefix = f"{uri}/"
names = {
@@ -206,47 +214,96 @@ async def test_phase2_cancellation_marks_archive_only_when_requested(
@pytest.mark.asyncio
-async def test_resume_queued_commit_continues_phase2(monkeypatch):
+async def test_resume_queued_commit_preserves_live_state_when_statistics_are_unavailable(
+ monkeypatch,
+):
session_uri = "viking://user/default/sessions/session-1"
archive_uri = f"{session_uri}/history/archive_001"
- archived = Message(id="archived", role="user", parts=[TextPart("old")])
- retained = Message(id="retained", role="assistant", parts=[TextPart("new")])
+ meta_uri = f"{session_uri}/.meta.json"
+ live_uri = f"{session_uri}/messages.jsonl"
+ archived = Message(id="archived", role="user", parts=[TextPart("archive input")])
+ latest_meta = SessionMeta(
+ session_id="session-1",
+ message_count=7,
+ total_message_count=50,
+ commit_count=2,
+ auto_commit_policy={"keep_recent_count": 3},
+ )
+ latest_meta.llm_token_usage["total_tokens"] = 100
+ original_meta = json.dumps(latest_meta.to_dict())
files = {
- f"{session_uri}/messages.jsonl": f"{retained.to_jsonl()}\n",
- f"{session_uri}/.meta.json": json.dumps(
- {"session_id": "session-1", "message_count": 1, "commit_count": 1}
- ),
- f"{archive_uri}/messages.jsonl": f"{archived.to_jsonl()}\n",
+ live_uri: "{live file being rewritten",
+ meta_uri: original_meta,
+ f"{archive_uri}/messages.jsonl": archived.to_jsonl() + "\n",
+ f"{archive_uri}/.meta.json": json.dumps({"phase1": {"status": "ready"}}),
}
- viking_fs = _MemoryVikingFS(files)
- session = Session(viking_fs=viking_fs, session_id="session-1", session_uri=session_uri)
+ storage = _MemoryVikingFS(files)
+ original_read = storage.read_file
+
+ async def read(uri, ctx=None):
+ if uri == meta_uri:
+ raise AGFSConnectionError("metadata unavailable")
+ return await original_read(uri, ctx)
+
+ reads = AsyncMock(side_effect=read)
+ monkeypatch.setattr(storage, "read_file", reads)
+ extract = AsyncMock(return_value={"contexts": [SimpleNamespace(category="profile")]})
+ summarize = AsyncMock(return_value="# Session Summary\n\nArchive summarized.")
+ session = Session(
+ viking_fs=storage,
+ session_id="session-1",
+ session_uri=session_uri,
+ session_compressor=SimpleNamespace(extract_long_term_memories=extract),
+ )
+ # Neither persisted statistics nor a stale instance may supply the fallback.
+ session.meta.llm_token_usage["total_tokens"] = 999
tracker = TaskTracker(_TaskStore())
- set_task_tracker(tracker)
- monkeypatch.setattr(session, "_run_memory_extraction", AsyncMock())
+ monkeypatch.setattr("openviking.service.task_tracker._instance", tracker)
+ monkeypatch.setattr(session, "_generate_archive_summary_async", summarize)
+ monkeypatch.setattr(
+ "openviking.session.session.get_openviking_config",
+ lambda: SimpleNamespace(
+ memory=SimpleNamespace(extraction_enabled=True, session_skill_extraction_enabled=False)
+ ),
+ )
message = SessionCommitMsg(
task_id="task-1",
- session_id="session-1",
+ session_id=session.session_id,
session_uri=session_uri,
archive_uri=archive_uri,
- user={"account_id": "default", "user_id": "default"},
- memory_policy={"memory_types": []},
- auto_commit_policy={"pending_token_threshold": 8000},
+ user=session.user.to_dict(),
+ memory_policy=MemoryPolicy(memory_types={"profile"}, enable_working_memory=True).to_dict(),
)
-
- try:
- await session.resume_queued_commit(message)
- finally:
- set_task_tracker(None)
-
- session._run_memory_extraction.assert_awaited_once()
- assert session._run_memory_extraction.await_args.kwargs["task_id"] == "task-1"
- assert session._run_memory_extraction.await_args.kwargs["agent_evolution_enabled"] is True
- assert session._run_memory_extraction.await_args.kwargs["auto_commit_policy"] == {
- "pending_token_threshold": 8000
+ processor = SessionCommitProcessor(SimpleNamespace(session=lambda *args, **kwargs: session))
+
+ await processor.on_dequeue(message.to_dict())
+
+ task = await tracker.get(message.task_id)
+ assert task.status is TaskStatus.COMPLETED
+ assert task.result["memories_extracted"] == {"profile": 1}
+ assert task.result["token_usage"] == {
+ "llm": {
+ "prompt_tokens": 0,
+ "completion_tokens": 0,
+ "total_tokens": 0,
+ "cached_tokens": 0,
+ "reasoning_tokens": 0,
+ },
+ "embedding": {"total_tokens": 0},
+ "total": {"total_tokens": 0, "cached_tokens": 0, "reasoning_tokens": 0},
}
- assert [item.id for item in session._run_memory_extraction.await_args.kwargs["messages"]] == [
- "archived"
- ]
+ assert files[meta_uri] == original_meta
+ assert f"{archive_uri}/.done" in files
+ assert [message.id for message in summarize.await_args.args[0]] == ["archived"]
+ assert [message.id for message in extract.await_args.kwargs["messages"]] == ["archived"]
+ assert extract.await_args.kwargs["allowed_memory_types"] == {"profile"}
+
+ await processor.on_dequeue(message.to_dict())
+
+ extract.assert_awaited_once()
+ summarize.assert_awaited_once()
+ assert all(call.args[0] != live_uri for call in reads.await_args_list)
+ assert files[meta_uri] == original_meta
@pytest.mark.asyncio