Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix(session): keep token rebuild off event loop (#4557)
bot-of-qin-ctxed4bb192d85b
diff --git a/openviking/session/session.py b/openviking/session/session.pyindex 7e88a1c48..e6eed9ecd 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -761,16 +761,21 @@ class Session:# counter stays consistent across restarts and is also backfilled for# legacy sessions whose .meta.json predates these fields. O(n) once,# subsequent add_message() maintains it in O(1).- self._rebuild_pending_tokens()+ await self._rebuild_pending_tokens()self._loaded = True- def _rebuild_pending_tokens(self) -> None:+ async def _rebuild_pending_tokens(self) -> None:"""Recompute ``pending_tokens`` from the current message list.- Used on load and as a safety net after rollbacks. Respects the- currently remembered ``keep_recent_count`` from meta.+ Used on load, turn-budget appends, and recovery. Respects the current+ retention settings from meta."""+ pending_tokens = await asyncio.to_thread(self._calculate_pending_tokens)+ self._meta.pending_tokens = max(0, pending_tokens)++ def _calculate_pending_tokens(self) -> int:+ """Calculate pending tokens without mutating session state."""if (self._meta.retention_mode == RETENTION_MODE_TURN_BUDGETand self._meta.keep_recent_turn_count > 0@@ -783,25 +788,19 @@ class Session:min_raw_tail_steps=self._meta.min_raw_tail_steps,)retained_ids = {message.id for message in plan.retained_messages}- self._meta.pending_tokens = sum(+ return sum(int(message.estimated_tokens or 0)for message in plan.archive_messagesif message.id not in retained_ids)- self._meta.pending_tokens = max(0, self._meta.pending_tokens)- returnkeep = max(0, int(self._meta.keep_recent_count or 0))total = len(self._messages)if keep <= 0:- self._meta.pending_tokens = sum(int(m.estimated_tokens or 0) for m in self._messages)- elif total > keep:- self._meta.pending_tokens = sum(- int(m.estimated_tokens or 0) for m in self._messages[: total - keep]- )- else:- self._meta.pending_tokens = 0- self._meta.pending_tokens = max(0, self._meta.pending_tokens)+ return sum(int(m.estimated_tokens or 0) for m in self._messages)+ if total > keep:+ return sum(int(m.estimated_tokens or 0) for m in self._messages[: total - keep])+ return 0async def exists(self) -> bool:"""Check whether this session already exists in storage."""@@ -1295,7 +1294,7 @@ class Session:if not messages:returnif not self._viking_fs:- self._apply_appended_messages_to_state(messages)+ await self._apply_appended_messages_to_state(messages)returnsession_path = self._viking_fs._uri_to_path(self._session_uri, ctx=self.ctx)@@ -1323,7 +1322,7 @@ class Session:# append. Message correctness remains rooted in messages.jsonl.self._meta = in_memory_meta- self._apply_appended_messages_to_state(messages)+ await self._apply_appended_messages_to_state(messages)batch_content = "".join(message.to_jsonl() + "\n" for message in messages)if live_messages_missing:await self._viking_fs.write_file(@@ -1341,7 +1340,7 @@ class Session:finally:await self._viking_fs._async_agfs.pathlock_release(lease)- def _apply_appended_messages_to_state(self, messages: List[Message]) -> None:+ async def _apply_appended_messages_to_state(self, messages: List[Message]) -> None:"""Update in-memory counters after an authoritative root reload."""for msg in messages:self._messages.append(msg)@@ -1360,7 +1359,7 @@ class Session:self._meta.pending_tokens += int(pushed_out.estimated_tokens or 0)if self._meta.retention_mode == RETENTION_MODE_TURN_BUDGET:- self._rebuild_pending_tokens()+ await self._rebuild_pending_tokens()self._meta.message_count = len(self._messages)if self._meta.total_message_count is not None:@@ -1804,7 +1803,7 @@ class Session:self._archive_index_from_uri(archive_uri),)self._meta.last_commit_at = get_current_timestamp()- self._rebuild_pending_tokens()+ await self._rebuild_pending_tokens()await self._save_meta()await self._write_phase1_ready_marker(archive_uri)logger.warning("Recovered interrupted Session Phase 1: %s", archive_uri)diff --git a/openviking/utils/token_estimation.py b/openviking/utils/token_estimation.pyindex 16caf2fe2..03e8751dd 100644--- a/openviking/utils/token_estimation.py+++ b/openviking/utils/token_estimation.py@@ -5,39 +5,43 @@from __future__ import annotations-import mathfrom typing import Any-def _is_cjk_code_point(code_point: int) -> bool:- return (- 0x3400 <= code_point <= 0x4DBF- or 0x4E00 <= code_point <= 0x9FFF- or 0xF900 <= code_point <= 0xFAFF- or 0x20000 <= code_point <= 0x2EBEF- or 0x3040 <= code_point <= 0x30FF- or 0x31F0 <= code_point <= 0x31FF- or 0xAC00 <= code_point <= 0xD7AF- or 0x1100 <= code_point <= 0x11FF- or 0x3130 <= code_point <= 0x318F- or 0xFF00 <= code_point <= 0xFFEF- or 0x3000 <= code_point <= 0x303F- )---def _code_point_weight(code_point: int) -> float:- if _is_cjk_code_point(code_point):- return 1.5- if code_point > 0xFFFF:- return 2.0- return 0.25--def estimate_text_tokens(text: str | None) -> int:"""Estimate tokens with a CJK-aware fallback."""if not text:return 0- return math.ceil(sum(_code_point_weight(ord(char)) for char in text))+ if text.isascii():+ return (len(text) + 3) // 4++ # Count quarter-token units to avoid a float sum and helper calls for every+ # character. The weights remain exactly the same: ordinary BMP characters+ # are 1 unit, CJK characters are 6, and other astral characters are 8.+ units = 0+ for char in text:+ code_point = ord(char)+ if 0x4E00 <= code_point <= 0x9FFF:+ units += 6+ elif code_point < 0x1100:+ units += 1+ elif (+ code_point <= 0x11FF+ or 0x3000 <= code_point <= 0x30FF+ or 0x3130 <= code_point <= 0x318F+ or 0x31F0 <= code_point <= 0x31FF+ or 0x3400 <= code_point <= 0x4DBF+ or 0xAC00 <= code_point <= 0xD7AF+ or 0xF900 <= code_point <= 0xFAFF+ or 0xFF00 <= code_point <= 0xFFEF+ or 0x20000 <= code_point <= 0x2EBEF+ ):+ units += 6+ elif code_point > 0xFFFF:+ units += 8+ else:+ units += 1+ return (units + 3) // 4def estimate_serialized_tokens(value: Any) -> int:diff --git a/tests/test_token_estimation.py b/tests/test_token_estimation.pyindex 97a4f1464..908e9e688 100644--- a/tests/test_token_estimation.py+++ b/tests/test_token_estimation.py@@ -16,6 +16,8 @@ def test_message_estimated_tokens_is_cjk_aware():msg = Message(id="msg-cjk", role="user", parts=[TextPart("\u4f60\u597d\u4e16\u754c")])assert msg.estimated_tokens == 6+ assert estimate_text_tokens("abcd") == 1+ assert estimate_text_tokens("\U0001f600") == 2def test_truncate_text_to_token_budget_preserves_head_and_tail():