Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix(session): keep token rebuild off event loop (#4557)

bot-of-qin-ctxed4bb192d85b
diff --git a/openviking/session/session.py b/openviking/session/session.py
index 7e88a1c48..e6eed9ecd 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -761,16 +761,21 @@ class Session:
# counter stays consistent across restarts and is also backfilled for
# legacy sessions whose .meta.json predates these fields. O(n) once,
# subsequent add_message() maintains it in O(1).
- self._rebuild_pending_tokens()
+ await self._rebuild_pending_tokens()
self._loaded = True
- def _rebuild_pending_tokens(self) -> None:
+ async def _rebuild_pending_tokens(self) -> None:
"""Recompute ``pending_tokens`` from the current message list.
- Used on load and as a safety net after rollbacks. Respects the
- currently remembered ``keep_recent_count`` from meta.
+ Used on load, turn-budget appends, and recovery. Respects the current
+ retention settings from meta.
"""
+ pending_tokens = await asyncio.to_thread(self._calculate_pending_tokens)
+ self._meta.pending_tokens = max(0, pending_tokens)
+
+ def _calculate_pending_tokens(self) -> int:
+ """Calculate pending tokens without mutating session state."""
if (
self._meta.retention_mode == RETENTION_MODE_TURN_BUDGET
and self._meta.keep_recent_turn_count > 0
@@ -783,25 +788,19 @@ class Session:
min_raw_tail_steps=self._meta.min_raw_tail_steps,
)
retained_ids = {message.id for message in plan.retained_messages}
- self._meta.pending_tokens = sum(
+ return sum(
int(message.estimated_tokens or 0)
for message in plan.archive_messages
if message.id not in retained_ids
)
- self._meta.pending_tokens = max(0, self._meta.pending_tokens)
- return
keep = max(0, int(self._meta.keep_recent_count or 0))
total = len(self._messages)
if keep <= 0:
- self._meta.pending_tokens = sum(int(m.estimated_tokens or 0) for m in self._messages)
- elif total > keep:
- self._meta.pending_tokens = sum(
- int(m.estimated_tokens or 0) for m in self._messages[: total - keep]
- )
- else:
- self._meta.pending_tokens = 0
- self._meta.pending_tokens = max(0, self._meta.pending_tokens)
+ return sum(int(m.estimated_tokens or 0) for m in self._messages)
+ if total > keep:
+ return sum(int(m.estimated_tokens or 0) for m in self._messages[: total - keep])
+ return 0
async def exists(self) -> bool:
"""Check whether this session already exists in storage."""
@@ -1295,7 +1294,7 @@ class Session:
if not messages:
return
if not self._viking_fs:
- self._apply_appended_messages_to_state(messages)
+ await self._apply_appended_messages_to_state(messages)
return
session_path = self._viking_fs._uri_to_path(self._session_uri, ctx=self.ctx)
@@ -1323,7 +1322,7 @@ class Session:
# append. Message correctness remains rooted in messages.jsonl.
self._meta = in_memory_meta
- self._apply_appended_messages_to_state(messages)
+ await self._apply_appended_messages_to_state(messages)
batch_content = "".join(message.to_jsonl() + "\n" for message in messages)
if live_messages_missing:
await self._viking_fs.write_file(
@@ -1341,7 +1340,7 @@ class Session:
finally:
await self._viking_fs._async_agfs.pathlock_release(lease)
- def _apply_appended_messages_to_state(self, messages: List[Message]) -> None:
+ async def _apply_appended_messages_to_state(self, messages: List[Message]) -> None:
"""Update in-memory counters after an authoritative root reload."""
for msg in messages:
self._messages.append(msg)
@@ -1360,7 +1359,7 @@ class Session:
self._meta.pending_tokens += int(pushed_out.estimated_tokens or 0)
if self._meta.retention_mode == RETENTION_MODE_TURN_BUDGET:
- self._rebuild_pending_tokens()
+ await self._rebuild_pending_tokens()
self._meta.message_count = len(self._messages)
if self._meta.total_message_count is not None:
@@ -1804,7 +1803,7 @@ class Session:
self._archive_index_from_uri(archive_uri),
)
self._meta.last_commit_at = get_current_timestamp()
- self._rebuild_pending_tokens()
+ await self._rebuild_pending_tokens()
await self._save_meta()
await self._write_phase1_ready_marker(archive_uri)
logger.warning("Recovered interrupted Session Phase 1: %s", archive_uri)
diff --git a/openviking/utils/token_estimation.py b/openviking/utils/token_estimation.py
index 16caf2fe2..03e8751dd 100644
--- a/openviking/utils/token_estimation.py
+++ b/openviking/utils/token_estimation.py
@@ -5,39 +5,43 @@
from __future__ import annotations
-import math
from typing import Any
-def _is_cjk_code_point(code_point: int) -> bool:
- return (
- 0x3400 <= code_point <= 0x4DBF
- or 0x4E00 <= code_point <= 0x9FFF
- or 0xF900 <= code_point <= 0xFAFF
- or 0x20000 <= code_point <= 0x2EBEF
- or 0x3040 <= code_point <= 0x30FF
- or 0x31F0 <= code_point <= 0x31FF
- or 0xAC00 <= code_point <= 0xD7AF
- or 0x1100 <= code_point <= 0x11FF
- or 0x3130 <= code_point <= 0x318F
- or 0xFF00 <= code_point <= 0xFFEF
- or 0x3000 <= code_point <= 0x303F
- )
-
-
-def _code_point_weight(code_point: int) -> float:
- if _is_cjk_code_point(code_point):
- return 1.5
- if code_point > 0xFFFF:
- return 2.0
- return 0.25
-
-
def estimate_text_tokens(text: str | None) -> int:
"""Estimate tokens with a CJK-aware fallback."""
if not text:
return 0
- return math.ceil(sum(_code_point_weight(ord(char)) for char in text))
+ if text.isascii():
+ return (len(text) + 3) // 4
+
+ # Count quarter-token units to avoid a float sum and helper calls for every
+ # character. The weights remain exactly the same: ordinary BMP characters
+ # are 1 unit, CJK characters are 6, and other astral characters are 8.
+ units = 0
+ for char in text:
+ code_point = ord(char)
+ if 0x4E00 <= code_point <= 0x9FFF:
+ units += 6
+ elif code_point < 0x1100:
+ units += 1
+ elif (
+ code_point <= 0x11FF
+ or 0x3000 <= code_point <= 0x30FF
+ or 0x3130 <= code_point <= 0x318F
+ or 0x31F0 <= code_point <= 0x31FF
+ or 0x3400 <= code_point <= 0x4DBF
+ or 0xAC00 <= code_point <= 0xD7AF
+ or 0xF900 <= code_point <= 0xFAFF
+ or 0xFF00 <= code_point <= 0xFFEF
+ or 0x20000 <= code_point <= 0x2EBEF
+ ):
+ units += 6
+ elif code_point > 0xFFFF:
+ units += 8
+ else:
+ units += 1
+ return (units + 3) // 4
def estimate_serialized_tokens(value: Any) -> int:
diff --git a/tests/test_token_estimation.py b/tests/test_token_estimation.py
index 97a4f1464..908e9e688 100644
--- a/tests/test_token_estimation.py
+++ b/tests/test_token_estimation.py
@@ -16,6 +16,8 @@ def test_message_estimated_tokens_is_cjk_aware():
msg = Message(id="msg-cjk", role="user", parts=[TextPart("\u4f60\u597d\u4e16\u754c")])
assert msg.estimated_tokens == 6
+ assert estimate_text_tokens("abcd") == 1
+ assert estimate_text_tokens("\U0001f600") == 2
def test_truncate_text_to_token_budget_preserves_head_and_tail():