Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: add cjk-aware token estimation (#2348)
Eurakaxunab855a43b054
diff --git a/examples/openclaw-plugin/auto-recall.ts b/examples/openclaw-plugin/auto-recall.tsindex 8e537be77..04037ad59 100644--- a/examples/openclaw-plugin/auto-recall.ts+++ b/examples/openclaw-plugin/auto-recall.ts@@ -8,6 +8,7 @@ import {} from "./memory-ranking.js";import { quickRecallPrecheck, withTimeout } from "./process-manager.js";import { sanitizeUserTextForCapture } from "./text-utils.js";+import { estimateTextTokens } from "./token-estimator.js";const AUTO_RECALL_TIMEOUT_MS = 5_000;const RECALL_QUERY_MAX_CHARS = 4_000;@@ -51,10 +52,9 @@ export function prepareRecallQuery(rawText: string): PreparedRecallQuery {};}-/** Estimate token count using chars/4 heuristic for diagnostics. */+/** Estimate token count using the shared CJK-aware fallback for diagnostics. */export function estimateTokenCount(text: string): number {- if (!text) return 0;- return Math.ceil(text.length / 4);+ return estimateTextTokens(text);}export type BuildMemoryLinesOptions = {diff --git a/examples/openclaw-plugin/context-engine.ts b/examples/openclaw-plugin/context-engine.tsindex 29d7321ca..9ebea3516 100644--- a/examples/openclaw-plugin/context-engine.ts+++ b/examples/openclaw-plugin/context-engine.ts@@ -18,6 +18,11 @@ import {toJsonLog,} from "./memory-ranking.js";import { sanitizeToolUseResultPairing } from "./session-transcript-repair.js";+import {+ estimateAgentMessageTokens,+ estimateAgentMessagesTokens,+ estimateTextTokens,+} from "./token-estimator.js";type AgentMessage = {role?: string;@@ -156,14 +161,11 @@ function allocateContextBudget(totalBudget: number, instructionTokens = 0): Cont}function roughEstimate(messages: AgentMessage[]): number {- return Math.ceil(JSON.stringify(messages).length / 4);+ return estimateAgentMessagesTokens(messages);}function msgTokenEstimate(msg: AgentMessage): number {- const raw = (msg as Record<string, unknown>).content;- if (typeof raw === "string") return Math.ceil(raw.length / 4);- if (Array.isArray(raw)) return Math.ceil(JSON.stringify(raw).length / 4);- return 1;+ return estimateAgentMessageTokens(msg);}function normalizeTimestamp(value: unknown): string | undefined {@@ -671,7 +673,7 @@ function buildSystemPromptAddition(): string {function buildInstructionPrompt(): { text: string; tokens: number } {const text = buildSystemPromptAddition();- return { text, tokens: Math.ceil(text.length / 4) };+ return { text, tokens: estimateTextTokens(text) };}function buildArchiveMemory(diff --git a/examples/openclaw-plugin/index.ts b/examples/openclaw-plugin/index.tsindex 20bd87ab3..733440d48 100644--- a/examples/openclaw-plugin/index.ts+++ b/examples/openclaw-plugin/index.ts@@ -43,6 +43,12 @@ export {estimateTokenCount,prepareRecallQuery,};+export {+ estimateAgentMessageTokens,+ estimateAgentMessagesTokens,+ estimateSerializedTokens,+ estimateTextTokens,+} from "./token-estimator.js";export type {BuildMemoryLinesOptions,BuildMemoryLinesWithBudgetOptions,diff --git a/examples/openclaw-plugin/install-manifest.json b/examples/openclaw-plugin/install-manifest.jsonindex 180e4be26..bbbb8cea5 100644--- a/examples/openclaw-plugin/install-manifest.json+++ b/examples/openclaw-plugin/install-manifest.json@@ -15,6 +15,7 @@"client.ts","process-manager.ts","memory-ranking.ts",+ "token-estimator.ts","text-utils.ts","tool-call-id.ts","session-transcript-repair.ts",diff --git a/examples/openclaw-plugin/tests/context-bloat-730.test.ts b/examples/openclaw-plugin/tests/context-bloat-730.test.tsindex ca78ddcc0..e2be3078f 100644--- a/examples/openclaw-plugin/tests/context-bloat-730.test.ts+++ b/examples/openclaw-plugin/tests/context-bloat-730.test.ts@@ -139,12 +139,14 @@ describe("Slice E: character budget enforcement", () => {expect(estimatedTokens).toBeLessThanOrEqual(53);});- it("should estimate tokens as ceil(chars/4)", async () => {+ it("should estimate tokens with CJK-aware fallback weights", async () => {const { estimateTokenCount } = await import("../index.js");expect(estimateTokenCount("")).toBe(0);expect(estimateTokenCount("abcd")).toBe(1);expect(estimateTokenCount("abcde")).toBe(2);expect(estimateTokenCount("A".repeat(100))).toBe(25);+ expect(estimateTokenCount("\u4f60\u597d\u4e16\u754c")).toBe(6);+ expect(estimateTokenCount("A\u4f60\ud83d\ude42")).toBe(4);});it("should have recallMaxInjectedChars in parsed config with default 4000-character budget", () => {diff --git a/examples/openclaw-plugin/tests/context-engine-assemble.test.ts b/examples/openclaw-plugin/tests/context-engine-assemble.test.tsindex 8aefd7db3..874d4ccf1 100644--- a/examples/openclaw-plugin/tests/context-engine-assemble.test.ts+++ b/examples/openclaw-plugin/tests/context-engine-assemble.test.ts@@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest";import type { OpenVikingClient } from "../client.js";import { memoryOpenVikingConfigSchema } from "../config.js";import { createMemoryOpenVikingContextEngine } from "../context-engine.js";+import { estimateAgentMessagesTokens, estimateTextTokens } from "../token-estimator.js";const cfg = memoryOpenVikingConfigSchema.parse({mode: "remote",@@ -14,11 +15,11 @@ const cfg = memoryOpenVikingConfigSchema.parse({});function roughEstimate(messages: unknown[]): number {- return Math.ceil(JSON.stringify(messages).length / 4);+ return estimateAgentMessagesTokens(messages);}function systemPromptTokens(text?: string): number {- return text ? Math.ceil(text.length / 4) : 0;+ return estimateTextTokens(text);}function makeLogger() {diff --git a/examples/openclaw-plugin/tests/e2e/test-cjk-token-estimation.py b/examples/openclaw-plugin/tests/e2e/test-cjk-token-estimation.pynew file mode 100644index 000000000..57d9c663a--- /dev/null+++ b/examples/openclaw-plugin/tests/e2e/test-cjk-token-estimation.py@@ -0,0 +1,269 @@+#!/usr/bin/env python3+"""OpenClaw plugin E2E for CJK-aware token estimation.++This test drives the real OpenClaw Gateway, then verifies the OpenViking plugin's+own assemble diagnostics. It intentionally checks the plugin-side token estimate,+not only the OV REST session counters.+"""++from __future__ import annotations++import argparse+import json+import math+import os+import pathlib+import time+import uuid+from typing import Any++import requests+++DEFAULT_GATEWAY_URL = os.environ.get("OPENCLAW_GATEWAY_URL", "http://127.0.0.1:19830")+DEFAULT_OPENVIKING_URL = os.environ.get("OPENVIKING_BASE_URL", "http://127.0.0.1:2948")+DEFAULT_CJK_REPEAT = 120+++def default_gateway_log_path() -> pathlib.Path:+ state_dir = os.environ.get("OPENCLAW_STATE_DIR")+ if state_dir:+ return pathlib.Path(state_dir) / "logs" / "gateway.stdout.log"+ return pathlib.Path.cwd() / "config" / ".openclaw" / "logs" / "gateway.stdout.log"+++def discover_gateway_token() -> str:+ env_token = os.environ.get("OPENCLAW_GATEWAY_TOKEN", "").strip()+ if env_token:+ return env_token++ candidates: list[pathlib.Path] = []+ config_path = os.environ.get("OPENCLAW_CONFIG_PATH")+ if config_path:+ candidates.append(pathlib.Path(config_path))++ state_dir = os.environ.get("OPENCLAW_STATE_DIR")+ if state_dir:+ candidates.append(pathlib.Path(state_dir) / "openclaw.json")++ candidates.extend(+ [+ pathlib.Path.cwd() / "config" / ".openclaw" / "openclaw.json",+ pathlib.Path.home() / ".openclaw" / "openclaw.json",+ ]+ )++ for path in candidates:+ try:+ cfg = json.loads(path.read_text(encoding="utf-8"))+ except Exception:+ continue+ token = str(cfg.get("gateway", {}).get("auth", {}).get("token", "")).strip()+ if token:+ return token+ return ""+++def send_gateway_message(+ gateway_url: str,+ token: str,+ user_id: str,+ message: str,+ timeout: int,+) -> dict[str, Any]:+ headers = {"Content-Type": "application/json"}+ if token:+ headers["Authorization"] = f"Bearer {token}"++ response = requests.post(+ f"{gateway_url.rstrip('/')}/v1/responses",+ headers=headers,+ json={"model": "openclaw", "input": message, "user": user_id},+ timeout=timeout,+ )+ response.raise_for_status()+ return response.json()+++def iter_new_log_lines(path: pathlib.Path, offset: int) -> list[str]:+ if not path.exists():+ return []+ with path.open("rb") as handle:+ handle.seek(min(offset, path.stat().st_size))+ return handle.read().decode("utf-8", errors="ignore").splitlines()+++def parse_diag(line: str) -> dict[str, Any] | None:+ marker = "openviking: diag "+ pos = line.find(marker)+ if pos < 0:+ return None+ try:+ return json.loads(line[pos + len(marker) :].strip())+ except json.JSONDecodeError:+ return None+++def find_assemble_diag_with_marker(+ log_path: pathlib.Path,+ offset: int,+ marker: str,+ timeout: float,+) -> tuple[dict[str, Any], dict[str, Any]]:+ deadline = time.time() + timeout+ last_seen = ""++ while time.time() < deadline:+ for line in iter_new_log_lines(log_path, offset):+ diag = parse_diag(line)+ if not diag or diag.get("stage") != "assemble_entry":+ continue+ data = diag.get("data", {})+ for message in data.get("messages", []):+ content = str(message.get("content", ""))+ if marker in content:+ return diag, message+ last_seen = json.dumps(data, ensure_ascii=False)[:500]+ time.sleep(0.5)++ raise AssertionError(+ f"no assemble_entry diagnostic contained marker {marker!r}; last_seen={last_seen}"+ )+++def flatten_message_text(value: Any) -> str:+ if value is None:+ return ""+ if isinstance(value, str):+ return value+ if isinstance(value, list):+ return "\n".join(flatten_message_text(item) for item in value)+ if isinstance(value, dict):+ chunks: list[str] = []+ for key in ("text", "abstract", "content"):+ if key in value:+ chunks.append(flatten_message_text(value.get(key)))+ for key in ("parts", "messages", "pre_archive_abstracts"):+ raw = value.get(key)+ if isinstance(raw, list):+ chunks.append(flatten_message_text(raw))+ return "\n".join(chunk for chunk in chunks if chunk)+ return ""+++def ov_get(base_url: str, path: str) -> Any:+ response = requests.get(f"{base_url.rstrip('/')}{path}", timeout=10)+ response.raise_for_status()+ data = response.json()+ return data.get("result", data)+++def find_ov_context_with_marker(+ openviking_url: str,+ marker: str,+ timeout: float,+) -> tuple[str, dict[str, Any]]:+ deadline = time.time() + timeout+ last_session_count = 0++ while time.time() < deadline:+ sessions = ov_get(openviking_url, "/api/v1/sessions")+ if not isinstance(sessions, list):+ sessions = []+ last_session_count = len(sessions)+ ordered = sorted(+ [s for s in sessions if isinstance(s, dict)],+ key=lambda s: str(s.get("updated_at", "")),+ reverse=True,+ )[:40]++ for session in ordered:+ session_id = str(session.get("session_id", ""))+ if not session_id or session_id.startswith("memory-store-"):+ continue+ try:+ ctx = ov_get(openviking_url, f"/api/v1/sessions/{session_id}/context?token_budget=128000")+ except requests.RequestException:+ continue+ if marker in flatten_message_text(ctx):+ return session_id, ctx++ time.sleep(1)++ raise AssertionError(+ f"marker {marker!r} was not found in OV contexts; checked sessions={last_session_count}"+ )+++def main() -> int:+ parser = argparse.ArgumentParser(description="OpenClaw plugin CJK token estimation E2E")+ parser.add_argument("--gateway", default=DEFAULT_GATEWAY_URL)+ parser.add_argument("--openviking", default=DEFAULT_OPENVIKING_URL)+ parser.add_argument("--gateway-log", type=pathlib.Path, default=default_gateway_log_path())+ parser.add_argument("--repeat", type=int, default=DEFAULT_CJK_REPEAT)+ parser.add_argument("--gateway-timeout", type=int, default=300)+ parser.add_argument("--diag-timeout", type=float, default=45)+ parser.add_argument("--ov-timeout", type=float, default=45)+ args = parser.parse_args()++ marker = f"CJK_TOKEN_E2E_{uuid.uuid4().hex[:10]}"+ user_id = f"cjk-token-e2e-{uuid.uuid4().hex[:10]}"+ cjk_text = "你好世界" * args.repeat+ expected_cjk_tokens = math.ceil(len(cjk_text) * 1.5)+ naive_chars_div_4 = math.ceil(len(cjk_text) / 4)+ message = (+ f"{marker}\n"+ "请只回复 OK。下面这段中文只用于 OpenViking token 估算端到端回归测试:\n"+ f"{cjk_text}"+ )++ log_offset = args.gateway_log.stat().st_size if args.gateway_log.exists() else 0+ token = discover_gateway_token()++ print(f"Gateway: {args.gateway}")+ print(f"OpenViking: {args.openviking}")+ print(f"Gateway log: {args.gateway_log}")+ print(f"Marker: {marker}")+ print(+ "Expected CJK token floor: "+ f"{expected_cjk_tokens} (old chars/4 would be {naive_chars_div_4})"+ )++ send_gateway_message(args.gateway, token, user_id, message, args.gateway_timeout)++ diag, marked_message = find_assemble_diag_with_marker(+ args.gateway_log,+ log_offset,+ marker,+ args.diag_timeout,+ )+ observed_tokens = int(marked_message.get("tokens", 0) or 0)+ input_tokens = int(diag.get("data", {}).get("inputTokenEstimate", 0) or 0)+ if observed_tokens < expected_cjk_tokens:+ raise AssertionError(+ "plugin assemble token estimate under-counted CJK message: "+ f"observed={observed_tokens}, expected>={expected_cjk_tokens}, "+ f"old chars/4={naive_chars_div_4}, inputTokenEstimate={input_tokens}"+ )++ session_id, ctx = find_ov_context_with_marker(args.openviking, marker, args.ov_timeout)+ estimated_tokens = int(ctx.get("estimatedTokens", 0) or 0)+ active_tokens = int(ctx.get("stats", {}).get("activeTokens", 0) or 0)+ if max(estimated_tokens, active_tokens) < expected_cjk_tokens:+ raise AssertionError(+ "OV context after plugin afterTurn under-counted CJK message: "+ f"estimatedTokens={estimated_tokens}, activeTokens={active_tokens}, "+ f"expected>={expected_cjk_tokens}, session={session_id}"+ )++ print(+ "PASS: plugin CJK estimate "+ f"messageTokens={observed_tokens}, inputTokenEstimate={input_tokens}, "+ f"ovEstimatedTokens={estimated_tokens}, ovActiveTokens={active_tokens}, "+ f"session={session_id}"+ )+ return 0+++if __name__ == "__main__":+ raise SystemExit(main())diff --git a/examples/openclaw-plugin/tests/ut/build-memory-lines.test.ts b/examples/openclaw-plugin/tests/ut/build-memory-lines.test.tsindex efeb88782..6d077d12f 100644--- a/examples/openclaw-plugin/tests/ut/build-memory-lines.test.ts+++ b/examples/openclaw-plugin/tests/ut/build-memory-lines.test.ts@@ -23,12 +23,17 @@ describe("estimateTokenCount", () => {expect(estimateTokenCount("")).toBe(0);});- it("estimates tokens as ceil(chars/4)", () => {+ it("keeps the legacy ASCII estimate close to ceil(chars/4)", () => {expect(estimateTokenCount("hello")).toBe(2); // ceil(5/4)expect(estimateTokenCount("abcd")).toBe(1); // ceil(4/4)expect(estimateTokenCount("abcde")).toBe(2); // ceil(5/4)});+ it("uses CJK-aware weighting for non-Latin text and emoji", () => {+ expect(estimateTokenCount("\u4f60\u597d\u4e16\u754c")).toBe(6);+ expect(estimateTokenCount("A\u4f60\ud83d\ude42")).toBe(4);+ });+it("handles long text", () => {const text = "a".repeat(1000);expect(estimateTokenCount(text)).toBe(250);diff --git a/examples/openclaw-plugin/tests/ut/context-engine-assemble.test.ts b/examples/openclaw-plugin/tests/ut/context-engine-assemble.test.tsindex 312b38459..212313eae 100644--- a/examples/openclaw-plugin/tests/ut/context-engine-assemble.test.ts+++ b/examples/openclaw-plugin/tests/ut/context-engine-assemble.test.ts@@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest";import type { OpenVikingClient } from "../../client.js";import { memoryOpenVikingConfigSchema } from "../../config.js";import { createMemoryOpenVikingContextEngine } from "../../context-engine.js";+import { estimateAgentMessagesTokens, estimateTextTokens } from "../../token-estimator.js";const cfg = memoryOpenVikingConfigSchema.parse({mode: "remote",@@ -12,11 +13,11 @@ const cfg = memoryOpenVikingConfigSchema.parse({});function roughEstimate(messages: unknown[]): number {- return Math.ceil(JSON.stringify(messages).length / 4);+ return estimateAgentMessagesTokens(messages);}function systemPromptTokens(text?: string): number {- return text ? Math.ceil(text.length / 4) : 0;+ return estimateTextTokens(text);}function makeLogger() {@@ -214,6 +215,29 @@ describe("context-engine assemble()", () => {expect(result.estimatedTokens).toBe(roughEstimate(sourceMessages));});+ it("does not underestimate CJK messages when passing through transformContext", async () => {+ const { engine, getClient } = makeEngine({+ latest_archive_overview: "unused",+ pre_archive_abstracts: [],+ messages: [],+ estimatedTokens: 0,+ stats: makeStats(),+ });+ const sourceMessages = [+ { role: "assistant", content: [{ type: "text", text: "Previous answer." }] },+ { role: "user", content: "\u4f60\u597d".repeat(100) },+ ];++ const result = await engine.assemble({+ sessionId: "session-cjk-estimate",+ messages: sourceMessages,+ });++ expect(getClient).not.toHaveBeenCalled();+ expect(result.messages).toBe(sourceMessages);+ expect(result.estimatedTokens).toBeGreaterThanOrEqual(300);+ });+it("treats prompt-less assemble with availableTools as main assemble", async () => {const { engine, client, resolveAgentId } = makeEngine({latest_archive_overview: "# Session Summary\nPreviously discussed repository setup.",diff --git a/examples/openclaw-plugin/token-estimator.ts b/examples/openclaw-plugin/token-estimator.tsnew file mode 100644index 000000000..b91b7e400--- /dev/null+++ b/examples/openclaw-plugin/token-estimator.ts@@ -0,0 +1,65 @@+type TokenCountableAgentMessage = {+ role?: string;+ content?: unknown;+};++function isCjkCodePoint(codePoint: number): boolean {+ return (+ (codePoint >= 0x3400 && codePoint <= 0x4dbf) ||+ (codePoint >= 0x4e00 && codePoint <= 0x9fff) ||+ (codePoint >= 0xf900 && codePoint <= 0xfaff) ||+ (codePoint >= 0x20000 && codePoint <= 0x2ebef) ||+ (codePoint >= 0x3040 && codePoint <= 0x30ff) ||+ (codePoint >= 0x31f0 && codePoint <= 0x31ff) ||+ (codePoint >= 0xac00 && codePoint <= 0xd7af) ||+ (codePoint >= 0x1100 && codePoint <= 0x11ff) ||+ (codePoint >= 0x3130 && codePoint <= 0x318f) ||+ (codePoint >= 0xff00 && codePoint <= 0xffef) ||+ (codePoint >= 0x3000 && codePoint <= 0x303f)+ );+}++function codePointWeight(codePoint: number): number {+ if (isCjkCodePoint(codePoint)) {+ return 1.5;+ }+ if (codePoint > 0xffff) {+ return 2;+ }+ return 0.25;+}++export function estimateTextTokens(text: string | null | undefined): number {+ if (!text) {+ return 0;+ }++ let weightedTokens = 0;+ for (const char of text) {+ weightedTokens += codePointWeight(char.codePointAt(0) ?? 0);+ }+ return Math.ceil(weightedTokens);+}++export function estimateSerializedTokens(value: unknown): number {+ if (value === null || value === undefined) {+ return 0;+ }+ if (typeof value === "string") {+ return estimateTextTokens(value);+ }+ if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") {+ return estimateTextTokens(String(value));+ }++ const serialized = JSON.stringify(value);+ return estimateTextTokens(serialized ?? "");+}++export function estimateAgentMessageTokens(message: TokenCountableAgentMessage): number {+ return estimateSerializedTokens(message);+}++export function estimateAgentMessagesTokens(messages: TokenCountableAgentMessage[]): number {+ return estimateSerializedTokens(messages);+}diff --git a/openviking/integrations/langchain/testing.py b/openviking/integrations/langchain/testing.pyindex 3450efe52..1113177b4 100644--- a/openviking/integrations/langchain/testing.py+++ b/openviking/integrations/langchain/testing.py@@ -11,6 +11,8 @@ from collections import defaultdictfrom datetime import datetime, timezonefrom typing import Any+from openviking.utils.token_estimation import estimate_text_tokens+class InMemoryOpenVikingClient:"""Small OpenViking-compatible client for examples and CI smoke tests.@@ -240,7 +242,7 @@ class InMemoryOpenVikingClient:if role_id is not None:message["role_id"] = role_idself.sessions.setdefault(session_id, []).append(message)- self.pending_tokens[session_id] += max(1, len(_message_text(message)) // 4)+ self.pending_tokens[session_id] += max(1, estimate_text_tokens(_message_text(message)))return {"session_id": session_id,"role": role,@@ -289,8 +291,10 @@ class InMemoryOpenVikingClient:latest_archive = self.archives.get(session_id, [])[-1:] or []latest = latest_archive[0] if latest_archive else {}messages = list(self.sessions.get(session_id, []))- active_tokens = sum(max(1, len(_message_text(message)) // 4) for message in messages)- archive_tokens = max(0, len(str(latest.get("overview", ""))) // 4)+ active_tokens = sum(+ max(1, estimate_text_tokens(_message_text(message))) for message in messages+ )+ archive_tokens = max(0, estimate_text_tokens(str(latest.get("overview", ""))))return {"latest_archive_overview": latest.get("overview", ""),"pre_archive_abstracts": [diff --git a/openviking/message/message.py b/openviking/message/message.pyindex d6263a424..638facb6f 100644--- a/openviking/message/message.py+++ b/openviking/message/message.py@@ -11,6 +11,7 @@ from datetime import datetime, timezonefrom typing import List, Literal, Optionalfrom openviking.message.part import ContextPart, Part, TextPart, ToolPart+from openviking.utils.token_estimation import estimate_text_tokens@dataclass@@ -33,7 +34,7 @@ class Message:@propertydef estimated_tokens(self) -> int:- """Estimate token count from all parts (ceil(len/4) heuristic).+ """Estimate token count from all parts using a CJK-aware fallback.Counts fields that actually appear in the assembled prompt:- TextPart.text: always emitted@@ -48,19 +49,19 @@ class Message:(8k/16k) or tool-dense sessions, consider adding a conservative per-toolbuffer instead of mirroring the full convertToAgentMessages logic."""- total_chars = 0+ token_text = []for p in self.parts:if isinstance(p, TextPart):- total_chars += len(p.text)+ token_text.append(p.text)elif isinstance(p, ContextPart):- total_chars += len(p.abstract)+ token_text.append(p.abstract)elif isinstance(p, ToolPart):- total_chars += len(p.tool_id) + len(p.tool_name)+ token_text.extend([p.tool_id, p.tool_name])if p.tool_input:- total_chars += len(json.dumps(p.tool_input, ensure_ascii=False))+ token_text.append(json.dumps(p.tool_input, ensure_ascii=False))if p.tool_output:- total_chars += len(p.tool_output)- return -(-total_chars // 4) # ceil division+ token_text.append(p.tool_output)+ return estimate_text_tokens("".join(token_text))def to_dict(self) -> dict:"""Serialize to JSONL."""diff --git a/openviking/session/session.py b/openviking/session/session.pyindex 9b602dcb7..f13aae78b 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -28,6 +28,7 @@ from openviking.session.tool_result_store import ()from openviking.telemetry import get_current_telemetry, tracerfrom openviking.telemetry.request_wait_tracker import get_request_wait_tracker+from openviking.utils.token_estimation import estimate_text_tokensfrom openviking.utils.time_utils import get_current_timestampfrom openviking_cli.exceptions import FailedPreconditionErrorfrom openviking_cli.session.user_id import UserIdentifier@@ -1287,8 +1288,8 @@ class Session:uri=f"{archive_uri}/.meta.json",content=json.dumps({- "overview_tokens": -(-len(summary) // 4),- "abstract_tokens": -(-len(abstract) // 4),+ "overview_tokens": estimate_text_tokens(summary),+ "abstract_tokens": estimate_text_tokens(abstract),}),ctx=self.ctx,@@ -1726,7 +1727,7 @@ class Session:{"archive_id": archive["archive_id"],"abstract": abstract,- "tokens": -(-len(abstract) // 4),+ "tokens": estimate_text_tokens(abstract),})else:@@ -1831,12 +1832,13 @@ class Session:async def _read_archive_overview_tokens(self, archive_uri: str, overview: str) -> int:"""Read overview token estimate from archive metadata."""- overview_tokens = -(-len(overview) // 4)+ overview_tokens = estimate_text_tokens(overview)try:meta_content = await self._viking_fs.read_file(f"{archive_uri}/.meta.json", ctx=self.ctx)- overview_tokens = json.loads(meta_content).get("overview_tokens", overview_tokens)+ meta_tokens = int(json.loads(meta_content).get("overview_tokens", overview_tokens))+ overview_tokens = max(overview_tokens, meta_tokens)except Exception:passreturn overview_tokens@@ -2348,7 +2350,7 @@ class Session:if name in Session._WM_APPEND_ONLY_SECTIONS:continueitems = Session._wm_extract_bullet_items(body)- est_tokens = len(body) // 4+ est_tokens = estimate_text_tokens(body)if (len(items) > Session._WM_SECTION_BULLET_THRESHOLDor est_tokens > Session._WM_SECTION_TOKEN_THRESHOLD@@ -2730,7 +2732,7 @@ class Session:return opif op_name == "APPEND":old_items = Session._wm_extract_bullet_items(old_content or "")- est_tokens = len(old_content or "") // 4+ est_tokens = estimate_text_tokens(old_content or "")bullet_over = len(old_items) > Session._WM_SECTION_BULLET_THRESHOLDtoken_over = est_tokens > Session._WM_SECTION_TOKEN_THRESHOLDif not bullet_over and not token_over:@@ -2784,7 +2786,7 @@ class Session:if not old_items:return op- est_tokens = len(old_content or "") // 4+ est_tokens = estimate_text_tokens(old_content or "")is_emergency = (len(old_items) > Session._WM_SECTION_BULLET_THRESHOLD * 2or est_tokens > Session._WM_SECTION_TOKEN_THRESHOLD * 2diff --git a/openviking/utils/token_estimation.py b/openviking/utils/token_estimation.pynew file mode 100644index 000000000..d1d2465eb--- /dev/null+++ b/openviking/utils/token_estimation.py@@ -0,0 +1,49 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++"""Shared conservative token estimation helpers."""++from __future__ import annotations++import math+from typing import Any+++def _is_cjk_code_point(code_point: int) -> bool:+ return (+ 0x3400 <= code_point <= 0x4DBF+ or 0x4E00 <= code_point <= 0x9FFF+ or 0xF900 <= code_point <= 0xFAFF+ or 0x20000 <= code_point <= 0x2EBEF+ or 0x3040 <= code_point <= 0x30FF+ or 0x31F0 <= code_point <= 0x31FF+ or 0xAC00 <= code_point <= 0xD7AF+ or 0x1100 <= code_point <= 0x11FF+ or 0x3130 <= code_point <= 0x318F+ or 0xFF00 <= code_point <= 0xFFEF+ or 0x3000 <= code_point <= 0x303F+ )+++def _code_point_weight(code_point: int) -> float:+ if _is_cjk_code_point(code_point):+ return 1.5+ if code_point > 0xFFFF:+ return 2.0+ return 0.25+++def estimate_text_tokens(text: str | None) -> int:+ """Estimate tokens with a CJK-aware fallback."""+ if not text:+ return 0+ return math.ceil(sum(_code_point_weight(ord(char)) for char in text))+++def estimate_serialized_tokens(value: Any) -> int:+ """Estimate tokens for already-structured prompt-like values."""+ if value is None:+ return 0+ if isinstance(value, str):+ return estimate_text_tokens(value)+ return estimate_text_tokens(str(value))diff --git a/tests/api_test/sessions/test_session_messages.py b/tests/api_test/sessions/test_session_messages.pyindex 51d1ba71c..43feb2fef 100644--- a/tests/api_test/sessions/test_session_messages.py+++ b/tests/api_test/sessions/test_session_messages.py@@ -1,4 +1,5 @@import uuid+from math import ceilclass TestSessionMessages:@@ -173,6 +174,61 @@ class TestSessionMessages:if session_id:api_client.delete_session(session_id)+ def test_cjk_message_uses_cjk_aware_token_estimate(self, api_client):+ cjk_session_id = None+ ascii_session_id = None+ try:+ cjk_text = "你好世界" * 100+ ascii_text = "abcd" * 100+ cjk_expected_min = ceil(len(cjk_text) * 1.5)+ naive_chars_div_4 = ceil(len(cjk_text) / 4)++ cjk_resp = api_client.create_session()+ assert cjk_resp.status_code == 200+ cjk_session_id = cjk_resp.json()["result"]["session_id"]+ add_cjk = api_client.add_message(cjk_session_id, "user", cjk_text)+ assert add_cjk.status_code == 200++ cjk_detail = api_client.get_session(cjk_session_id)+ assert cjk_detail.status_code == 200+ cjk_pending = cjk_detail.json().get("result", {}).get("pending_tokens", 0)+ assert cjk_pending >= cjk_expected_min, (+ "pending_tokens should use the CJK-aware estimate; "+ f"got {cjk_pending}, expected at least {cjk_expected_min}, "+ f"old chars/4 estimate would be {naive_chars_div_4}"+ )++ cjk_ctx = api_client.get_session_context(cjk_session_id, token_budget=128000)+ assert cjk_ctx.status_code == 200+ cjk_result = cjk_ctx.json().get("result", {})+ cjk_estimated = cjk_result.get("estimatedTokens", 0)+ cjk_active = cjk_result.get("stats", {}).get("activeTokens", 0)+ assert cjk_estimated >= cjk_expected_min, (+ f"context estimatedTokens should preserve CJK-aware estimate, got {cjk_estimated}"+ )+ assert cjk_active >= cjk_expected_min, (+ f"context stats.activeTokens should preserve CJK-aware estimate, got {cjk_active}"+ )++ ascii_resp = api_client.create_session()+ assert ascii_resp.status_code == 200+ ascii_session_id = ascii_resp.json()["result"]["session_id"]+ add_ascii = api_client.add_message(ascii_session_id, "user", ascii_text)+ assert add_ascii.status_code == 200++ ascii_detail = api_client.get_session(ascii_session_id)+ assert ascii_detail.status_code == 200+ ascii_pending = ascii_detail.json().get("result", {}).get("pending_tokens", 0)+ assert cjk_pending >= ascii_pending * 4, (+ "same-length CJK should be estimated much higher than ASCII; "+ f"cjk={cjk_pending}, ascii={ascii_pending}"+ )+ finally:+ if cjk_session_id:+ api_client.delete_session(cjk_session_id)+ if ascii_session_id:+ api_client.delete_session(ascii_session_id)+def test_multiline_content_preserved(self, api_client):session_id = Nonetry:diff --git a/tests/test_token_estimation.py b/tests/test_token_estimation.pynew file mode 100644index 000000000..baf6f2f7f--- /dev/null+++ b/tests/test_token_estimation.py@@ -0,0 +1,48 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: AGPL-3.0++"""Shared token estimation tests."""++from openviking.message import Message+from openviking.session import Session+++def test_message_estimated_tokens_is_cjk_aware():+ """Chinese text should not be estimated with the ASCII chars/4 fallback."""+ msg = Message.create_user("\u4f60\u597d\u4e16\u754c")++ assert msg.estimated_tokens == 6+++async def test_archive_overview_tokens_do_not_trust_stale_low_metadata():+ class FakeFS:+ async def read_file(self, uri, ctx=None):+ del uri, ctx+ return '{"overview_tokens": 1}'++ fake_session = type("FakeSession", (), {"_viking_fs": FakeFS(), "ctx": None})()++ tokens = await Session._read_archive_overview_tokens(+ fake_session,+ "viking://session/test/history/archive_001",+ "\u4f60\u597d\u4e16\u754c",+ )++ assert tokens == 6+++async def test_archive_overview_tokens_keep_higher_metadata_estimate():+ class FakeFS:+ async def read_file(self, uri, ctx=None):+ del uri, ctx+ return '{"overview_tokens": 5}'++ fake_session = type("FakeSession", (), {"_viking_fs": FakeFS(), "ctx": None})()++ tokens = await Session._read_archive_overview_tokens(+ fake_session,+ "viking://session/test/history/archive_001",+ "abcd",+ )++ assert tokens == 5