Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix: add cjk-aware token estimation (#2348)

Eurakaxunab855a43b054
diff --git a/examples/openclaw-plugin/auto-recall.ts b/examples/openclaw-plugin/auto-recall.ts
index 8e537be77..04037ad59 100644
--- a/examples/openclaw-plugin/auto-recall.ts
+++ b/examples/openclaw-plugin/auto-recall.ts
@@ -8,6 +8,7 @@ import {
} from "./memory-ranking.js";
import { quickRecallPrecheck, withTimeout } from "./process-manager.js";
import { sanitizeUserTextForCapture } from "./text-utils.js";
+import { estimateTextTokens } from "./token-estimator.js";
const AUTO_RECALL_TIMEOUT_MS = 5_000;
const RECALL_QUERY_MAX_CHARS = 4_000;
@@ -51,10 +52,9 @@ export function prepareRecallQuery(rawText: string): PreparedRecallQuery {
};
}
-/** Estimate token count using chars/4 heuristic for diagnostics. */
+/** Estimate token count using the shared CJK-aware fallback for diagnostics. */
export function estimateTokenCount(text: string): number {
- if (!text) return 0;
- return Math.ceil(text.length / 4);
+ return estimateTextTokens(text);
}
export type BuildMemoryLinesOptions = {
diff --git a/examples/openclaw-plugin/context-engine.ts b/examples/openclaw-plugin/context-engine.ts
index 29d7321ca..9ebea3516 100644
--- a/examples/openclaw-plugin/context-engine.ts
+++ b/examples/openclaw-plugin/context-engine.ts
@@ -18,6 +18,11 @@ import {
toJsonLog,
} from "./memory-ranking.js";
import { sanitizeToolUseResultPairing } from "./session-transcript-repair.js";
+import {
+ estimateAgentMessageTokens,
+ estimateAgentMessagesTokens,
+ estimateTextTokens,
+} from "./token-estimator.js";
type AgentMessage = {
role?: string;
@@ -156,14 +161,11 @@ function allocateContextBudget(totalBudget: number, instructionTokens = 0): Cont
}
function roughEstimate(messages: AgentMessage[]): number {
- return Math.ceil(JSON.stringify(messages).length / 4);
+ return estimateAgentMessagesTokens(messages);
}
function msgTokenEstimate(msg: AgentMessage): number {
- const raw = (msg as Record<string, unknown>).content;
- if (typeof raw === "string") return Math.ceil(raw.length / 4);
- if (Array.isArray(raw)) return Math.ceil(JSON.stringify(raw).length / 4);
- return 1;
+ return estimateAgentMessageTokens(msg);
}
function normalizeTimestamp(value: unknown): string | undefined {
@@ -671,7 +673,7 @@ function buildSystemPromptAddition(): string {
function buildInstructionPrompt(): { text: string; tokens: number } {
const text = buildSystemPromptAddition();
- return { text, tokens: Math.ceil(text.length / 4) };
+ return { text, tokens: estimateTextTokens(text) };
}
function buildArchiveMemory(
diff --git a/examples/openclaw-plugin/index.ts b/examples/openclaw-plugin/index.ts
index 20bd87ab3..733440d48 100644
--- a/examples/openclaw-plugin/index.ts
+++ b/examples/openclaw-plugin/index.ts
@@ -43,6 +43,12 @@ export {
estimateTokenCount,
prepareRecallQuery,
};
+export {
+ estimateAgentMessageTokens,
+ estimateAgentMessagesTokens,
+ estimateSerializedTokens,
+ estimateTextTokens,
+} from "./token-estimator.js";
export type {
BuildMemoryLinesOptions,
BuildMemoryLinesWithBudgetOptions,
diff --git a/examples/openclaw-plugin/install-manifest.json b/examples/openclaw-plugin/install-manifest.json
index 180e4be26..bbbb8cea5 100644
--- a/examples/openclaw-plugin/install-manifest.json
+++ b/examples/openclaw-plugin/install-manifest.json
@@ -15,6 +15,7 @@
"client.ts",
"process-manager.ts",
"memory-ranking.ts",
+ "token-estimator.ts",
"text-utils.ts",
"tool-call-id.ts",
"session-transcript-repair.ts",
diff --git a/examples/openclaw-plugin/tests/context-bloat-730.test.ts b/examples/openclaw-plugin/tests/context-bloat-730.test.ts
index ca78ddcc0..e2be3078f 100644
--- a/examples/openclaw-plugin/tests/context-bloat-730.test.ts
+++ b/examples/openclaw-plugin/tests/context-bloat-730.test.ts
@@ -139,12 +139,14 @@ describe("Slice E: character budget enforcement", () => {
expect(estimatedTokens).toBeLessThanOrEqual(53);
});
- it("should estimate tokens as ceil(chars/4)", async () => {
+ it("should estimate tokens with CJK-aware fallback weights", async () => {
const { estimateTokenCount } = await import("../index.js");
expect(estimateTokenCount("")).toBe(0);
expect(estimateTokenCount("abcd")).toBe(1);
expect(estimateTokenCount("abcde")).toBe(2);
expect(estimateTokenCount("A".repeat(100))).toBe(25);
+ expect(estimateTokenCount("\u4f60\u597d\u4e16\u754c")).toBe(6);
+ expect(estimateTokenCount("A\u4f60\ud83d\ude42")).toBe(4);
});
it("should have recallMaxInjectedChars in parsed config with default 4000-character budget", () => {
diff --git a/examples/openclaw-plugin/tests/context-engine-assemble.test.ts b/examples/openclaw-plugin/tests/context-engine-assemble.test.ts
index 8aefd7db3..874d4ccf1 100644
--- a/examples/openclaw-plugin/tests/context-engine-assemble.test.ts
+++ b/examples/openclaw-plugin/tests/context-engine-assemble.test.ts
@@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest";
import type { OpenVikingClient } from "../client.js";
import { memoryOpenVikingConfigSchema } from "../config.js";
import { createMemoryOpenVikingContextEngine } from "../context-engine.js";
+import { estimateAgentMessagesTokens, estimateTextTokens } from "../token-estimator.js";
const cfg = memoryOpenVikingConfigSchema.parse({
mode: "remote",
@@ -14,11 +15,11 @@ const cfg = memoryOpenVikingConfigSchema.parse({
});
function roughEstimate(messages: unknown[]): number {
- return Math.ceil(JSON.stringify(messages).length / 4);
+ return estimateAgentMessagesTokens(messages);
}
function systemPromptTokens(text?: string): number {
- return text ? Math.ceil(text.length / 4) : 0;
+ return estimateTextTokens(text);
}
function makeLogger() {
diff --git a/examples/openclaw-plugin/tests/e2e/test-cjk-token-estimation.py b/examples/openclaw-plugin/tests/e2e/test-cjk-token-estimation.py
new file mode 100644
index 000000000..57d9c663a
--- /dev/null
+++ b/examples/openclaw-plugin/tests/e2e/test-cjk-token-estimation.py
@@ -0,0 +1,269 @@
+#!/usr/bin/env python3
+"""OpenClaw plugin E2E for CJK-aware token estimation.
+
+This test drives the real OpenClaw Gateway, then verifies the OpenViking plugin's
+own assemble diagnostics. It intentionally checks the plugin-side token estimate,
+not only the OV REST session counters.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import pathlib
+import time
+import uuid
+from typing import Any
+
+import requests
+
+
+DEFAULT_GATEWAY_URL = os.environ.get("OPENCLAW_GATEWAY_URL", "http://127.0.0.1:19830")
+DEFAULT_OPENVIKING_URL = os.environ.get("OPENVIKING_BASE_URL", "http://127.0.0.1:2948")
+DEFAULT_CJK_REPEAT = 120
+
+
+def default_gateway_log_path() -> pathlib.Path:
+ state_dir = os.environ.get("OPENCLAW_STATE_DIR")
+ if state_dir:
+ return pathlib.Path(state_dir) / "logs" / "gateway.stdout.log"
+ return pathlib.Path.cwd() / "config" / ".openclaw" / "logs" / "gateway.stdout.log"
+
+
+def discover_gateway_token() -> str:
+ env_token = os.environ.get("OPENCLAW_GATEWAY_TOKEN", "").strip()
+ if env_token:
+ return env_token
+
+ candidates: list[pathlib.Path] = []
+ config_path = os.environ.get("OPENCLAW_CONFIG_PATH")
+ if config_path:
+ candidates.append(pathlib.Path(config_path))
+
+ state_dir = os.environ.get("OPENCLAW_STATE_DIR")
+ if state_dir:
+ candidates.append(pathlib.Path(state_dir) / "openclaw.json")
+
+ candidates.extend(
+ [
+ pathlib.Path.cwd() / "config" / ".openclaw" / "openclaw.json",
+ pathlib.Path.home() / ".openclaw" / "openclaw.json",
+ ]
+ )
+
+ for path in candidates:
+ try:
+ cfg = json.loads(path.read_text(encoding="utf-8"))
+ except Exception:
+ continue
+ token = str(cfg.get("gateway", {}).get("auth", {}).get("token", "")).strip()
+ if token:
+ return token
+ return ""
+
+
+def send_gateway_message(
+ gateway_url: str,
+ token: str,
+ user_id: str,
+ message: str,
+ timeout: int,
+) -> dict[str, Any]:
+ headers = {"Content-Type": "application/json"}
+ if token:
+ headers["Authorization"] = f"Bearer {token}"
+
+ response = requests.post(
+ f"{gateway_url.rstrip('/')}/v1/responses",
+ headers=headers,
+ json={"model": "openclaw", "input": message, "user": user_id},
+ timeout=timeout,
+ )
+ response.raise_for_status()
+ return response.json()
+
+
+def iter_new_log_lines(path: pathlib.Path, offset: int) -> list[str]:
+ if not path.exists():
+ return []
+ with path.open("rb") as handle:
+ handle.seek(min(offset, path.stat().st_size))
+ return handle.read().decode("utf-8", errors="ignore").splitlines()
+
+
+def parse_diag(line: str) -> dict[str, Any] | None:
+ marker = "openviking: diag "
+ pos = line.find(marker)
+ if pos < 0:
+ return None
+ try:
+ return json.loads(line[pos + len(marker) :].strip())
+ except json.JSONDecodeError:
+ return None
+
+
+def find_assemble_diag_with_marker(
+ log_path: pathlib.Path,
+ offset: int,
+ marker: str,
+ timeout: float,
+) -> tuple[dict[str, Any], dict[str, Any]]:
+ deadline = time.time() + timeout
+ last_seen = ""
+
+ while time.time() < deadline:
+ for line in iter_new_log_lines(log_path, offset):
+ diag = parse_diag(line)
+ if not diag or diag.get("stage") != "assemble_entry":
+ continue
+ data = diag.get("data", {})
+ for message in data.get("messages", []):
+ content = str(message.get("content", ""))
+ if marker in content:
+ return diag, message
+ last_seen = json.dumps(data, ensure_ascii=False)[:500]
+ time.sleep(0.5)
+
+ raise AssertionError(
+ f"no assemble_entry diagnostic contained marker {marker!r}; last_seen={last_seen}"
+ )
+
+
+def flatten_message_text(value: Any) -> str:
+ if value is None:
+ return ""
+ if isinstance(value, str):
+ return value
+ if isinstance(value, list):
+ return "\n".join(flatten_message_text(item) for item in value)
+ if isinstance(value, dict):
+ chunks: list[str] = []
+ for key in ("text", "abstract", "content"):
+ if key in value:
+ chunks.append(flatten_message_text(value.get(key)))
+ for key in ("parts", "messages", "pre_archive_abstracts"):
+ raw = value.get(key)
+ if isinstance(raw, list):
+ chunks.append(flatten_message_text(raw))
+ return "\n".join(chunk for chunk in chunks if chunk)
+ return ""
+
+
+def ov_get(base_url: str, path: str) -> Any:
+ response = requests.get(f"{base_url.rstrip('/')}{path}", timeout=10)
+ response.raise_for_status()
+ data = response.json()
+ return data.get("result", data)
+
+
+def find_ov_context_with_marker(
+ openviking_url: str,
+ marker: str,
+ timeout: float,
+) -> tuple[str, dict[str, Any]]:
+ deadline = time.time() + timeout
+ last_session_count = 0
+
+ while time.time() < deadline:
+ sessions = ov_get(openviking_url, "/api/v1/sessions")
+ if not isinstance(sessions, list):
+ sessions = []
+ last_session_count = len(sessions)
+ ordered = sorted(
+ [s for s in sessions if isinstance(s, dict)],
+ key=lambda s: str(s.get("updated_at", "")),
+ reverse=True,
+ )[:40]
+
+ for session in ordered:
+ session_id = str(session.get("session_id", ""))
+ if not session_id or session_id.startswith("memory-store-"):
+ continue
+ try:
+ ctx = ov_get(openviking_url, f"/api/v1/sessions/{session_id}/context?token_budget=128000")
+ except requests.RequestException:
+ continue
+ if marker in flatten_message_text(ctx):
+ return session_id, ctx
+
+ time.sleep(1)
+
+ raise AssertionError(
+ f"marker {marker!r} was not found in OV contexts; checked sessions={last_session_count}"
+ )
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description="OpenClaw plugin CJK token estimation E2E")
+ parser.add_argument("--gateway", default=DEFAULT_GATEWAY_URL)
+ parser.add_argument("--openviking", default=DEFAULT_OPENVIKING_URL)
+ parser.add_argument("--gateway-log", type=pathlib.Path, default=default_gateway_log_path())
+ parser.add_argument("--repeat", type=int, default=DEFAULT_CJK_REPEAT)
+ parser.add_argument("--gateway-timeout", type=int, default=300)
+ parser.add_argument("--diag-timeout", type=float, default=45)
+ parser.add_argument("--ov-timeout", type=float, default=45)
+ args = parser.parse_args()
+
+ marker = f"CJK_TOKEN_E2E_{uuid.uuid4().hex[:10]}"
+ user_id = f"cjk-token-e2e-{uuid.uuid4().hex[:10]}"
+ cjk_text = "你好世界" * args.repeat
+ expected_cjk_tokens = math.ceil(len(cjk_text) * 1.5)
+ naive_chars_div_4 = math.ceil(len(cjk_text) / 4)
+ message = (
+ f"{marker}\n"
+ "请只回复 OK。下面这段中文只用于 OpenViking token 估算端到端回归测试:\n"
+ f"{cjk_text}"
+ )
+
+ log_offset = args.gateway_log.stat().st_size if args.gateway_log.exists() else 0
+ token = discover_gateway_token()
+
+ print(f"Gateway: {args.gateway}")
+ print(f"OpenViking: {args.openviking}")
+ print(f"Gateway log: {args.gateway_log}")
+ print(f"Marker: {marker}")
+ print(
+ "Expected CJK token floor: "
+ f"{expected_cjk_tokens} (old chars/4 would be {naive_chars_div_4})"
+ )
+
+ send_gateway_message(args.gateway, token, user_id, message, args.gateway_timeout)
+
+ diag, marked_message = find_assemble_diag_with_marker(
+ args.gateway_log,
+ log_offset,
+ marker,
+ args.diag_timeout,
+ )
+ observed_tokens = int(marked_message.get("tokens", 0) or 0)
+ input_tokens = int(diag.get("data", {}).get("inputTokenEstimate", 0) or 0)
+ if observed_tokens < expected_cjk_tokens:
+ raise AssertionError(
+ "plugin assemble token estimate under-counted CJK message: "
+ f"observed={observed_tokens}, expected>={expected_cjk_tokens}, "
+ f"old chars/4={naive_chars_div_4}, inputTokenEstimate={input_tokens}"
+ )
+
+ session_id, ctx = find_ov_context_with_marker(args.openviking, marker, args.ov_timeout)
+ estimated_tokens = int(ctx.get("estimatedTokens", 0) or 0)
+ active_tokens = int(ctx.get("stats", {}).get("activeTokens", 0) or 0)
+ if max(estimated_tokens, active_tokens) < expected_cjk_tokens:
+ raise AssertionError(
+ "OV context after plugin afterTurn under-counted CJK message: "
+ f"estimatedTokens={estimated_tokens}, activeTokens={active_tokens}, "
+ f"expected>={expected_cjk_tokens}, session={session_id}"
+ )
+
+ print(
+ "PASS: plugin CJK estimate "
+ f"messageTokens={observed_tokens}, inputTokenEstimate={input_tokens}, "
+ f"ovEstimatedTokens={estimated_tokens}, ovActiveTokens={active_tokens}, "
+ f"session={session_id}"
+ )
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/examples/openclaw-plugin/tests/ut/build-memory-lines.test.ts b/examples/openclaw-plugin/tests/ut/build-memory-lines.test.ts
index efeb88782..6d077d12f 100644
--- a/examples/openclaw-plugin/tests/ut/build-memory-lines.test.ts
+++ b/examples/openclaw-plugin/tests/ut/build-memory-lines.test.ts
@@ -23,12 +23,17 @@ describe("estimateTokenCount", () => {
expect(estimateTokenCount("")).toBe(0);
});
- it("estimates tokens as ceil(chars/4)", () => {
+ it("keeps the legacy ASCII estimate close to ceil(chars/4)", () => {
expect(estimateTokenCount("hello")).toBe(2); // ceil(5/4)
expect(estimateTokenCount("abcd")).toBe(1); // ceil(4/4)
expect(estimateTokenCount("abcde")).toBe(2); // ceil(5/4)
});
+ it("uses CJK-aware weighting for non-Latin text and emoji", () => {
+ expect(estimateTokenCount("\u4f60\u597d\u4e16\u754c")).toBe(6);
+ expect(estimateTokenCount("A\u4f60\ud83d\ude42")).toBe(4);
+ });
+
it("handles long text", () => {
const text = "a".repeat(1000);
expect(estimateTokenCount(text)).toBe(250);
diff --git a/examples/openclaw-plugin/tests/ut/context-engine-assemble.test.ts b/examples/openclaw-plugin/tests/ut/context-engine-assemble.test.ts
index 312b38459..212313eae 100644
--- a/examples/openclaw-plugin/tests/ut/context-engine-assemble.test.ts
+++ b/examples/openclaw-plugin/tests/ut/context-engine-assemble.test.ts
@@ -3,6 +3,7 @@ import { describe, expect, it, vi } from "vitest";
import type { OpenVikingClient } from "../../client.js";
import { memoryOpenVikingConfigSchema } from "../../config.js";
import { createMemoryOpenVikingContextEngine } from "../../context-engine.js";
+import { estimateAgentMessagesTokens, estimateTextTokens } from "../../token-estimator.js";
const cfg = memoryOpenVikingConfigSchema.parse({
mode: "remote",
@@ -12,11 +13,11 @@ const cfg = memoryOpenVikingConfigSchema.parse({
});
function roughEstimate(messages: unknown[]): number {
- return Math.ceil(JSON.stringify(messages).length / 4);
+ return estimateAgentMessagesTokens(messages);
}
function systemPromptTokens(text?: string): number {
- return text ? Math.ceil(text.length / 4) : 0;
+ return estimateTextTokens(text);
}
function makeLogger() {
@@ -214,6 +215,29 @@ describe("context-engine assemble()", () => {
expect(result.estimatedTokens).toBe(roughEstimate(sourceMessages));
});
+ it("does not underestimate CJK messages when passing through transformContext", async () => {
+ const { engine, getClient } = makeEngine({
+ latest_archive_overview: "unused",
+ pre_archive_abstracts: [],
+ messages: [],
+ estimatedTokens: 0,
+ stats: makeStats(),
+ });
+ const sourceMessages = [
+ { role: "assistant", content: [{ type: "text", text: "Previous answer." }] },
+ { role: "user", content: "\u4f60\u597d".repeat(100) },
+ ];
+
+ const result = await engine.assemble({
+ sessionId: "session-cjk-estimate",
+ messages: sourceMessages,
+ });
+
+ expect(getClient).not.toHaveBeenCalled();
+ expect(result.messages).toBe(sourceMessages);
+ expect(result.estimatedTokens).toBeGreaterThanOrEqual(300);
+ });
+
it("treats prompt-less assemble with availableTools as main assemble", async () => {
const { engine, client, resolveAgentId } = makeEngine({
latest_archive_overview: "# Session Summary\nPreviously discussed repository setup.",
diff --git a/examples/openclaw-plugin/token-estimator.ts b/examples/openclaw-plugin/token-estimator.ts
new file mode 100644
index 000000000..b91b7e400
--- /dev/null
+++ b/examples/openclaw-plugin/token-estimator.ts
@@ -0,0 +1,65 @@
+type TokenCountableAgentMessage = {
+ role?: string;
+ content?: unknown;
+};
+
+function isCjkCodePoint(codePoint: number): boolean {
+ return (
+ (codePoint >= 0x3400 && codePoint <= 0x4dbf) ||
+ (codePoint >= 0x4e00 && codePoint <= 0x9fff) ||
+ (codePoint >= 0xf900 && codePoint <= 0xfaff) ||
+ (codePoint >= 0x20000 && codePoint <= 0x2ebef) ||
+ (codePoint >= 0x3040 && codePoint <= 0x30ff) ||
+ (codePoint >= 0x31f0 && codePoint <= 0x31ff) ||
+ (codePoint >= 0xac00 && codePoint <= 0xd7af) ||
+ (codePoint >= 0x1100 && codePoint <= 0x11ff) ||
+ (codePoint >= 0x3130 && codePoint <= 0x318f) ||
+ (codePoint >= 0xff00 && codePoint <= 0xffef) ||
+ (codePoint >= 0x3000 && codePoint <= 0x303f)
+ );
+}
+
+function codePointWeight(codePoint: number): number {
+ if (isCjkCodePoint(codePoint)) {
+ return 1.5;
+ }
+ if (codePoint > 0xffff) {
+ return 2;
+ }
+ return 0.25;
+}
+
+export function estimateTextTokens(text: string | null | undefined): number {
+ if (!text) {
+ return 0;
+ }
+
+ let weightedTokens = 0;
+ for (const char of text) {
+ weightedTokens += codePointWeight(char.codePointAt(0) ?? 0);
+ }
+ return Math.ceil(weightedTokens);
+}
+
+export function estimateSerializedTokens(value: unknown): number {
+ if (value === null || value === undefined) {
+ return 0;
+ }
+ if (typeof value === "string") {
+ return estimateTextTokens(value);
+ }
+ if (typeof value === "number" || typeof value === "boolean" || typeof value === "bigint") {
+ return estimateTextTokens(String(value));
+ }
+
+ const serialized = JSON.stringify(value);
+ return estimateTextTokens(serialized ?? "");
+}
+
+export function estimateAgentMessageTokens(message: TokenCountableAgentMessage): number {
+ return estimateSerializedTokens(message);
+}
+
+export function estimateAgentMessagesTokens(messages: TokenCountableAgentMessage[]): number {
+ return estimateSerializedTokens(messages);
+}
diff --git a/openviking/integrations/langchain/testing.py b/openviking/integrations/langchain/testing.py
index 3450efe52..1113177b4 100644
--- a/openviking/integrations/langchain/testing.py
+++ b/openviking/integrations/langchain/testing.py
@@ -11,6 +11,8 @@ from collections import defaultdict
from datetime import datetime, timezone
from typing import Any
+from openviking.utils.token_estimation import estimate_text_tokens
+
class InMemoryOpenVikingClient:
"""Small OpenViking-compatible client for examples and CI smoke tests.
@@ -240,7 +242,7 @@ class InMemoryOpenVikingClient:
if role_id is not None:
message["role_id"] = role_id
self.sessions.setdefault(session_id, []).append(message)
- self.pending_tokens[session_id] += max(1, len(_message_text(message)) // 4)
+ self.pending_tokens[session_id] += max(1, estimate_text_tokens(_message_text(message)))
return {
"session_id": session_id,
"role": role,
@@ -289,8 +291,10 @@ class InMemoryOpenVikingClient:
latest_archive = self.archives.get(session_id, [])[-1:] or []
latest = latest_archive[0] if latest_archive else {}
messages = list(self.sessions.get(session_id, []))
- active_tokens = sum(max(1, len(_message_text(message)) // 4) for message in messages)
- archive_tokens = max(0, len(str(latest.get("overview", ""))) // 4)
+ active_tokens = sum(
+ max(1, estimate_text_tokens(_message_text(message))) for message in messages
+ )
+ archive_tokens = max(0, estimate_text_tokens(str(latest.get("overview", ""))))
return {
"latest_archive_overview": latest.get("overview", ""),
"pre_archive_abstracts": [
diff --git a/openviking/message/message.py b/openviking/message/message.py
index d6263a424..638facb6f 100644
--- a/openviking/message/message.py
+++ b/openviking/message/message.py
@@ -11,6 +11,7 @@ from datetime import datetime, timezone
from typing import List, Literal, Optional
from openviking.message.part import ContextPart, Part, TextPart, ToolPart
+from openviking.utils.token_estimation import estimate_text_tokens
@dataclass
@@ -33,7 +34,7 @@ class Message:
@property
def estimated_tokens(self) -> int:
- """Estimate token count from all parts (ceil(len/4) heuristic).
+ """Estimate token count from all parts using a CJK-aware fallback.
Counts fields that actually appear in the assembled prompt:
- TextPart.text: always emitted
@@ -48,19 +49,19 @@ class Message:
(8k/16k) or tool-dense sessions, consider adding a conservative per-tool
buffer instead of mirroring the full convertToAgentMessages logic.
"""
- total_chars = 0
+ token_text = []
for p in self.parts:
if isinstance(p, TextPart):
- total_chars += len(p.text)
+ token_text.append(p.text)
elif isinstance(p, ContextPart):
- total_chars += len(p.abstract)
+ token_text.append(p.abstract)
elif isinstance(p, ToolPart):
- total_chars += len(p.tool_id) + len(p.tool_name)
+ token_text.extend([p.tool_id, p.tool_name])
if p.tool_input:
- total_chars += len(json.dumps(p.tool_input, ensure_ascii=False))
+ token_text.append(json.dumps(p.tool_input, ensure_ascii=False))
if p.tool_output:
- total_chars += len(p.tool_output)
- return -(-total_chars // 4) # ceil division
+ token_text.append(p.tool_output)
+ return estimate_text_tokens("".join(token_text))
def to_dict(self) -> dict:
"""Serialize to JSONL."""
diff --git a/openviking/session/session.py b/openviking/session/session.py
index 9b602dcb7..f13aae78b 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -28,6 +28,7 @@ from openviking.session.tool_result_store import (
)
from openviking.telemetry import get_current_telemetry, tracer
from openviking.telemetry.request_wait_tracker import get_request_wait_tracker
+from openviking.utils.token_estimation import estimate_text_tokens
from openviking.utils.time_utils import get_current_timestamp
from openviking_cli.exceptions import FailedPreconditionError
from openviking_cli.session.user_id import UserIdentifier
@@ -1287,8 +1288,8 @@ class Session:
uri=f"{archive_uri}/.meta.json",
content=json.dumps(
{
- "overview_tokens": -(-len(summary) // 4),
- "abstract_tokens": -(-len(abstract) // 4),
+ "overview_tokens": estimate_text_tokens(summary),
+ "abstract_tokens": estimate_text_tokens(abstract),
}
),
ctx=self.ctx,
@@ -1726,7 +1727,7 @@ class Session:
{
"archive_id": archive["archive_id"],
"abstract": abstract,
- "tokens": -(-len(abstract) // 4),
+ "tokens": estimate_text_tokens(abstract),
}
)
else:
@@ -1831,12 +1832,13 @@ class Session:
async def _read_archive_overview_tokens(self, archive_uri: str, overview: str) -> int:
"""Read overview token estimate from archive metadata."""
- overview_tokens = -(-len(overview) // 4)
+ overview_tokens = estimate_text_tokens(overview)
try:
meta_content = await self._viking_fs.read_file(
f"{archive_uri}/.meta.json", ctx=self.ctx
)
- overview_tokens = json.loads(meta_content).get("overview_tokens", overview_tokens)
+ meta_tokens = int(json.loads(meta_content).get("overview_tokens", overview_tokens))
+ overview_tokens = max(overview_tokens, meta_tokens)
except Exception:
pass
return overview_tokens
@@ -2348,7 +2350,7 @@ class Session:
if name in Session._WM_APPEND_ONLY_SECTIONS:
continue
items = Session._wm_extract_bullet_items(body)
- est_tokens = len(body) // 4
+ est_tokens = estimate_text_tokens(body)
if (
len(items) > Session._WM_SECTION_BULLET_THRESHOLD
or est_tokens > Session._WM_SECTION_TOKEN_THRESHOLD
@@ -2730,7 +2732,7 @@ class Session:
return op
if op_name == "APPEND":
old_items = Session._wm_extract_bullet_items(old_content or "")
- est_tokens = len(old_content or "") // 4
+ est_tokens = estimate_text_tokens(old_content or "")
bullet_over = len(old_items) > Session._WM_SECTION_BULLET_THRESHOLD
token_over = est_tokens > Session._WM_SECTION_TOKEN_THRESHOLD
if not bullet_over and not token_over:
@@ -2784,7 +2786,7 @@ class Session:
if not old_items:
return op
- est_tokens = len(old_content or "") // 4
+ est_tokens = estimate_text_tokens(old_content or "")
is_emergency = (
len(old_items) > Session._WM_SECTION_BULLET_THRESHOLD * 2
or est_tokens > Session._WM_SECTION_TOKEN_THRESHOLD * 2
diff --git a/openviking/utils/token_estimation.py b/openviking/utils/token_estimation.py
new file mode 100644
index 000000000..d1d2465eb
--- /dev/null
+++ b/openviking/utils/token_estimation.py
@@ -0,0 +1,49 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+"""Shared conservative token estimation helpers."""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+
+def _is_cjk_code_point(code_point: int) -> bool:
+ return (
+ 0x3400 <= code_point <= 0x4DBF
+ or 0x4E00 <= code_point <= 0x9FFF
+ or 0xF900 <= code_point <= 0xFAFF
+ or 0x20000 <= code_point <= 0x2EBEF
+ or 0x3040 <= code_point <= 0x30FF
+ or 0x31F0 <= code_point <= 0x31FF
+ or 0xAC00 <= code_point <= 0xD7AF
+ or 0x1100 <= code_point <= 0x11FF
+ or 0x3130 <= code_point <= 0x318F
+ or 0xFF00 <= code_point <= 0xFFEF
+ or 0x3000 <= code_point <= 0x303F
+ )
+
+
+def _code_point_weight(code_point: int) -> float:
+ if _is_cjk_code_point(code_point):
+ return 1.5
+ if code_point > 0xFFFF:
+ return 2.0
+ return 0.25
+
+
+def estimate_text_tokens(text: str | None) -> int:
+ """Estimate tokens with a CJK-aware fallback."""
+ if not text:
+ return 0
+ return math.ceil(sum(_code_point_weight(ord(char)) for char in text))
+
+
+def estimate_serialized_tokens(value: Any) -> int:
+ """Estimate tokens for already-structured prompt-like values."""
+ if value is None:
+ return 0
+ if isinstance(value, str):
+ return estimate_text_tokens(value)
+ return estimate_text_tokens(str(value))
diff --git a/tests/api_test/sessions/test_session_messages.py b/tests/api_test/sessions/test_session_messages.py
index 51d1ba71c..43feb2fef 100644
--- a/tests/api_test/sessions/test_session_messages.py
+++ b/tests/api_test/sessions/test_session_messages.py
@@ -1,4 +1,5 @@
import uuid
+from math import ceil
class TestSessionMessages:
@@ -173,6 +174,61 @@ class TestSessionMessages:
if session_id:
api_client.delete_session(session_id)
+ def test_cjk_message_uses_cjk_aware_token_estimate(self, api_client):
+ cjk_session_id = None
+ ascii_session_id = None
+ try:
+ cjk_text = "你好世界" * 100
+ ascii_text = "abcd" * 100
+ cjk_expected_min = ceil(len(cjk_text) * 1.5)
+ naive_chars_div_4 = ceil(len(cjk_text) / 4)
+
+ cjk_resp = api_client.create_session()
+ assert cjk_resp.status_code == 200
+ cjk_session_id = cjk_resp.json()["result"]["session_id"]
+ add_cjk = api_client.add_message(cjk_session_id, "user", cjk_text)
+ assert add_cjk.status_code == 200
+
+ cjk_detail = api_client.get_session(cjk_session_id)
+ assert cjk_detail.status_code == 200
+ cjk_pending = cjk_detail.json().get("result", {}).get("pending_tokens", 0)
+ assert cjk_pending >= cjk_expected_min, (
+ "pending_tokens should use the CJK-aware estimate; "
+ f"got {cjk_pending}, expected at least {cjk_expected_min}, "
+ f"old chars/4 estimate would be {naive_chars_div_4}"
+ )
+
+ cjk_ctx = api_client.get_session_context(cjk_session_id, token_budget=128000)
+ assert cjk_ctx.status_code == 200
+ cjk_result = cjk_ctx.json().get("result", {})
+ cjk_estimated = cjk_result.get("estimatedTokens", 0)
+ cjk_active = cjk_result.get("stats", {}).get("activeTokens", 0)
+ assert cjk_estimated >= cjk_expected_min, (
+ f"context estimatedTokens should preserve CJK-aware estimate, got {cjk_estimated}"
+ )
+ assert cjk_active >= cjk_expected_min, (
+ f"context stats.activeTokens should preserve CJK-aware estimate, got {cjk_active}"
+ )
+
+ ascii_resp = api_client.create_session()
+ assert ascii_resp.status_code == 200
+ ascii_session_id = ascii_resp.json()["result"]["session_id"]
+ add_ascii = api_client.add_message(ascii_session_id, "user", ascii_text)
+ assert add_ascii.status_code == 200
+
+ ascii_detail = api_client.get_session(ascii_session_id)
+ assert ascii_detail.status_code == 200
+ ascii_pending = ascii_detail.json().get("result", {}).get("pending_tokens", 0)
+ assert cjk_pending >= ascii_pending * 4, (
+ "same-length CJK should be estimated much higher than ASCII; "
+ f"cjk={cjk_pending}, ascii={ascii_pending}"
+ )
+ finally:
+ if cjk_session_id:
+ api_client.delete_session(cjk_session_id)
+ if ascii_session_id:
+ api_client.delete_session(ascii_session_id)
+
def test_multiline_content_preserved(self, api_client):
session_id = None
try:
diff --git a/tests/test_token_estimation.py b/tests/test_token_estimation.py
new file mode 100644
index 000000000..baf6f2f7f
--- /dev/null
+++ b/tests/test_token_estimation.py
@@ -0,0 +1,48 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+
+"""Shared token estimation tests."""
+
+from openviking.message import Message
+from openviking.session import Session
+
+
+def test_message_estimated_tokens_is_cjk_aware():
+ """Chinese text should not be estimated with the ASCII chars/4 fallback."""
+ msg = Message.create_user("\u4f60\u597d\u4e16\u754c")
+
+ assert msg.estimated_tokens == 6
+
+
+async def test_archive_overview_tokens_do_not_trust_stale_low_metadata():
+ class FakeFS:
+ async def read_file(self, uri, ctx=None):
+ del uri, ctx
+ return '{"overview_tokens": 1}'
+
+ fake_session = type("FakeSession", (), {"_viking_fs": FakeFS(), "ctx": None})()
+
+ tokens = await Session._read_archive_overview_tokens(
+ fake_session,
+ "viking://session/test/history/archive_001",
+ "\u4f60\u597d\u4e16\u754c",
+ )
+
+ assert tokens == 6
+
+
+async def test_archive_overview_tokens_keep_higher_metadata_estimate():
+ class FakeFS:
+ async def read_file(self, uri, ctx=None):
+ del uri, ctx
+ return '{"overview_tokens": 5}'
+
+ fake_session = type("FakeSession", (), {"_viking_fs": FakeFS(), "ctx": None})()
+
+ tokens = await Session._read_archive_overview_tokens(
+ fake_session,
+ "viking://session/test/history/archive_001",
+ "abcd",
+ )
+
+ assert tokens == 5