Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
#!/usr/bin/env python3
"""OpenClaw plugin E2E for CJK-aware token estimation.
This test drives the real OpenClaw Gateway, then verifies the OpenViking plugin's
own assemble diagnostics. It intentionally checks the plugin-side token estimate,
not only the OV REST session counters.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import pathlib
import time
import uuid
from typing import Any
import requests
DEFAULT_GATEWAY_URL = os.environ.get("OPENCLAW_GATEWAY_URL", "http://127.0.0.1:19830")
DEFAULT_OPENVIKING_URL = os.environ.get("OPENVIKING_BASE_URL", "http://127.0.0.1:2948")
DEFAULT_CJK_REPEAT = 120
def default_gateway_log_path() -> pathlib.Path:
state_dir = os.environ.get("OPENCLAW_STATE_DIR")
if state_dir:
return pathlib.Path(state_dir) / "logs" / "gateway.stdout.log"
return pathlib.Path.cwd() / "config" / ".openclaw" / "logs" / "gateway.stdout.log"
def discover_gateway_token() -> str:
env_token = os.environ.get("OPENCLAW_GATEWAY_TOKEN", "").strip()
if env_token:
return env_token
candidates: list[pathlib.Path] = []
config_path = os.environ.get("OPENCLAW_CONFIG_PATH")
if config_path:
candidates.append(pathlib.Path(config_path))
state_dir = os.environ.get("OPENCLAW_STATE_DIR")
if state_dir:
candidates.append(pathlib.Path(state_dir) / "openclaw.json")
candidates.extend(
[
pathlib.Path.cwd() / "config" / ".openclaw" / "openclaw.json",
pathlib.Path.home() / ".openclaw" / "openclaw.json",
]
)
for path in candidates:
try:
cfg = json.loads(path.read_text(encoding="utf-8"))
except Exception:
continue
token = str(cfg.get("gateway", {}).get("auth", {}).get("token", "")).strip()
if token:
return token
return ""
def send_gateway_message(
gateway_url: str,
token: str,
user_id: str,
message: str,
timeout: int,
) -> dict[str, Any]:
headers = {"Content-Type": "application/json"}
if token:
headers["Authorization"] = f"Bearer {token}"
response = requests.post(
f"{gateway_url.rstrip('/')}/v1/responses",
headers=headers,
json={"model": "openclaw", "input": message, "user": user_id},
timeout=timeout,
)
response.raise_for_status()
return response.json()
def iter_new_log_lines(path: pathlib.Path, offset: int) -> list[str]:
if not path.exists():
return []
with path.open("rb") as handle:
handle.seek(min(offset, path.stat().st_size))
return handle.read().decode("utf-8", errors="ignore").splitlines()
def parse_diag(line: str) -> dict[str, Any] | None:
marker = "openviking: diag "
pos = line.find(marker)
if pos < 0:
return None
try:
return json.loads(line[pos + len(marker) :].strip())
except json.JSONDecodeError:
return None
def find_assemble_diag_with_marker(
log_path: pathlib.Path,
offset: int,
marker: str,
timeout: float,
) -> tuple[dict[str, Any], dict[str, Any]]:
deadline = time.time() + timeout
last_seen = ""
while time.time() < deadline:
for line in iter_new_log_lines(log_path, offset):
diag = parse_diag(line)
if not diag or diag.get("stage") != "assemble_entry":
continue
data = diag.get("data", {})
for message in data.get("messages", []):
content = str(message.get("content", ""))
if marker in content:
return diag, message
last_seen = json.dumps(data, ensure_ascii=False)[:500]
time.sleep(0.5)
raise AssertionError(
f"no assemble_entry diagnostic contained marker {marker!r}; last_seen={last_seen}"
)
def flatten_message_text(value: Any) -> str:
if value is None:
return ""
if isinstance(value, str):
return value
if isinstance(value, list):
return "\n".join(flatten_message_text(item) for item in value)
if isinstance(value, dict):
chunks: list[str] = []
for key in ("text", "abstract", "content"):
if key in value:
chunks.append(flatten_message_text(value.get(key)))
for key in ("parts", "messages", "pre_archive_abstracts"):
raw = value.get(key)
if isinstance(raw, list):
chunks.append(flatten_message_text(raw))
return "\n".join(chunk for chunk in chunks if chunk)
return ""
def ov_get(base_url: str, path: str) -> Any:
response = requests.get(f"{base_url.rstrip('/')}{path}", timeout=10)
response.raise_for_status()
data = response.json()
return data.get("result", data)
def find_ov_context_with_marker(
openviking_url: str,
marker: str,
timeout: float,
) -> tuple[str, dict[str, Any]]:
deadline = time.time() + timeout
last_session_count = 0
while time.time() < deadline:
sessions = ov_get(openviking_url, "/api/v1/sessions")
if not isinstance(sessions, list):
sessions = []
last_session_count = len(sessions)
ordered = sorted(
[s for s in sessions if isinstance(s, dict)],
key=lambda s: str(s.get("updated_at", "")),
reverse=True,
)[:40]
for session in ordered:
session_id = str(session.get("session_id", ""))
if not session_id or session_id.startswith("memory-store-"):
continue
try:
ctx = ov_get(
openviking_url, f"/api/v1/sessions/{session_id}/context?token_budget=128000"
)
except requests.RequestException:
continue
if marker in flatten_message_text(ctx):
return session_id, ctx
time.sleep(1)
raise AssertionError(
f"marker {marker!r} was not found in OV contexts; checked sessions={last_session_count}"
)
def main() -> int:
parser = argparse.ArgumentParser(description="OpenClaw plugin CJK token estimation E2E")
parser.add_argument("--gateway", default=DEFAULT_GATEWAY_URL)
parser.add_argument("--openviking", default=DEFAULT_OPENVIKING_URL)
parser.add_argument("--gateway-log", type=pathlib.Path, default=default_gateway_log_path())
parser.add_argument("--repeat", type=int, default=DEFAULT_CJK_REPEAT)
parser.add_argument("--gateway-timeout", type=int, default=300)
parser.add_argument("--diag-timeout", type=float, default=45)
parser.add_argument("--ov-timeout", type=float, default=45)
args = parser.parse_args()
marker = f"CJK_TOKEN_E2E_{uuid.uuid4().hex[:10]}"
user_id = f"cjk-token-e2e-{uuid.uuid4().hex[:10]}"
cjk_text = "你好世界" * args.repeat
expected_cjk_tokens = math.ceil(len(cjk_text) * 1.5)
naive_chars_div_4 = math.ceil(len(cjk_text) / 4)
message = (
f"{marker}\n"
"请只回复 OK。下面这段中文只用于 OpenViking token 估算端到端回归测试:\n"
f"{cjk_text}"
)
log_offset = args.gateway_log.stat().st_size if args.gateway_log.exists() else 0
token = discover_gateway_token()
print(f"Gateway: {args.gateway}")
print(f"OpenViking: {args.openviking}")
print(f"Gateway log: {args.gateway_log}")
print(f"Marker: {marker}")
print(
"Expected CJK token floor: "
f"{expected_cjk_tokens} (old chars/4 would be {naive_chars_div_4})"
)
send_gateway_message(args.gateway, token, user_id, message, args.gateway_timeout)
diag, marked_message = find_assemble_diag_with_marker(
args.gateway_log,
log_offset,
marker,
args.diag_timeout,
)
observed_tokens = int(marked_message.get("tokens", 0) or 0)
input_tokens = int(diag.get("data", {}).get("inputTokenEstimate", 0) or 0)
if observed_tokens < expected_cjk_tokens:
raise AssertionError(
"plugin assemble token estimate under-counted CJK message: "
f"observed={observed_tokens}, expected>={expected_cjk_tokens}, "
f"old chars/4={naive_chars_div_4}, inputTokenEstimate={input_tokens}"
)
session_id, ctx = find_ov_context_with_marker(args.openviking, marker, args.ov_timeout)
estimated_tokens = int(ctx.get("estimatedTokens", 0) or 0)
active_tokens = int(ctx.get("stats", {}).get("activeTokens", 0) or 0)
if max(estimated_tokens, active_tokens) < expected_cjk_tokens:
raise AssertionError(
"OV context after plugin afterTurn under-counted CJK message: "
f"estimatedTokens={estimated_tokens}, activeTokens={active_tokens}, "
f"expected>={expected_cjk_tokens}, session={session_id}"
)
print(
"PASS: plugin CJK estimate "
f"messageTokens={observed_tokens}, inputTokenEstimate={input_tokens}, "
f"ovEstimatedTokens={estimated_tokens}, ovActiveTokens={active_tokens}, "
f"session={session_id}"
)
return 0
if __name__ == "__main__":
raise SystemExit(main())