Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(eval):Opt memory eval script (#2563)

DuTao07326bd827f2
diff --git a/benchmark/locomo/vikingbot/preflight_eval_runtime.py b/benchmark/locomo/vikingbot/preflight_eval_runtime.py
index 72b515998..a1fcece5d 100644
--- a/benchmark/locomo/vikingbot/preflight_eval_runtime.py
+++ b/benchmark/locomo/vikingbot/preflight_eval_runtime.py
@@ -6,13 +6,14 @@ import os
import shlex
import sys
import urllib.error
+import urllib.parse
import urllib.request
from pathlib import Path
from openviking_cli.utils.config.config_loader import resolve_config_path
from openviking_cli.utils.config.consts import (
- DEFAULT_OVCLI_CONF,
DEFAULT_OV_CONF,
+ DEFAULT_OVCLI_CONF,
OPENVIKING_CLI_CONFIG_ENV,
OPENVIKING_CONFIG_ENV,
)
@@ -26,6 +27,10 @@ def _error(message: str) -> None:
print(f"[preflight] {message}", file=sys.stderr)
+class UserKeyValidationError(RuntimeError):
+ """Raised when the configured OpenViking key is not a usable User key."""
+
+
def _load_json(path: Path) -> dict:
with open(path, "r", encoding="utf-8-sig") as f:
return json.load(f)
@@ -38,8 +43,7 @@ def _is_interactive() -> bool:
return sys.stdin.isatty() and sys.stdout.isatty()
-def _prompt_api_key() -> str:
- prompt = "[preflight] 请输入 OpenViking User API key: "
+def _prompt_text(prompt: str) -> str:
try:
with open("/dev/tty", "r", encoding="utf-8") as tty_in:
print(prompt, end="", flush=True)
@@ -48,6 +52,26 @@ def _prompt_api_key() -> str:
return input(prompt).strip()
+def _prompt_api_key() -> str:
+ return _prompt_text("[preflight] 请输入 OpenViking User API key: ")
+
+
+def _prompt_root_api_key() -> str:
+ return _prompt_text(
+ "[preflight] 请输入 OpenViking Root API key,用于自动创建 default User key: "
+ )
+
+
+def _prompt_yes_no(prompt: str, default: bool = False) -> bool:
+ if not _is_interactive():
+ return False
+ suffix = "Y/n" if default else "y/N"
+ answer = _prompt_text(f"{prompt} [{suffix}]: ").strip().lower()
+ if not answer:
+ return default
+ return answer in {"y", "yes", "1", "true"}
+
+
def _resolve_configured_account_hint() -> str:
path = resolve_config_path(None, OPENVIKING_CLI_CONFIG_ENV, DEFAULT_OVCLI_CONF)
if path is None:
@@ -134,7 +158,10 @@ def _sync_bot_identity(account_id: str, user_id: str, api_key: str) -> None:
changed = True
if changed:
_write_json_with_backup(path, ov_data)
- _log(f"已同步 bot.ov_server: api_key_type=user, account_id={account_id}, admin_user_id={user_id}")
+ _log(
+ "已同步 bot.ov_server: "
+ f"api_key_type=user, account_id={account_id}, admin_user_id={user_id}"
+ )
def _resolve_openviking_api_key() -> tuple[str, str]:
@@ -158,26 +185,157 @@ def _resolve_openviking_api_key() -> tuple[str, str]:
if key:
return key, "interactive input"
- _error("未配置 OpenViking User API key,请设置 bot.ov_server.api_key 或 ovcli.conf.api_key 后重试。")
- raise SystemExit(1)
+ return "", "not configured"
+
+
+def _resolve_root_api_key() -> tuple[str, str]:
+ ov_data = _load_ov_conf()
+ server_key = str((ov_data.get("server") or {}).get("root_api_key") or "").strip()
+ if server_key:
+ return server_key, "server.root_api_key"
+
+ bot_legacy_key = str(
+ ((ov_data.get("bot") or {}).get("ov_server") or {}).get("root_api_key") or ""
+ ).strip()
+ if bot_legacy_key:
+ return bot_legacy_key, "bot.ov_server.root_api_key"
+
+ if _is_interactive():
+ key = _prompt_root_api_key()
+ if key:
+ return key, "interactive input"
+
+ return "", "not configured"
def _parse_health_identity(body: str) -> tuple[str, str, str]:
try:
payload = json.loads(body)
except Exception as exc:
- _error(f"/health 返回非 JSON: {exc}")
- raise SystemExit(1)
+ raise UserKeyValidationError(f"/health 返回非 JSON: {exc}") from exc
account_id = str(payload.get("account_id") or "").strip()
user_id = str(payload.get("user_id") or "").strip()
role = str(payload.get("role") or "").strip().lower()
if not account_id or not user_id or not role:
- _error(f"API key 未解析出有效身份,请检查 key 是否正确。/health 返回: {payload}")
- raise SystemExit(1)
+ raise UserKeyValidationError(
+ f"API key 未解析出有效身份,请检查 key 是否正确。/health 返回: {payload}"
+ )
return account_id, user_id, role
+def _read_http_error(exc: urllib.error.HTTPError) -> str:
+ return exc.read().decode("utf-8", errors="replace")
+
+
+def _admin_request(
+ url: str,
+ root_api_key: str,
+ method: str,
+ path: str,
+ payload: dict | None = None,
+) -> dict:
+ body = None
+ headers = {
+ "X-API-Key": root_api_key,
+ "Content-Type": "application/json",
+ }
+ if payload is not None:
+ body = json.dumps(payload).encode("utf-8")
+ req = urllib.request.Request(
+ f"{url}{path}",
+ data=body,
+ headers=headers,
+ method=method,
+ )
+ try:
+ with urllib.request.urlopen(req, timeout=10) as resp:
+ raw = resp.read().decode("utf-8", errors="replace")
+ except urllib.error.HTTPError as exc:
+ detail = _read_http_error(exc)
+ raise UserKeyValidationError(f"Admin API 请求失败(HTTP {exc.code}): {detail}") from exc
+ except Exception as exc:
+ raise UserKeyValidationError(f"Admin API 请求失败: {exc}") from exc
+
+ try:
+ parsed = json.loads(raw)
+ except Exception as exc:
+ raise UserKeyValidationError(f"Admin API 返回非 JSON: {raw}") from exc
+ if parsed.get("status") != "ok":
+ raise UserKeyValidationError(f"Admin API 返回失败: {parsed}")
+ result = parsed.get("result")
+ return result if isinstance(result, dict) else {"items": result}
+
+
+def _ensure_default_user_key(url: str, root_api_key: str) -> tuple[str, str, str]:
+ account_id = "default"
+ user_id = "default"
+ admin_user_id = "admin"
+ quoted_account = urllib.parse.quote(account_id, safe="")
+ quoted_user = urllib.parse.quote(user_id, safe="")
+
+ accounts = _admin_request(url, root_api_key, "GET", "/api/v1/admin/accounts").get("items") or []
+ account_exists = any(
+ item.get("account_id") == account_id for item in accounts if isinstance(item, dict)
+ )
+ if not account_exists:
+ _log("default account 不存在,将使用 Root key 创建 account=default。")
+ _admin_request(
+ url,
+ root_api_key,
+ "POST",
+ "/api/v1/admin/accounts",
+ {"account_id": account_id, "admin_user_id": admin_user_id},
+ )
+
+ users = (
+ _admin_request(
+ url,
+ root_api_key,
+ "GET",
+ f"/api/v1/admin/accounts/{quoted_account}/users",
+ ).get("items")
+ or []
+ )
+ default_user = next(
+ (item for item in users if isinstance(item, dict) and item.get("user_id") == user_id),
+ None,
+ )
+ if default_user is None:
+ _log("default User 不存在,将注册 account=default, user=default, role=user。")
+ created = _admin_request(
+ url,
+ root_api_key,
+ "POST",
+ f"/api/v1/admin/accounts/{quoted_account}/users",
+ {"user_id": user_id, "role": "user"},
+ )
+ user_key = str(created.get("user_key") or "").strip()
+ else:
+ role = str(default_user.get("role") or "").strip().lower()
+ if role != "user":
+ _log(f"default User 当前 role={role or 'unknown'},将调整为 role=user。")
+ _admin_request(
+ url,
+ root_api_key,
+ "PUT",
+ f"/api/v1/admin/accounts/{quoted_account}/users/{quoted_user}/role",
+ {"role": "user"},
+ )
+ regenerated = _admin_request(
+ url,
+ root_api_key,
+ "POST",
+ f"/api/v1/admin/accounts/{quoted_account}/users/{quoted_user}/key",
+ {},
+ )
+ user_key = str(regenerated.get("user_key") or "").strip()
+
+ if not user_key:
+ raise UserKeyValidationError("Admin API 未返回 user_key,无法继续评测。")
+ return account_id, user_id, user_key
+
+
def _ensure_server_and_user_key_ready(
url: str, selected_account: str, api_key: str
) -> tuple[str, str]:
@@ -194,28 +352,78 @@ def _ensure_server_and_user_key_ready(
with urllib.request.urlopen(req, timeout=10) as resp:
body = resp.read().decode("utf-8", errors="replace")
except urllib.error.HTTPError as e:
- detail = e.read().decode("utf-8", errors="replace")
- _error(f"OpenViking server 检查失败(HTTP {e.code}): {detail}")
- raise SystemExit(1)
+ detail = _read_http_error(e)
+ raise UserKeyValidationError(
+ f"OpenViking server 检查失败(HTTP {e.code}): {detail}"
+ ) from e
except Exception as exc:
- _error(f"OpenViking server 不可用: {exc}")
- raise SystemExit(1)
+ raise UserKeyValidationError(f"OpenViking server 不可用: {exc}") from exc
account_id, user_id, role = _parse_health_identity(body)
if role != "user":
- _error(f"当前 API key 解析为 role={role}。评测需要普通 User key,请更换后重试。")
- raise SystemExit(1)
+ raise UserKeyValidationError(f"当前 API key 解析为 role={role}。评测需要普通 User key。")
if selected_account and selected_account != "default" and selected_account != account_id:
_log(
- f"ovcli.conf.account={selected_account} 与 API key 归属 account={account_id} 不一致;"
+ f"ovcli.conf.account={selected_account} "
+ f"与 API key 归属 account={account_id} 不一致;"
"本次评测使用 API key 归属 account。"
)
- _log(f"OpenViking server 可用,User key 身份: account={account_id}, user={user_id}, role={role}。")
+ _log(
+ "OpenViking server 可用,User key 身份: "
+ f"account={account_id}, user={user_id}, role={role}。"
+ )
return account_id, user_id
+def _resolve_ready_user_identity(
+ openviking_url: str,
+ selected_account: str,
+ api_key: str,
+ key_source: str,
+) -> tuple[str, str, str]:
+ if api_key:
+ _log(f"使用 {key_source} 校验 OpenViking User key")
+ try:
+ account, user_id = _ensure_server_and_user_key_ready(
+ openviking_url, selected_account, api_key
+ )
+ return account, user_id, api_key
+ except UserKeyValidationError as exc:
+ _error(str(exc))
+ prompt = (
+ "[preflight] 当前 User key 不可用,是否使用 Root key 自动生成 default User API key"
+ )
+ else:
+ _error("未配置 OpenViking User API key。")
+ prompt = "[preflight] 是否使用 Root key 自动生成 default User API key"
+
+ if not _prompt_yes_no(prompt, default=False):
+ _error("请配置可用的 bot.ov_server.api_key 或 ovcli.conf.api_key 后重试。")
+ raise SystemExit(1)
+
+ root_api_key, root_key_source = _resolve_root_api_key()
+ if not root_api_key:
+ _error(
+ "未配置 OpenViking Root API key,无法自动生成 User key。"
+ "请设置 server.root_api_key 后重试。"
+ )
+ raise SystemExit(1)
+
+ try:
+ _log(f"使用 {root_key_source} 自动生成/刷新 default User key。")
+ account, user_id, user_key = _ensure_default_user_key(openviking_url, root_api_key)
+ checked_account, checked_user_id = _ensure_server_and_user_key_ready(
+ openviking_url, selected_account, user_key
+ )
+ except UserKeyValidationError as exc:
+ _error(str(exc))
+ raise SystemExit(1) from exc
+ _log("已生成可用的 default User API key。")
+ return checked_account or account, checked_user_id or user_id, user_key
+
+
def _write_env_file(
path: Path, account: str, openviking_url: str, api_key: str, user_id: str
) -> None:
@@ -227,18 +435,25 @@ def _write_env_file(
def main() -> int:
- parser = argparse.ArgumentParser(description="Resolve runtime eval account/url and validate OpenViking readiness")
- parser.add_argument("--output-env-file", required=True, help="File path to write ACCOUNT/OPENVIKING_URL exports")
+ parser = argparse.ArgumentParser(
+ description="Resolve runtime eval account/url and validate OpenViking readiness"
+ )
+ parser.add_argument(
+ "--output-env-file",
+ required=True,
+ help="File path to write ACCOUNT/OPENVIKING_URL exports",
+ )
args = parser.parse_args()
selected_account = _resolve_configured_account_hint()
openviking_url = _resolve_openviking_url()
api_key, key_source = _resolve_openviking_api_key()
- _log(f"使用 {key_source} 校验 OpenViking User key")
_log(f"本次导入使用 OpenViking URL: {openviking_url}")
- account, user_id = _ensure_server_and_user_key_ready(openviking_url, selected_account, api_key)
+ account, user_id, api_key = _resolve_ready_user_identity(
+ openviking_url, selected_account, api_key, key_source
+ )
_sync_bot_identity(account, user_id, api_key)
_write_env_file(Path(args.output_env_file), account, openviking_url, api_key, user_id)
return 0
diff --git a/benchmark/locomo/vikingbot/run_eval.py b/benchmark/locomo/vikingbot/run_eval.py
index 45129f173..1fe40d3cc 100644
--- a/benchmark/locomo/vikingbot/run_eval.py
+++ b/benchmark/locomo/vikingbot/run_eval.py
@@ -2,6 +2,7 @@ import argparse
import csv
import json
import os
+import re
import subprocess
import threading
import time
@@ -240,8 +241,13 @@ def run_vikingbot_chat(
question_id: str | None = None,
config: str | None = None,
memory_peer_ids: list[str] | None = None,
-) -> tuple[str, dict, float, int, list]:
+) -> tuple[str, dict, float, int, list, str]:
"""执行vikingbot chat命令,返回回答、token使用情况、耗时(秒)、迭代次数、使用的工具列表"""
+ bot_log_file = build_bot_log_file(question_id)
+ env = os.environ.copy()
+ if bot_log_file:
+ env["VIKINGBOT_LOG_FILE"] = bot_log_file
+
# 先执行 /new 命令清除会话
if sender_peer_id:
new_cmd = ["vikingbot", "chat"]
@@ -263,7 +269,7 @@ def run_vikingbot_chat(
new_cmd.extend(["--memory-peer", peer_id])
try:
# print(f'new_cmd={new_cmd}')
- subprocess.run(new_cmd, capture_output=True, text=True, timeout=300)
+ subprocess.run(new_cmd, capture_output=True, text=True, timeout=300, env=env)
except Exception:
# 忽略 /new 命令的错误
pass
@@ -288,7 +294,9 @@ def run_vikingbot_chat(
start_time = time.time()
try:
# print(f'cmd={cmd}')
- result = subprocess.run(cmd, capture_output=True, text=True, check=True, timeout=600)
+ result = subprocess.run(
+ cmd, capture_output=True, text=True, check=True, timeout=600, env=env
+ )
end_time = time.time()
time_cost = end_time - start_time
@@ -308,7 +316,7 @@ def run_vikingbot_chat(
token_usage = {"prompt_tokens": 0, "completion_tokens": 0, "total_tokens": 0}
iteration = 0
tools_used_names = []
- return response, token_usage, time_cost, iteration, tools_used_names
+ return response, token_usage, time_cost, iteration, tools_used_names, bot_log_file
except subprocess.CalledProcessError as e:
return (
f"[CMD ERROR] {e.stderr}",
@@ -316,6 +324,7 @@ def run_vikingbot_chat(
0,
0,
[],
+ bot_log_file,
)
except subprocess.TimeoutExpired:
time_cost = 0
@@ -325,9 +334,25 @@ def run_vikingbot_chat(
time_cost,
0,
[],
+ bot_log_file,
)
+def sanitize_log_name(value: str | None) -> str:
+ text = value or f"qa_{int(time.time() * 1000)}"
+ return re.sub(r"[^A-Za-z0-9_.-]+", "_", text).strip("._") or "qa"
+
+
+def build_bot_log_file(question_id: str | None) -> str:
+ log_dir = os.environ.get("LOCOMO_VIKINGBOT_LOG_DIR")
+ if not log_dir:
+ return ""
+
+ path = Path(log_dir).expanduser()
+ path.mkdir(parents=True, exist_ok=True)
+ return str(path / f"vikingbot.{sanitize_log_name(question_id)}.log")
+
+
def load_processed_questions(output_path: str, skip_done: bool = False) -> set[str]:
"""加载已处理的问题集合。"""
if not skip_done or not os.path.exists(output_path):
@@ -346,6 +371,20 @@ def load_processed_questions(output_path: str, skip_done: bool = False) -> set[s
def append_row_to_csv(output_path: str, fieldnames: list[str], row: dict) -> None:
"""追加单行结果到 CSV。"""
file_exists = os.path.exists(output_path)
+ if file_exists:
+ with open(output_path, "r", encoding="utf-8", newline="") as f:
+ reader = csv.DictReader(f)
+ existing_fieldnames = reader.fieldnames or []
+ existing_rows = list(reader)
+ missing_fields = [field for field in fieldnames if field not in existing_fieldnames]
+ if missing_fields:
+ merged_fieldnames = [*existing_fieldnames, *missing_fields]
+ with open(output_path, "w", encoding="utf-8", newline="") as f:
+ writer = csv.DictWriter(f, fieldnames=merged_fieldnames)
+ writer.writeheader()
+ writer.writerows(existing_rows)
+ fieldnames = merged_fieldnames
+
with open(output_path, "a", encoding="utf-8", newline="") as f:
writer = csv.DictWriter(f, fieldnames=fieldnames)
if not file_exists:
@@ -506,6 +545,7 @@ def main():
"time_cost",
"iteration",
"tools_used_names",
+ "bot_log_file",
]
# 创建线程锁,确保多线程写文件安全
@@ -551,13 +591,15 @@ def main():
if memory_peer_ids:
print(f" [memory peers: {memory_peer_ids}]")
- response, token_usage, time_cost, iteration, tools_used_names = run_vikingbot_chat(
- question,
- question_time,
- sender_peer_id,
- question_id,
- args.config,
- memory_peer_ids,
+ response, token_usage, time_cost, iteration, tools_used_names, bot_log_file = (
+ run_vikingbot_chat(
+ question,
+ question_time,
+ sender_peer_id,
+ question_id,
+ args.config,
+ memory_peer_ids,
+ )
)
row = {
@@ -575,6 +617,7 @@ def main():
"time_cost": round(time_cost, 2),
"iteration": iteration,
"tools_used_names": json.dumps(tools_used_names, ensure_ascii=False),
+ "bot_log_file": bot_log_file,
"is_invalid": qa_item.get("is_invalid", False),
}
@@ -587,6 +630,8 @@ def main():
reader = csv.DictReader(f)
existing_rows = list(reader)
existing_fieldnames = reader.fieldnames or fieldnames
+ if "bot_log_file" not in existing_fieldnames:
+ existing_fieldnames.append("bot_log_file")
q_idx = str(row.get("question_index", ""))
found = False
diff --git a/benchmark/locomo/vikingbot/run_full_eval.sh b/benchmark/locomo/vikingbot/run_full_eval.sh
index 4cb1f97b8..010c0350a 100755
--- a/benchmark/locomo/vikingbot/run_full_eval.sh
+++ b/benchmark/locomo/vikingbot/run_full_eval.sh
@@ -297,6 +297,14 @@ print(render_table(["metric", "value"], summary_rows, align_right={1}))
PY
}
+prepare_bot_log_dir() {
+ local output_file="$1"
+ local base="${output_file%.csv}"
+ export LOCOMO_VIKINGBOT_LOG_DIR="${base}_bot_logs"
+ mkdir -p "$LOCOMO_VIKINGBOT_LOG_DIR"
+ echo "[eval] vikingbot logs: $LOCOMO_VIKINGBOT_LOG_DIR"
+}
+
# ========== 重跑错题模式(优先) ==========
if [ -n "$RETRY_WRONG" ]; then
if [ ! -f "$RETRY_WRONG" ]; then
@@ -331,6 +339,7 @@ if [ -n "$RETRY_WRONG" ]; then
# 评估错题
echo "[2/3] 重新评估错题..."
+ prepare_bot_log_dir "$RESULT_FILE"
"$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" \
"$INPUT_FILE" \
--output "$RESULT_FILE" \
@@ -377,6 +386,7 @@ if [ -z "$SAMPLE" ]; then
# 评估
echo "[2/4] 评估..."
+ prepare_bot_log_dir "$RESULT_FILE"
"$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" "$INPUT_FILE" --output "$RESULT_FILE" --config "$OPENVIKING_CONFIG_FILE" "${COMMON_OPTS[@]}"
# 裁判打分
@@ -464,6 +474,7 @@ if [ -n "$QUESTION_INDEX" ]; then
else
OUTPUT_FILE=./result/locomo_${SAMPLE}_${QUESTION_INDEX}_result_${TIMESTAMP}.csv
fi
+ prepare_bot_log_dir "$OUTPUT_FILE"
"$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" \
"$INPUT_FILE" \
--sample "$SAMPLE_ID_FOR_CMD" \
@@ -569,11 +580,12 @@ PY
else
OUTPUT_FILE=./result/locomo_${SAMPLE}_result_${TIMESTAMP}.csv
fi
+ prepare_bot_log_dir "$OUTPUT_FILE"
"$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" \
"$INPUT_FILE" \
--sample "$SAMPLE_ID_FOR_CMD" \
--output "$OUTPUT_FILE" \
- --threads 5 \
+ --threads 10 \
--config "$OPENVIKING_CONFIG_FILE" \
"${COMMON_OPTS[@]}"
diff --git a/bot/vikingbot/cli/commands.py b/bot/vikingbot/cli/commands.py
index db596e241..30931e672 100644
--- a/bot/vikingbot/cli/commands.py
+++ b/bot/vikingbot/cli/commands.py
@@ -714,7 +714,13 @@ def chat(
_init_bot_data(config)
logger.remove()
- log_file = get_data_dir() / "log" / f"vikingbot.debug.{os.getpid()}.log"
+ configured_log_file = os.environ.get("VIKINGBOT_LOG_FILE")
+ log_file = (
+ Path(configured_log_file).expanduser()
+ if configured_log_file
+ else get_data_dir() / "log" / f"vikingbot.debug.{os.getpid()}.log"
+ )
+ log_file.parent.mkdir(parents=True, exist_ok=True)
logger.add(
log_file,
level="DEBUG",
diff --git a/bot/vikingbot/providers/vlm_adapter.py b/bot/vikingbot/providers/vlm_adapter.py
index 91c02e5c6..5f91f31bd 100644
--- a/bot/vikingbot/providers/vlm_adapter.py
+++ b/bot/vikingbot/providers/vlm_adapter.py
@@ -5,6 +5,7 @@ LLMProvider interface, so that bot.agents.provider / bot.agents.model
configuration semantics are consistent with openviking server's vlm section.
"""
+import time
from collections.abc import AsyncIterator
from typing import Any
@@ -147,6 +148,7 @@ class VLMProviderAdapter(LLMProvider):
"type": "enabled" if getattr(self._vlm, "thinking", False) else "disabled"
},
"stream": True,
+ "stream_options": {"include_usage": True},
}
if tools:
kwargs["tools"] = tools
@@ -156,11 +158,17 @@ class VLMProviderAdapter(LLMProvider):
reasoning_parts: list[str] = []
tool_calls: dict[int, dict[str, Any]] = {}
finish_reason = "stop"
+ usage: dict[str, int] = {}
+ start_time = time.perf_counter()
try:
client = self._vlm.get_async_client()
response = await client.chat.completions.create(**kwargs)
async for chunk in response:
+ chunk_usage = self._parse_usage(getattr(chunk, "usage", None))
+ if chunk_usage:
+ usage = chunk_usage
+
choices = getattr(chunk, "choices", None) or []
if not choices:
continue
@@ -184,6 +192,9 @@ class VLMProviderAdapter(LLMProvider):
for delta_tool_call in getattr(delta, "tool_calls", None) or []:
merge_stream_tool_call_delta(tool_calls, delta_tool_call)
+ if usage:
+ self._record_vlm_usage(usage, time.perf_counter() - start_time)
+
yield LLMStreamEvent(
type="response",
response=build_stream_response(
@@ -191,6 +202,7 @@ class VLMProviderAdapter(LLMProvider):
reasoning_content="".join(reasoning_parts),
raw_tool_calls=tool_calls,
finish_reason=finish_reason,
+ usage=usage,
),
)
except Exception as e:
@@ -202,6 +214,61 @@ class VLMProviderAdapter(LLMProvider):
),
)
+ @staticmethod
+ def _usage_value(usage: Any, name: str) -> int:
+ if isinstance(usage, dict):
+ return int(usage.get(name, 0) or 0)
+ return int(getattr(usage, name, 0) or 0)
+
+ @classmethod
+ def _parse_usage(cls, raw_usage: Any) -> dict[str, int]:
+ if not raw_usage:
+ return {}
+
+ usage = {
+ "prompt_tokens": cls._usage_value(raw_usage, "prompt_tokens"),
+ "completion_tokens": cls._usage_value(raw_usage, "completion_tokens"),
+ "total_tokens": cls._usage_value(raw_usage, "total_tokens"),
+ }
+ prompt_details = (
+ raw_usage.get("prompt_tokens_details")
+ if isinstance(raw_usage, dict)
+ else getattr(raw_usage, "prompt_tokens_details", None)
+ )
+ completion_details = (
+ raw_usage.get("completion_tokens_details")
+ if isinstance(raw_usage, dict)
+ else getattr(raw_usage, "completion_tokens_details", None)
+ )
+ cached = cls._usage_value(prompt_details, "cached_tokens") if prompt_details else 0
+ reasoning = (
+ cls._usage_value(completion_details, "reasoning_tokens")
+ if completion_details
+ else 0
+ )
+ if cached:
+ usage["cache_read_input_tokens"] = cached
+ if reasoning:
+ usage["reasoning_tokens"] = reasoning
+ return usage
+
+ def _record_vlm_usage(self, usage: dict[str, int], duration_seconds: float) -> None:
+ update_token_usage = getattr(self._vlm, "update_token_usage", None)
+ if not callable(update_token_usage):
+ return
+ try:
+ update_token_usage(
+ model_name=getattr(self._vlm, "model", None) or self._default_model,
+ provider=getattr(self._vlm, "provider", None) or "volcengine",
+ prompt_tokens=usage.get("prompt_tokens", 0),
+ completion_tokens=usage.get("completion_tokens", 0),
+ duration_seconds=duration_seconds,
+ prompt_cached_tokens=usage.get("cache_read_input_tokens", 0),
+ completion_reasoning_tokens=usage.get("reasoning_tokens", 0),
+ )
+ except Exception as e:
+ logger.debug(f"[VLM] Failed to record stream token usage: {e}")
+
def _convert_response(self, result) -> LLMResponse:
"""Convert VLMResponse (or str) to LLMResponse."""
if isinstance(result, str):