Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(eval)Opt vikingbot eval script (#2305)

DuTaobe1e7fc482c8
diff --git a/benchmark/locomo/vikingbot/import_to_ov.py b/benchmark/locomo/vikingbot/import_to_ov.py
index 0a1759935..6bbc6441c 100644
--- a/benchmark/locomo/vikingbot/import_to_ov.py
+++ b/benchmark/locomo/vikingbot/import_to_ov.py
@@ -132,7 +132,7 @@ def build_session_messages(
return sessions
-# ---------------------------------------------------------------------------
+# --------------------------------------------------------------------------
# Ingest record helpers (avoid duplicate ingestion)
# ---------------------------------------------------------------------------
@@ -157,14 +157,17 @@ def write_success_record(
fieldnames = [
"timestamp",
"sample_id",
+ "display_id",
"session",
"date_time",
"speakers",
"embedding_tokens",
"vlm_tokens",
- "llm_input_tokens",
+ "cache_tokens",
+ "reasoning_tokens",
"llm_output_tokens",
"total_tokens",
+ "duration_seconds",
]
with open(csv_path, "a", encoding="utf-8", newline="") as f:
@@ -176,14 +179,17 @@ def write_success_record(
{
"timestamp": record["timestamp"],
"sample_id": record["sample_id"],
+ "display_id": record.get("display_id", ""),
"session": record["session"],
"date_time": record.get("meta", {}).get("date_time", ""),
"speakers": record.get("meta", {}).get("speakers", ""),
"embedding_tokens": record["token_usage"].get("embedding", 0),
"vlm_tokens": record["token_usage"].get("vlm", 0),
- "llm_input_tokens": record["token_usage"].get("llm_input", 0),
+ "cache_tokens": record["token_usage"].get("cache", 0),
+ "reasoning_tokens": record["token_usage"].get("reasoning", 0),
"llm_output_tokens": record["token_usage"].get("llm_output", 0),
"total_tokens": record["token_usage"].get("total", 0),
+ "duration_seconds": record.get("duration_seconds", 0),
}
)
@@ -260,22 +266,33 @@ def _parse_token_usage(commit_result: Dict[str, Any]) -> Dict[str, int]:
# embedding 格式可能是 {"total": N} 或 {"total_tokens": N}
embed_total = embedding.get("total", embedding.get("total_tokens", 0))
llm_total = llm.get("total", llm.get("total_tokens", 0))
+ llm_input = llm.get("input", llm.get("prompt_tokens", 0))
+ llm_output = llm.get("output", llm.get("completion_tokens", 0))
+ cache_tokens = llm.get("cached_tokens", llm.get("prompt_cached", 0))
+ reasoning_tokens = llm.get(
+ "reasoning_tokens", llm.get("completion_reasoning", 0)
+ )
return {
"embedding": embed_total,
- "vlm": llm_total,
- "llm_input": llm.get("input", 0),
- "llm_output": llm.get("output", 0),
+ "vlm": llm_input,
+ "llm_input": llm_input,
+ "llm_output": llm_output,
+ "cache": cache_tokens,
+ "reasoning": reasoning_tokens,
"total": tu.get("total", {}).get("total_tokens", embed_total + llm_total),
}
# 从 commit 响应的 telemetry 中提取
telemetry = commit_result.get("telemetry", {}).get("summary", {})
tokens = telemetry.get("tokens", {})
+ llm = tokens.get("llm", {})
return {
"embedding": tokens.get("embedding", {}).get("total", 0),
- "vlm": tokens.get("llm", {}).get("total", 0),
- "llm_input": tokens.get("llm", {}).get("input", 0),
- "llm_output": tokens.get("llm", {}).get("output", 0),
+ "vlm": llm.get("total", 0),
+ "llm_input": llm.get("input", llm.get("prompt_tokens", 0)),
+ "llm_output": llm.get("output", llm.get("completion_tokens", 0)),
+ "cache": llm.get("prompt_cached", llm.get("cached_tokens", 0)),
+ "reasoning": llm.get("completion_reasoning", llm.get("reasoning_tokens", 0)),
"total": tokens.get("total", 0),
}
@@ -287,6 +304,7 @@ async def viking_ingest(
user_id: Optional[str] = None,
agent_id: Optional[str] = None,
account: str = "default",
+ api_key: Optional[str] = None,
) -> Dict[str, int]:
"""Save messages to OpenViking via OpenViking SDK client.
Returns token usage dict with embedding and vlm token counts.
@@ -298,6 +316,7 @@ async def viking_ingest(
user_id: User identifier for separate userspace (e.g., "conv-26")
agent_id: Agent identifier for separate agentspace (e.g., "conv-26")
account: OpenViking account identifier
+ api_key: Optional API key for OpenViking client authentication
"""
# 解析 session_time - 为每条消息计算递增的时间戳
base_datetime = None
@@ -313,6 +332,7 @@ async def viking_ingest(
user=user_id,
agent_id=agent_id,
account=account,
+ api_key=api_key,
timeout=600,
)
await client.initialize()
@@ -350,7 +370,7 @@ async def viking_ingest(
trace_id = result.get("trace_id", "")
if task_id:
# 轮询任务状态直到完成
- max_attempts = 1200 # 最多等待20分钟
+ max_attempts = 2400 # 最多等待40分钟
for attempt in range(max_attempts):
task = await client.get_task(task_id)
status = task.get("status") if task else "unknown"
@@ -394,10 +414,12 @@ async def process_single_session(
meta = meta or {}
ingest_record = ingest_record or {}
csv_id = display_id or str(sample_id)
+ source_sample_id = str(sample_id)
try:
+ started_at = time.perf_counter()
user_id = str(sample_id) if args.separate_user_by_sample else ""
agent_id = str(sample_id) if args.separate_user_by_sample else ""
- account = args.account
+ account = args.account if args.separate_user_by_sample else ""
result = await viking_ingest(
messages,
args.openviking_url,
@@ -405,27 +427,37 @@ async def process_single_session(
user_id=user_id,
agent_id=agent_id,
account=account,
+ api_key=args.api_key,
)
+ duration_seconds = round(time.perf_counter() - started_at, 3)
token_usage = result["token_usage"]
task_id = result.get("task_id")
trace_id = result.get("trace_id", "")
embedding_tokens = token_usage.get("embedding", 0)
- vlm_tokens = token_usage.get("vlm", 0)
+ vlm_tokens = token_usage.get("llm_input", 0)
+ cache_tokens = token_usage.get("cache", 0)
+ reasoning_tokens = token_usage.get("reasoning", 0)
+ llm_output_tokens = token_usage.get("llm_output", 0)
print(
- f" -> [COMPLETED] [{csv_id}/{session_key}] embed={embedding_tokens}, vlm={vlm_tokens}, task_id={task_id}, trace_id={trace_id}",
+ f" -> [COMPLETED] [{csv_id}/{session_key}] duration={duration_seconds}s, embed={embedding_tokens}, vlm={vlm_tokens}, cache={cache_tokens}, reasoning={reasoning_tokens}, completion={llm_output_tokens}, task_id={task_id}, trace_id={trace_id}",
file=sys.stderr,
)
# Write success record
result = {
"timestamp": run_time,
- "sample_id": csv_id,
+ "sample_id": source_sample_id,
+ "display_id": csv_id,
"session": session_key,
"status": "success",
"meta": meta,
"token_usage": token_usage,
+ "duration_seconds": duration_seconds,
"embedding_tokens": embedding_tokens,
"vlm_tokens": vlm_tokens,
+ "cache_tokens": cache_tokens,
+ "reasoning_tokens": reasoning_tokens,
+ "llm_output_tokens": llm_output_tokens,
"task_id": task_id,
"trace_id": trace_id,
}
@@ -446,7 +478,8 @@ async def process_single_session(
# Write error record
result = {
"timestamp": run_time,
- "sample_id": csv_id,
+ "sample_id": source_sample_id,
+ "display_id": csv_id,
"session": session_key,
"status": "error",
"error": str(e),
@@ -569,6 +602,9 @@ async def run_import(args: argparse.Namespace) -> None:
error_count = 0
total_embedding_tokens = 0
total_vlm_tokens = 0
+ total_cache_tokens = 0
+ total_reasoning_tokens = 0
+ total_llm_output_tokens = 0
tasks = []
if args.input.endswith(".json"):
@@ -601,7 +637,7 @@ async def run_import(args: argparse.Namespace) -> None:
# 为每个 sample 创建独立的处理协程
async def process_sample(item, sample_index):
- nonlocal success_count, error_count, total_embedding_tokens, total_vlm_tokens
+ nonlocal success_count, error_count, total_embedding_tokens, total_vlm_tokens, total_cache_tokens, total_reasoning_tokens, total_llm_output_tokens
sample_id = item["sample_id"]
display_id = f"sample_{sample_index}"
@@ -655,6 +691,9 @@ async def run_import(args: argparse.Namespace) -> None:
success_count += 1
total_embedding_tokens += res.get("embedding_tokens", 0)
total_vlm_tokens += res.get("vlm_tokens", 0)
+ total_cache_tokens += res.get("cache_tokens", 0)
+ total_reasoning_tokens += res.get("reasoning_tokens", 0)
+ total_llm_output_tokens += res.get("llm_output_tokens", 0)
elif res.get("status") == "error":
error_count += 1
@@ -734,6 +773,7 @@ async def run_import(args: argparse.Namespace) -> None:
success_count += 1
total_embedding_tokens += r.get("embedding_tokens", 0)
total_vlm_tokens += r.get("vlm_tokens", 0)
+ total_llm_output_tokens += r.get("llm_output_tokens", 0)
elif r.get("status") == "error":
error_count += 1
@@ -747,12 +787,24 @@ async def run_import(args: argparse.Namespace) -> None:
print(f"\n=== Token usage summary ===", file=sys.stderr)
print(f"Total Embedding tokens: {total_embedding_tokens}", file=sys.stderr)
print(f"Total VLM tokens: {total_vlm_tokens}", file=sys.stderr)
+ print(f"Total Cache tokens: {total_cache_tokens}", file=sys.stderr)
+ print(f"Total Reasoning tokens: {total_reasoning_tokens}", file=sys.stderr)
+ print(f"Total Completion tokens: {total_llm_output_tokens}", file=sys.stderr)
if success_count > 0:
print(
f"Average Embedding per session: {total_embedding_tokens / success_count:.1f}",
file=sys.stderr,
)
print(f"Average VLM per session: {total_vlm_tokens / success_count:.1f}", file=sys.stderr)
+ print(f"Average Cache per session: {total_cache_tokens / success_count:.1f}", file=sys.stderr)
+ print(
+ f"Average Reasoning per session: {total_reasoning_tokens / success_count:.1f}",
+ file=sys.stderr,
+ )
+ print(
+ f"Average Completion per session: {total_llm_output_tokens / success_count:.1f}",
+ file=sys.stderr,
+ )
print(f"\nResults saved to:", file=sys.stderr)
print(f" - Success records: {args.success_csv}", file=sys.stderr)
print(f" - Error logs: {args.error_log}", file=sys.stderr)
@@ -789,6 +841,11 @@ def main():
default="http://localhost:1933",
help="OpenViking service URL (default: http://localhost:1933)",
)
+ parser.add_argument(
+ "--api-key",
+ default=None,
+ help="OpenViking API key to pass to AsyncHTTPClient",
+ )
parser.add_argument(
"--account",
default="default",
diff --git a/benchmark/locomo/vikingbot/run_full_eval.sh b/benchmark/locomo/vikingbot/run_full_eval.sh
index 775d9d51b..c446df113 100755
--- a/benchmark/locomo/vikingbot/run_full_eval.sh
+++ b/benchmark/locomo/vikingbot/run_full_eval.sh
@@ -166,6 +166,126 @@ if [ "$AUTO_COMMIT" = "true" ]; then
fi
GIT_COMMIT_ID=$(git rev-parse --short HEAD)
TIMESTAMP=$(date +%Y%m%d%H%M%S)
+IMPORT_SUCCESS_CSV="./result/import_success.csv"
+IMPORT_ROW_START=0
+IMPORT_PERFORMED=false
+
+count_import_rows() {
+ IMPORT_SUCCESS_CSV="$IMPORT_SUCCESS_CSV" "$PYTHON_BIN" - <<'PY'
+import csv
+import os
+from pathlib import Path
+
+path = Path(os.environ["IMPORT_SUCCESS_CSV"])
+if not path.exists():
+ print(0)
+else:
+ with open(path, "r", encoding="utf-8", newline="") as f:
+ print(sum(1 for _ in csv.DictReader(f)))
+PY
+}
+
+capture_import_row_start() {
+ IMPORT_ROW_START=$(count_import_rows)
+ IMPORT_PERFORMED=false
+}
+
+print_import_summary_table() {
+ if [ "$SKIP_IMPORT" = "true" ] || [ "$IMPORT_PERFORMED" != "true" ]; then
+ return
+ fi
+
+ echo ""
+ IMPORT_SUCCESS_CSV="$IMPORT_SUCCESS_CSV" IMPORT_ROW_START="$IMPORT_ROW_START" "$PYTHON_BIN" - <<'PY'
+import csv
+import os
+from pathlib import Path
+
+
+def to_int(value: str) -> int:
+ try:
+ return int(float(value or 0))
+ except (TypeError, ValueError):
+ return 0
+
+
+def to_float(value: str) -> float:
+ try:
+ return float(value or 0)
+ except (TypeError, ValueError):
+ return 0.0
+
+
+def render_table(headers: list[str], rows: list[list[str]], align_right: set[int] | None = None) -> str:
+ align_right = align_right or set()
+ widths = [len(header) for header in headers]
+ for row in rows:
+ for i, cell in enumerate(row):
+ widths[i] = max(widths[i], len(cell))
+
+ def format_row(row: list[str]) -> str:
+ cells = []
+ for i, cell in enumerate(row):
+ cells.append(cell.rjust(widths[i]) if i in align_right else cell.ljust(widths[i]))
+ return "| " + " | ".join(cells) + " |"
+
+ sep = "+-" + "-+-".join("-" * width for width in widths) + "-+"
+ lines = [sep, format_row(headers), sep]
+ for row in rows:
+ lines.append(format_row(row))
+ lines.append(sep)
+ return "\n".join(lines)
+
+
+path = Path(os.environ["IMPORT_SUCCESS_CSV"])
+start = int(os.environ.get("IMPORT_ROW_START", "0"))
+print("=== Import Summary ===")
+if not path.exists():
+ print("No import success CSV found.")
+ raise SystemExit(0)
+
+with open(path, "r", encoding="utf-8", newline="") as f:
+ rows = list(csv.DictReader(f))
+
+rows = rows[start:]
+if not rows:
+ print("No new import records were written in this run.")
+ raise SystemExit(0)
+
+totals = {
+ "sessions": len(rows),
+ "embedding_tokens": 0,
+ "vlm_tokens": 0,
+ "cache_tokens": 0,
+ "reasoning_tokens": 0,
+ "llm_output_tokens": 0,
+ "total_tokens": 0,
+ "duration_seconds": 0.0,
+}
+for row in rows:
+ totals["embedding_tokens"] += to_int(row.get("embedding_tokens"))
+ totals["vlm_tokens"] += to_int(row.get("vlm_tokens"))
+ totals["cache_tokens"] += to_int(row.get("cache_tokens"))
+ totals["reasoning_tokens"] += to_int(row.get("reasoning_tokens"))
+ totals["llm_output_tokens"] += to_int(row.get("llm_output_tokens"))
+ totals["total_tokens"] += to_int(row.get("total_tokens"))
+ totals["duration_seconds"] += to_float(row.get("duration_seconds"))
+
+avg_duration = totals["duration_seconds"] / totals["sessions"] if totals["sessions"] else 0.0
+summary_rows = [
+ ["sessions", str(totals["sessions"])],
+ ["embedding_tokens", str(totals["embedding_tokens"])],
+ ["vlm_tokens", str(totals["vlm_tokens"])],
+ ["cache_tokens", str(totals["cache_tokens"])],
+ ["reasoning_tokens", str(totals["reasoning_tokens"])],
+ ["llm_output_tokens", str(totals["llm_output_tokens"])],
+ ["total_tokens", str(totals["total_tokens"])],
+ ["total_duration_s", f"{totals['duration_seconds']:.3f}"],
+ ["avg_duration_s", f"{avg_duration:.3f}"],
+]
+print(render_table(["metric", "value"], summary_rows, align_right={1}))
+PY
+}
# ========== 重跑错题模式(优先) ==========
if [ -n "$RETRY_WRONG" ]; then
@@ -185,6 +305,7 @@ if [ -n "$RETRY_WRONG" ]; then
# 从错题 CSV 中提取需要导入的对话(复用 import_to_ov.py 的并行逻辑)
echo "[1/3] 导入错题相关对话..."
+ capture_import_row_start
"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \
--input "$INPUT_FILE" \
--retry-wrong "$RETRY_WRONG" \
@@ -192,6 +313,7 @@ if [ -n "$RETRY_WRONG" ]; then
--account "$ACCOUNT" \
--openviking-url "$OPENVIKING_URL" \
"${COMMON_OPTS[@]}"
+ IMPORT_PERFORMED=true
echo "等待数据处理完成..."
sleep 30
@@ -203,6 +325,7 @@ if [ -n "$RETRY_WRONG" ]; then
--output "$RESULT_FILE" \
--retry-wrong "$RETRY_WRONG" \
--threads 20 \
+ --config "$OPENVIKING_CONFIG_FILE" \
"${COMMON_OPTS[@]}"
# 裁判打分
@@ -211,6 +334,7 @@ if [ -n "$RETRY_WRONG" ]; then
# 统计结果
"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$RESULT_FILE"
+ print_import_summary_table
echo ""
echo "=== 错题重跑完成 ==="
@@ -233,14 +357,16 @@ if [ -z "$SAMPLE" ]; then
echo "[1/4] 跳过导入数据..."
else
echo "[1/4] 导入数据..."
+ capture_import_row_start
"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" --input "$INPUT_FILE" --force-ingest --account "$ACCOUNT" --openviking-url "$OPENVIKING_URL" "${COMMON_OPTS[@]}"
+ IMPORT_PERFORMED=true
echo "等待 1 分钟..."
sleep 60
fi
# 评估
echo "[2/4] 评估..."
- "$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" "$INPUT_FILE" --output "$RESULT_FILE" "${COMMON_OPTS[@]}"
+ "$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" "$INPUT_FILE" --output "$RESULT_FILE" --config "$OPENVIKING_CONFIG_FILE" "${COMMON_OPTS[@]}"
# 裁判打分
echo "[3/4] 裁判打分..."
@@ -249,6 +375,7 @@ if [ -z "$SAMPLE" ]; then
# 计算结果
echo "[4/4] 计算结果..."
"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$RESULT_FILE"
+ print_import_summary_table
echo ""
echo "=== 全量评测完成 ==="
@@ -299,6 +426,7 @@ if [ -n "$QUESTION_INDEX" ]; then
echo "[1/3] Skipping import (--skip-import)"
else
echo "[1/3] Importing sample $SAMPLE_INDEX, question $QUESTION_INDEX..."
+ capture_import_row_start
"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \
--input "$INPUT_FILE" \
--sample "$SAMPLE_INDEX" \
@@ -307,6 +435,7 @@ if [ -n "$QUESTION_INDEX" ]; then
--account "$ACCOUNT" \
--openviking-url "$OPENVIKING_URL" \
"${COMMON_OPTS[@]}"
+ IMPORT_PERFORMED=true
echo "Waiting for data processing..."
sleep 3
@@ -329,6 +458,7 @@ if [ -n "$QUESTION_INDEX" ]; then
--question-index "$QUESTION_INDEX" \
--count 1 \
--output "$OUTPUT_FILE" \
+ --config "$OPENVIKING_CONFIG_FILE" \
"${COMMON_OPTS[@]}"
# 运行 Judge 评分
@@ -342,6 +472,7 @@ if [ -n "$QUESTION_INDEX" ]; then
# 输出结果
echo ""
echo "=== 评测结果 ==="
+ print_import_summary_table
OUTPUT_FILE="$OUTPUT_FILE" QUESTION_INDEX="$QUESTION_INDEX" "$PYTHON_BIN" - <<'PY'
import csv
import json
@@ -400,6 +531,7 @@ PY
echo "[1/4] Skipping import (--skip-import)"
else
echo "[1/4] Importing all sessions for sample $SAMPLE_INDEX..."
+ capture_import_row_start
"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \
--input "$INPUT_FILE" \
--sample "$SAMPLE_INDEX" \
@@ -407,6 +539,7 @@ PY
--account "$ACCOUNT" \
--openviking-url "$OPENVIKING_URL" \
"${COMMON_OPTS[@]}"
+ IMPORT_PERFORMED=true
echo "Waiting for data processing..."
sleep 10
@@ -428,6 +561,7 @@ PY
--sample "$SAMPLE_ID_FOR_CMD" \
--output "$OUTPUT_FILE" \
--threads 5 \
+ --config "$OPENVIKING_CONFIG_FILE" \
"${COMMON_OPTS[@]}"
# 运行 Judge 评分
@@ -445,6 +579,7 @@ PY
echo "[4/4] Calculating statistics..."
fi
"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$OUTPUT_FILE"
+ print_import_summary_table
echo ""
echo "=== 批量评测完成 ==="
diff --git a/bot/vikingbot/channels/feishu.py b/bot/vikingbot/channels/feishu.py
index 4587bff42..0f8be9e15 100644
--- a/bot/vikingbot/channels/feishu.py
+++ b/bot/vikingbot/channels/feishu.py
@@ -759,6 +759,8 @@ class FeishuChannel(BaseChannel):
except Exception as e:
logger.warning(f"Failed to process {msg_type} message: {e}")
+ elif msg_type =="interactive":
+ content = message.content
else:
content = MSG_TYPE_MAP.get(msg_type, f"[{msg_type}]")
diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.py
index 81a833238..012adcbf3 100644
--- a/openviking/models/vlm/backends/openai_vlm.py
+++ b/openviking/models/vlm/backends/openai_vlm.py
@@ -158,12 +158,20 @@ class OpenAIVLM(VLMBase):
tracer.info(f"response.usage={response.usage}")
prompt_tokens = response.usage.prompt_tokens
completion_tokens = response.usage.completion_tokens
+ prompt_tokens_details = getattr(response.usage, "prompt_tokens_details", None)
+ completion_tokens_details = getattr(response.usage, "completion_tokens_details", None)
+ prompt_cached_tokens = getattr(prompt_tokens_details, "cached_tokens", 0) or 0
+ completion_reasoning_tokens = (
+ getattr(completion_tokens_details, "reasoning_tokens", 0) or 0
+ )
self.update_token_usage(
model_name=self.model or "gpt-4o-mini",
provider=self.provider,
prompt_tokens=prompt_tokens,
completion_tokens=completion_tokens,
duration_seconds=duration_seconds,
+ prompt_cached_tokens=prompt_cached_tokens,
+ completion_reasoning_tokens=completion_reasoning_tokens,
)
return
diff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.py
index 5c59b719e..1aab3231e 100644
--- a/openviking/models/vlm/base.py
+++ b/openviking/models/vlm/base.py
@@ -190,6 +190,8 @@ class VLMBase(ABC):
prompt_tokens: int,
completion_tokens: int,
duration_seconds: float = 0.0,
+ prompt_cached_tokens: int = 0,
+ completion_reasoning_tokens: int = 0,
) -> None:
"""Update token usage
@@ -199,6 +201,8 @@ class VLMBase(ABC):
prompt_tokens: Number of prompt tokens
completion_tokens: Number of completion tokens
duration_seconds: Wall-clock duration of the VLM call in seconds
+ prompt_cached_tokens: Number of cached prompt tokens from provider usage details
+ completion_reasoning_tokens: Number of reasoning completion tokens from provider usage details
"""
self._token_tracker.update(
model_name=model_name,
@@ -210,10 +214,14 @@ class VLMBase(ABC):
try:
from openviking.telemetry import get_current_telemetry, get_current_telemetry_stage
- get_current_telemetry().add_token_usage(
+ telemetry = get_current_telemetry()
+ stage = get_current_telemetry_stage() or "vlm"
+ telemetry.add_token_usage(
prompt_tokens,
completion_tokens,
- stage=get_current_telemetry_stage() or "vlm",
+ stage=stage,
+ prompt_cached_tokens=max(int(prompt_cached_tokens), 0),
+ completion_reasoning_tokens=max(int(completion_reasoning_tokens), 0),
)
except Exception as e:
# Telemetry must never break model inference.
diff --git a/openviking/session/session.py b/openviking/session/session.py
index 193d05ad1..9b602dcb7 100644
--- a/openviking/session/session.py
+++ b/openviking/session/session.py
@@ -194,6 +194,8 @@ class SessionMeta:
"prompt_tokens": 0,
"completion_tokens": 0,
"total_tokens": 0,
+ "cached_tokens": 0,
+ "reasoning_tokens": 0,
}
)
embedding_token_usage: Dict[str, int] = field(
@@ -265,6 +267,8 @@ class SessionMeta:
"prompt_tokens": llm_token_usage.get("prompt_tokens", 0),
"completion_tokens": llm_token_usage.get("completion_tokens", 0),
"total_tokens": llm_token_usage.get("total_tokens", 0),
+ "cached_tokens": llm_token_usage.get("cached_tokens", 0),
+ "reasoning_tokens": llm_token_usage.get("reasoning_tokens", 0),
},
embedding_token_usage={
"total_tokens": embedding_token_usage.get("total_tokens", 0),
@@ -1474,7 +1478,9 @@ class Session:
"embedding": dict(self._meta.embedding_token_usage),
"total": {
"total_tokens": self._meta.llm_token_usage["total_tokens"]
- + self._meta.embedding_token_usage["total_tokens"]
+ + self._meta.embedding_token_usage["total_tokens"],
+ "cached_tokens": self._meta.llm_token_usage["cached_tokens"],
+ "reasoning_tokens": self._meta.llm_token_usage["reasoning_tokens"],
},
},
},
@@ -1956,6 +1962,8 @@ class Session:
latest_meta.llm_token_usage["prompt_tokens"] += llm.get("input", 0)
latest_meta.llm_token_usage["completion_tokens"] += llm.get("output", 0)
latest_meta.llm_token_usage["total_tokens"] += llm.get("total", 0)
+ latest_meta.llm_token_usage["cached_tokens"] += llm.get("prompt_cached", 0)
+ latest_meta.llm_token_usage["reasoning_tokens"] += llm.get("completion_reasoning", 0)
embedding = telemetry_snapshot.summary.get("tokens", {}).get("embedding", {})
latest_meta.embedding_token_usage["total_tokens"] += embedding.get("total", 0)
diff --git a/openviking/telemetry/operation.py b/openviking/telemetry/operation.py
index d5fb4d268..9d5343e2d 100644
--- a/openviking/telemetry/operation.py
+++ b/openviking/telemetry/operation.py
@@ -162,6 +162,10 @@ class TelemetrySummaryBuilder:
llm_input_tokens = cls._i(counters.get("tokens.llm.input"), 0)
llm_output_tokens = cls._i(counters.get("tokens.llm.output"), 0)
llm_total_tokens = cls._i(counters.get("tokens.llm.total"), 0)
+ llm_prompt_cached_tokens = cls._i(counters.get("tokens.llm.prompt_cached"), 0)
+ llm_completion_reasoning_tokens = cls._i(
+ counters.get("tokens.llm.completion_reasoning"), 0
+ )
embedding_total_tokens = cls._i(counters.get("tokens.embedding.total"), 0)
rerank_total_tokens = cls._i(counters.get("tokens.rerank.total"), 0)
stage_token_summary = cls._build_stage_token_summary(counters)
@@ -183,6 +187,8 @@ class TelemetrySummaryBuilder:
"input": llm_input_tokens,
"output": llm_output_tokens,
"total": llm_total_tokens,
+ "prompt_cached": llm_prompt_cached_tokens,
+ "completion_reasoning": llm_completion_reasoning_tokens,
},
"embedding": {"total": embedding_total_tokens},
"rerank": {"total": rerank_total_tokens},
@@ -365,10 +371,23 @@ class OperationTelemetry:
self.add_duration(key, (time.perf_counter() - start) * 1000)
def add_token_usage(
- self, input_tokens: int, output_tokens: int, *, stage: str | None = None
+ self,
+ input_tokens: int,
+ output_tokens: int,
+ *,
+ stage: str | None = None,
+ prompt_cached_tokens: int = 0,
+ completion_reasoning_tokens: int = 0,
) -> None:
"""Record LLM token usage into aggregate and optional stage-specific counters."""
- self.add_token_usage_by_source("llm", input_tokens, output_tokens, stage=stage)
+ self.add_token_usage_by_source(
+ "llm",
+ input_tokens,
+ output_tokens,
+ stage=stage,
+ prompt_cached_tokens=prompt_cached_tokens,
+ completion_reasoning_tokens=completion_reasoning_tokens,
+ )
def record_token_usage(
self,
@@ -377,9 +396,18 @@ class OperationTelemetry:
output_tokens: int = 0,
*,
stage: str | None = None,
+ prompt_cached_tokens: int = 0,
+ completion_reasoning_tokens: int = 0,
) -> None:
"""Record source-scoped token usage into aggregate and optional stage-specific counters."""
- self.add_token_usage_by_source(source, input_tokens, output_tokens, stage=stage)
+ self.add_token_usage_by_source(
+ source,
+ input_tokens,
+ output_tokens,
+ stage=stage,
+ prompt_cached_tokens=prompt_cached_tokens,
+ completion_reasoning_tokens=completion_reasoning_tokens,
+ )
def add_token_usage_by_source(
self,
@@ -388,6 +416,8 @@ class OperationTelemetry:
output_tokens: int = 0,
*,
stage: str | None = None,
+ prompt_cached_tokens: int = 0,
+ completion_reasoning_tokens: int = 0,
) -> None:
"""Record token usage for one source and optionally mirror it into a fixed stage bucket."""
if not self.enabled:
@@ -396,6 +426,8 @@ class OperationTelemetry:
normalized_input = max(input_tokens, 0)
normalized_output = max(output_tokens, 0)
normalized_total = normalized_input + normalized_output
+ normalized_prompt_cached = max(prompt_cached_tokens, 0)
+ normalized_completion_reasoning = max(completion_reasoning_tokens, 0)
self.count("tokens.input", normalized_input)
self.count("tokens.output", normalized_output)
@@ -403,6 +435,12 @@ class OperationTelemetry:
self.count(f"tokens.{source}.input", normalized_input)
self.count(f"tokens.{source}.output", normalized_output)
self.count(f"tokens.{source}.total", normalized_total)
+ if source == "llm":
+ self.count(f"tokens.{source}.prompt_cached", normalized_prompt_cached)
+ self.count(
+ f"tokens.{source}.completion_reasoning",
+ normalized_completion_reasoning,
+ )
if stage is None:
try:
from .context import get_current_telemetry_stage
@@ -416,6 +454,15 @@ class OperationTelemetry:
self.count(f"tokens.stages.{normalized_stage}.{source}.input", normalized_input)
self.count(f"tokens.stages.{normalized_stage}.{source}.output", normalized_output)
self.count(f"tokens.stages.{normalized_stage}.{source}.total", normalized_total)
+ if source == "llm":
+ self.count(
+ f"tokens.stages.{normalized_stage}.{source}.prompt_cached",
+ normalized_prompt_cached,
+ )
+ self.count(
+ f"tokens.stages.{normalized_stage}.{source}.completion_reasoning",
+ normalized_completion_reasoning,
+ )
def set_error(self, stage: str, code: str, message: str) -> None:
if not self.enabled: