Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat(eval)Opt vikingbot eval script (#2305)
DuTaobe1e7fc482c8
diff --git a/benchmark/locomo/vikingbot/import_to_ov.py b/benchmark/locomo/vikingbot/import_to_ov.pyindex 0a1759935..6bbc6441c 100644--- a/benchmark/locomo/vikingbot/import_to_ov.py+++ b/benchmark/locomo/vikingbot/import_to_ov.py@@ -132,7 +132,7 @@ def build_session_messages(return sessions-# ---------------------------------------------------------------------------+# --------------------------------------------------------------------------# Ingest record helpers (avoid duplicate ingestion)# ---------------------------------------------------------------------------@@ -157,14 +157,17 @@ def write_success_record(fieldnames = ["timestamp","sample_id",+ "display_id","session","date_time","speakers","embedding_tokens","vlm_tokens",- "llm_input_tokens",+ "cache_tokens",+ "reasoning_tokens","llm_output_tokens","total_tokens",+ "duration_seconds",]with open(csv_path, "a", encoding="utf-8", newline="") as f:@@ -176,14 +179,17 @@ def write_success_record({"timestamp": record["timestamp"],"sample_id": record["sample_id"],+ "display_id": record.get("display_id", ""),"session": record["session"],"date_time": record.get("meta", {}).get("date_time", ""),"speakers": record.get("meta", {}).get("speakers", ""),"embedding_tokens": record["token_usage"].get("embedding", 0),"vlm_tokens": record["token_usage"].get("vlm", 0),- "llm_input_tokens": record["token_usage"].get("llm_input", 0),+ "cache_tokens": record["token_usage"].get("cache", 0),+ "reasoning_tokens": record["token_usage"].get("reasoning", 0),"llm_output_tokens": record["token_usage"].get("llm_output", 0),"total_tokens": record["token_usage"].get("total", 0),+ "duration_seconds": record.get("duration_seconds", 0),})@@ -260,22 +266,33 @@ def _parse_token_usage(commit_result: Dict[str, Any]) -> Dict[str, int]:# embedding 格式可能是 {"total": N} 或 {"total_tokens": N}embed_total = embedding.get("total", embedding.get("total_tokens", 0))llm_total = llm.get("total", llm.get("total_tokens", 0))+ llm_input = llm.get("input", llm.get("prompt_tokens", 0))+ llm_output = llm.get("output", llm.get("completion_tokens", 0))+ cache_tokens = llm.get("cached_tokens", llm.get("prompt_cached", 0))+ reasoning_tokens = llm.get(+ "reasoning_tokens", llm.get("completion_reasoning", 0)+ )return {"embedding": embed_total,- "vlm": llm_total,- "llm_input": llm.get("input", 0),- "llm_output": llm.get("output", 0),+ "vlm": llm_input,+ "llm_input": llm_input,+ "llm_output": llm_output,+ "cache": cache_tokens,+ "reasoning": reasoning_tokens,"total": tu.get("total", {}).get("total_tokens", embed_total + llm_total),}# 从 commit 响应的 telemetry 中提取telemetry = commit_result.get("telemetry", {}).get("summary", {})tokens = telemetry.get("tokens", {})+ llm = tokens.get("llm", {})return {"embedding": tokens.get("embedding", {}).get("total", 0),- "vlm": tokens.get("llm", {}).get("total", 0),- "llm_input": tokens.get("llm", {}).get("input", 0),- "llm_output": tokens.get("llm", {}).get("output", 0),+ "vlm": llm.get("total", 0),+ "llm_input": llm.get("input", llm.get("prompt_tokens", 0)),+ "llm_output": llm.get("output", llm.get("completion_tokens", 0)),+ "cache": llm.get("prompt_cached", llm.get("cached_tokens", 0)),+ "reasoning": llm.get("completion_reasoning", llm.get("reasoning_tokens", 0)),"total": tokens.get("total", 0),}@@ -287,6 +304,7 @@ async def viking_ingest(user_id: Optional[str] = None,agent_id: Optional[str] = None,account: str = "default",+ api_key: Optional[str] = None,) -> Dict[str, int]:"""Save messages to OpenViking via OpenViking SDK client.Returns token usage dict with embedding and vlm token counts.@@ -298,6 +316,7 @@ async def viking_ingest(user_id: User identifier for separate userspace (e.g., "conv-26")agent_id: Agent identifier for separate agentspace (e.g., "conv-26")account: OpenViking account identifier+ api_key: Optional API key for OpenViking client authentication"""# 解析 session_time - 为每条消息计算递增的时间戳base_datetime = None@@ -313,6 +332,7 @@ async def viking_ingest(user=user_id,agent_id=agent_id,account=account,+ api_key=api_key,timeout=600,)await client.initialize()@@ -350,7 +370,7 @@ async def viking_ingest(trace_id = result.get("trace_id", "")if task_id:# 轮询任务状态直到完成- max_attempts = 1200 # 最多等待20分钟+ max_attempts = 2400 # 最多等待40分钟for attempt in range(max_attempts):task = await client.get_task(task_id)status = task.get("status") if task else "unknown"@@ -394,10 +414,12 @@ async def process_single_session(meta = meta or {}ingest_record = ingest_record or {}csv_id = display_id or str(sample_id)+ source_sample_id = str(sample_id)try:+ started_at = time.perf_counter()user_id = str(sample_id) if args.separate_user_by_sample else ""agent_id = str(sample_id) if args.separate_user_by_sample else ""- account = args.account+ account = args.account if args.separate_user_by_sample else ""result = await viking_ingest(messages,args.openviking_url,@@ -405,27 +427,37 @@ async def process_single_session(user_id=user_id,agent_id=agent_id,account=account,+ api_key=args.api_key,)+ duration_seconds = round(time.perf_counter() - started_at, 3)token_usage = result["token_usage"]task_id = result.get("task_id")trace_id = result.get("trace_id", "")embedding_tokens = token_usage.get("embedding", 0)- vlm_tokens = token_usage.get("vlm", 0)+ vlm_tokens = token_usage.get("llm_input", 0)+ cache_tokens = token_usage.get("cache", 0)+ reasoning_tokens = token_usage.get("reasoning", 0)+ llm_output_tokens = token_usage.get("llm_output", 0)print(- f" -> [COMPLETED] [{csv_id}/{session_key}] embed={embedding_tokens}, vlm={vlm_tokens}, task_id={task_id}, trace_id={trace_id}",+ f" -> [COMPLETED] [{csv_id}/{session_key}] duration={duration_seconds}s, embed={embedding_tokens}, vlm={vlm_tokens}, cache={cache_tokens}, reasoning={reasoning_tokens}, completion={llm_output_tokens}, task_id={task_id}, trace_id={trace_id}",file=sys.stderr,)# Write success recordresult = {"timestamp": run_time,- "sample_id": csv_id,+ "sample_id": source_sample_id,+ "display_id": csv_id,"session": session_key,"status": "success","meta": meta,"token_usage": token_usage,+ "duration_seconds": duration_seconds,"embedding_tokens": embedding_tokens,"vlm_tokens": vlm_tokens,+ "cache_tokens": cache_tokens,+ "reasoning_tokens": reasoning_tokens,+ "llm_output_tokens": llm_output_tokens,"task_id": task_id,"trace_id": trace_id,}@@ -446,7 +478,8 @@ async def process_single_session(# Write error recordresult = {"timestamp": run_time,- "sample_id": csv_id,+ "sample_id": source_sample_id,+ "display_id": csv_id,"session": session_key,"status": "error","error": str(e),@@ -569,6 +602,9 @@ async def run_import(args: argparse.Namespace) -> None:error_count = 0total_embedding_tokens = 0total_vlm_tokens = 0+ total_cache_tokens = 0+ total_reasoning_tokens = 0+ total_llm_output_tokens = 0tasks = []if args.input.endswith(".json"):@@ -601,7 +637,7 @@ async def run_import(args: argparse.Namespace) -> None:# 为每个 sample 创建独立的处理协程async def process_sample(item, sample_index):- nonlocal success_count, error_count, total_embedding_tokens, total_vlm_tokens+ nonlocal success_count, error_count, total_embedding_tokens, total_vlm_tokens, total_cache_tokens, total_reasoning_tokens, total_llm_output_tokenssample_id = item["sample_id"]display_id = f"sample_{sample_index}"@@ -655,6 +691,9 @@ async def run_import(args: argparse.Namespace) -> None:success_count += 1total_embedding_tokens += res.get("embedding_tokens", 0)total_vlm_tokens += res.get("vlm_tokens", 0)+ total_cache_tokens += res.get("cache_tokens", 0)+ total_reasoning_tokens += res.get("reasoning_tokens", 0)+ total_llm_output_tokens += res.get("llm_output_tokens", 0)elif res.get("status") == "error":error_count += 1@@ -734,6 +773,7 @@ async def run_import(args: argparse.Namespace) -> None:success_count += 1total_embedding_tokens += r.get("embedding_tokens", 0)total_vlm_tokens += r.get("vlm_tokens", 0)+ total_llm_output_tokens += r.get("llm_output_tokens", 0)elif r.get("status") == "error":error_count += 1@@ -747,12 +787,24 @@ async def run_import(args: argparse.Namespace) -> None:print(f"\n=== Token usage summary ===", file=sys.stderr)print(f"Total Embedding tokens: {total_embedding_tokens}", file=sys.stderr)print(f"Total VLM tokens: {total_vlm_tokens}", file=sys.stderr)+ print(f"Total Cache tokens: {total_cache_tokens}", file=sys.stderr)+ print(f"Total Reasoning tokens: {total_reasoning_tokens}", file=sys.stderr)+ print(f"Total Completion tokens: {total_llm_output_tokens}", file=sys.stderr)if success_count > 0:print(f"Average Embedding per session: {total_embedding_tokens / success_count:.1f}",file=sys.stderr,)print(f"Average VLM per session: {total_vlm_tokens / success_count:.1f}", file=sys.stderr)+ print(f"Average Cache per session: {total_cache_tokens / success_count:.1f}", file=sys.stderr)+ print(+ f"Average Reasoning per session: {total_reasoning_tokens / success_count:.1f}",+ file=sys.stderr,+ )+ print(+ f"Average Completion per session: {total_llm_output_tokens / success_count:.1f}",+ file=sys.stderr,+ )print(f"\nResults saved to:", file=sys.stderr)print(f" - Success records: {args.success_csv}", file=sys.stderr)print(f" - Error logs: {args.error_log}", file=sys.stderr)@@ -789,6 +841,11 @@ def main():default="http://localhost:1933",help="OpenViking service URL (default: http://localhost:1933)",)+ parser.add_argument(+ "--api-key",+ default=None,+ help="OpenViking API key to pass to AsyncHTTPClient",+ )parser.add_argument("--account",default="default",diff --git a/benchmark/locomo/vikingbot/run_full_eval.sh b/benchmark/locomo/vikingbot/run_full_eval.shindex 775d9d51b..c446df113 100755--- a/benchmark/locomo/vikingbot/run_full_eval.sh+++ b/benchmark/locomo/vikingbot/run_full_eval.sh@@ -166,6 +166,126 @@ if [ "$AUTO_COMMIT" = "true" ]; thenfiGIT_COMMIT_ID=$(git rev-parse --short HEAD)TIMESTAMP=$(date +%Y%m%d%H%M%S)+IMPORT_SUCCESS_CSV="./result/import_success.csv"+IMPORT_ROW_START=0+IMPORT_PERFORMED=false++count_import_rows() {+ IMPORT_SUCCESS_CSV="$IMPORT_SUCCESS_CSV" "$PYTHON_BIN" - <<'PY'+import csv+import os+from pathlib import Path++path = Path(os.environ["IMPORT_SUCCESS_CSV"])+if not path.exists():+ print(0)+else:+ with open(path, "r", encoding="utf-8", newline="") as f:+ print(sum(1 for _ in csv.DictReader(f)))+PY+}++capture_import_row_start() {+ IMPORT_ROW_START=$(count_import_rows)+ IMPORT_PERFORMED=false+}++print_import_summary_table() {+ if [ "$SKIP_IMPORT" = "true" ] || [ "$IMPORT_PERFORMED" != "true" ]; then+ return+ fi++ echo ""+ IMPORT_SUCCESS_CSV="$IMPORT_SUCCESS_CSV" IMPORT_ROW_START="$IMPORT_ROW_START" "$PYTHON_BIN" - <<'PY'+import csv+import os+from pathlib import Path+++def to_int(value: str) -> int:+ try:+ return int(float(value or 0))+ except (TypeError, ValueError):+ return 0+++def to_float(value: str) -> float:+ try:+ return float(value or 0)+ except (TypeError, ValueError):+ return 0.0+++def render_table(headers: list[str], rows: list[list[str]], align_right: set[int] | None = None) -> str:+ align_right = align_right or set()+ widths = [len(header) for header in headers]+ for row in rows:+ for i, cell in enumerate(row):+ widths[i] = max(widths[i], len(cell))++ def format_row(row: list[str]) -> str:+ cells = []+ for i, cell in enumerate(row):+ cells.append(cell.rjust(widths[i]) if i in align_right else cell.ljust(widths[i]))+ return "| " + " | ".join(cells) + " |"++ sep = "+-" + "-+-".join("-" * width for width in widths) + "-+"+ lines = [sep, format_row(headers), sep]+ for row in rows:+ lines.append(format_row(row))+ lines.append(sep)+ return "\n".join(lines)+++path = Path(os.environ["IMPORT_SUCCESS_CSV"])+start = int(os.environ.get("IMPORT_ROW_START", "0"))+print("=== Import Summary ===")+if not path.exists():+ print("No import success CSV found.")+ raise SystemExit(0)++with open(path, "r", encoding="utf-8", newline="") as f:+ rows = list(csv.DictReader(f))++rows = rows[start:]+if not rows:+ print("No new import records were written in this run.")+ raise SystemExit(0)++totals = {+ "sessions": len(rows),+ "embedding_tokens": 0,+ "vlm_tokens": 0,+ "cache_tokens": 0,+ "reasoning_tokens": 0,+ "llm_output_tokens": 0,+ "total_tokens": 0,+ "duration_seconds": 0.0,+}+for row in rows:+ totals["embedding_tokens"] += to_int(row.get("embedding_tokens"))+ totals["vlm_tokens"] += to_int(row.get("vlm_tokens"))+ totals["cache_tokens"] += to_int(row.get("cache_tokens"))+ totals["reasoning_tokens"] += to_int(row.get("reasoning_tokens"))+ totals["llm_output_tokens"] += to_int(row.get("llm_output_tokens"))+ totals["total_tokens"] += to_int(row.get("total_tokens"))+ totals["duration_seconds"] += to_float(row.get("duration_seconds"))++avg_duration = totals["duration_seconds"] / totals["sessions"] if totals["sessions"] else 0.0+summary_rows = [+ ["sessions", str(totals["sessions"])],+ ["embedding_tokens", str(totals["embedding_tokens"])],+ ["vlm_tokens", str(totals["vlm_tokens"])],+ ["cache_tokens", str(totals["cache_tokens"])],+ ["reasoning_tokens", str(totals["reasoning_tokens"])],+ ["llm_output_tokens", str(totals["llm_output_tokens"])],+ ["total_tokens", str(totals["total_tokens"])],+ ["total_duration_s", f"{totals['duration_seconds']:.3f}"],+ ["avg_duration_s", f"{avg_duration:.3f}"],+]+print(render_table(["metric", "value"], summary_rows, align_right={1}))+PY+}# ========== 重跑错题模式(优先) ==========if [ -n "$RETRY_WRONG" ]; then@@ -185,6 +305,7 @@ if [ -n "$RETRY_WRONG" ]; then# 从错题 CSV 中提取需要导入的对话(复用 import_to_ov.py 的并行逻辑)echo "[1/3] 导入错题相关对话..."+ capture_import_row_start"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \--input "$INPUT_FILE" \--retry-wrong "$RETRY_WRONG" \@@ -192,6 +313,7 @@ if [ -n "$RETRY_WRONG" ]; then--account "$ACCOUNT" \--openviking-url "$OPENVIKING_URL" \"${COMMON_OPTS[@]}"+ IMPORT_PERFORMED=trueecho "等待数据处理完成..."sleep 30@@ -203,6 +325,7 @@ if [ -n "$RETRY_WRONG" ]; then--output "$RESULT_FILE" \--retry-wrong "$RETRY_WRONG" \--threads 20 \+ --config "$OPENVIKING_CONFIG_FILE" \"${COMMON_OPTS[@]}"# 裁判打分@@ -211,6 +334,7 @@ if [ -n "$RETRY_WRONG" ]; then# 统计结果"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$RESULT_FILE"+ print_import_summary_tableecho ""echo "=== 错题重跑完成 ==="@@ -233,14 +357,16 @@ if [ -z "$SAMPLE" ]; thenecho "[1/4] 跳过导入数据..."elseecho "[1/4] 导入数据..."+ capture_import_row_start"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" --input "$INPUT_FILE" --force-ingest --account "$ACCOUNT" --openviking-url "$OPENVIKING_URL" "${COMMON_OPTS[@]}"+ IMPORT_PERFORMED=trueecho "等待 1 分钟..."sleep 60fi# 评估echo "[2/4] 评估..."- "$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" "$INPUT_FILE" --output "$RESULT_FILE" "${COMMON_OPTS[@]}"+ "$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" "$INPUT_FILE" --output "$RESULT_FILE" --config "$OPENVIKING_CONFIG_FILE" "${COMMON_OPTS[@]}"# 裁判打分echo "[3/4] 裁判打分..."@@ -249,6 +375,7 @@ if [ -z "$SAMPLE" ]; then# 计算结果echo "[4/4] 计算结果...""$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$RESULT_FILE"+ print_import_summary_tableecho ""echo "=== 全量评测完成 ==="@@ -299,6 +426,7 @@ if [ -n "$QUESTION_INDEX" ]; thenecho "[1/3] Skipping import (--skip-import)"elseecho "[1/3] Importing sample $SAMPLE_INDEX, question $QUESTION_INDEX..."+ capture_import_row_start"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \--input "$INPUT_FILE" \--sample "$SAMPLE_INDEX" \@@ -307,6 +435,7 @@ if [ -n "$QUESTION_INDEX" ]; then--account "$ACCOUNT" \--openviking-url "$OPENVIKING_URL" \"${COMMON_OPTS[@]}"+ IMPORT_PERFORMED=trueecho "Waiting for data processing..."sleep 3@@ -329,6 +458,7 @@ if [ -n "$QUESTION_INDEX" ]; then--question-index "$QUESTION_INDEX" \--count 1 \--output "$OUTPUT_FILE" \+ --config "$OPENVIKING_CONFIG_FILE" \"${COMMON_OPTS[@]}"# 运行 Judge 评分@@ -342,6 +472,7 @@ if [ -n "$QUESTION_INDEX" ]; then# 输出结果echo ""echo "=== 评测结果 ==="+ print_import_summary_tableOUTPUT_FILE="$OUTPUT_FILE" QUESTION_INDEX="$QUESTION_INDEX" "$PYTHON_BIN" - <<'PY'import csvimport json@@ -400,6 +531,7 @@ PYecho "[1/4] Skipping import (--skip-import)"elseecho "[1/4] Importing all sessions for sample $SAMPLE_INDEX..."+ capture_import_row_start"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \--input "$INPUT_FILE" \--sample "$SAMPLE_INDEX" \@@ -407,6 +539,7 @@ PY--account "$ACCOUNT" \--openviking-url "$OPENVIKING_URL" \"${COMMON_OPTS[@]}"+ IMPORT_PERFORMED=trueecho "Waiting for data processing..."sleep 10@@ -428,6 +561,7 @@ PY--sample "$SAMPLE_ID_FOR_CMD" \--output "$OUTPUT_FILE" \--threads 5 \+ --config "$OPENVIKING_CONFIG_FILE" \"${COMMON_OPTS[@]}"# 运行 Judge 评分@@ -445,6 +579,7 @@ PYecho "[4/4] Calculating statistics..."fi"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$OUTPUT_FILE"+ print_import_summary_tableecho ""echo "=== 批量评测完成 ==="diff --git a/bot/vikingbot/channels/feishu.py b/bot/vikingbot/channels/feishu.pyindex 4587bff42..0f8be9e15 100644--- a/bot/vikingbot/channels/feishu.py+++ b/bot/vikingbot/channels/feishu.py@@ -759,6 +759,8 @@ class FeishuChannel(BaseChannel):except Exception as e:logger.warning(f"Failed to process {msg_type} message: {e}")+ elif msg_type =="interactive":+ content = message.contentelse:content = MSG_TYPE_MAP.get(msg_type, f"[{msg_type}]")diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.pyindex 81a833238..012adcbf3 100644--- a/openviking/models/vlm/backends/openai_vlm.py+++ b/openviking/models/vlm/backends/openai_vlm.py@@ -158,12 +158,20 @@ class OpenAIVLM(VLMBase):tracer.info(f"response.usage={response.usage}")prompt_tokens = response.usage.prompt_tokenscompletion_tokens = response.usage.completion_tokens+ prompt_tokens_details = getattr(response.usage, "prompt_tokens_details", None)+ completion_tokens_details = getattr(response.usage, "completion_tokens_details", None)+ prompt_cached_tokens = getattr(prompt_tokens_details, "cached_tokens", 0) or 0+ completion_reasoning_tokens = (+ getattr(completion_tokens_details, "reasoning_tokens", 0) or 0+ )self.update_token_usage(model_name=self.model or "gpt-4o-mini",provider=self.provider,prompt_tokens=prompt_tokens,completion_tokens=completion_tokens,duration_seconds=duration_seconds,+ prompt_cached_tokens=prompt_cached_tokens,+ completion_reasoning_tokens=completion_reasoning_tokens,)returndiff --git a/openviking/models/vlm/base.py b/openviking/models/vlm/base.pyindex 5c59b719e..1aab3231e 100644--- a/openviking/models/vlm/base.py+++ b/openviking/models/vlm/base.py@@ -190,6 +190,8 @@ class VLMBase(ABC):prompt_tokens: int,completion_tokens: int,duration_seconds: float = 0.0,+ prompt_cached_tokens: int = 0,+ completion_reasoning_tokens: int = 0,) -> None:"""Update token usage@@ -199,6 +201,8 @@ class VLMBase(ABC):prompt_tokens: Number of prompt tokenscompletion_tokens: Number of completion tokensduration_seconds: Wall-clock duration of the VLM call in seconds+ prompt_cached_tokens: Number of cached prompt tokens from provider usage details+ completion_reasoning_tokens: Number of reasoning completion tokens from provider usage details"""self._token_tracker.update(model_name=model_name,@@ -210,10 +214,14 @@ class VLMBase(ABC):try:from openviking.telemetry import get_current_telemetry, get_current_telemetry_stage- get_current_telemetry().add_token_usage(+ telemetry = get_current_telemetry()+ stage = get_current_telemetry_stage() or "vlm"+ telemetry.add_token_usage(prompt_tokens,completion_tokens,- stage=get_current_telemetry_stage() or "vlm",+ stage=stage,+ prompt_cached_tokens=max(int(prompt_cached_tokens), 0),+ completion_reasoning_tokens=max(int(completion_reasoning_tokens), 0),)except Exception as e:# Telemetry must never break model inference.diff --git a/openviking/session/session.py b/openviking/session/session.pyindex 193d05ad1..9b602dcb7 100644--- a/openviking/session/session.py+++ b/openviking/session/session.py@@ -194,6 +194,8 @@ class SessionMeta:"prompt_tokens": 0,"completion_tokens": 0,"total_tokens": 0,+ "cached_tokens": 0,+ "reasoning_tokens": 0,})embedding_token_usage: Dict[str, int] = field(@@ -265,6 +267,8 @@ class SessionMeta:"prompt_tokens": llm_token_usage.get("prompt_tokens", 0),"completion_tokens": llm_token_usage.get("completion_tokens", 0),"total_tokens": llm_token_usage.get("total_tokens", 0),+ "cached_tokens": llm_token_usage.get("cached_tokens", 0),+ "reasoning_tokens": llm_token_usage.get("reasoning_tokens", 0),},embedding_token_usage={"total_tokens": embedding_token_usage.get("total_tokens", 0),@@ -1474,7 +1478,9 @@ class Session:"embedding": dict(self._meta.embedding_token_usage),"total": {"total_tokens": self._meta.llm_token_usage["total_tokens"]- + self._meta.embedding_token_usage["total_tokens"]+ + self._meta.embedding_token_usage["total_tokens"],+ "cached_tokens": self._meta.llm_token_usage["cached_tokens"],+ "reasoning_tokens": self._meta.llm_token_usage["reasoning_tokens"],},},},@@ -1956,6 +1962,8 @@ class Session:latest_meta.llm_token_usage["prompt_tokens"] += llm.get("input", 0)latest_meta.llm_token_usage["completion_tokens"] += llm.get("output", 0)latest_meta.llm_token_usage["total_tokens"] += llm.get("total", 0)+ latest_meta.llm_token_usage["cached_tokens"] += llm.get("prompt_cached", 0)+ latest_meta.llm_token_usage["reasoning_tokens"] += llm.get("completion_reasoning", 0)embedding = telemetry_snapshot.summary.get("tokens", {}).get("embedding", {})latest_meta.embedding_token_usage["total_tokens"] += embedding.get("total", 0)diff --git a/openviking/telemetry/operation.py b/openviking/telemetry/operation.pyindex d5fb4d268..9d5343e2d 100644--- a/openviking/telemetry/operation.py+++ b/openviking/telemetry/operation.py@@ -162,6 +162,10 @@ class TelemetrySummaryBuilder:llm_input_tokens = cls._i(counters.get("tokens.llm.input"), 0)llm_output_tokens = cls._i(counters.get("tokens.llm.output"), 0)llm_total_tokens = cls._i(counters.get("tokens.llm.total"), 0)+ llm_prompt_cached_tokens = cls._i(counters.get("tokens.llm.prompt_cached"), 0)+ llm_completion_reasoning_tokens = cls._i(+ counters.get("tokens.llm.completion_reasoning"), 0+ )embedding_total_tokens = cls._i(counters.get("tokens.embedding.total"), 0)rerank_total_tokens = cls._i(counters.get("tokens.rerank.total"), 0)stage_token_summary = cls._build_stage_token_summary(counters)@@ -183,6 +187,8 @@ class TelemetrySummaryBuilder:"input": llm_input_tokens,"output": llm_output_tokens,"total": llm_total_tokens,+ "prompt_cached": llm_prompt_cached_tokens,+ "completion_reasoning": llm_completion_reasoning_tokens,},"embedding": {"total": embedding_total_tokens},"rerank": {"total": rerank_total_tokens},@@ -365,10 +371,23 @@ class OperationTelemetry:self.add_duration(key, (time.perf_counter() - start) * 1000)def add_token_usage(- self, input_tokens: int, output_tokens: int, *, stage: str | None = None+ self,+ input_tokens: int,+ output_tokens: int,+ *,+ stage: str | None = None,+ prompt_cached_tokens: int = 0,+ completion_reasoning_tokens: int = 0,) -> None:"""Record LLM token usage into aggregate and optional stage-specific counters."""- self.add_token_usage_by_source("llm", input_tokens, output_tokens, stage=stage)+ self.add_token_usage_by_source(+ "llm",+ input_tokens,+ output_tokens,+ stage=stage,+ prompt_cached_tokens=prompt_cached_tokens,+ completion_reasoning_tokens=completion_reasoning_tokens,+ )def record_token_usage(self,@@ -377,9 +396,18 @@ class OperationTelemetry:output_tokens: int = 0,*,stage: str | None = None,+ prompt_cached_tokens: int = 0,+ completion_reasoning_tokens: int = 0,) -> None:"""Record source-scoped token usage into aggregate and optional stage-specific counters."""- self.add_token_usage_by_source(source, input_tokens, output_tokens, stage=stage)+ self.add_token_usage_by_source(+ source,+ input_tokens,+ output_tokens,+ stage=stage,+ prompt_cached_tokens=prompt_cached_tokens,+ completion_reasoning_tokens=completion_reasoning_tokens,+ )def add_token_usage_by_source(self,@@ -388,6 +416,8 @@ class OperationTelemetry:output_tokens: int = 0,*,stage: str | None = None,+ prompt_cached_tokens: int = 0,+ completion_reasoning_tokens: int = 0,) -> None:"""Record token usage for one source and optionally mirror it into a fixed stage bucket."""if not self.enabled:@@ -396,6 +426,8 @@ class OperationTelemetry:normalized_input = max(input_tokens, 0)normalized_output = max(output_tokens, 0)normalized_total = normalized_input + normalized_output+ normalized_prompt_cached = max(prompt_cached_tokens, 0)+ normalized_completion_reasoning = max(completion_reasoning_tokens, 0)self.count("tokens.input", normalized_input)self.count("tokens.output", normalized_output)@@ -403,6 +435,12 @@ class OperationTelemetry:self.count(f"tokens.{source}.input", normalized_input)self.count(f"tokens.{source}.output", normalized_output)self.count(f"tokens.{source}.total", normalized_total)+ if source == "llm":+ self.count(f"tokens.{source}.prompt_cached", normalized_prompt_cached)+ self.count(+ f"tokens.{source}.completion_reasoning",+ normalized_completion_reasoning,+ )if stage is None:try:from .context import get_current_telemetry_stage@@ -416,6 +454,15 @@ class OperationTelemetry:self.count(f"tokens.stages.{normalized_stage}.{source}.input", normalized_input)self.count(f"tokens.stages.{normalized_stage}.{source}.output", normalized_output)self.count(f"tokens.stages.{normalized_stage}.{source}.total", normalized_total)+ if source == "llm":+ self.count(+ f"tokens.stages.{normalized_stage}.{source}.prompt_cached",+ normalized_prompt_cached,+ )+ self.count(+ f"tokens.stages.{normalized_stage}.{source}.completion_reasoning",+ normalized_completion_reasoning,+ )def set_error(self, stage: str, code: str, message: str) -> None:if not self.enabled: