Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
1. 优化评测参数化; (#3332)
DuTao55a9d12cd6d6
diff --git a/benchmark/locomo/vikingbot/preflight_eval_config.py b/benchmark/locomo/vikingbot/preflight_eval_config.pyindex 90c805d6f..f6dc0f083 100644--- a/benchmark/locomo/vikingbot/preflight_eval_config.py+++ b/benchmark/locomo/vikingbot/preflight_eval_config.py@@ -11,7 +11,12 @@ from openviking_cli.utils.config.consts import (OPENVIKING_CONFIG_ENV,)-_USE_COLOR = hasattr(sys.stdout, "isatty") and sys.stdout.isatty()+_USE_COLOR = (+ hasattr(sys.stdout, "isatty")+ and sys.stdout.isatty()+ and os.environ.get("TERM", "dumb") != "dumb"+ and "NO_COLOR" not in os.environ+)def _color(text: str, code: str) -> str:@@ -20,24 +25,20 @@ def _color(text: str, code: str) -> str:return f"\033[{code}m{text}\033[0m"-def _prefix() -> str:- return _color("[preflight]", "36")--def _log_info(message: str) -> None:- print(f"{_prefix()} {_color('[INFO]', '34')} {message}")+ print(_color(f" │ {message}", "2"))def _log_warn(message: str) -> None:- print(f"{_prefix()} {_color('[WARN]', '33')} {message}")+ print(f" {_color('!', '33')} {message}")def _log_ok(message: str) -> None:- print(f"{_prefix()} {_color('[OK]', '32')} {message}")+ print(f" {_color('✓', '32')} {message}")def _log_error(message: str) -> None:- print(f"{_prefix()} {_color('[ERROR]', '31')} {message}", file=sys.stderr)+ print(f" {_color('✗', '31')} {message}", file=sys.stderr)def _is_interactive() -> bool:@@ -45,8 +46,10 @@ def _is_interactive() -> bool:def _prompt_text(prompt: str, default: str | None = None) -> str:- suffix = f" [{default}]" if default else ""- raw = input(f"{_prefix()} {prompt}{suffix}: ").strip()+ print(f"\n {_color('?', '33')} {prompt}")+ if default:+ print(f" {_color('默认:', '2')} {default}")+ raw = input(f" {_color('>', '32')} ").strip()if not raw and default is not None:return defaultreturn rawdiff --git a/benchmark/locomo/vikingbot/preflight_eval_runtime.py b/benchmark/locomo/vikingbot/preflight_eval_runtime.pyindex 31c255545..0efdf94a5 100644--- a/benchmark/locomo/vikingbot/preflight_eval_runtime.py+++ b/benchmark/locomo/vikingbot/preflight_eval_runtime.py@@ -1,6 +1,7 @@#!/usr/bin/env python3import argparse+import getpassimport jsonimport osimport shlex@@ -18,13 +19,38 @@ from openviking_cli.utils.config.consts import (OPENVIKING_CONFIG_ENV,)+_USE_COLOR = (+ hasattr(sys.stdout, "isatty")+ and sys.stdout.isatty()+ and os.environ.get("TERM", "dumb") != "dumb"+ and "NO_COLOR" not in os.environ+)+++def _color(text: str, code: str) -> str:+ if not _USE_COLOR:+ return text+ return f"\033[{code}m{text}\033[0m"+def _log(message: str) -> None:- print(f"[preflight] {message}")+ print(_color(f" │ {message}", "2"))+++def _fact(message: str) -> None:+ print(f" {_color('◆', '36')} {message}")+++def _ok(message: str) -> None:+ print(f" {_color('✓', '32')} {message}")+++def _warn(message: str) -> None:+ print(f" {_color('!', '33')} {message}")def _error(message: str) -> None:- print(f"[preflight] {message}", file=sys.stderr)+ print(f" {_color('✗', '31')} {message}", file=sys.stderr)class UserKeyValidationError(RuntimeError):@@ -44,21 +70,30 @@ def _is_interactive() -> bool:def _prompt_text(prompt: str) -> str:+ print(f"\n {_color('?', '33')} {prompt}")try:with open("/dev/tty", "r", encoding="utf-8") as tty_in:- print(prompt, end="", flush=True)+ print(f" {_color('>', '32')} ", end="", flush=True)return tty_in.readline().strip()except Exception:- return input(prompt).strip()+ return input(f" {_color('>', '32')} ").strip()+++def _prompt_secret(prompt: str) -> str:+ print(f"\n {_color('?', '33')} {prompt}")+ try:+ return getpass.getpass(f" {_color('>', '32')} ").strip()+ except Exception:+ return _prompt_text("请输入密钥").strip()def _prompt_api_key() -> str:- return _prompt_text("[preflight] 请输入 OpenViking User API key: ")+ return _prompt_secret("请输入 OpenViking User API key(输入内容不会显示)")def _prompt_root_api_key() -> str:- return _prompt_text(- "[preflight] 请输入 OpenViking Root API key,用于自动创建 default User key: "+ return _prompt_secret(+ "请输入 OpenViking Root API key,用于自动创建 default User key(输入内容不会显示)")@@ -66,7 +101,7 @@ def _prompt_yes_no(prompt: str, default: bool = False) -> bool:if not _is_interactive():return Falsesuffix = "Y/n" if default else "y/N"- answer = _prompt_text(f"{prompt} [{suffix}]: ").strip().lower()+ answer = _prompt_text(f"{prompt} [{suffix}]").strip().lower()if not answer:return defaultreturn answer in {"y", "yes", "1", "true"}@@ -170,7 +205,7 @@ def _sync_bot_identity(account_id: str, user_id: str, api_key: str, *, auth_modechanged = Trueif changed:_write_json_with_backup(path, ov_data)- _log(+ _ok("已同步 bot.ov_server: "f"api_key_type={desired['api_key_type']}, account_id={account_id}, admin_user_id={user_id}")@@ -337,7 +372,7 @@ def _ensure_default_user_key(url: str, root_api_key: str) -> tuple[str, str, stritem.get("account_id") == account_id for item in accounts if isinstance(item, dict))if not account_exists:- _log("default account 不存在,将使用 Root key 创建 account=default。")+ _warn("default account 不存在,将使用 Root key 创建 account=default。")_admin_request(url,root_api_key,@@ -360,7 +395,7 @@ def _ensure_default_user_key(url: str, root_api_key: str) -> tuple[str, str, strNone,)if default_user is None:- _log("default User 不存在,将注册 account=default, user=default, role=user。")+ _warn("default User 不存在,将注册 account=default, user=default, role=user。")created = _admin_request(url,root_api_key,@@ -372,7 +407,7 @@ def _ensure_default_user_key(url: str, root_api_key: str) -> tuple[str, str, strelse:role = str(default_user.get("role") or "").strip().lower()if role != "user":- _log(f"default User 当前 role={role or 'unknown'},将调整为 role=user。")+ _warn(f"default User 当前 role={role or 'unknown'},将调整为 role=user。")_admin_request(url,root_api_key,@@ -403,13 +438,13 @@ def _ensure_server_and_user_key_ready(raise UserKeyValidationError(f"当前 API key 解析为 role={role}。评测需要普通 User key。")if selected_account and selected_account != "default" and selected_account != account_id:- _log(+ _warn(f"ovcli.conf.account={selected_account} "f"与 API key 归属 account={account_id} 不一致;""本次评测使用 API key 归属 account。")- _log(+ _ok("OpenViking server 可用,User key 身份: "f"account={account_id}, user={user_id}, role={role}。")@@ -422,9 +457,11 @@ def _ensure_trusted_server_ready(payload = _request_health(url, api_key, account_id=account_id, user_id=user_id)health_account, health_user, role = _parse_health_identity_payload(payload)if role != "user":- raise UserKeyValidationError(f"当前 trusted 身份解析为 role={role}。评测需要普通 User 身份。")+ raise UserKeyValidationError(+ f"当前 trusted 身份解析为 role={role}。评测需要普通 User 身份。"+ )- _log(+ _ok("OpenViking server 可用,trusted 身份: "f"account={health_account}, user={health_user}, role={role}。")@@ -465,12 +502,10 @@ def _resolve_ready_user_identity(return account, user_id, api_key, "api_key"except UserKeyValidationError as exc:_error(str(exc))- prompt = (- "[preflight] 当前 User key 不可用,是否使用 Root key 自动生成 default User API key"- )+ prompt = "当前 User key 不可用,是否使用 Root key 自动生成 default User API key"else:_error("未配置 OpenViking API key。")- prompt = "[preflight] 是否使用 Root key 自动生成 default User API key"+ prompt = "是否使用 Root key 自动生成 default User API key"if not _prompt_yes_no(prompt, default=False):_error("请配置可用的 bot.ov_server.api_key 或 ovcli.conf.api_key 后重试。")@@ -493,7 +528,7 @@ def _resolve_ready_user_identity(except UserKeyValidationError as exc:_error(str(exc))raise SystemExit(1) from exc- _log("已生成可用的 default User API key。")+ _ok("已生成可用的 default User API key。")return checked_account or account, checked_user_id or user_id, user_key, "api_key"@@ -523,13 +558,15 @@ def main() -> int:openviking_url = _resolve_openviking_url()api_key, key_source = _resolve_openviking_api_key()- _log(f"本次导入使用 OpenViking URL: {openviking_url}")+ _fact(f"本次导入使用 OpenViking URL: {openviking_url}")account, user_id, api_key, auth_mode = _resolve_ready_user_identity(openviking_url, selected_account, selected_user, api_key, key_source)_sync_bot_identity(account, user_id, api_key, auth_mode=auth_mode)- _write_env_file(Path(args.output_env_file), account, openviking_url, api_key, user_id, auth_mode)+ _write_env_file(+ Path(args.output_env_file), account, openviking_url, api_key, user_id, auth_mode+ )return 0diff --git a/benchmark/locomo/vikingbot/run_full_eval.sh b/benchmark/locomo/vikingbot/run_full_eval.shindex 0f5191c4a..5d8aa3a80 100755--- a/benchmark/locomo/vikingbot/run_full_eval.sh+++ b/benchmark/locomo/vikingbot/run_full_eval.sh@@ -12,9 +12,64 @@# ./run_full_eval.sh --skip-import --auto-commit # 评测全部,跳过导入,自动提交# ./run_full_eval.sh --retry-wrong result/locomo_result_xxx.csv # 只重跑错题# ./run_full_eval.sh --parallel-import-sessions 20 0 1 # 覆盖默认 session 导入并发数+# ./run_full_eval.sh --parallel-import-sessions 50 --parallel-run-eval 20 --parallel-judge 40 # 分别设置导入、评测和裁判并发数set -e+UI_RESET=""+UI_BOLD=""+UI_DIM=""+UI_RED=""+UI_GREEN=""+UI_YELLOW=""+UI_CYAN=""+if [ -t 1 ] && [ "${TERM:-dumb}" != "dumb" ] && [ -z "${NO_COLOR+x}" ]; then+ UI_RESET=$'\033[0m'+ UI_BOLD=$'\033[1m'+ UI_DIM=$'\033[2m'+ UI_RED=$'\033[31m'+ UI_GREEN=$'\033[32m'+ UI_YELLOW=$'\033[33m'+ UI_CYAN=$'\033[36m'+fi++ui_banner() {+ printf "\n%b%s%b\n" "${UI_BOLD}${UI_CYAN}" "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" "$UI_RESET"+ printf "%b %s%b\n" "${UI_BOLD}" "$1" "$UI_RESET"+ printf "%b%s%b\n" "${UI_BOLD}${UI_CYAN}" "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" "$UI_RESET"+}++ui_section() {+ printf "\n%b%s%b\n" "${UI_BOLD}${UI_CYAN}" "━━ $1" "$UI_RESET"+}++ui_info() {+ printf " %b│ %s%b\n" "$UI_DIM" "$1" "$UI_RESET"+}++ui_success() {+ printf " %b✓%b %s\n" "$UI_GREEN" "$UI_RESET" "$1"+}++ui_warn() {+ printf " %b!%b %s\n" "$UI_YELLOW" "$UI_RESET" "$1"+}++ui_error() {+ printf " %b✗%b %s\n" "$UI_RED" "$UI_RESET" "$1" >&2+}++ui_kv() {+ printf " %b%s:%b %s\n" "$UI_DIM" "$1" "$UI_RESET" "$2"+}++ui_step() {+ local current="$1"+ local total="$2"+ local title="$3"+ printf "\n %b▶ [%s/%s] %s%b\n" "${UI_BOLD}${UI_CYAN}" "$current" "$total" "$title" "$UI_RESET"+}+# --help 提前处理,避免触发 Python preflightfor arg in "$@"; doif [ "$arg" = "--help" ] || [ "$arg" = "-h" ]; then@@ -31,7 +86,9 @@ for arg in "$@"; doecho " --no-group-chat 非群聊模式(默认),使用 sample_id 作为 Peer"echo " --auto-commit 自动提交未提交的代码变更,结果文件名带 commit id 和时间戳"echo " --retry-wrong CSV 只重跑指定结果文件中的有效错题(导入相关对话+重新问答)"- echo " --parallel-import-sessions N 单 sample 内并发导入 sessions(默认 200)"+ echo " --parallel-import-sessions N 导入 session 并发数(默认 50)"+ echo " --parallel-run-eval N run_eval 并发线程数(默认 100)"+ echo " --parallel-judge N judge 并发请求数(默认 100)"exit 0fidone@@ -41,17 +98,22 @@ SKIP_IMPORT=falseGROUP_CHAT=falseAUTO_COMMIT=falseRETRY_WRONG=""-PARALLEL_IMPORT_SESSIONS="200"+PARALLEL_IMPORT_SESSIONS="50"+PARALLEL_RUN_EVAL="100"+PARALLEL_JUDGE="100"if command -v python3 >/dev/null 2>&1; thenPYTHON_BIN="python3"elif command -v python >/dev/null 2>&1; thenPYTHON_BIN="python"else- echo "未找到 python3/python,请先安装 Python。" >&2+ ui_error "未找到 python3/python,请先安装 Python。"exit 1fi+ui_banner "LoCoMo · VikingBot Evaluation"+ui_section "1. 环境预检"+DEFAULT_OV_CONF_PATH="$($PYTHON_BIN - <<'PY'from pathlib import Path@@ -64,8 +126,10 @@ PY)"if [ -t 0 ] && [ -t 1 ]; then- echo "[preflight] OpenViking 配置默认路径: $DEFAULT_OV_CONF_PATH"- printf "[preflight] 直接回车使用默认,或输入新路径 [%s]: " "$DEFAULT_OV_CONF_PATH"+ ui_kv "默认配置" "$DEFAULT_OV_CONF_PATH"+ printf "\n %b?%b 请选择 OpenViking 配置文件\n" "$UI_YELLOW" "$UI_RESET"+ printf " %b直接回车使用默认路径%b\n" "$UI_DIM" "$UI_RESET"+ printf " %b>%b " "$UI_GREEN" "$UI_RESET"if ! read -r OV_CONF_PATH < /dev/tty; thenOV_CONF_PATH="$DEFAULT_OV_CONF_PATH"fi@@ -83,14 +147,16 @@ elif [[ "$OV_CONF_PATH" == ~/* ]]; thenfiexport OPENVIKING_CONFIG_FILE="$OV_CONF_PATH"-echo "[preflight] 本次使用 ov.conf: $OPENVIKING_CONFIG_FILE"+printf "\n"+ui_kv "本次配置" "$OPENVIKING_CONFIG_FILE"+ui_info "正在检查本地配置…"# 评测前预检配置PRECHECK_STATUS=0"$PYTHON_BIN" "$SCRIPT_DIR/preflight_eval_config.py" || PRECHECK_STATUS=$?if [ "$PRECHECK_STATUS" -ne 0 ]; thenif [ "$PRECHECK_STATUS" -eq 2 ]; then- echo "[preflight] 已完成 OpenViking API key 初始化,请重新执行评测脚本。" >&2+ ui_warn "已完成 OpenViking API key 初始化,请重新执行评测脚本。"fiexit "$PRECHECK_STATUS"fi@@ -107,6 +173,7 @@ fiINTERACTIVE="$INTERACTIVE" "$PYTHON_BIN" "$SCRIPT_DIR/preflight_eval_runtime.py" --output-env-file "$RUNTIME_ENV_FILE"# shellcheck disable=SC1090source "$RUNTIME_ENV_FILE"+ui_success "环境预检完成"# 解析参数PREV_ARG=""@@ -121,6 +188,16 @@ for arg in "$@"; doPREV_ARG=""continuefi+ if [ "$PREV_ARG" = "--parallel-run-eval" ]; then+ PARALLEL_RUN_EVAL="$arg"+ PREV_ARG=""+ continue+ fi+ if [ "$PREV_ARG" = "--parallel-judge" ]; then+ PARALLEL_JUDGE="$arg"+ PREV_ARG=""+ continue+ fiif [ "$arg" = "--skip-import" ]; thenSKIP_IMPORT=trueelif [ "$arg" = "--group-chat" ]; then@@ -135,11 +212,17 @@ for arg in "$@"; doelif [ "$arg" = "--parallel-import-sessions" ]; thenPREV_ARG="$arg"continue+ elif [ "$arg" = "--parallel-run-eval" ]; then+ PREV_ARG="$arg"+ continue+ elif [ "$arg" = "--parallel-judge" ]; then+ PREV_ARG="$arg"+ continuefiPREV_ARG=""doneif [ -n "$PREV_ARG" ]; then- echo "Error: $PREV_ARG requires a value" >&2+ ui_error "$PREV_ARG requires a value"exit 1fi@@ -151,7 +234,7 @@ for arg in "$@"; doSKIP_NEXT=falsecontinuefi- if [ "$arg" = "--retry-wrong" ] || [ "$arg" = "--parallel-import-sessions" ]; then+ if [ "$arg" = "--retry-wrong" ] || [ "$arg" = "--parallel-import-sessions" ] || [ "$arg" = "--parallel-run-eval" ] || [ "$arg" = "--parallel-judge" ]; thenSKIP_NEXT=truecontinuefi@@ -177,28 +260,47 @@ if [ -n "${OPENVIKING_API_KEY:-}" ]; thenfiif [ -n "${PARALLEL_IMPORT_SESSIONS:-}" ]; thenif ! [[ "$PARALLEL_IMPORT_SESSIONS" =~ ^[1-9][0-9]*$ ]]; then- echo "Error: --parallel-import-sessions requires a positive integer" >&2+ ui_error "--parallel-import-sessions requires a positive integer"exit 1fiIMPORT_OPTS+=("--parallel-sessions" "$PARALLEL_IMPORT_SESSIONS")- echo "[import] 单 sample 内 session 并发已开启: $PARALLEL_IMPORT_SESSIONS"fi+if ! [[ "$PARALLEL_RUN_EVAL" =~ ^[1-9][0-9]*$ ]]; then+ ui_error "--parallel-run-eval requires a positive integer"+ exit 1+fi+if ! [[ "$PARALLEL_JUDGE" =~ ^[1-9][0-9]*$ ]]; then+ ui_error "--parallel-judge requires a positive integer"+ exit 1+fi+RUN_EVAL_OPTS=("--threads" "$PARALLEL_RUN_EVAL")+JUDGE_OPTS=("--parallel" "$PARALLEL_JUDGE")SAMPLE=${ARGS[0]}QUESTION_INDEX=${ARGS[1]}INPUT_FILE="$SCRIPT_DIR/../data/locomo10.json"+ui_section "2. 运行配置"+ui_kv "配置文件" "$OPENVIKING_CONFIG_FILE"+ui_kv "OpenViking" "$OPENVIKING_URL"+ui_kv "运行身份" "account=$ACCOUNT · user=$OPENVIKING_USER · auth=$OPENVIKING_AUTH_MODE"+ui_kv "会话模式" "$([ "$GROUP_CHAT" = "true" ] && printf '群聊' || printf '非群聊')"+ui_kv "导入并发" "$PARALLEL_IMPORT_SESSIONS sessions"+ui_kv "评测并发" "$PARALLEL_RUN_EVAL threads"+ui_kv "裁判并发" "$PARALLEL_JUDGE requests"+ui_kv "导入策略" "$([ "$SKIP_IMPORT" = "true" ] && printf '跳过导入' || printf '强制导入')"+# Export for inline Python usageexport SCRIPT_DIR INPUT_FILE RETRY_WRONG PARALLEL_IMPORT_SESSIONS ACCOUNT OPENVIKING_URL OPENVIKING_API_KEY OPENVIKING_USER OPENVIKING_AUTH_MODE GROUP_CHAT# auto-commit 逻辑if [ "$AUTO_COMMIT" = "true" ]; thenif [ -n "$(git status --porcelain)" ]; then- echo "[auto-commit] 检测到未提交变更,正在提交..."+ ui_info "检测到未提交变更,正在自动提交…"git add -Agit commit -m "auto-commit before eval $(date +%Y%m%d_%H%M%S)"else- echo "[auto-commit] 工作区干净,无需提交"+ ui_success "工作区干净,无需自动提交"fifiGIT_COMMIT_ID=$(git rev-parse --short HEAD)@@ -232,7 +334,7 @@ print_import_summary_table() {returnfi- echo ""+ ui_section "导入摘要"IMPORT_SUCCESS_CSV="$IMPORT_SUCCESS_CSV" IMPORT_ROW_START="$IMPORT_ROW_START" "$PYTHON_BIN" - <<'PY'import csvimport os@@ -276,7 +378,6 @@ def render_table(headers: list[str], rows: list[list[str]], align_right: set[intpath = Path(os.environ["IMPORT_SUCCESS_CSV"])start = int(os.environ.get("IMPORT_ROW_START", "0"))-print("=== Import Summary ===")if not path.exists():print("No import success CSV found.")raise SystemExit(0)@@ -329,18 +430,18 @@ prepare_bot_log_dir() {local base="${output_file%.csv}"export LOCOMO_VIKINGBOT_LOG_DIR="${base}_bot_logs"mkdir -p "$LOCOMO_VIKINGBOT_LOG_DIR"- echo "[eval] vikingbot logs: $LOCOMO_VIKINGBOT_LOG_DIR"+ ui_kv "VikingBot 日志" "$LOCOMO_VIKINGBOT_LOG_DIR"}# ========== 重跑错题模式(优先) ==========if [ -n "$RETRY_WRONG" ]; thenif [ ! -f "$RETRY_WRONG" ]; then- echo "Error: --retry-wrong file not found: $RETRY_WRONG" >&2+ ui_error "--retry-wrong file not found: $RETRY_WRONG"exit 1fi- echo "=== 重跑错题模式 ==="- echo "源文件: $RETRY_WRONG"+ ui_section "3. 执行评测 · 错题重跑"+ ui_kv "错题文件" "$RETRY_WRONG"if [ "$AUTO_COMMIT" = "true" ]; thenRESULT_FILE="./result/locomo/locomo_retry_${TIMESTAMP}_${GIT_COMMIT_ID}.csv"@@ -349,7 +450,7 @@ if [ -n "$RETRY_WRONG" ]; thenfi# 从错题 CSV 中提取需要导入的对话(复用 import_to_ov.py 的并行逻辑)- echo "[1/3] 导入错题相关对话..."+ ui_step 1 3 "导入错题相关对话"capture_import_row_start"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \--input "$INPUT_FILE" \@@ -361,37 +462,37 @@ if [ -n "$RETRY_WRONG" ]; then"${COMMON_OPTS[@]}"IMPORT_PERFORMED=true- echo "等待数据处理完成..."+ ui_info "等待数据处理完成(30 秒)…"sleep 30# 评估错题- echo "[2/3] 重新评估错题..."+ ui_step 2 3 "重新评估错题"prepare_bot_log_dir "$RESULT_FILE""$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" \"$INPUT_FILE" \--output "$RESULT_FILE" \--retry-wrong "$RETRY_WRONG" \- --threads 100 \--config "$OPENVIKING_CONFIG_FILE" \+ "${RUN_EVAL_OPTS[@]}" \"${COMMON_OPTS[@]}"# 裁判打分- echo "[3/3] 裁判打分..."- "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$RESULT_FILE" --parallel 100+ ui_step 3 3 "裁判打分"+ "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$RESULT_FILE" "${JUDGE_OPTS[@]}"# 统计结果"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$RESULT_FILE"print_import_summary_table- echo ""- echo "=== 错题重跑完成 ==="- echo "结果文件: $RESULT_FILE"+ ui_section "完成"+ ui_success "错题重跑完成"+ ui_kv "结果文件" "$RESULT_FILE"exit 0fi# ========== 全量评测模式 ==========if [ -z "$SAMPLE" ]; then- echo "=== 全量评测模式 ==="+ ui_section "3. 执行评测 · 全量模式"if [ "$AUTO_COMMIT" = "true" ]; thenRESULT_FILE="./result/locomo/locomo_result_${TIMESTAMP}_${GIT_COMMIT_ID}.csv"@@ -401,33 +502,45 @@ if [ -z "$SAMPLE" ]; then# 导入数据if [ "$SKIP_IMPORT" = "true" ]; then- echo "[1/4] 跳过导入数据..."+ ui_warn "已通过 --skip-import 跳过导入数据"else- echo "[1/4] 导入数据..."+ ui_step 1 4 "导入数据"capture_import_row_start"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" --input "$INPUT_FILE" --force-ingest --account "$ACCOUNT" --openviking-url "$OPENVIKING_URL" "${IMPORT_OPTS[@]}" "${COMMON_OPTS[@]}"IMPORT_PERFORMED=true- echo "等待 1 分钟..."+ ui_info "等待数据处理完成(60 秒)…"sleep 60fi# 评估- echo "[2/4] 评估..."+ if [ "$SKIP_IMPORT" = "true" ]; then+ ui_step 1 3 "运行评估"+ else+ ui_step 2 4 "运行评估"+ fiprepare_bot_log_dir "$RESULT_FILE"- "$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" "$INPUT_FILE" --output "$RESULT_FILE" --config "$OPENVIKING_CONFIG_FILE" "${COMMON_OPTS[@]}"+ "$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" "$INPUT_FILE" --output "$RESULT_FILE" --config "$OPENVIKING_CONFIG_FILE" "${RUN_EVAL_OPTS[@]}" "${COMMON_OPTS[@]}"# 裁判打分- echo "[3/4] 裁判打分..."- "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$RESULT_FILE" --parallel 100+ if [ "$SKIP_IMPORT" = "true" ]; then+ ui_step 2 3 "裁判打分"+ else+ ui_step 3 4 "裁判打分"+ fi+ "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$RESULT_FILE" "${JUDGE_OPTS[@]}"# 计算结果- echo "[4/4] 计算结果..."+ if [ "$SKIP_IMPORT" = "true" ]; then+ ui_step 3 3 "汇总结果"+ else+ ui_step 4 4 "汇总结果"+ fi"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$RESULT_FILE"print_import_summary_table- echo ""- echo "=== 全量评测完成 ==="- echo "结果文件: $RESULT_FILE"+ ui_section "完成"+ ui_success "全量评测完成"+ ui_kv "结果文件" "$RESULT_FILE"exit 0fi@@ -436,7 +549,7 @@ fiif [[ "$SAMPLE" =~ ^-?[0-9]+$ ]]; thenSAMPLE_INDEX=$SAMPLESAMPLE_ID_FOR_CMD=$SAMPLE_INDEX- echo "Using sample index: $SAMPLE_INDEX"+ ui_kv "Sample" "index=$SAMPLE_INDEX"elseSAMPLE_INDEX=$(SAMPLE="$SAMPLE" INPUT_FILE="$INPUT_FILE" "$PYTHON_BIN" - <<'PY'import json@@ -457,23 +570,24 @@ else:PY)if [ "$SAMPLE_INDEX" = "NOT_FOUND" ]; then- echo "Error: sample_id '$SAMPLE' not found"+ ui_error "sample_id '$SAMPLE' not found"exit 1fiSAMPLE_ID_FOR_CMD=$SAMPLE- echo "Using sample_id: $SAMPLE (index: $SAMPLE_INDEX)"+ ui_kv "Sample" "id=$SAMPLE · index=$SAMPLE_INDEX"fi# 判断是单题模式还是批量模式if [ -n "$QUESTION_INDEX" ]; then# ========== 单题模式 ==========- echo "=== 单题模式: sample $SAMPLE, question $QUESTION_INDEX ==="+ ui_section "3. 执行评测 · 单题模式"+ ui_kv "评测范围" "sample=$SAMPLE · question=$QUESTION_INDEX"# 导入对话if [ "$SKIP_IMPORT" = "true" ]; then- echo "[1/3] Skipping import (--skip-import)"+ ui_warn "已通过 --skip-import 跳过导入对话"else- echo "[1/3] Importing sample $SAMPLE_INDEX, question $QUESTION_INDEX..."+ ui_step 1 3 "导入 sample $SAMPLE_INDEX · question $QUESTION_INDEX"capture_import_row_start"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \--input "$INPUT_FILE" \@@ -486,15 +600,15 @@ if [ -n "$QUESTION_INDEX" ]; then"${COMMON_OPTS[@]}"IMPORT_PERFORMED=true- echo "Waiting for data processing..."+ ui_info "等待数据处理完成(3 秒)…"sleep 3fi# 运行评测if [ "$SKIP_IMPORT" = "true" ]; then- echo "[1/2] Running evaluation (skip-import mode)..."+ ui_step 1 2 "运行评估"else- echo "[2/3] Running evaluation..."+ ui_step 2 3 "运行评估"fiif [ "$AUTO_COMMIT" = "true" ]; thenOUTPUT_FILE=./result/locomo/locomo_${SAMPLE}_${QUESTION_INDEX}_result_${TIMESTAMP}_${GIT_COMMIT_ID}.csv@@ -509,19 +623,19 @@ if [ -n "$QUESTION_INDEX" ]; then--count 1 \--output "$OUTPUT_FILE" \--config "$OPENVIKING_CONFIG_FILE" \+ "${RUN_EVAL_OPTS[@]}" \"${COMMON_OPTS[@]}"# 运行 Judge 评分if [ "$SKIP_IMPORT" = "true" ]; then- echo "[2/2] Running judge..."+ ui_step 2 2 "裁判打分"else- echo "[3/3] Running judge..."+ ui_step 3 3 "裁判打分"fi- "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$OUTPUT_FILE" --parallel 100+ "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$OUTPUT_FILE" "${JUDGE_OPTS[@]}"# 输出结果- echo ""- echo "=== 评测结果 ==="+ ui_section "评测结果"print_import_summary_tableOUTPUT_FILE="$OUTPUT_FILE" QUESTION_INDEX="$QUESTION_INDEX" "$PYTHON_BIN" - <<'PY'import csv@@ -557,7 +671,8 @@ PYelse# ========== 批量模式 ==========- echo "=== 批量模式: sample $SAMPLE, 所有问题 ==="+ ui_section "3. 执行评测 · Sample 批量模式"+ ui_kv "评测范围" "sample=$SAMPLE · 所有问题"# 获取该 sample 的问题数量QUESTION_COUNT=$(SAMPLE_INDEX="$SAMPLE_INDEX" INPUT_FILE="$INPUT_FILE" "$PYTHON_BIN" - <<'PY'@@ -574,13 +689,13 @@ sample = data[sample_index]print(len(sample.get("qa", [])))PY)- echo "Found $QUESTION_COUNT questions for sample $SAMPLE"+ ui_kv "问题数量" "$QUESTION_COUNT"# 导入所有 sessionsif [ "$SKIP_IMPORT" = "true" ]; then- echo "[1/4] Skipping import (--skip-import)"+ ui_warn "已通过 --skip-import 跳过导入所有 Sessions"else- echo "[1/4] Importing all sessions for sample $SAMPLE_INDEX..."+ ui_step 1 4 "导入 sample $SAMPLE_INDEX 的所有 Sessions"capture_import_row_start"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \--input "$INPUT_FILE" \@@ -592,15 +707,15 @@ PY"${COMMON_OPTS[@]}"IMPORT_PERFORMED=true- echo "Waiting for data processing..."+ ui_info "等待数据处理完成(10 秒)…"sleep 10fi# 运行评测(所有问题)if [ "$SKIP_IMPORT" = "true" ]; then- echo "[1/3] Running evaluation for all questions (skip-import mode)..."+ ui_step 1 3 "评估所有问题"else- echo "[2/4] Running evaluation for all questions..."+ ui_step 2 4 "评估所有问题"fiif [ "$AUTO_COMMIT" = "true" ]; thenOUTPUT_FILE=./result/locomo/locomo_${SAMPLE}_result_${TIMESTAMP}_${GIT_COMMIT_ID}.csv@@ -612,28 +727,28 @@ PY"$INPUT_FILE" \--sample "$SAMPLE_ID_FOR_CMD" \--output "$OUTPUT_FILE" \- --threads 100 \--config "$OPENVIKING_CONFIG_FILE" \+ "${RUN_EVAL_OPTS[@]}" \"${COMMON_OPTS[@]}"# 运行 Judge 评分if [ "$SKIP_IMPORT" = "true" ]; then- echo "[2/3] Running judge..."+ ui_step 2 3 "裁判打分"else- echo "[3/4] Running judge..."+ ui_step 3 4 "裁判打分"fi- "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$OUTPUT_FILE" --parallel 100+ "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$OUTPUT_FILE" "${JUDGE_OPTS[@]}"# 输出统计结果if [ "$SKIP_IMPORT" = "true" ]; then- echo "[3/3] Calculating statistics..."+ ui_step 3 3 "汇总结果"else- echo "[4/4] Calculating statistics..."+ ui_step 4 4 "汇总结果"fi"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$OUTPUT_FILE"print_import_summary_table- echo ""- echo "=== 批量评测完成 ==="- echo "结果文件: $OUTPUT_FILE"+ ui_section "完成"+ ui_success "批量评测完成"+ ui_kv "结果文件" "$OUTPUT_FILE"fidiff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.pyindex 65b5de41f..05da60d8a 100644--- a/openviking/models/vlm/backends/litellm_vlm.py+++ b/openviking/models/vlm/backends/litellm_vlm.py@@ -352,6 +352,9 @@ class LiteLLMVLMProvider(VLMBase):def _build_vlm_response(self, response, has_tools: bool) -> Union[str, VLMResponse]:"""Build response from LiteLLM response. Returns str or VLMResponse based on has_tools."""+ if isinstance(response, str):+ return VLMResponse(content=response) if has_tools else response+choice = response.choices[0]message = choice.messagediff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.pyindex 3d8a54f8e..e06de8fbe 100644--- a/openviking/models/vlm/backends/openai_vlm.py+++ b/openviking/models/vlm/backends/openai_vlm.py@@ -191,6 +191,9 @@ class OpenAIVLM(VLMBase):def _build_vlm_response(self, response, has_tools: bool) -> Union[str, VLMResponse]:"""Build response from OpenAI response. Returns str or VLMResponse based on has_tools."""+ if isinstance(response, str):+ return VLMResponse(content=response) if has_tools else response+choice = response.choices[0]message = choice.messagetracer.info(f"result={message.content}")diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.pyindex 076631c59..485b50b06 100644--- a/openviking/models/vlm/backends/volcengine_vlm.py+++ b/openviking/models/vlm/backends/volcengine_vlm.py@@ -58,6 +58,9 @@ class VolcEngineVLM(OpenAIVLM):def _build_vlm_response(self, response, has_tools: bool) -> Union[str, VLMResponse]:"""Build response from Chat Completions response. Returns str or VLMResponse based on has_tools."""+ if isinstance(response, str):+ return VLMResponse(content=response) if has_tools else response+choice = response.choices[0]message = choice.messageif hasattr(message, "tool_calls") and message.tool_calls:diff --git a/tests/unit/test_vikingbot_vlm_adapter_retry.py b/tests/unit/test_vikingbot_vlm_adapter_retry.pyindex 6350d84b2..90bc362a6 100644--- a/tests/unit/test_vikingbot_vlm_adapter_retry.py+++ b/tests/unit/test_vikingbot_vlm_adapter_retry.py@@ -6,9 +6,8 @@ import vikingbot.providers.vlm_adapter as vlm_adapterfrom vikingbot.providers.vlm_adapter import VLMProviderAdapterfrom volcenginesdkarkruntime._exceptions import ArkRateLimitError-from openviking.utils.model_retry import (- is_retryable_rate_limit_error,-)+from openviking.models.vlm.backends.openai_vlm import OpenAIVLM+from openviking.utils.model_retry import is_retryable_rate_limit_errorclass _DisabledLangfuse:@@ -112,6 +111,37 @@ async def test_chat_does_not_retry_errors_without_rate_limit_markers(monkeypatchassert fake_vlm.calls == 1+async def test_chat_accepts_string_response_from_openai_backend_with_tools(monkeypatch):+ async def create(**_kwargs):+ return "plain string response"++ client = SimpleNamespace(+ chat=SimpleNamespace(completions=SimpleNamespace(create=create)),+ )+ vlm = OpenAIVLM({"provider": "openai", "model": "gpt-5.6-terra"})+ monkeypatch.setattr(vlm, "get_async_client", lambda: client)+ adapter = VLMProviderAdapter(vlm, "gpt-5.6-terra", langfuse_client=_DisabledLangfuse())++ response = await adapter.chat(+ messages=[{"role": "user", "content": "hello"}],+ tools=[+ {+ "type": "function",+ "function": {+ "name": "test_tool",+ "description": "A test tool",+ "parameters": {"type": "object", "properties": {}},+ },+ }+ ],+ )++ assert response.content == "plain string response"+ assert response.tool_calls == []+ assert response.finish_reason == "stop"++@pytest.mark.asyncioasync def test_chat_stream_retries_rate_limit_until_success(monkeypatch):sleep_delays: list[float] = []diff --git a/tests/unit/test_vlm_response_formats.py b/tests/unit/test_vlm_response_formats.pyindex 3d0eaa360..8ff35ef58 100644--- a/tests/unit/test_vlm_response_formats.py+++ b/tests/unit/test_vlm_response_formats.py@@ -6,7 +6,10 @@ from types import SimpleNamespaceimport pytest-from openviking.models.vlm.base import VLMBase+from openviking.models.vlm.backends.litellm_vlm import LiteLLMVLMProvider+from openviking.models.vlm.backends.openai_vlm import OpenAIVLM+from openviking.models.vlm.backends.volcengine_vlm import VolcEngineVLM+from openviking.models.vlm.base import VLMBase, VLMResponseclass TestVLMBaseResponseFormats:@@ -57,3 +60,54 @@ class TestVLMBaseResponseFormats:choices=[SimpleNamespace(message=SimpleNamespace(content="standard response content"))])assert vlm._extract_content_from_response(response) == "standard response content"+++ ("vlm_type", "config"),+ [+ (OpenAIVLM, {"provider": "openai", "model": "gpt-5.6-terra"}),+ (LiteLLMVLMProvider, {"provider": "litellm", "model": "gpt-5.6-terra"}),+ (VolcEngineVLM, {"provider": "volcengine", "model": "test-model"}),+ ],+ ids=["openai", "litellm", "volcengine"],+)+def test_build_vlm_response_from_str_with_tools(vlm_type, config):+ vlm = vlm_type(config)+ response = vlm._build_vlm_response("plain string response", has_tools=True)++ assert isinstance(response, VLMResponse)+ assert response.content == "plain string response"+ assert response.tool_calls == []+ assert response.finish_reason == "stop"+ assert response.usage == {}+++async def test_openai_async_completion_from_str_with_tools(monkeypatch):+ async def create(**_kwargs):+ return "plain string response"++ client = SimpleNamespace(+ chat=SimpleNamespace(completions=SimpleNamespace(create=create)),+ )+ vlm = OpenAIVLM({"provider": "openai", "model": "gpt-5.6-terra"})+ monkeypatch.setattr(vlm, "get_async_client", lambda: client)++ response = await vlm.get_completion_async(+ messages=[{"role": "user", "content": "hello"}],+ tools=[+ {+ "type": "function",+ "function": {+ "name": "test_tool",+ "description": "A test tool",+ "parameters": {"type": "object", "properties": {}},+ },+ }+ ],+ )++ assert isinstance(response, VLMResponse)+ assert response.content == "plain string response"+ assert response.tool_calls == []+ assert response.finish_reason == "stop"