Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

1. 优化评测参数化; (#3332)

DuTao55a9d12cd6d6
diff --git a/benchmark/locomo/vikingbot/preflight_eval_config.py b/benchmark/locomo/vikingbot/preflight_eval_config.py
index 90c805d6f..f6dc0f083 100644
--- a/benchmark/locomo/vikingbot/preflight_eval_config.py
+++ b/benchmark/locomo/vikingbot/preflight_eval_config.py
@@ -11,7 +11,12 @@ from openviking_cli.utils.config.consts import (
OPENVIKING_CONFIG_ENV,
)
-_USE_COLOR = hasattr(sys.stdout, "isatty") and sys.stdout.isatty()
+_USE_COLOR = (
+ hasattr(sys.stdout, "isatty")
+ and sys.stdout.isatty()
+ and os.environ.get("TERM", "dumb") != "dumb"
+ and "NO_COLOR" not in os.environ
+)
def _color(text: str, code: str) -> str:
@@ -20,24 +25,20 @@ def _color(text: str, code: str) -> str:
return f"\033[{code}m{text}\033[0m"
-def _prefix() -> str:
- return _color("[preflight]", "36")
-
-
def _log_info(message: str) -> None:
- print(f"{_prefix()} {_color('[INFO]', '34')} {message}")
+ print(_color(f" │ {message}", "2"))
def _log_warn(message: str) -> None:
- print(f"{_prefix()} {_color('[WARN]', '33')} {message}")
+ print(f" {_color('!', '33')} {message}")
def _log_ok(message: str) -> None:
- print(f"{_prefix()} {_color('[OK]', '32')} {message}")
+ print(f" {_color('✓', '32')} {message}")
def _log_error(message: str) -> None:
- print(f"{_prefix()} {_color('[ERROR]', '31')} {message}", file=sys.stderr)
+ print(f" {_color('✗', '31')} {message}", file=sys.stderr)
def _is_interactive() -> bool:
@@ -45,8 +46,10 @@ def _is_interactive() -> bool:
def _prompt_text(prompt: str, default: str | None = None) -> str:
- suffix = f" [{default}]" if default else ""
- raw = input(f"{_prefix()} {prompt}{suffix}: ").strip()
+ print(f"\n {_color('?', '33')} {prompt}")
+ if default:
+ print(f" {_color('默认:', '2')} {default}")
+ raw = input(f" {_color('>', '32')} ").strip()
if not raw and default is not None:
return default
return raw
diff --git a/benchmark/locomo/vikingbot/preflight_eval_runtime.py b/benchmark/locomo/vikingbot/preflight_eval_runtime.py
index 31c255545..0efdf94a5 100644
--- a/benchmark/locomo/vikingbot/preflight_eval_runtime.py
+++ b/benchmark/locomo/vikingbot/preflight_eval_runtime.py
@@ -1,6 +1,7 @@
#!/usr/bin/env python3
import argparse
+import getpass
import json
import os
import shlex
@@ -18,13 +19,38 @@ from openviking_cli.utils.config.consts import (
OPENVIKING_CONFIG_ENV,
)
+_USE_COLOR = (
+ hasattr(sys.stdout, "isatty")
+ and sys.stdout.isatty()
+ and os.environ.get("TERM", "dumb") != "dumb"
+ and "NO_COLOR" not in os.environ
+)
+
+
+def _color(text: str, code: str) -> str:
+ if not _USE_COLOR:
+ return text
+ return f"\033[{code}m{text}\033[0m"
+
def _log(message: str) -> None:
- print(f"[preflight] {message}")
+ print(_color(f" │ {message}", "2"))
+
+
+def _fact(message: str) -> None:
+ print(f" {_color('◆', '36')} {message}")
+
+
+def _ok(message: str) -> None:
+ print(f" {_color('✓', '32')} {message}")
+
+
+def _warn(message: str) -> None:
+ print(f" {_color('!', '33')} {message}")
def _error(message: str) -> None:
- print(f"[preflight] {message}", file=sys.stderr)
+ print(f" {_color('✗', '31')} {message}", file=sys.stderr)
class UserKeyValidationError(RuntimeError):
@@ -44,21 +70,30 @@ def _is_interactive() -> bool:
def _prompt_text(prompt: str) -> str:
+ print(f"\n {_color('?', '33')} {prompt}")
try:
with open("/dev/tty", "r", encoding="utf-8") as tty_in:
- print(prompt, end="", flush=True)
+ print(f" {_color('>', '32')} ", end="", flush=True)
return tty_in.readline().strip()
except Exception:
- return input(prompt).strip()
+ return input(f" {_color('>', '32')} ").strip()
+
+
+def _prompt_secret(prompt: str) -> str:
+ print(f"\n {_color('?', '33')} {prompt}")
+ try:
+ return getpass.getpass(f" {_color('>', '32')} ").strip()
+ except Exception:
+ return _prompt_text("请输入密钥").strip()
def _prompt_api_key() -> str:
- return _prompt_text("[preflight] 请输入 OpenViking User API key: ")
+ return _prompt_secret("请输入 OpenViking User API key(输入内容不会显示)")
def _prompt_root_api_key() -> str:
- return _prompt_text(
- "[preflight] 请输入 OpenViking Root API key,用于自动创建 default User key: "
+ return _prompt_secret(
+ "请输入 OpenViking Root API key,用于自动创建 default User key(输入内容不会显示)"
)
@@ -66,7 +101,7 @@ def _prompt_yes_no(prompt: str, default: bool = False) -> bool:
if not _is_interactive():
return False
suffix = "Y/n" if default else "y/N"
- answer = _prompt_text(f"{prompt} [{suffix}]: ").strip().lower()
+ answer = _prompt_text(f"{prompt} [{suffix}]").strip().lower()
if not answer:
return default
return answer in {"y", "yes", "1", "true"}
@@ -170,7 +205,7 @@ def _sync_bot_identity(account_id: str, user_id: str, api_key: str, *, auth_mode
changed = True
if changed:
_write_json_with_backup(path, ov_data)
- _log(
+ _ok(
"已同步 bot.ov_server: "
f"api_key_type={desired['api_key_type']}, account_id={account_id}, admin_user_id={user_id}"
)
@@ -337,7 +372,7 @@ def _ensure_default_user_key(url: str, root_api_key: str) -> tuple[str, str, str
item.get("account_id") == account_id for item in accounts if isinstance(item, dict)
)
if not account_exists:
- _log("default account 不存在,将使用 Root key 创建 account=default。")
+ _warn("default account 不存在,将使用 Root key 创建 account=default。")
_admin_request(
url,
root_api_key,
@@ -360,7 +395,7 @@ def _ensure_default_user_key(url: str, root_api_key: str) -> tuple[str, str, str
None,
)
if default_user is None:
- _log("default User 不存在,将注册 account=default, user=default, role=user。")
+ _warn("default User 不存在,将注册 account=default, user=default, role=user。")
created = _admin_request(
url,
root_api_key,
@@ -372,7 +407,7 @@ def _ensure_default_user_key(url: str, root_api_key: str) -> tuple[str, str, str
else:
role = str(default_user.get("role") or "").strip().lower()
if role != "user":
- _log(f"default User 当前 role={role or 'unknown'},将调整为 role=user。")
+ _warn(f"default User 当前 role={role or 'unknown'},将调整为 role=user。")
_admin_request(
url,
root_api_key,
@@ -403,13 +438,13 @@ def _ensure_server_and_user_key_ready(
raise UserKeyValidationError(f"当前 API key 解析为 role={role}。评测需要普通 User key。")
if selected_account and selected_account != "default" and selected_account != account_id:
- _log(
+ _warn(
f"ovcli.conf.account={selected_account} "
f"与 API key 归属 account={account_id} 不一致;"
"本次评测使用 API key 归属 account。"
)
- _log(
+ _ok(
"OpenViking server 可用,User key 身份: "
f"account={account_id}, user={user_id}, role={role}。"
)
@@ -422,9 +457,11 @@ def _ensure_trusted_server_ready(
payload = _request_health(url, api_key, account_id=account_id, user_id=user_id)
health_account, health_user, role = _parse_health_identity_payload(payload)
if role != "user":
- raise UserKeyValidationError(f"当前 trusted 身份解析为 role={role}。评测需要普通 User 身份。")
+ raise UserKeyValidationError(
+ f"当前 trusted 身份解析为 role={role}。评测需要普通 User 身份。"
+ )
- _log(
+ _ok(
"OpenViking server 可用,trusted 身份: "
f"account={health_account}, user={health_user}, role={role}。"
)
@@ -465,12 +502,10 @@ def _resolve_ready_user_identity(
return account, user_id, api_key, "api_key"
except UserKeyValidationError as exc:
_error(str(exc))
- prompt = (
- "[preflight] 当前 User key 不可用,是否使用 Root key 自动生成 default User API key"
- )
+ prompt = "当前 User key 不可用,是否使用 Root key 自动生成 default User API key"
else:
_error("未配置 OpenViking API key。")
- prompt = "[preflight] 是否使用 Root key 自动生成 default User API key"
+ prompt = "是否使用 Root key 自动生成 default User API key"
if not _prompt_yes_no(prompt, default=False):
_error("请配置可用的 bot.ov_server.api_key 或 ovcli.conf.api_key 后重试。")
@@ -493,7 +528,7 @@ def _resolve_ready_user_identity(
except UserKeyValidationError as exc:
_error(str(exc))
raise SystemExit(1) from exc
- _log("已生成可用的 default User API key。")
+ _ok("已生成可用的 default User API key。")
return checked_account or account, checked_user_id or user_id, user_key, "api_key"
@@ -523,13 +558,15 @@ def main() -> int:
openviking_url = _resolve_openviking_url()
api_key, key_source = _resolve_openviking_api_key()
- _log(f"本次导入使用 OpenViking URL: {openviking_url}")
+ _fact(f"本次导入使用 OpenViking URL: {openviking_url}")
account, user_id, api_key, auth_mode = _resolve_ready_user_identity(
openviking_url, selected_account, selected_user, api_key, key_source
)
_sync_bot_identity(account, user_id, api_key, auth_mode=auth_mode)
- _write_env_file(Path(args.output_env_file), account, openviking_url, api_key, user_id, auth_mode)
+ _write_env_file(
+ Path(args.output_env_file), account, openviking_url, api_key, user_id, auth_mode
+ )
return 0
diff --git a/benchmark/locomo/vikingbot/run_full_eval.sh b/benchmark/locomo/vikingbot/run_full_eval.sh
index 0f5191c4a..5d8aa3a80 100755
--- a/benchmark/locomo/vikingbot/run_full_eval.sh
+++ b/benchmark/locomo/vikingbot/run_full_eval.sh
@@ -12,9 +12,64 @@
# ./run_full_eval.sh --skip-import --auto-commit # 评测全部,跳过导入,自动提交
# ./run_full_eval.sh --retry-wrong result/locomo_result_xxx.csv # 只重跑错题
# ./run_full_eval.sh --parallel-import-sessions 20 0 1 # 覆盖默认 session 导入并发数
+# ./run_full_eval.sh --parallel-import-sessions 50 --parallel-run-eval 20 --parallel-judge 40 # 分别设置导入、评测和裁判并发数
set -e
+UI_RESET=""
+UI_BOLD=""
+UI_DIM=""
+UI_RED=""
+UI_GREEN=""
+UI_YELLOW=""
+UI_CYAN=""
+if [ -t 1 ] && [ "${TERM:-dumb}" != "dumb" ] && [ -z "${NO_COLOR+x}" ]; then
+ UI_RESET=$'\033[0m'
+ UI_BOLD=$'\033[1m'
+ UI_DIM=$'\033[2m'
+ UI_RED=$'\033[31m'
+ UI_GREEN=$'\033[32m'
+ UI_YELLOW=$'\033[33m'
+ UI_CYAN=$'\033[36m'
+fi
+
+ui_banner() {
+ printf "\n%b%s%b\n" "${UI_BOLD}${UI_CYAN}" "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" "$UI_RESET"
+ printf "%b %s%b\n" "${UI_BOLD}" "$1" "$UI_RESET"
+ printf "%b%s%b\n" "${UI_BOLD}${UI_CYAN}" "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━" "$UI_RESET"
+}
+
+ui_section() {
+ printf "\n%b%s%b\n" "${UI_BOLD}${UI_CYAN}" "━━ $1" "$UI_RESET"
+}
+
+ui_info() {
+ printf " %b│ %s%b\n" "$UI_DIM" "$1" "$UI_RESET"
+}
+
+ui_success() {
+ printf " %b✓%b %s\n" "$UI_GREEN" "$UI_RESET" "$1"
+}
+
+ui_warn() {
+ printf " %b!%b %s\n" "$UI_YELLOW" "$UI_RESET" "$1"
+}
+
+ui_error() {
+ printf " %b✗%b %s\n" "$UI_RED" "$UI_RESET" "$1" >&2
+}
+
+ui_kv() {
+ printf " %b%s:%b %s\n" "$UI_DIM" "$1" "$UI_RESET" "$2"
+}
+
+ui_step() {
+ local current="$1"
+ local total="$2"
+ local title="$3"
+ printf "\n %b▶ [%s/%s] %s%b\n" "${UI_BOLD}${UI_CYAN}" "$current" "$total" "$title" "$UI_RESET"
+}
+
# --help 提前处理,避免触发 Python preflight
for arg in "$@"; do
if [ "$arg" = "--help" ] || [ "$arg" = "-h" ]; then
@@ -31,7 +86,9 @@ for arg in "$@"; do
echo " --no-group-chat 非群聊模式(默认),使用 sample_id 作为 Peer"
echo " --auto-commit 自动提交未提交的代码变更,结果文件名带 commit id 和时间戳"
echo " --retry-wrong CSV 只重跑指定结果文件中的有效错题(导入相关对话+重新问答)"
- echo " --parallel-import-sessions N 单 sample 内并发导入 sessions(默认 200)"
+ echo " --parallel-import-sessions N 导入 session 并发数(默认 50)"
+ echo " --parallel-run-eval N run_eval 并发线程数(默认 100)"
+ echo " --parallel-judge N judge 并发请求数(默认 100)"
exit 0
fi
done
@@ -41,17 +98,22 @@ SKIP_IMPORT=false
GROUP_CHAT=false
AUTO_COMMIT=false
RETRY_WRONG=""
-PARALLEL_IMPORT_SESSIONS="200"
+PARALLEL_IMPORT_SESSIONS="50"
+PARALLEL_RUN_EVAL="100"
+PARALLEL_JUDGE="100"
if command -v python3 >/dev/null 2>&1; then
PYTHON_BIN="python3"
elif command -v python >/dev/null 2>&1; then
PYTHON_BIN="python"
else
- echo "未找到 python3/python,请先安装 Python。" >&2
+ ui_error "未找到 python3/python,请先安装 Python。"
exit 1
fi
+ui_banner "LoCoMo · VikingBot Evaluation"
+ui_section "1. 环境预检"
+
DEFAULT_OV_CONF_PATH="$($PYTHON_BIN - <<'PY'
from pathlib import Path
@@ -64,8 +126,10 @@ PY
)"
if [ -t 0 ] && [ -t 1 ]; then
- echo "[preflight] OpenViking 配置默认路径: $DEFAULT_OV_CONF_PATH"
- printf "[preflight] 直接回车使用默认,或输入新路径 [%s]: " "$DEFAULT_OV_CONF_PATH"
+ ui_kv "默认配置" "$DEFAULT_OV_CONF_PATH"
+ printf "\n %b?%b 请选择 OpenViking 配置文件\n" "$UI_YELLOW" "$UI_RESET"
+ printf " %b直接回车使用默认路径%b\n" "$UI_DIM" "$UI_RESET"
+ printf " %b>%b " "$UI_GREEN" "$UI_RESET"
if ! read -r OV_CONF_PATH < /dev/tty; then
OV_CONF_PATH="$DEFAULT_OV_CONF_PATH"
fi
@@ -83,14 +147,16 @@ elif [[ "$OV_CONF_PATH" == ~/* ]]; then
fi
export OPENVIKING_CONFIG_FILE="$OV_CONF_PATH"
-echo "[preflight] 本次使用 ov.conf: $OPENVIKING_CONFIG_FILE"
+printf "\n"
+ui_kv "本次配置" "$OPENVIKING_CONFIG_FILE"
+ui_info "正在检查本地配置…"
# 评测前预检配置
PRECHECK_STATUS=0
"$PYTHON_BIN" "$SCRIPT_DIR/preflight_eval_config.py" || PRECHECK_STATUS=$?
if [ "$PRECHECK_STATUS" -ne 0 ]; then
if [ "$PRECHECK_STATUS" -eq 2 ]; then
- echo "[preflight] 已完成 OpenViking API key 初始化,请重新执行评测脚本。" >&2
+ ui_warn "已完成 OpenViking API key 初始化,请重新执行评测脚本。"
fi
exit "$PRECHECK_STATUS"
fi
@@ -107,6 +173,7 @@ fi
INTERACTIVE="$INTERACTIVE" "$PYTHON_BIN" "$SCRIPT_DIR/preflight_eval_runtime.py" --output-env-file "$RUNTIME_ENV_FILE"
# shellcheck disable=SC1090
source "$RUNTIME_ENV_FILE"
+ui_success "环境预检完成"
# 解析参数
PREV_ARG=""
@@ -121,6 +188,16 @@ for arg in "$@"; do
PREV_ARG=""
continue
fi
+ if [ "$PREV_ARG" = "--parallel-run-eval" ]; then
+ PARALLEL_RUN_EVAL="$arg"
+ PREV_ARG=""
+ continue
+ fi
+ if [ "$PREV_ARG" = "--parallel-judge" ]; then
+ PARALLEL_JUDGE="$arg"
+ PREV_ARG=""
+ continue
+ fi
if [ "$arg" = "--skip-import" ]; then
SKIP_IMPORT=true
elif [ "$arg" = "--group-chat" ]; then
@@ -135,11 +212,17 @@ for arg in "$@"; do
elif [ "$arg" = "--parallel-import-sessions" ]; then
PREV_ARG="$arg"
continue
+ elif [ "$arg" = "--parallel-run-eval" ]; then
+ PREV_ARG="$arg"
+ continue
+ elif [ "$arg" = "--parallel-judge" ]; then
+ PREV_ARG="$arg"
+ continue
fi
PREV_ARG=""
done
if [ -n "$PREV_ARG" ]; then
- echo "Error: $PREV_ARG requires a value" >&2
+ ui_error "$PREV_ARG requires a value"
exit 1
fi
@@ -151,7 +234,7 @@ for arg in "$@"; do
SKIP_NEXT=false
continue
fi
- if [ "$arg" = "--retry-wrong" ] || [ "$arg" = "--parallel-import-sessions" ]; then
+ if [ "$arg" = "--retry-wrong" ] || [ "$arg" = "--parallel-import-sessions" ] || [ "$arg" = "--parallel-run-eval" ] || [ "$arg" = "--parallel-judge" ]; then
SKIP_NEXT=true
continue
fi
@@ -177,28 +260,47 @@ if [ -n "${OPENVIKING_API_KEY:-}" ]; then
fi
if [ -n "${PARALLEL_IMPORT_SESSIONS:-}" ]; then
if ! [[ "$PARALLEL_IMPORT_SESSIONS" =~ ^[1-9][0-9]*$ ]]; then
- echo "Error: --parallel-import-sessions requires a positive integer" >&2
+ ui_error "--parallel-import-sessions requires a positive integer"
exit 1
fi
IMPORT_OPTS+=("--parallel-sessions" "$PARALLEL_IMPORT_SESSIONS")
- echo "[import] 单 sample 内 session 并发已开启: $PARALLEL_IMPORT_SESSIONS"
fi
+if ! [[ "$PARALLEL_RUN_EVAL" =~ ^[1-9][0-9]*$ ]]; then
+ ui_error "--parallel-run-eval requires a positive integer"
+ exit 1
+fi
+if ! [[ "$PARALLEL_JUDGE" =~ ^[1-9][0-9]*$ ]]; then
+ ui_error "--parallel-judge requires a positive integer"
+ exit 1
+fi
+RUN_EVAL_OPTS=("--threads" "$PARALLEL_RUN_EVAL")
+JUDGE_OPTS=("--parallel" "$PARALLEL_JUDGE")
SAMPLE=${ARGS[0]}
QUESTION_INDEX=${ARGS[1]}
INPUT_FILE="$SCRIPT_DIR/../data/locomo10.json"
+ui_section "2. 运行配置"
+ui_kv "配置文件" "$OPENVIKING_CONFIG_FILE"
+ui_kv "OpenViking" "$OPENVIKING_URL"
+ui_kv "运行身份" "account=$ACCOUNT · user=$OPENVIKING_USER · auth=$OPENVIKING_AUTH_MODE"
+ui_kv "会话模式" "$([ "$GROUP_CHAT" = "true" ] && printf '群聊' || printf '非群聊')"
+ui_kv "导入并发" "$PARALLEL_IMPORT_SESSIONS sessions"
+ui_kv "评测并发" "$PARALLEL_RUN_EVAL threads"
+ui_kv "裁判并发" "$PARALLEL_JUDGE requests"
+ui_kv "导入策略" "$([ "$SKIP_IMPORT" = "true" ] && printf '跳过导入' || printf '强制导入')"
+
# Export for inline Python usage
export SCRIPT_DIR INPUT_FILE RETRY_WRONG PARALLEL_IMPORT_SESSIONS ACCOUNT OPENVIKING_URL OPENVIKING_API_KEY OPENVIKING_USER OPENVIKING_AUTH_MODE GROUP_CHAT
# auto-commit 逻辑
if [ "$AUTO_COMMIT" = "true" ]; then
if [ -n "$(git status --porcelain)" ]; then
- echo "[auto-commit] 检测到未提交变更,正在提交..."
+ ui_info "检测到未提交变更,正在自动提交…"
git add -A
git commit -m "auto-commit before eval $(date +%Y%m%d_%H%M%S)"
else
- echo "[auto-commit] 工作区干净,无需提交"
+ ui_success "工作区干净,无需自动提交"
fi
fi
GIT_COMMIT_ID=$(git rev-parse --short HEAD)
@@ -232,7 +334,7 @@ print_import_summary_table() {
return
fi
- echo ""
+ ui_section "导入摘要"
IMPORT_SUCCESS_CSV="$IMPORT_SUCCESS_CSV" IMPORT_ROW_START="$IMPORT_ROW_START" "$PYTHON_BIN" - <<'PY'
import csv
import os
@@ -276,7 +378,6 @@ def render_table(headers: list[str], rows: list[list[str]], align_right: set[int
path = Path(os.environ["IMPORT_SUCCESS_CSV"])
start = int(os.environ.get("IMPORT_ROW_START", "0"))
-print("=== Import Summary ===")
if not path.exists():
print("No import success CSV found.")
raise SystemExit(0)
@@ -329,18 +430,18 @@ prepare_bot_log_dir() {
local base="${output_file%.csv}"
export LOCOMO_VIKINGBOT_LOG_DIR="${base}_bot_logs"
mkdir -p "$LOCOMO_VIKINGBOT_LOG_DIR"
- echo "[eval] vikingbot logs: $LOCOMO_VIKINGBOT_LOG_DIR"
+ ui_kv "VikingBot 日志" "$LOCOMO_VIKINGBOT_LOG_DIR"
}
# ========== 重跑错题模式(优先) ==========
if [ -n "$RETRY_WRONG" ]; then
if [ ! -f "$RETRY_WRONG" ]; then
- echo "Error: --retry-wrong file not found: $RETRY_WRONG" >&2
+ ui_error "--retry-wrong file not found: $RETRY_WRONG"
exit 1
fi
- echo "=== 重跑错题模式 ==="
- echo "源文件: $RETRY_WRONG"
+ ui_section "3. 执行评测 · 错题重跑"
+ ui_kv "错题文件" "$RETRY_WRONG"
if [ "$AUTO_COMMIT" = "true" ]; then
RESULT_FILE="./result/locomo/locomo_retry_${TIMESTAMP}_${GIT_COMMIT_ID}.csv"
@@ -349,7 +450,7 @@ if [ -n "$RETRY_WRONG" ]; then
fi
# 从错题 CSV 中提取需要导入的对话(复用 import_to_ov.py 的并行逻辑)
- echo "[1/3] 导入错题相关对话..."
+ ui_step 1 3 "导入错题相关对话"
capture_import_row_start
"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \
--input "$INPUT_FILE" \
@@ -361,37 +462,37 @@ if [ -n "$RETRY_WRONG" ]; then
"${COMMON_OPTS[@]}"
IMPORT_PERFORMED=true
- echo "等待数据处理完成..."
+ ui_info "等待数据处理完成(30 秒)…"
sleep 30
# 评估错题
- echo "[2/3] 重新评估错题..."
+ ui_step 2 3 "重新评估错题"
prepare_bot_log_dir "$RESULT_FILE"
"$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" \
"$INPUT_FILE" \
--output "$RESULT_FILE" \
--retry-wrong "$RETRY_WRONG" \
- --threads 100 \
--config "$OPENVIKING_CONFIG_FILE" \
+ "${RUN_EVAL_OPTS[@]}" \
"${COMMON_OPTS[@]}"
# 裁判打分
- echo "[3/3] 裁判打分..."
- "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$RESULT_FILE" --parallel 100
+ ui_step 3 3 "裁判打分"
+ "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$RESULT_FILE" "${JUDGE_OPTS[@]}"
# 统计结果
"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$RESULT_FILE"
print_import_summary_table
- echo ""
- echo "=== 错题重跑完成 ==="
- echo "结果文件: $RESULT_FILE"
+ ui_section "完成"
+ ui_success "错题重跑完成"
+ ui_kv "结果文件" "$RESULT_FILE"
exit 0
fi
# ========== 全量评测模式 ==========
if [ -z "$SAMPLE" ]; then
- echo "=== 全量评测模式 ==="
+ ui_section "3. 执行评测 · 全量模式"
if [ "$AUTO_COMMIT" = "true" ]; then
RESULT_FILE="./result/locomo/locomo_result_${TIMESTAMP}_${GIT_COMMIT_ID}.csv"
@@ -401,33 +502,45 @@ if [ -z "$SAMPLE" ]; then
# 导入数据
if [ "$SKIP_IMPORT" = "true" ]; then
- echo "[1/4] 跳过导入数据..."
+ ui_warn "已通过 --skip-import 跳过导入数据"
else
- echo "[1/4] 导入数据..."
+ ui_step 1 4 "导入数据"
capture_import_row_start
"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" --input "$INPUT_FILE" --force-ingest --account "$ACCOUNT" --openviking-url "$OPENVIKING_URL" "${IMPORT_OPTS[@]}" "${COMMON_OPTS[@]}"
IMPORT_PERFORMED=true
- echo "等待 1 分钟..."
+ ui_info "等待数据处理完成(60 秒)…"
sleep 60
fi
# 评估
- echo "[2/4] 评估..."
+ if [ "$SKIP_IMPORT" = "true" ]; then
+ ui_step 1 3 "运行评估"
+ else
+ ui_step 2 4 "运行评估"
+ fi
prepare_bot_log_dir "$RESULT_FILE"
- "$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" "$INPUT_FILE" --output "$RESULT_FILE" --config "$OPENVIKING_CONFIG_FILE" "${COMMON_OPTS[@]}"
+ "$PYTHON_BIN" "$SCRIPT_DIR/run_eval.py" "$INPUT_FILE" --output "$RESULT_FILE" --config "$OPENVIKING_CONFIG_FILE" "${RUN_EVAL_OPTS[@]}" "${COMMON_OPTS[@]}"
# 裁判打分
- echo "[3/4] 裁判打分..."
- "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$RESULT_FILE" --parallel 100
+ if [ "$SKIP_IMPORT" = "true" ]; then
+ ui_step 2 3 "裁判打分"
+ else
+ ui_step 3 4 "裁判打分"
+ fi
+ "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$RESULT_FILE" "${JUDGE_OPTS[@]}"
# 计算结果
- echo "[4/4] 计算结果..."
+ if [ "$SKIP_IMPORT" = "true" ]; then
+ ui_step 3 3 "汇总结果"
+ else
+ ui_step 4 4 "汇总结果"
+ fi
"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$RESULT_FILE"
print_import_summary_table
- echo ""
- echo "=== 全量评测完成 ==="
- echo "结果文件: $RESULT_FILE"
+ ui_section "完成"
+ ui_success "全量评测完成"
+ ui_kv "结果文件" "$RESULT_FILE"
exit 0
fi
@@ -436,7 +549,7 @@ fi
if [[ "$SAMPLE" =~ ^-?[0-9]+$ ]]; then
SAMPLE_INDEX=$SAMPLE
SAMPLE_ID_FOR_CMD=$SAMPLE_INDEX
- echo "Using sample index: $SAMPLE_INDEX"
+ ui_kv "Sample" "index=$SAMPLE_INDEX"
else
SAMPLE_INDEX=$(SAMPLE="$SAMPLE" INPUT_FILE="$INPUT_FILE" "$PYTHON_BIN" - <<'PY'
import json
@@ -457,23 +570,24 @@ else:
PY
)
if [ "$SAMPLE_INDEX" = "NOT_FOUND" ]; then
- echo "Error: sample_id '$SAMPLE' not found"
+ ui_error "sample_id '$SAMPLE' not found"
exit 1
fi
SAMPLE_ID_FOR_CMD=$SAMPLE
- echo "Using sample_id: $SAMPLE (index: $SAMPLE_INDEX)"
+ ui_kv "Sample" "id=$SAMPLE · index=$SAMPLE_INDEX"
fi
# 判断是单题模式还是批量模式
if [ -n "$QUESTION_INDEX" ]; then
# ========== 单题模式 ==========
- echo "=== 单题模式: sample $SAMPLE, question $QUESTION_INDEX ==="
+ ui_section "3. 执行评测 · 单题模式"
+ ui_kv "评测范围" "sample=$SAMPLE · question=$QUESTION_INDEX"
# 导入对话
if [ "$SKIP_IMPORT" = "true" ]; then
- echo "[1/3] Skipping import (--skip-import)"
+ ui_warn "已通过 --skip-import 跳过导入对话"
else
- echo "[1/3] Importing sample $SAMPLE_INDEX, question $QUESTION_INDEX..."
+ ui_step 1 3 "导入 sample $SAMPLE_INDEX · question $QUESTION_INDEX"
capture_import_row_start
"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \
--input "$INPUT_FILE" \
@@ -486,15 +600,15 @@ if [ -n "$QUESTION_INDEX" ]; then
"${COMMON_OPTS[@]}"
IMPORT_PERFORMED=true
- echo "Waiting for data processing..."
+ ui_info "等待数据处理完成(3 秒)…"
sleep 3
fi
# 运行评测
if [ "$SKIP_IMPORT" = "true" ]; then
- echo "[1/2] Running evaluation (skip-import mode)..."
+ ui_step 1 2 "运行评估"
else
- echo "[2/3] Running evaluation..."
+ ui_step 2 3 "运行评估"
fi
if [ "$AUTO_COMMIT" = "true" ]; then
OUTPUT_FILE=./result/locomo/locomo_${SAMPLE}_${QUESTION_INDEX}_result_${TIMESTAMP}_${GIT_COMMIT_ID}.csv
@@ -509,19 +623,19 @@ if [ -n "$QUESTION_INDEX" ]; then
--count 1 \
--output "$OUTPUT_FILE" \
--config "$OPENVIKING_CONFIG_FILE" \
+ "${RUN_EVAL_OPTS[@]}" \
"${COMMON_OPTS[@]}"
# 运行 Judge 评分
if [ "$SKIP_IMPORT" = "true" ]; then
- echo "[2/2] Running judge..."
+ ui_step 2 2 "裁判打分"
else
- echo "[3/3] Running judge..."
+ ui_step 3 3 "裁判打分"
fi
- "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$OUTPUT_FILE" --parallel 100
+ "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$OUTPUT_FILE" "${JUDGE_OPTS[@]}"
# 输出结果
- echo ""
- echo "=== 评测结果 ==="
+ ui_section "评测结果"
print_import_summary_table
OUTPUT_FILE="$OUTPUT_FILE" QUESTION_INDEX="$QUESTION_INDEX" "$PYTHON_BIN" - <<'PY'
import csv
@@ -557,7 +671,8 @@ PY
else
# ========== 批量模式 ==========
- echo "=== 批量模式: sample $SAMPLE, 所有问题 ==="
+ ui_section "3. 执行评测 · Sample 批量模式"
+ ui_kv "评测范围" "sample=$SAMPLE · 所有问题"
# 获取该 sample 的问题数量
QUESTION_COUNT=$(SAMPLE_INDEX="$SAMPLE_INDEX" INPUT_FILE="$INPUT_FILE" "$PYTHON_BIN" - <<'PY'
@@ -574,13 +689,13 @@ sample = data[sample_index]
print(len(sample.get("qa", [])))
PY
)
- echo "Found $QUESTION_COUNT questions for sample $SAMPLE"
+ ui_kv "问题数量" "$QUESTION_COUNT"
# 导入所有 sessions
if [ "$SKIP_IMPORT" = "true" ]; then
- echo "[1/4] Skipping import (--skip-import)"
+ ui_warn "已通过 --skip-import 跳过导入所有 Sessions"
else
- echo "[1/4] Importing all sessions for sample $SAMPLE_INDEX..."
+ ui_step 1 4 "导入 sample $SAMPLE_INDEX 的所有 Sessions"
capture_import_row_start
"$PYTHON_BIN" "$SCRIPT_DIR/import_to_ov.py" \
--input "$INPUT_FILE" \
@@ -592,15 +707,15 @@ PY
"${COMMON_OPTS[@]}"
IMPORT_PERFORMED=true
- echo "Waiting for data processing..."
+ ui_info "等待数据处理完成(10 秒)…"
sleep 10
fi
# 运行评测(所有问题)
if [ "$SKIP_IMPORT" = "true" ]; then
- echo "[1/3] Running evaluation for all questions (skip-import mode)..."
+ ui_step 1 3 "评估所有问题"
else
- echo "[2/4] Running evaluation for all questions..."
+ ui_step 2 4 "评估所有问题"
fi
if [ "$AUTO_COMMIT" = "true" ]; then
OUTPUT_FILE=./result/locomo/locomo_${SAMPLE}_result_${TIMESTAMP}_${GIT_COMMIT_ID}.csv
@@ -612,28 +727,28 @@ PY
"$INPUT_FILE" \
--sample "$SAMPLE_ID_FOR_CMD" \
--output "$OUTPUT_FILE" \
- --threads 100 \
--config "$OPENVIKING_CONFIG_FILE" \
+ "${RUN_EVAL_OPTS[@]}" \
"${COMMON_OPTS[@]}"
# 运行 Judge 评分
if [ "$SKIP_IMPORT" = "true" ]; then
- echo "[2/3] Running judge..."
+ ui_step 2 3 "裁判打分"
else
- echo "[3/4] Running judge..."
+ ui_step 3 4 "裁判打分"
fi
- "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$OUTPUT_FILE" --parallel 100
+ "$PYTHON_BIN" "$SCRIPT_DIR/judge.py" --input "$OUTPUT_FILE" "${JUDGE_OPTS[@]}"
# 输出统计结果
if [ "$SKIP_IMPORT" = "true" ]; then
- echo "[3/3] Calculating statistics..."
+ ui_step 3 3 "汇总结果"
else
- echo "[4/4] Calculating statistics..."
+ ui_step 4 4 "汇总结果"
fi
"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$OUTPUT_FILE"
print_import_summary_table
- echo ""
- echo "=== 批量评测完成 ==="
- echo "结果文件: $OUTPUT_FILE"
+ ui_section "完成"
+ ui_success "批量评测完成"
+ ui_kv "结果文件" "$OUTPUT_FILE"
fi
diff --git a/openviking/models/vlm/backends/litellm_vlm.py b/openviking/models/vlm/backends/litellm_vlm.py
index 65b5de41f..05da60d8a 100644
--- a/openviking/models/vlm/backends/litellm_vlm.py
+++ b/openviking/models/vlm/backends/litellm_vlm.py
@@ -352,6 +352,9 @@ class LiteLLMVLMProvider(VLMBase):
def _build_vlm_response(self, response, has_tools: bool) -> Union[str, VLMResponse]:
"""Build response from LiteLLM response. Returns str or VLMResponse based on has_tools."""
+ if isinstance(response, str):
+ return VLMResponse(content=response) if has_tools else response
+
choice = response.choices[0]
message = choice.message
diff --git a/openviking/models/vlm/backends/openai_vlm.py b/openviking/models/vlm/backends/openai_vlm.py
index 3d8a54f8e..e06de8fbe 100644
--- a/openviking/models/vlm/backends/openai_vlm.py
+++ b/openviking/models/vlm/backends/openai_vlm.py
@@ -191,6 +191,9 @@ class OpenAIVLM(VLMBase):
def _build_vlm_response(self, response, has_tools: bool) -> Union[str, VLMResponse]:
"""Build response from OpenAI response. Returns str or VLMResponse based on has_tools."""
+ if isinstance(response, str):
+ return VLMResponse(content=response) if has_tools else response
+
choice = response.choices[0]
message = choice.message
tracer.info(f"result={message.content}")
diff --git a/openviking/models/vlm/backends/volcengine_vlm.py b/openviking/models/vlm/backends/volcengine_vlm.py
index 076631c59..485b50b06 100644
--- a/openviking/models/vlm/backends/volcengine_vlm.py
+++ b/openviking/models/vlm/backends/volcengine_vlm.py
@@ -58,6 +58,9 @@ class VolcEngineVLM(OpenAIVLM):
def _build_vlm_response(self, response, has_tools: bool) -> Union[str, VLMResponse]:
"""Build response from Chat Completions response. Returns str or VLMResponse based on has_tools."""
+ if isinstance(response, str):
+ return VLMResponse(content=response) if has_tools else response
+
choice = response.choices[0]
message = choice.message
if hasattr(message, "tool_calls") and message.tool_calls:
diff --git a/tests/unit/test_vikingbot_vlm_adapter_retry.py b/tests/unit/test_vikingbot_vlm_adapter_retry.py
index 6350d84b2..90bc362a6 100644
--- a/tests/unit/test_vikingbot_vlm_adapter_retry.py
+++ b/tests/unit/test_vikingbot_vlm_adapter_retry.py
@@ -6,9 +6,8 @@ import vikingbot.providers.vlm_adapter as vlm_adapter
from vikingbot.providers.vlm_adapter import VLMProviderAdapter
from volcenginesdkarkruntime._exceptions import ArkRateLimitError
-from openviking.utils.model_retry import (
- is_retryable_rate_limit_error,
-)
+from openviking.models.vlm.backends.openai_vlm import OpenAIVLM
+from openviking.utils.model_retry import is_retryable_rate_limit_error
class _DisabledLangfuse:
@@ -112,6 +111,37 @@ async def test_chat_does_not_retry_errors_without_rate_limit_markers(monkeypatch
assert fake_vlm.calls == 1
+async def test_chat_accepts_string_response_from_openai_backend_with_tools(monkeypatch):
+ async def create(**_kwargs):
+ return "plain string response"
+
+ client = SimpleNamespace(
+ chat=SimpleNamespace(completions=SimpleNamespace(create=create)),
+ )
+ vlm = OpenAIVLM({"provider": "openai", "model": "gpt-5.6-terra"})
+ monkeypatch.setattr(vlm, "get_async_client", lambda: client)
+ adapter = VLMProviderAdapter(vlm, "gpt-5.6-terra", langfuse_client=_DisabledLangfuse())
+
+ response = await adapter.chat(
+ messages=[{"role": "user", "content": "hello"}],
+ tools=[
+ {
+ "type": "function",
+ "function": {
+ "name": "test_tool",
+ "description": "A test tool",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ],
+ )
+
+ assert response.content == "plain string response"
+ assert response.tool_calls == []
+ assert response.finish_reason == "stop"
+
+
@pytest.mark.asyncio
async def test_chat_stream_retries_rate_limit_until_success(monkeypatch):
sleep_delays: list[float] = []
diff --git a/tests/unit/test_vlm_response_formats.py b/tests/unit/test_vlm_response_formats.py
index 3d0eaa360..8ff35ef58 100644
--- a/tests/unit/test_vlm_response_formats.py
+++ b/tests/unit/test_vlm_response_formats.py
@@ -6,7 +6,10 @@ from types import SimpleNamespace
import pytest
-from openviking.models.vlm.base import VLMBase
+from openviking.models.vlm.backends.litellm_vlm import LiteLLMVLMProvider
+from openviking.models.vlm.backends.openai_vlm import OpenAIVLM
+from openviking.models.vlm.backends.volcengine_vlm import VolcEngineVLM
+from openviking.models.vlm.base import VLMBase, VLMResponse
class TestVLMBaseResponseFormats:
@@ -57,3 +60,54 @@ class TestVLMBaseResponseFormats:
choices=[SimpleNamespace(message=SimpleNamespace(content="standard response content"))]
)
assert vlm._extract_content_from_response(response) == "standard response content"
+
+
+ ("vlm_type", "config"),
+ [
+ (OpenAIVLM, {"provider": "openai", "model": "gpt-5.6-terra"}),
+ (LiteLLMVLMProvider, {"provider": "litellm", "model": "gpt-5.6-terra"}),
+ (VolcEngineVLM, {"provider": "volcengine", "model": "test-model"}),
+ ],
+ ids=["openai", "litellm", "volcengine"],
+)
+def test_build_vlm_response_from_str_with_tools(vlm_type, config):
+ vlm = vlm_type(config)
+ response = vlm._build_vlm_response("plain string response", has_tools=True)
+
+ assert isinstance(response, VLMResponse)
+ assert response.content == "plain string response"
+ assert response.tool_calls == []
+ assert response.finish_reason == "stop"
+ assert response.usage == {}
+
+
+async def test_openai_async_completion_from_str_with_tools(monkeypatch):
+ async def create(**_kwargs):
+ return "plain string response"
+
+ client = SimpleNamespace(
+ chat=SimpleNamespace(completions=SimpleNamespace(create=create)),
+ )
+ vlm = OpenAIVLM({"provider": "openai", "model": "gpt-5.6-terra"})
+ monkeypatch.setattr(vlm, "get_async_client", lambda: client)
+
+ response = await vlm.get_completion_async(
+ messages=[{"role": "user", "content": "hello"}],
+ tools=[
+ {
+ "type": "function",
+ "function": {
+ "name": "test_tool",
+ "description": "A test tool",
+ "parameters": {"type": "object", "properties": {}},
+ },
+ }
+ ],
+ )
+
+ assert isinstance(response, VLMResponse)
+ assert response.content == "plain string response"
+ assert response.tool_calls == []
+ assert response.finish_reason == "stop"