Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

fix emb (#1825)

DuTaod4a5e5ea3958
diff --git a/benchmark/locomo/vikingbot/import_and_eval_one.sh b/benchmark/locomo/vikingbot/import_and_eval_one.sh
index dcc3a4a1e..549f6561b 100755
--- a/benchmark/locomo/vikingbot/import_and_eval_one.sh
+++ b/benchmark/locomo/vikingbot/import_and_eval_one.sh
@@ -7,11 +7,14 @@
# ./import_and_eval_one.sh conv-26 # sample_id conv-26, 所有问题 (批量)
# ./import_and_eval_one.sh conv-26 2 --skip-import # 跳过导入,直接评测
# ./import_and_eval_one.sh conv-26 --skip-import # 跳过导入,批量评测
+# ./import_and_eval_one.sh conv-26 --skip-done # 跳过结果文件中已存在的问题
+# ./import_and_eval_one.sh conv-26 --skip-import --skip-done # 跳过导入并跳过已完成题目
set -e
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
SKIP_IMPORT=false
+SKIP_DONE=false
if command -v python3 >/dev/null 2>&1; then
PYTHON_BIN="python3"
@@ -82,13 +85,15 @@ source "$RUNTIME_ENV_FILE"
for arg in "$@"; do
if [ "$arg" = "--skip-import" ]; then
SKIP_IMPORT=true
+ elif [ "$arg" = "--skip-done" ]; then
+ SKIP_DONE=true
fi
done
-# 过滤掉 --skip-import 获取实际参数
+# 过滤掉 --skip-import/--skip-done 获取实际参数
ARGS=()
for arg in "$@"; do
- if [ "$arg" != "--skip-import" ]; then
+ if [ "$arg" != "--skip-import" ] && [ "$arg" != "--skip-done" ]; then
ARGS+=("$arg")
fi
done
@@ -96,12 +101,17 @@ done
SAMPLE=${ARGS[0]}
QUESTION_INDEX=${ARGS[1]}
INPUT_FILE="$SCRIPT_DIR/../data/locomo10.json"
+RUN_EVAL_EXTRA_ARGS=()
+if [ "$SKIP_DONE" = "true" ]; then
+ RUN_EVAL_EXTRA_ARGS+=("--skip-done")
+fi
if [ -z "$SAMPLE" ]; then
- echo "Usage: $0 <sample_index|sample_id> [question_index] [--skip-import]"
+ echo "Usage: $0 <sample_index|sample_id> [question_index] [--skip-import] [--skip-done]"
echo " sample_index: 数字索引 (0,1,2...) 或 sample_id (conv-26)"
echo " question_index: 问题索引 (可选),不传则测试该 sample 的所有问题"
echo " --skip-import: 跳过导入步骤,直接使用已导入的数据进行评测"
+ echo " --skip-done: 跳过输出结果文件中已存在的问题"
exit 1
fi
@@ -173,7 +183,8 @@ if [ -n "$QUESTION_INDEX" ]; then
"$INPUT_FILE" \
--sample "$SAMPLE_ID_FOR_CMD" \
--question-index "$QUESTION_INDEX" \
- --count 1
+ --count 1 \
+ "${RUN_EVAL_EXTRA_ARGS[@]}"
else
# sample_id 模式直接更新批量结果文件
OUTPUT_FILE=./result/locomo_${SAMPLE}_result.csv
@@ -183,7 +194,8 @@ if [ -n "$QUESTION_INDEX" ]; then
--question-index "$QUESTION_INDEX" \
--count 1 \
--output "$OUTPUT_FILE" \
- --update-mode
+ --update-mode \
+ "${RUN_EVAL_EXTRA_ARGS[@]}"
fi
# 运行 Judge 评分
@@ -277,7 +289,9 @@ PY
"$INPUT_FILE" \
--sample "$SAMPLE_ID_FOR_CMD" \
--output "$OUTPUT_FILE" \
- --threads 5
+ --threads 5 \
+ --update-mode \
+ "${RUN_EVAL_EXTRA_ARGS[@]}"
# 运行 Judge 评分
if [ "$SKIP_IMPORT" = "true" ]; then
@@ -295,6 +309,58 @@ PY
fi
"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$OUTPUT_FILE"
+ echo ""
+ SAMPLE="$SAMPLE" IMPORT_SUCCESS_FILE="$SCRIPT_DIR/result/import_success.csv" "$PYTHON_BIN" - <<'PY'
+import csv
+import os
+from pathlib import Path
+
+
+def make_table(title, rows):
+ metric_width = max(len("Metric"), *(len(metric) for metric, _ in rows))
+ value_width = max(len("Value"), *(len(value) for _, value in rows))
+ border = f"+-{'-' * (metric_width + 2)}-+-{'-' * (value_width + 2)}-+"
+
+ lines = [title, border]
+ lines.append(f"| {'Metric'.center(metric_width)} | {'Value'.center(value_width)} |")
+ lines.append(border)
+ for metric, value in rows:
+ lines.append(f"| {metric.ljust(metric_width)} | {value.rjust(value_width)} |")
+ lines.append(border)
+ return lines
+
+
+sample = os.environ["SAMPLE"]
+import_success_file = Path(os.environ["IMPORT_SUCCESS_FILE"])
+
+embedding_total = 0
+vlm_total = 0
+all_total = 0
+matched_rows = 0
+
+if import_success_file.exists():
+ with import_success_file.open("r", encoding="utf-8", newline="") as f:
+ reader = csv.DictReader(f)
+ for row in reader:
+ if row.get("sample_id") != sample:
+ continue
+ matched_rows += 1
+ embedding_total += int(row.get("embedding_tokens") or 0)
+ vlm_total += int(row.get("vlm_tokens") or 0)
+ all_total += int(row.get("total_tokens") or 0)
+
+rows = [
+ ("Sample", sample),
+ ("Imported sessions", f"{matched_rows:,}"),
+ ("Total embedding tokens", f"{embedding_total:,}"),
+ ("Total vlm tokens", f"{vlm_total:,}"),
+ ("Total tokens", f"{all_total:,}"),
+]
+
+for line in make_table("=== Import Token Usage ===", rows):
+ print(line)
+PY
+
echo ""
echo "=== 批量评测完成 ==="
echo "结果文件: $OUTPUT_FILE"
diff --git a/benchmark/locomo/vikingbot/run_eval.py b/benchmark/locomo/vikingbot/run_eval.py
index 2d38a0454..3d3e1c7df 100644
--- a/benchmark/locomo/vikingbot/run_eval.py
+++ b/benchmark/locomo/vikingbot/run_eval.py
@@ -231,7 +231,7 @@ def run_vikingbot_chat(
]
try:
# print(f'new_cmd={new_cmd}')
- subprocess.run(new_cmd, capture_output=True, text=True, timeout=60)
+ subprocess.run(new_cmd, capture_output=True, text=True, timeout=300)
except Exception:
# 忽略 /new 命令的错误
pass
@@ -289,10 +289,19 @@ def run_vikingbot_chat(
)
-def load_processed_questions(output_path: str) -> set:
- """加载已处理的问题集合(已禁用,每次重新运行)"""
- # 注意:去重逻辑已禁用,每次运行都会重新执行所有问题
- return set()
+def load_processed_questions(output_path: str, skip_done: bool = False) -> set[str]:
+ """加载已处理的问题集合。"""
+ if not skip_done or not os.path.exists(output_path):
+ return set()
+
+ processed_questions = set()
+ with open(output_path, "r", encoding="utf-8", newline="") as f:
+ reader = csv.DictReader(f)
+ for row in reader:
+ question = row.get("question")
+ if question:
+ processed_questions.add(question)
+ return processed_questions
def main():
@@ -341,6 +350,11 @@ def main():
action="store_true",
help="Update mode: if output file exists, update matching question_index rows instead of overwriting",
)
+ parser.add_argument(
+ "--skip-done",
+ action="store_true",
+ help="Skip questions already present in the output file",
+ )
args = parser.parse_args()
# 如果指定了 question-index,自动设置 count=1
@@ -380,7 +394,7 @@ def main():
print(f"Filtered to {len(qa_list)} questions after removing category=5")
# 加载已处理的问题
- processed_questions = load_processed_questions(args.output)
+ processed_questions = load_processed_questions(args.output, skip_done=args.skip_done)
remaining = total - len(processed_questions)
print(
f"Loaded {total} QA questions, {len(processed_questions)} already processed, {remaining} remaining"
diff --git a/benchmark/locomo/vikingbot/stat_judge_result.py b/benchmark/locomo/vikingbot/stat_judge_result.py
index 298d0c708..19577a497 100644
--- a/benchmark/locomo/vikingbot/stat_judge_result.py
+++ b/benchmark/locomo/vikingbot/stat_judge_result.py
@@ -4,6 +4,24 @@ import json
import os
+def make_table(title: str, rows: list[tuple[str, str]]) -> list[str]:
+ metric_width = max(len("Metric"), *(len(metric) for metric, _ in rows))
+ value_width = max(len("Value"), *(len(value) for _, value in rows))
+ border = f"+-{'-' * (metric_width + 2)}-+-{'-' * (value_width + 2)}-+"
+
+ lines = [title, border]
+ lines.append(f"| {'Metric'.center(metric_width)} | {'Value'.center(value_width)} |")
+ lines.append(border)
+ for metric, value in rows:
+ lines.append(f"| {metric.ljust(metric_width)} | {value.rjust(value_width)} |")
+ lines.append(border)
+ return lines
+
+
+def format_int(value: int) -> str:
+ return f"{value:,}"
+
+
def main():
parser = argparse.ArgumentParser(description="Statistics for judge result csv")
parser.add_argument(
@@ -17,7 +35,6 @@ def main():
print(f"Error: File not found: {args.input}")
exit(1)
- # 统计所有题目 (排除 category=5)
correct = 0
wrong = 0
total_time = 0.0
@@ -27,7 +44,6 @@ def main():
valid_rows = 0
total_iteration = 0
- # 统计 is_valid=True 的题目 (排除 category=5)
valid_only_correct = 0
valid_only_wrong = 0
valid_only_total_time = 0.0
@@ -40,18 +56,15 @@ def main():
with open(args.input, "r", encoding="utf-8", newline="") as f:
reader = csv.DictReader(f)
for row in reader:
- # 检查 category 是否为 5,跳过
category = row.get("category", "")
if category == "5":
continue
valid_rows += 1
- # 检查是否是无效题目
is_invalid = row.get("is_invalid", "").lower() == "true"
is_valid = not is_invalid
- # 统计结果
result = row.get("result", "").strip().upper()
if result == "CORRECT":
correct += 1
@@ -66,7 +79,6 @@ def main():
if is_valid:
valid_only_total_iteration += int(row.get("iteration", "0"))
- # 统计耗时
time_cost = row.get("time_cost", "")
if time_cost:
try:
@@ -77,7 +89,6 @@ def main():
except (ValueError, TypeError):
pass
- # 统计token
token_usage = row.get("token_usage", "")
if token_usage and token_usage.strip():
try:
@@ -100,14 +111,12 @@ def main():
accuracy = correct / total_graded if total_graded > 0 else 0.0
avg_time = total_time / valid_rows if valid_rows > 0 else 0.0
- # is_valid=True 题目的统计 (排除 category=5)
valid_only_total_graded = valid_only_correct + valid_only_wrong
valid_only_accuracy = (
valid_only_correct / valid_only_total_graded if valid_only_total_graded > 0 else 0.0
)
valid_only_avg_time = valid_only_total_time / valid_only_rows if valid_only_rows > 0 else 0.0
- # 平均 token 消耗
avg_prompt_tokens = total_prompt_tokens / valid_rows if valid_rows > 0 else 0.0
avg_completion_tokens = total_completion_tokens / valid_rows if valid_rows > 0 else 0.0
avg_total_tokens = total_tokens / valid_rows if valid_rows > 0 else 0.0
@@ -122,45 +131,50 @@ def main():
valid_only_total_tokens / valid_only_rows if valid_only_rows > 0 else 0.0
)
+ all_rows = [
+ ("Total rows", format_int(valid_rows)),
+ ("Graded rows", format_int(total_graded)),
+ ("Correct", format_int(correct)),
+ ("Wrong", format_int(wrong)),
+ ("Accuracy", f"{accuracy:.2%}"),
+ ("Avg time cost", f"{avg_time:.2f}s"),
+ ("Avg iteration", f"{total_iteration / valid_rows if valid_rows > 0 else 0.0:.2f}"),
+ ("Total prompt tokens", format_int(total_prompt_tokens)),
+ ("Total completion tokens", format_int(total_completion_tokens)),
+ ("Total tokens", format_int(total_tokens)),
+ ("Avg prompt tokens", f"{avg_prompt_tokens:.2f}"),
+ ("Avg completion tokens", f"{avg_completion_tokens:.2f}"),
+ ("Avg total tokens", f"{avg_total_tokens:.2f}"),
+ ]
+
+ valid_rows_table = [
+ ("Valid rows", format_int(valid_only_rows)),
+ ("Valid graded rows", format_int(valid_only_total_graded)),
+ ("Valid correct", format_int(valid_only_correct)),
+ ("Valid wrong", format_int(valid_only_wrong)),
+ ("Valid accuracy", f"{valid_only_accuracy:.2%}"),
+ ("Avg time cost", f"{valid_only_avg_time:.2f}s"),
+ (
+ "Avg iteration",
+ f"{valid_only_total_iteration / valid_only_rows if valid_only_rows > 0 else 0.0:.2f}",
+ ),
+ ("Total prompt tokens", format_int(valid_only_total_prompt_tokens)),
+ ("Total completion tokens", format_int(valid_only_total_completion_tokens)),
+ ("Total tokens", format_int(valid_only_total_tokens)),
+ ("Avg prompt tokens", f"{valid_only_avg_prompt_tokens:.2f}"),
+ ("Avg completion tokens", f"{valid_only_avg_completion_tokens:.2f}"),
+ ("Avg total tokens", f"{valid_only_avg_total_tokens:.2f}"),
+ ]
+
output_lines = [
- "=== Judge Result Statistics (excluding category=5) ===",
- f"Total rows: {valid_rows}",
- f"Graded rows: {total_graded}",
- f"Correct: {correct}",
- f"Wrong: {wrong}",
- f"Accuracy: {accuracy:.2%}",
- f"\nAverage time cost: {avg_time:.2f}s",
- f"\nAverage iteration: {total_iteration / valid_rows if valid_rows > 0 else 0.0:.2f}",
- f"\nToken usage:",
- f" Total prompt tokens: {total_prompt_tokens}",
- f" Total completion tokens: {total_completion_tokens}",
- f" Total tokens: {total_tokens}",
- f" Avg prompt tokens: {avg_prompt_tokens:.2f}",
- f" Avg completion tokens: {avg_completion_tokens:.2f}",
- f" Avg total tokens: {avg_total_tokens:.2f}",
+ *make_table("=== Judge Result Statistics (excluding category=5) ===", all_rows),
"",
- "=== Valid Questions Only (is_valid=True, excluding category=5) ===",
- f"Valid rows: {valid_only_rows}",
- f"Valid graded rows: {valid_only_total_graded}",
- f"Valid correct: {valid_only_correct}",
- f"Valid wrong: {valid_only_wrong}",
- f"Valid accuracy: {valid_only_accuracy:.2%}",
- f"\nAverage time cost: {valid_only_avg_time:.2f}s",
- f"\nAverage iteration: {valid_only_total_iteration / valid_only_rows if valid_only_rows > 0 else 0.0:.2f}",
- f"\nToken usage:",
- f" Total prompt tokens: {valid_only_total_prompt_tokens}",
- f" Total completion tokens: {valid_only_total_completion_tokens}",
- f" Total tokens: {valid_only_total_tokens}",
- f" Avg prompt tokens: {valid_only_avg_prompt_tokens:.2f}",
- f" Avg completion tokens: {valid_only_avg_completion_tokens:.2f}",
- f" Avg total tokens: {valid_only_avg_total_tokens:.2f}",
+ *make_table("=== Valid Questions Only (is_valid=True, excluding category=5) ===", valid_rows_table),
]
- # 打印到控制台
for line in output_lines:
print(line)
- # 写入summary.txt
summary_path = os.path.join(os.path.dirname(args.input), "summary.txt")
with open(summary_path, "w", encoding="utf-8") as f:
f.write("\n".join(output_lines) + "\n")
diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.py
index 16e40c9c8..cd9c0bb9e 100644
--- a/openviking_cli/utils/config/embedding_config.py
+++ b/openviking_cli/utils/config/embedding_config.py
@@ -239,6 +239,16 @@ class EmbeddingModelConfig(BaseModel):
return self.dimension
provider = (self.provider or "").lower()
+ if provider in {"openai", "azure"}:
+ openai_model_dimensions = {
+ "text-embedding-ada-002": 1536,
+ "text-embedding-3-small": 1536,
+ "text-embedding-3-large": 3072,
+ }
+ model_lower = (self.model or "").lower()
+ if model_lower in openai_model_dimensions:
+ return openai_model_dimensions[model_lower]
+
if provider == "voyage":
from openviking.models.embedder.voyage_embedders import (
get_voyage_model_default_dimension,