Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix emb (#1825)
DuTaod4a5e5ea3958
diff --git a/benchmark/locomo/vikingbot/import_and_eval_one.sh b/benchmark/locomo/vikingbot/import_and_eval_one.shindex dcc3a4a1e..549f6561b 100755--- a/benchmark/locomo/vikingbot/import_and_eval_one.sh+++ b/benchmark/locomo/vikingbot/import_and_eval_one.sh@@ -7,11 +7,14 @@# ./import_and_eval_one.sh conv-26 # sample_id conv-26, 所有问题 (批量)# ./import_and_eval_one.sh conv-26 2 --skip-import # 跳过导入,直接评测# ./import_and_eval_one.sh conv-26 --skip-import # 跳过导入,批量评测+# ./import_and_eval_one.sh conv-26 --skip-done # 跳过结果文件中已存在的问题+# ./import_and_eval_one.sh conv-26 --skip-import --skip-done # 跳过导入并跳过已完成题目set -eSCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"SKIP_IMPORT=false+SKIP_DONE=falseif command -v python3 >/dev/null 2>&1; thenPYTHON_BIN="python3"@@ -82,13 +85,15 @@ source "$RUNTIME_ENV_FILE"for arg in "$@"; doif [ "$arg" = "--skip-import" ]; thenSKIP_IMPORT=true+ elif [ "$arg" = "--skip-done" ]; then+ SKIP_DONE=truefidone-# 过滤掉 --skip-import 获取实际参数+# 过滤掉 --skip-import/--skip-done 获取实际参数ARGS=()for arg in "$@"; do- if [ "$arg" != "--skip-import" ]; then+ if [ "$arg" != "--skip-import" ] && [ "$arg" != "--skip-done" ]; thenARGS+=("$arg")fidone@@ -96,12 +101,17 @@ doneSAMPLE=${ARGS[0]}QUESTION_INDEX=${ARGS[1]}INPUT_FILE="$SCRIPT_DIR/../data/locomo10.json"+RUN_EVAL_EXTRA_ARGS=()+if [ "$SKIP_DONE" = "true" ]; then+ RUN_EVAL_EXTRA_ARGS+=("--skip-done")+fiif [ -z "$SAMPLE" ]; then- echo "Usage: $0 <sample_index|sample_id> [question_index] [--skip-import]"+ echo "Usage: $0 <sample_index|sample_id> [question_index] [--skip-import] [--skip-done]"echo " sample_index: 数字索引 (0,1,2...) 或 sample_id (conv-26)"echo " question_index: 问题索引 (可选),不传则测试该 sample 的所有问题"echo " --skip-import: 跳过导入步骤,直接使用已导入的数据进行评测"+ echo " --skip-done: 跳过输出结果文件中已存在的问题"exit 1fi@@ -173,7 +183,8 @@ if [ -n "$QUESTION_INDEX" ]; then"$INPUT_FILE" \--sample "$SAMPLE_ID_FOR_CMD" \--question-index "$QUESTION_INDEX" \- --count 1+ --count 1 \+ "${RUN_EVAL_EXTRA_ARGS[@]}"else# sample_id 模式直接更新批量结果文件OUTPUT_FILE=./result/locomo_${SAMPLE}_result.csv@@ -183,7 +194,8 @@ if [ -n "$QUESTION_INDEX" ]; then--question-index "$QUESTION_INDEX" \--count 1 \--output "$OUTPUT_FILE" \- --update-mode+ --update-mode \+ "${RUN_EVAL_EXTRA_ARGS[@]}"fi# 运行 Judge 评分@@ -277,7 +289,9 @@ PY"$INPUT_FILE" \--sample "$SAMPLE_ID_FOR_CMD" \--output "$OUTPUT_FILE" \- --threads 5+ --threads 5 \+ --update-mode \+ "${RUN_EVAL_EXTRA_ARGS[@]}"# 运行 Judge 评分if [ "$SKIP_IMPORT" = "true" ]; then@@ -295,6 +309,58 @@ PYfi"$PYTHON_BIN" "$SCRIPT_DIR/stat_judge_result.py" --input "$OUTPUT_FILE"+ echo ""+ SAMPLE="$SAMPLE" IMPORT_SUCCESS_FILE="$SCRIPT_DIR/result/import_success.csv" "$PYTHON_BIN" - <<'PY'+import csv+import os+from pathlib import Path+++def make_table(title, rows):+ metric_width = max(len("Metric"), *(len(metric) for metric, _ in rows))+ value_width = max(len("Value"), *(len(value) for _, value in rows))+ border = f"+-{'-' * (metric_width + 2)}-+-{'-' * (value_width + 2)}-+"++ lines = [title, border]+ lines.append(f"| {'Metric'.center(metric_width)} | {'Value'.center(value_width)} |")+ lines.append(border)+ for metric, value in rows:+ lines.append(f"| {metric.ljust(metric_width)} | {value.rjust(value_width)} |")+ lines.append(border)+ return lines+++sample = os.environ["SAMPLE"]+import_success_file = Path(os.environ["IMPORT_SUCCESS_FILE"])++embedding_total = 0+vlm_total = 0+all_total = 0+matched_rows = 0++if import_success_file.exists():+ with import_success_file.open("r", encoding="utf-8", newline="") as f:+ reader = csv.DictReader(f)+ for row in reader:+ if row.get("sample_id") != sample:+ continue+ matched_rows += 1+ embedding_total += int(row.get("embedding_tokens") or 0)+ vlm_total += int(row.get("vlm_tokens") or 0)+ all_total += int(row.get("total_tokens") or 0)++rows = [+ ("Sample", sample),+ ("Imported sessions", f"{matched_rows:,}"),+ ("Total embedding tokens", f"{embedding_total:,}"),+ ("Total vlm tokens", f"{vlm_total:,}"),+ ("Total tokens", f"{all_total:,}"),+]++for line in make_table("=== Import Token Usage ===", rows):+ print(line)+PY+echo ""echo "=== 批量评测完成 ==="echo "结果文件: $OUTPUT_FILE"diff --git a/benchmark/locomo/vikingbot/run_eval.py b/benchmark/locomo/vikingbot/run_eval.pyindex 2d38a0454..3d3e1c7df 100644--- a/benchmark/locomo/vikingbot/run_eval.py+++ b/benchmark/locomo/vikingbot/run_eval.py@@ -231,7 +231,7 @@ def run_vikingbot_chat(]try:# print(f'new_cmd={new_cmd}')- subprocess.run(new_cmd, capture_output=True, text=True, timeout=60)+ subprocess.run(new_cmd, capture_output=True, text=True, timeout=300)except Exception:# 忽略 /new 命令的错误pass@@ -289,10 +289,19 @@ def run_vikingbot_chat()-def load_processed_questions(output_path: str) -> set:- """加载已处理的问题集合(已禁用,每次重新运行)"""- # 注意:去重逻辑已禁用,每次运行都会重新执行所有问题- return set()+def load_processed_questions(output_path: str, skip_done: bool = False) -> set[str]:+ """加载已处理的问题集合。"""+ if not skip_done or not os.path.exists(output_path):+ return set()++ processed_questions = set()+ with open(output_path, "r", encoding="utf-8", newline="") as f:+ reader = csv.DictReader(f)+ for row in reader:+ question = row.get("question")+ if question:+ processed_questions.add(question)+ return processed_questionsdef main():@@ -341,6 +350,11 @@ def main():action="store_true",help="Update mode: if output file exists, update matching question_index rows instead of overwriting",)+ parser.add_argument(+ "--skip-done",+ action="store_true",+ help="Skip questions already present in the output file",+ )args = parser.parse_args()# 如果指定了 question-index,自动设置 count=1@@ -380,7 +394,7 @@ def main():print(f"Filtered to {len(qa_list)} questions after removing category=5")# 加载已处理的问题- processed_questions = load_processed_questions(args.output)+ processed_questions = load_processed_questions(args.output, skip_done=args.skip_done)remaining = total - len(processed_questions)print(f"Loaded {total} QA questions, {len(processed_questions)} already processed, {remaining} remaining"diff --git a/benchmark/locomo/vikingbot/stat_judge_result.py b/benchmark/locomo/vikingbot/stat_judge_result.pyindex 298d0c708..19577a497 100644--- a/benchmark/locomo/vikingbot/stat_judge_result.py+++ b/benchmark/locomo/vikingbot/stat_judge_result.py@@ -4,6 +4,24 @@ import jsonimport os+def make_table(title: str, rows: list[tuple[str, str]]) -> list[str]:+ metric_width = max(len("Metric"), *(len(metric) for metric, _ in rows))+ value_width = max(len("Value"), *(len(value) for _, value in rows))+ border = f"+-{'-' * (metric_width + 2)}-+-{'-' * (value_width + 2)}-+"++ lines = [title, border]+ lines.append(f"| {'Metric'.center(metric_width)} | {'Value'.center(value_width)} |")+ lines.append(border)+ for metric, value in rows:+ lines.append(f"| {metric.ljust(metric_width)} | {value.rjust(value_width)} |")+ lines.append(border)+ return lines+++def format_int(value: int) -> str:+ return f"{value:,}"++def main():parser = argparse.ArgumentParser(description="Statistics for judge result csv")parser.add_argument(@@ -17,7 +35,6 @@ def main():print(f"Error: File not found: {args.input}")exit(1)- # 统计所有题目 (排除 category=5)correct = 0wrong = 0total_time = 0.0@@ -27,7 +44,6 @@ def main():valid_rows = 0total_iteration = 0- # 统计 is_valid=True 的题目 (排除 category=5)valid_only_correct = 0valid_only_wrong = 0valid_only_total_time = 0.0@@ -40,18 +56,15 @@ def main():with open(args.input, "r", encoding="utf-8", newline="") as f:reader = csv.DictReader(f)for row in reader:- # 检查 category 是否为 5,跳过category = row.get("category", "")if category == "5":continuevalid_rows += 1- # 检查是否是无效题目is_invalid = row.get("is_invalid", "").lower() == "true"is_valid = not is_invalid- # 统计结果result = row.get("result", "").strip().upper()if result == "CORRECT":correct += 1@@ -66,7 +79,6 @@ def main():if is_valid:valid_only_total_iteration += int(row.get("iteration", "0"))- # 统计耗时time_cost = row.get("time_cost", "")if time_cost:try:@@ -77,7 +89,6 @@ def main():except (ValueError, TypeError):pass- # 统计tokentoken_usage = row.get("token_usage", "")if token_usage and token_usage.strip():try:@@ -100,14 +111,12 @@ def main():accuracy = correct / total_graded if total_graded > 0 else 0.0avg_time = total_time / valid_rows if valid_rows > 0 else 0.0- # is_valid=True 题目的统计 (排除 category=5)valid_only_total_graded = valid_only_correct + valid_only_wrongvalid_only_accuracy = (valid_only_correct / valid_only_total_graded if valid_only_total_graded > 0 else 0.0)valid_only_avg_time = valid_only_total_time / valid_only_rows if valid_only_rows > 0 else 0.0- # 平均 token 消耗avg_prompt_tokens = total_prompt_tokens / valid_rows if valid_rows > 0 else 0.0avg_completion_tokens = total_completion_tokens / valid_rows if valid_rows > 0 else 0.0avg_total_tokens = total_tokens / valid_rows if valid_rows > 0 else 0.0@@ -122,45 +131,50 @@ def main():valid_only_total_tokens / valid_only_rows if valid_only_rows > 0 else 0.0)+ all_rows = [+ ("Total rows", format_int(valid_rows)),+ ("Graded rows", format_int(total_graded)),+ ("Correct", format_int(correct)),+ ("Wrong", format_int(wrong)),+ ("Accuracy", f"{accuracy:.2%}"),+ ("Avg time cost", f"{avg_time:.2f}s"),+ ("Avg iteration", f"{total_iteration / valid_rows if valid_rows > 0 else 0.0:.2f}"),+ ("Total prompt tokens", format_int(total_prompt_tokens)),+ ("Total completion tokens", format_int(total_completion_tokens)),+ ("Total tokens", format_int(total_tokens)),+ ("Avg prompt tokens", f"{avg_prompt_tokens:.2f}"),+ ("Avg completion tokens", f"{avg_completion_tokens:.2f}"),+ ("Avg total tokens", f"{avg_total_tokens:.2f}"),+ ]++ valid_rows_table = [+ ("Valid rows", format_int(valid_only_rows)),+ ("Valid graded rows", format_int(valid_only_total_graded)),+ ("Valid correct", format_int(valid_only_correct)),+ ("Valid wrong", format_int(valid_only_wrong)),+ ("Valid accuracy", f"{valid_only_accuracy:.2%}"),+ ("Avg time cost", f"{valid_only_avg_time:.2f}s"),+ (+ "Avg iteration",+ f"{valid_only_total_iteration / valid_only_rows if valid_only_rows > 0 else 0.0:.2f}",+ ),+ ("Total prompt tokens", format_int(valid_only_total_prompt_tokens)),+ ("Total completion tokens", format_int(valid_only_total_completion_tokens)),+ ("Total tokens", format_int(valid_only_total_tokens)),+ ("Avg prompt tokens", f"{valid_only_avg_prompt_tokens:.2f}"),+ ("Avg completion tokens", f"{valid_only_avg_completion_tokens:.2f}"),+ ("Avg total tokens", f"{valid_only_avg_total_tokens:.2f}"),+ ]+output_lines = [- "=== Judge Result Statistics (excluding category=5) ===",- f"Total rows: {valid_rows}",- f"Graded rows: {total_graded}",- f"Correct: {correct}",- f"Wrong: {wrong}",- f"Accuracy: {accuracy:.2%}",- f"\nAverage time cost: {avg_time:.2f}s",- f"\nAverage iteration: {total_iteration / valid_rows if valid_rows > 0 else 0.0:.2f}",- f"\nToken usage:",- f" Total prompt tokens: {total_prompt_tokens}",- f" Total completion tokens: {total_completion_tokens}",- f" Total tokens: {total_tokens}",- f" Avg prompt tokens: {avg_prompt_tokens:.2f}",- f" Avg completion tokens: {avg_completion_tokens:.2f}",- f" Avg total tokens: {avg_total_tokens:.2f}",+ *make_table("=== Judge Result Statistics (excluding category=5) ===", all_rows),"",- "=== Valid Questions Only (is_valid=True, excluding category=5) ===",- f"Valid rows: {valid_only_rows}",- f"Valid graded rows: {valid_only_total_graded}",- f"Valid correct: {valid_only_correct}",- f"Valid wrong: {valid_only_wrong}",- f"Valid accuracy: {valid_only_accuracy:.2%}",- f"\nAverage time cost: {valid_only_avg_time:.2f}s",- f"\nAverage iteration: {valid_only_total_iteration / valid_only_rows if valid_only_rows > 0 else 0.0:.2f}",- f"\nToken usage:",- f" Total prompt tokens: {valid_only_total_prompt_tokens}",- f" Total completion tokens: {valid_only_total_completion_tokens}",- f" Total tokens: {valid_only_total_tokens}",- f" Avg prompt tokens: {valid_only_avg_prompt_tokens:.2f}",- f" Avg completion tokens: {valid_only_avg_completion_tokens:.2f}",- f" Avg total tokens: {valid_only_avg_total_tokens:.2f}",+ *make_table("=== Valid Questions Only (is_valid=True, excluding category=5) ===", valid_rows_table),]- # 打印到控制台for line in output_lines:print(line)- # 写入summary.txtsummary_path = os.path.join(os.path.dirname(args.input), "summary.txt")with open(summary_path, "w", encoding="utf-8") as f:f.write("\n".join(output_lines) + "\n")diff --git a/openviking_cli/utils/config/embedding_config.py b/openviking_cli/utils/config/embedding_config.pyindex 16e40c9c8..cd9c0bb9e 100644--- a/openviking_cli/utils/config/embedding_config.py+++ b/openviking_cli/utils/config/embedding_config.py@@ -239,6 +239,16 @@ class EmbeddingModelConfig(BaseModel):return self.dimensionprovider = (self.provider or "").lower()+ if provider in {"openai", "azure"}:+ openai_model_dimensions = {+ "text-embedding-ada-002": 1536,+ "text-embedding-3-small": 1536,+ "text-embedding-3-large": 3072,+ }+ model_lower = (self.model or "").lower()+ if model_lower in openai_model_dimensions:+ return openai_model_dimensions[model_lower]+if provider == "voyage":from openviking.models.embedder.voyage_embedders import (get_voyage_model_default_dimension,