Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
#!/usr/bin/env python3
# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
# SPDX-License-Identifier: AGPL-3.0
"""
RAG Evaluation CLI Tool for OpenViking.
Usage:
python -m openviking.eval.rag_eval --docs_dir ./docs --question_file ./questions.jsonl
python -m openviking.eval.rag_eval --docs_dir ./docs --code_dir ./code --question_file ./questions.jsonl
"""
import argparse
import asyncio
import json
import logging
import os
import sys
import time
from pathlib import Path
from typing import Any, Dict, List
from openviking_cli.utils.config import OPENVIKING_CONFIG_ENV
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
)
logger = logging.getLogger(__name__)
def load_questions(question_file: str) -> List[Dict[str, Any]]:
"""
Load questions from JSONL file.
Args:
question_file: Path to JSONL file with questions
Returns:
List of question dictionaries
"""
questions = []
with open(question_file, "r", encoding="utf-8") as f:
for line_num, line in enumerate(f, 1):
line = line.strip()
if not line:
continue
try:
item = json.loads(line)
if "question" not in item:
logger.warning(f"Line {line_num}: Missing 'question' field")
continue
questions.append(item)
except json.JSONDecodeError as e:
logger.warning(f"Line {line_num}: Invalid JSON - {e}")
return questions
class RAGEvaluator:
"""
RAG evaluator that uses OpenViking for retrieval and evaluation.
"""
def __init__(
self,
docs_dirs: List[str],
code_dirs: List[str],
config_path: str = "./ov.conf",
server_url: str = "http://127.0.0.1:1933",
):
"""
Initialize the RAG evaluator.
Args:
docs_dirs: List of document directories or files
code_dirs: List of code repository paths
config_path: Path to OpenViking config file
server_url: OpenViking HTTP server URL
"""
self.docs_dirs = docs_dirs
self.code_dirs = code_dirs
self.config_path = config_path
self.server_url = server_url
self._client = None
self._initialized = False
def _get_client(self):
"""Get or create OpenViking client."""
if self._client is None:
try:
from openviking_sdk import SyncHTTPClient
config_path = Path(self.config_path).expanduser()
if config_path.exists():
os.environ[OPENVIKING_CONFIG_ENV] = str(config_path)
logger.info(f"Using config file: {config_path}")
self._client = SyncHTTPClient(url=self.server_url)
self._client.initialize()
except Exception as e:
logger.error(f"Failed to create OpenViking client: {e}")
raise
return self._client
async def initialize(self):
"""Initialize the evaluator by adding resources."""
if self._initialized:
return
client = self._get_client()
for doc_path in self.docs_dirs:
path = Path(doc_path).expanduser()
if not path.exists():
logger.warning(f"Document path does not exist: {path}")
continue
logger.info(f"Adding document: {path}")
try:
result = client.add_resource(
str(path),
wait=True,
)
if result and "root_uri" in result:
logger.info(f"Added: {result['root_uri']}")
except Exception as e:
logger.error(f"Failed to add document {path}: {e}")
for code_path in self.code_dirs:
path = Path(code_path).expanduser()
if not path.exists():
logger.warning(f"Code path does not exist: {path}")
continue
logger.info(f"Adding code: {path}")
try:
result = client.add_resource(
str(path),
wait=True,
)
if result and "root_uri" in result:
logger.info(f"Added: {result['root_uri']}")
except Exception as e:
logger.error(f"Failed to add code {path}: {e}")
self._initialized = True
async def retrieve(self, query: str, top_k: int = 5) -> Dict[str, Any]:
"""
Retrieve relevant contexts for a query.
Args:
query: The query string
top_k: Number of results to retrieve
Returns:
Dict with contexts and timing info
"""
client = self._get_client()
start_time = time.time()
try:
result = client.search(query, limit=top_k)
contexts = []
if result:
items = (
result.get("memories", [])
+ result.get("resources", [])
+ result.get("skills", [])
)
for ctx in items:
contexts.append(
{
"uri": ctx.get("uri", ""),
"content": ctx.get("abstract", "") or ctx.get("overview", ""),
"score": ctx.get("score", 0.0),
}
)
retrieval_time = time.time() - start_time
return {
"contexts": contexts,
"retrieval_time": retrieval_time,
}
except Exception as e:
logger.error(f"Failed to retrieve for query '{query}': {e}")
return {
"contexts": [],
"retrieval_time": time.time() - start_time,
}
async def evaluate(
self,
questions: List[Dict[str, Any]],
top_k: int = 5,
) -> Dict[str, Any]:
"""
Evaluate RAG performance on a set of questions.
Args:
questions: List of question dictionaries
top_k: Number of contexts to retrieve per query
Returns:
Evaluation results dictionary
"""
await self.initialize()
results = []
total_questions = len(questions)
total_retrieval_time = 0.0
for i, q_item in enumerate(questions, 1):
question = q_item["question"]
logger.info(f"Processing question {i}/{total_questions}: {question[:50]}...")
retrieve_result = await self.retrieve(question, top_k=top_k)
contexts = retrieve_result["contexts"]
retrieval_time = retrieve_result["retrieval_time"]
total_retrieval_time += retrieval_time
result = {
"question": question,
"contexts": contexts,
"context_count": len(contexts),
"ground_truth": q_item.get("answer", ""),
"files": q_item.get("files", []),
"retrieval_time": retrieval_time,
}
results.append(result)
return {
"total_questions": total_questions,
"results": results,
"metrics": self._calculate_metrics(results, total_retrieval_time),
}
def _calculate_metrics(
self, results: List[Dict[str, Any]], total_retrieval_time: float
) -> Dict[str, Any]:
"""Calculate evaluation metrics."""
total = len(results)
if total == 0:
return {}
context_counts = [r["context_count"] for r in results]
avg_contexts = sum(context_counts) / total if total > 0 else 0
questions_with_contexts = sum(1 for c in context_counts if c > 0)
retrieval_rate = questions_with_contexts / total if total > 0 else 0
retrieval_times = [r["retrieval_time"] for r in results]
avg_retrieval_time = sum(retrieval_times) / total if total > 0 else 0
return {
"total_questions": total,
"avg_contexts_per_question": round(avg_contexts, 2),
"questions_with_contexts": questions_with_contexts,
"retrieval_success_rate": round(retrieval_rate, 2),
"avg_retrieval_time_ms": round(avg_retrieval_time * 1000, 2),
"total_retrieval_time_ms": round(total_retrieval_time * 1000, 2),
}
def print_report(eval_results: Dict[str, Any]):
"""Print evaluation report to console."""
print("\n" + "=" * 60)
print("RAG Evaluation Report")
print("=" * 60)
metrics = eval_results.get("metrics", {})
print("\nOverall Metrics:")
print(f" Total Questions: {metrics.get('total_questions', 0)}")
print(f" Avg Contexts/Question: {metrics.get('avg_contexts_per_question', 0)}")
print(f" Questions with Contexts: {metrics.get('questions_with_contexts', 0)}")
print(f" Retrieval Success Rate: {metrics.get('retrieval_success_rate', 0):.1%}")
print(f" Avg Retrieval Time: {metrics.get('avg_retrieval_time_ms', 0):.1f}ms")
print(f" Total Retrieval Time: {metrics.get('total_retrieval_time_ms', 0):.1f}ms")
print("\nDetailed Results:")
for i, result in enumerate(eval_results.get("results", []), 1):
print(f"\n[Q{i}] {result['question'][:80]}...")
print(f" Contexts Retrieved: {result['context_count']}")
print(f" Retrieval Time: {result['retrieval_time'] * 1000:.1f}ms")
if result["contexts"]:
for j, ctx in enumerate(result["contexts"][:2], 1):
print(f" [{j}] URI: {ctx['uri'][:60]}...")
print(f" Score: {ctx['score']:.3f}")
print("\n" + "=" * 60)
def save_report(eval_results: Dict[str, Any], output_path: str):
"""Save evaluation report to JSON file."""
with open(output_path, "w", encoding="utf-8") as f:
json.dump(eval_results, f, ensure_ascii=False, indent=2)
logger.info(f"Report saved to: {output_path}")
async def run_ragas_evaluation(eval_results: Dict[str, Any]):
"""Run RAGAS evaluation if available."""
try:
from openviking.eval.ragas.types import EvalDataset, EvalSample
from . import RagasEvaluator
print("\nRunning RAGAS evaluation...")
ragas_eval = RagasEvaluator()
samples = []
for result in eval_results["results"]:
sample = EvalSample(
query=result["question"],
context=[c["content"] for c in result["contexts"]],
response="",
ground_truth=result.get("ground_truth", ""),
)
samples.append(sample)
dataset = EvalDataset(name="rag_eval", samples=samples)
ragas_result = await ragas_eval.evaluate_dataset(dataset)
print("\nRAGAS Metrics:")
for metric, score in ragas_result.mean_scores.items():
print(f" {metric}: {score:.3f}")
return ragas_result
except ImportError as e:
logger.error("RAGAS not installed.", exc_info=e)
logger.info(" Install with: pip install ragas datasets")
return None
async def main_async(args):
"""Main async function."""
# if not args.docs_dir and not args.code_dir:
# logger.error("At least one --docs_dir or --code_dir must be specified")
# sys.exit(1)
if not args.question_file:
logger.error("--question_file is required")
sys.exit(1)
question_file = Path(args.question_file)
if not question_file.exists():
logger.error(f"Question file not found: {question_file}")
sys.exit(1)
print("Loading questions...")
questions = load_questions(str(question_file))
print(f" Loaded {len(questions)} questions")
evaluator = RAGEvaluator(
docs_dirs=args.docs_dir,
code_dirs=args.code_dir,
config_path=args.config,
server_url=args.url,
)
print("\nRunning RAG evaluation...")
eval_results = await evaluator.evaluate(
questions=questions,
top_k=args.top_k,
)
print_report(eval_results)
if args.output:
save_report(eval_results, args.output)
if args.ragas:
await run_ragas_evaluation(eval_results)
def main():
"""Main entry point."""
parser = argparse.ArgumentParser(
description="RAG Evaluation Tool for OpenViking",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
# Evaluate with documents
python -m openviking.eval.rag_eval --docs_dir ./docs --question_file ./questions.jsonl
# Evaluate with multiple document directories and code
python -m openviking.eval.rag_eval --docs_dir ./docs1 --docs_dir ./docs2 --code_dir ./code --question_file ./questions.jsonl
# With RAGAS metrics
python -m openviking.eval.rag_eval --docs_dir ./docs --question_file ./questions.jsonl --ragas
""",
)
parser.add_argument(
"--docs_dir",
action="append",
default=[],
help="Document directory or file path (can be specified multiple times)",
)
parser.add_argument(
"--code_dir",
action="append",
default=[],
help="Code repository path (can be specified multiple times)",
)
parser.add_argument(
"--question_file",
required=True,
help="Path to questions file (JSONL format)",
)
parser.add_argument(
"--config",
default="./ov.conf",
help="Path to OpenViking config file (default: ./ov.conf)",
)
parser.add_argument(
"--url",
default="http://127.0.0.1:1933",
help="OpenViking server URL (default: http://127.0.0.1:1933)",
)
parser.add_argument(
"--top_k",
type=int,
default=5,
help="Number of contexts to retrieve per query (default: 5)",
)
parser.add_argument(
"--output",
help="Path to save evaluation results (JSON format)",
)
parser.add_argument(
"--ragas",
action="store_true",
help="Run RAGAS evaluation (requires ragas package)",
)
args = parser.parse_args()
asyncio.run(main_async(args))
if __name__ == "__main__":
main()