Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
Feat(benchmark): Add benchmark/RAG : RAG system evaluation framework (#825)
zgy1b3a8f20a028
diff --git a/.gitignore b/.gitignoreindex e8bddfc20..92b164c30 100644--- a/.gitignore+++ b/.gitignore@@ -124,6 +124,15 @@ tests/api_test/api-test-report.htmltests/api_test/openviking-server.logtests/api_test/openviking-server.pid+# Benchmark outputs+examples/benchmark/outputs/+examples/benchmark/datasets/full/+examples/benchmark/*.log+RAGbenchmark/datasets/*/+!RAGbenchmark/datasets/Benchmark_Lite/+RAGbenchmark/Output/+RAGbenchmark/*.log+# AI CodingCLAUDE.md*.sodiff --git a/benchmark/RAG/.gitignore b/benchmark/RAG/.gitignorenew file mode 100644index 000000000..8f4f065fb--- /dev/null+++ b/benchmark/RAG/.gitignore@@ -0,0 +1,53 @@+# Raw datasets (downloaded from external sources)+raw_data/++# Processed datasets (sampled subsets)+datasets/+data/++# Processed documents and vector storage+ov_storage/++# Evaluation output results+Output/++# Python+__pycache__/+*.py[cod]+*$py.class+*.so+.Python+build/+develop-eggs/+dist/+downloads/+eggs/+.eggs/+lib/+lib64/+parts/+sdist/+var/+wheels/+*.egg-info/+.installed.cfg+*.egg++# Virtual Environment+.venv/+env/+ENV/++# IDE+.vscode/+.idea/+*.swp+*.swo+*~++# Logs+*.log++# Temporary files+*.tmp+*.tempdiff --git a/benchmark/RAG/README.md b/benchmark/RAG/README.mdnew file mode 100644index 000000000..da9318348--- /dev/null+++ b/benchmark/RAG/README.md@@ -0,0 +1,674 @@+# RAG++## English Version++[中文版 README](./README_zh.md)++RAG is an independent RAG (Retrieval-Augmented Generation) system evaluation framework, fully compatible with the latest version of OpenViking.++### Project Structure++```+benchmark/RAG/+├── src/ # Source code+│ ├── __init__.py+│ ├── pipeline.py # Evaluation core pipeline+│ ├── adapters/ # Dataset adapters+│ │ ├── __init__.py+│ │ ├── base.py # Base adapter class+│ │ ├── locomo_adapter.py # Locomo dataset adapter+│ │ ├── syllabusqa_adapter.py # SyllabusQA dataset adapter+│ │ ├── qasper_adapter.py # Qasper dataset adapter+│ │ └── financebench_adapter.py # FinanceBench dataset adapter+│ └── core/ # Core components+│ ├── __init__.py+│ ├── logger.py # Logging module+│ ├── vector_store.py # Vector store wrapper+│ ├── llm_client.py # LLM client wrapper+│ ├── metrics.py # Metrics calculation+│ ├── judge_util.py # LLM judge utility+│ └── monitor.py # Monitoring utility+├── config/ # Configuration files+│ ├── config.yaml # Main configuration file+│ ├── locomo_config.yaml # Locomo dataset configuration+│ ├── syllabusqa_config.yaml # SyllabusQA dataset configuration+│ ├── qasper_config.yaml # Qasper dataset configuration+│ └── financebench_config.yaml # FinanceBench dataset configuration+├── scripts/ # Utility scripts+│ ├── __init__.py+│ ├── download_dataset.py # Dataset download script+│ ├── sample_dataset.py # Dataset sampling script+│ ├── prepare_dataset.py # Unified dataset preparation script+│ └── run_sampling.py # Custom sampling script+├── raw_data/ # Raw dataset directory (downloaded)+├── datasets/ # Sampled dataset directory+├── Output/ # Output result directory+├── run.py # Main execution script+└── README.md+```++### Quick Start++#### 1. Install Dependencies++```bash+cd OpenViking+uv pip install -e ".[benchmark]"+source .venv/bin/activate+```++#### 2. Prepare Datasets++This project provides a complete dataset preparation workflow, including downloading, sampling, and configuration.++##### Dataset Preparation Workflow++Dataset preparation involves two main steps:++1. **Download**: Download raw datasets from official sources to the `raw_data/` directory+2. **Sample**: Sample from raw datasets (optional) to the `datasets/` directory++```+Raw data source → Download → raw_data/{dataset_name}/ → Sample → datasets/{dataset_name}/+```++##### Download Datasets++Use `download_dataset.py` to download datasets:++```bash+cd benchmark/RAG++# Download all configured datasets+python scripts/download_dataset.py++# Download a specific dataset+python scripts/download_dataset.py --dataset Locomo++# Force re-download even if already exists+python scripts/download_dataset.py --dataset Locomo --force+```++##### Sample Datasets++Use `sample_dataset.py` to sample datasets:++```bash+# Sample all datasets (use full dataset, no sampling)+python scripts/sample_dataset.py++# Sample a specific dataset (use full dataset, no sampling)+python scripts/sample_dataset.py --dataset Locomo++# Sample by QA count+python scripts/sample_dataset.py --dataset Locomo --sample-size 100++# Sample by document count (recommended)+python scripts/sample_dataset.py --dataset Locomo --num-docs 5++# Use full dataset (explicitly, no sampling)+python scripts/sample_dataset.py --dataset Locomo --full++# Specify random seed (reproducible)+python scripts/sample_dataset.py --dataset Locomo --num-docs 5 --seed 42+```++**Sampling Strategies:**++1. **Document-level sampling (recommended)**: Use `--num-docs N` to sample N documents first, preserving all QAs within documents+2. **QA-level sampling**: Use `--sample-size N` to randomly select documents until QA count reaches N+3. **Full dataset**: Use `--full` or no sampling parameters to use the complete dataset++##### One-click Preparation++Use `prepare_dataset.py` to complete downloading and sampling in one step:++```bash+# Prepare all datasets (use full dataset, no sampling)+python scripts/prepare_dataset.py++# Prepare a specific dataset, sample 5 documents+python scripts/prepare_dataset.py --dataset Locomo --num-docs 5++# Use full dataset (explicitly, no sampling)+python scripts/prepare_dataset.py --dataset Locomo --full++# Skip download, only sample existing data+python scripts/prepare_dataset.py --dataset Locomo --num-docs 5 --skip-download++# Skip sampling, only download+python scripts/prepare_dataset.py --dataset Locomo --skip-sampling+```++##### Update Configuration Files++After preparing the datasets, you need to update the `dataset_path` in the evaluation configuration files.++**Configuration File Locations:**++```+benchmark/RAG/config/+├── config.yaml # Main configuration file+├── locomo_config.yaml+├── syllabusqa_config.yaml+├── qasper_config.yaml+└── financebench_config.yaml+```++**Dataset Configuration Examples:**++- **Locomo**:+ ```yaml+ dataset_name: "Locomo"+ paths:+ dataset_path: "datasets/Locomo/locomo10.json"+ ```+- **SyllabusQA**:+ ```yaml+ dataset_name: "SyllabusQA"+ paths:+ dataset_path: "datasets/SyllabusQA"+ ```+- **Qasper**:+ ```yaml+ dataset_name: "Qasper"+ paths:+ dataset_path: "datasets/Qasper"+ ```+- **FinanceBench**:+ ```yaml+ dataset_name: "FinanceBench"+ paths:+ dataset_path: "datasets/FinanceBench/financebench_open_source.jsonl"+ ```++**Note:** For datasets with multiple files like SyllabusQA and Qasper, `dataset_path` should be set to the directory path, and the adapter will automatically find and load all relevant files.++#### 3. Configure LLM++Edit LLM configuration in `config/*.yaml`. This configuration is used for both:++- **Answer generation**: Generating answers from retrieved context+- **LLM-as-judge evaluation**: Using LLM to evaluate the quality of generated answers++#### 4. Configure OpenViking++If you need to use custom OpenViking configuration (for data ingestion and retrieval), create an `ov.conf` file in the benchmark/RAG directory. This will override the default OpenViking settings.++You can refer to `examples/ov.conf.example` in the OpenViking root directory for the configuration format.++#### 5. Run Evaluation++```bash+cd benchmark/RAG++# Run complete evaluation (data ingestion, answer generation, evaluation, and data deletion)+python run.py --config config/locomo_config.yaml++# Only run data ingestion and answer generation stage+python run.py --config config/locomo_config.yaml --step gen++# Only run evaluation stage (requires generated answers from previous step)+python run.py --config config/locomo_config.yaml --step eval++# Only run data deletion stage+python run.py --config config/locomo_config.yaml --step del+```++### Supported Datasets++| Dataset | Type | Docs | QAs | Characteristics |+| ---------------- | ---------- | ---- | ---- | ------------------------------------------------------------------------------------------------------------------------------ |+| **Locomo** | Multi-turn | 10 | 1540 | Long conversation understanding, 4 question types (factual, temporal, reasoning, understanding) |+| **SyllabusQA** | Syllabus | 39 | 5078 | Education domain, 6 question types (single factual, multi factual, single reasoning, multi reasoning, summarization, yes/no) |+| **Qasper** | Academic | 1585 | 5049 | Research domain, 1585 NLP papers, 3 answer types (extractive, free-form, yes/no) |+| **FinanceBench** | Financial | 84 | 150 | Financial domain, open-source subset with 150 QA pairs, 3 question types (domain-relevant, metrics-generated, novel-generated) |++### How to Use Different Datasets++Each dataset has its own configuration file in the `config/` directory. To use a specific dataset:++1. **Choose a dataset configuration file**:+ - `config/locomo_config.yaml` - For Locomo dataset+ - `config/syllabusqa_config.yaml` - For SyllabusQA dataset+ - `config/qasper_config.yaml` - For Qasper dataset+ - `config/financebench_config.yaml` - For FinanceBench dataset+2. **Run evaluation with the chosen configuration**:+ ```bash+ # Evaluate with Locomo dataset+ python run.py --config config/locomo_config.yaml++ # Evaluate with SyllabusQA dataset+ python run.py --config config/syllabusqa_config.yaml++ # Evaluate with Qasper dataset+ python run.py --config config/qasper_config.yaml++ # Evaluate with FinanceBench dataset+ python run.py --config config/financebench_config.yaml+ ```+3. **Customize configuration (optional)**:+ You can copy a dataset configuration file and modify it to suit your needs:+ ```bash+ cp config/locomo_config.yaml config/my_custom_config.yaml+ # Edit config/my_custom_config.yaml with your preferences+ python run.py --config config/my_custom_config.yaml+ ```++### Configuration Guide++RAG uses YAML configuration files to control the evaluation process. Each dataset has its own configuration file in the `config/` directory.++**Key Configuration Sections:**++1. **Basic Configuration**:+ - `dataset_name`: Name of the dataset being evaluated+2. **Adapter Configuration**:+ - `adapter.module`: Python module path for the dataset adapter+ - `adapter.class_name`: Class name of the dataset adapter+3. **Execution Configuration**:+ - `max_workers`: Number of concurrent worker threads+ - `ingest_workers`: Number of worker threads for document ingestion+ - `retrieval_topk`: Number of documents to retrieve+ - `max_queries`: Limit the number of queries to process (null = all)+ - `skip_ingestion`: Skip document ingestion (use existing index)+ - `ingest_mode`: Document ingestion mode ("directory" or "per\_file")+ - `retrieval_instruction`: Custom instruction for retrieval (empty by default)+4. **Path Configuration**:+ - `dataset_dir`: Path to dataset file or directory+ - `doc_output_dir`: Directory for processed documents+ - `vector_store`: Directory for vector index storage+ - `output_dir`: Directory for evaluation results+ - `log_file`: Path to log file+5. **LLM Configuration**:+ - `llm.model`: LLM model name+ - `llm.temperature`: Generation temperature+ - `llm.base_url`: API base URL+ - `llm.api_key`: API key (keep secure)++### Evaluation Process Overview++The evaluation process consists of 5 main stages:++1. **Data Preparation**+ - Convert raw dataset into OpenViking-friendly format+ - Process documents for ingestion+2. **Data Ingestion**+ - Ingest processed documents into OpenViking vector store+ - Create embeddings for documents+ - Store vector index for retrieval+3. **Answer Generation**+ - For each question, retrieve relevant documents from vector store+ - Build prompt with retrieved context and question+ - Generate answer using LLM+4. **Evaluation**+ - Use LLM-as-judge to evaluate generated answers against gold answers+ - Calculate metrics (Recall, F1, Accuracy)+5. **Data Deletion**+ - Clean up vector store and remove ingested documents++### Evaluation Metrics++- **Recall**: Retrieval recall rate+- **F1 Score**: Answer F1 score+- **Accuracy**: LLM judge score (0-4)+- **Latency**: Retrieval latency+- **Token Usage**: Token consumption++### Output Files++Evaluation results are saved in the `Output/` directory with the following structure:++```+Output/+└── {dataset_name}/+ └── experiment_{experiment_name}/+ ├── generated_answers.json # Generated answers from LLM+ ├── qa_eval_detailed_results.json # Detailed evaluation results+ ├── benchmark_metrics_report.json # Aggregated metrics report+ ├── docs/ # Processed documents (if skip_ingestion=false)+ └── benchmark.log # Log file+```++**Vector Store Database Location:**+The vector index (document database) is stored in the path specified by `vector_store` in the configuration file. By default, this is:++```+datasets/{dataset_name}/viking_store_index_dir+```++#### File descriptions and examples++**1.** **`benchmark_metrics_report.json`** **- Summary Report**++- **What it contains**: Aggregated metrics report with overall performance scores++Example:++```json+{+ "Insertion Efficiency (Total Dataset)": {+ "Total Insertion Time (s)": 131.98,+ "Total Input Tokens": 142849,+ "Total Output Tokens": 52077,+ "Total Embedding Tokens": 95626+ },+ "Query Efficiency (Average Per Query)": {+ "Average Retrieval Time (s)": 0.17,+ "Average Input Tokens": 3364.46,+ "Average Output Tokens": 15.5+ },+ "Dataset": "Locomo",+ "Total Queries Evaluated": 100,+ "Performance Metrics": {+ "Average F1 Score": 0.318,+ "Average Recall": 0.724,+ "Average Accuracy (Hit 0-4)": 2.36,+ "Average Accuracy (normalization)": 0.59+ }+}+```++**Field descriptions:**++- `Insertion Efficiency`: Document ingestion performance statistics+- `Query Efficiency`: Per-query performance averages+- `Performance Metrics`: Core evaluation scores (0-4 scale for Accuracy)++***++**2.** **`generated_answers.json`** **- Generated Answers**++- **What it contains**: All questions, retrieved contexts, and LLM-generated answers++Example (single result):++```json+{+ "_global_index": 0,+ "sample_id": "conv-26",+ "question": "Would Caroline pursue writing as a career option?",+ "gold_answers": ["LIkely no; though she likes reading, she wants to be a counselor"],+ "category": "3",+ "evidence": ["D7:5", "D7:9"],+ "retrieval": {+ "latency_sec": 0.288,+ "uris": ["viking://resources/...", "viking://resources/..."]+ },+ "llm": {+ "final_answer": "Not mentioned"+ },+ "metrics": {+ "Recall": 1.0+ },+ "token_usage": {+ "total_input_tokens": 2643,+ "llm_output_tokens": 2+ }+}+```++**Field descriptions:**++- `_global_index`: Unique query identifier+- `question`: The question being asked+- `gold_answers`: Ground truth answers+- `retrieval.uris`: URIs of retrieved documents+- `llm.final_answer`: Answer generated by LLM+- `metrics.Recall`: Retrieval recall score (0-1)+- `token_usage`: Token consumption statistics++***++**3.** **`qa_eval_detailed_results.json`** **- Detailed Evaluation**++- **What it contains**: Per-question evaluation including LLM judge reasoning and scores++Example (single result):++```json+{+ "_global_index": 2,+ "question": "When did Caroline go to the LGBTQ support group?",+ "gold_answers": ["7 May 2023"],+ "llm": {+ "final_answer": "7 May 2023 (the day before the chat at 1:56 pm on 8 May, 2023)"+ },+ "metrics": {+ "Recall": 1.0,+ "F1": 0.375,+ "Accuracy": 4+ },+ "llm_evaluation": {+ "prompt_used": "Locomo_0or4",+ "reasoning": "The generated answer explicitly includes the exact date 7 May 2023 that matches the gold answer...",+ "normalized_score": 4+ }+}+```++**Field descriptions:**++- `metrics.F1`: Answer F1 score (0-1)+- `metrics.Accuracy`: LLM judge score (0-4, 4 = perfect)+- `llm_evaluation.reasoning`: LLM judge's reasoning for the score+- `llm_evaluation.normalized_score`: Final normalized score++***++**4.** **`benchmark.log`** **- Execution Log**++- **What it contains**: Detailed execution log with timestamps, warnings, and errors+- **How to view**: Open directly in any text editor++***++**5.** **`docs/`** **- Processed Documents**++- **What it contains**: Processed documents in Markdown format (if `skip_ingestion=false`)+- **How to view**: Open `.md` files directly in any Markdown viewer or text editor++### Benchmark Results Reference++Below are the benchmark results (top-5 retrieval) for reference:++| Dataset | Queries Evaluated | Average F1 Score | Average Recall | Average Accuracy (0-4) | Normalized Accuracy |+| ---------------- | ----------------- | ---------------- | -------------- | ---------------------- | ------------------- |+| **FinanceBench** | 12 | 0.224 | 0.694 | 2.5 | 0.625 |+| **Locomo** | 80 | 0.254 | 0.592 | 2.4 | 0.600 |+| **Qasper** | 60 | 0.293 | 0.614 | 2.12 | 0.529 |+| **SyllabusQA** | 90 | 0.344 | 0.675 | 2.54 | 0.636 |++**Test Configuration Details:**++- **LLM Model:** `doubao-seed-2-0-pro-260215`+- **API Base URL:** `https://ark.cn-beijing.volces.com/api/v3`+- **Temperature:** 0 (deterministic)+- **Retrieval Top-K:** 5+- **Max Workers:** 8+- **Ingest Workers:** 8+- **Ingest Mode:** directory+- **Retrieval Instruction:** (empty)+- **Evaluation Metrics:** Recall, F1 Score, Accuracy (0-4 scale)++All datasets used the same LLM and execution configuration, with dataset-specific adapters and paths configured in their respective YAML files.++### Reproducing the Experiment++To reproduce the benchmark results, follow these steps:++```bash+cd OpenViking/benchmark/RAG++# 1. Install dependencies (if not already installed)+uv pip install -e ".[benchmark]"+source .venv/bin/activate++# 2. Download all datasets+python scripts/download_dataset.py++# 3. Run one-click sampling for all datasets with the same parameters as the benchmark+python scripts/run_sampling.py++# 4. Configure your LLM API key+# Edit the configuration files in config/ and set your API key in the llm.api_key field++# 5. Run evaluation for each dataset+python run.py --config config/locomo_config.yaml+python run.py --config config/syllabusqa_config.yaml+python run.py --config config/qasper_config.yaml+python run.py --config config/financebench_config.yaml++# 6. Check results in Output/{dataset_name}/experiment_test_top_5/+```++**Note:** The `run_sampling.py` script will sample the following:+- Locomo: 3 documents, 80 QAs+- SyllabusQA: 7 documents, 90 QAs+- Qasper: 8 documents, 60 QAs+- FinanceBench: 3 documents, 12 QAs+All with seed=42 for reproducibility.++### Advanced Configuration++#### Retrieval Instruction Configuration++You can configure a custom retrieval instruction in the `config.yaml` file to guide the retrieval process. This instruction is prepended to each query during retrieval.++**Configuration Example:**++```yaml+# ===========Execution Configuration============+# Instruction for retrieval, empty by default+# Recommended format: "Target_modality: xxx.\nInstruction:xxx.\nQuery:"+retrieval_instruction: "Target_modality: text.\nInstruction:Locate the part of the conversation where the speakers discuss.\nQuery:"+```++**Recommended Format:**++- `Target_modality: xxx.` - Specify the target modality (e.g., text, image, audio)+- `Instruction: xxx.` - Provide specific instructions for retrieval+- `Query:` - Mark the start of the actual query++When `retrieval_instruction` is empty, the system will use the raw question for retrieval.++#### Customizing Prompts++RAG uses dataset-specific and question-type-specific prompts to guide LLM answer generation. You can customize these prompts in the adapter files under `src/adapters/` to improve evaluation results.++##### Locomo Dataset Prompts (src/adapters/locomo\_adapter.py)++Locomo has 4 question categories, each with specific instructions:++- **Category 1 (Factual Extraction)**:+ ```+ Extract the exact factual answer from the conversation.+ - Use the exact words from the context when possible+ - If multiple items, separate with commas+ ```+- **Category 2 (Time-related)**:+ ```+ Answer the time-related question.+ - Pay close attention to DATE labels in the conversation+ - Calculate relative time (e.g., "10 years ago") when needed+ - Use the exact dates from the context+ ```+- **Category 3 (Reasoning)**:+ ```+ Reason and infer based on the conversation.+ - Use ONLY the facts in the context+ - State your conclusion clearly (e.g., "Likely yes", "Probably no")+ - Do NOT explain your reasoning or provide any basis/justification+ - Only output your final conclusion, nothing else+ - Do NOT invent information+ ```+- **Category 4 (Understanding/Significance)**:+ ```+ Understand the meaning and significance.+ - Focus on what the speakers mean, not just what they say+ - Identify symbolism or implied meaning+ - Use wording from the context when possible+ ```++##### SyllabusQA Dataset Prompts (src/adapters/syllabusqa\_adapter.py)++SyllabusQA has 6 question types:++- **single factual**: Extract single factual answer+- **multi factual**: Extract multiple factual answers+- **single reasoning**: Simple logical reasoning+- **multi reasoning**: Complex reasoning+- **summarization**: Summarize relevant information+- **yes/no**: Yes/No questions++##### Qasper Dataset Prompts (src/adapters/qasper\_adapter.py)++Qasper has 3 answer types:++- **extractive**: Extract exact answer from paper+- **free\_form**: Free-form answer in own words+- **yes\_no**: Yes/No questions++##### FinanceBench Dataset Prompts (src/adapters/financebench\_adapter.py)++FinanceBench has 3 question types:++- **domain-relevant**: Financial domain questions+- **metrics-generated**: Calculate financial metrics+- **novel-generated**: Novel financial questions++##### How to Customize Prompts++1. Open the adapter file for your dataset (e.g., `src/adapters/locomo_adapter.py`)+2. Locate the `CATEGORY_INSTRUCTIONS` dictionary+3. Modify the prompt text for the question type(s) you want to improve+4. Re-run the evaluation with the modified prompts++### Adding New Datasets++1. Create a new adapter class in `src/adapters/`, inheriting from `BaseAdapter`+2. Create corresponding configuration file in `config/`+3. Implement necessary methods:+ - `data_prepare()`: Data preprocessing+ - `load_and_transform()`: Load and transform data+ - `build_prompt()`: Build prompt+ - `post_process_answer()`: Post-process answer++### Integration with OpenViking++This project integrates with OpenViking through:++- Using `openviking` client for data ingestion and retrieval+- Configuring OpenViking connection via `ov.conf`+- Supporting dynamic loading of OpenViking's latest features++### Frequently Asked Questions (FAQ)++**Q: How do I skip the data ingestion stage if I already have a vector index?**+A: Set `skip_ingestion: true` in the configuration file. This will use the existing vector index.++**Q: Can I run only the evaluation stage without re-ingesting documents?**+A: Yes! First run `--step gen` to generate answers, then run `--step eval` to evaluate the generated answers.++**Q: What should I do if I get an API key error?**+A: Make sure you have set a valid API key in the `llm.api_key` field of your configuration file. Keep your API key secure and do not commit it to version control.++**Q: How can I limit the number of queries processed for testing?**+A: Set `max_queries` in the configuration file to the number of queries you want to process (e.g., `max_queries: 10`).++**Q: What's the difference between "directory" and "per\_file" ingest modes?**+A:++- "directory": Treats the entire directory as one document+- "per\_file": Treats each file as a separate document++**Q: How do I customize the retrieval instruction?**+A: Set `retrieval_instruction` in the configuration file. The recommended format is:+`"Target_modality: xxx.\nInstruction:xxx.\nQuery:"`++**Q: Where can I find the evaluation results?**+A: Results are saved in the directory specified by `output_dir` in the configuration file. By default, this is `Output/{dataset_name}/experiment_{experiment_name}/`.++### License++Same license as OpenViking.diff --git a/benchmark/RAG/README_zh.md b/benchmark/RAG/README_zh.mdnew file mode 100644index 000000000..4eb25e246--- /dev/null+++ b/benchmark/RAG/README_zh.md@@ -0,0 +1,674 @@+# RAG++## 中文版++[English Version README](./README.md)++RAG 是一个独立的 RAG(检索增强生成)系统评估框架,完全兼容最新版本的 OpenViking。++### 项目结构++```+benchmark/RAG/+├── src/ # 源代码+│ ├── __init__.py+│ ├── pipeline.py # 评估核心流水线+│ ├── adapters/ # 数据集适配器+│ │ ├── __init__.py+│ │ ├── base.py # 基础适配器类+│ │ ├── locomo_adapter.py # Locomo 数据集适配器+│ │ ├── syllabusqa_adapter.py # SyllabusQA 数据集适配器+│ │ ├── qasper_adapter.py # Qasper 数据集适配器+│ │ └── financebench_adapter.py # FinanceBench 数据集适配器+│ └── core/ # 核心组件+│ ├── __init__.py+│ ├── logger.py # 日志模块+│ ├── vector_store.py # 向量存储包装器+│ ├── llm_client.py # LLM 客户端包装器+│ ├── metrics.py # 指标计算+│ ├── judge_util.py # LLM 评判工具+│ └── monitor.py # 监控工具+├── config/ # 配置文件+│ ├── config.yaml # 主配置文件+│ ├── locomo_config.yaml # Locomo 数据集配置+│ ├── syllabusqa_config.yaml # SyllabusQA 数据集配置+│ ├── qasper_config.yaml # Qasper 数据集配置+│ └── financebench_config.yaml # FinanceBench 数据集配置+├── scripts/ # 工具脚本+│ ├── __init__.py+│ ├── download_dataset.py # 数据集下载脚本+│ ├── sample_dataset.py # 数据集抽样脚本+│ ├── prepare_dataset.py # 统一数据集准备脚本+│ └── run_sampling.py # 自定义抽样脚本+├── raw_data/ # 原始数据集目录(下载)+├── datasets/ # 抽样数据集目录+├── Output/ # 输出结果目录+├── run.py # 主执行脚本+└── README.md+```++### 快速开始++#### 1. 安装依赖++```bash+cd OpenViking+uv pip install -e ".[benchmark]"+source .venv/bin/activate+```++#### 2. 准备数据集++本项目提供完整的数据集准备工作流,包括下载、抽样和配置。++##### 数据集准备工作流++数据集准备包括两个主要步骤:++1. **下载**:从官方源下载原始数据集到 `raw_data/` 目录+2. **抽样**:从原始数据集抽样(可选)到 `datasets/` 目录++```+原始数据源 → 下载 → raw_data/{dataset_name}/ → 抽样 → datasets/{dataset_name}/+```++##### 下载数据集++使用 `download_dataset.py` 下载数据集:++```bash+cd benchmark/RAG++# 下载所有配置的数据集+python scripts/download_dataset.py++# 下载特定数据集+python scripts/download_dataset.py --dataset Locomo++# 强制重新下载,即使已存在+python scripts/download_dataset.py --dataset Locomo --force+```++##### 抽样数据集++使用 `sample_dataset.py` 抽样数据集:++```bash+# 抽样所有数据集(使用完整数据集,不抽样)+python scripts/sample_dataset.py++# 抽样特定数据集(使用完整数据集,不抽样)+python scripts/sample_dataset.py --dataset Locomo++# 按 QA 数量抽样+python scripts/sample_dataset.py --dataset Locomo --sample-size 100++# 按文档数量抽样(推荐)+python scripts/sample_dataset.py --dataset Locomo --num-docs 5++# 使用完整数据集(显式,不抽样)+python scripts/sample_dataset.py --dataset Locomo --full++# 指定随机种子(可重现)+python scripts/sample_dataset.py --dataset Locomo --num-docs 5 --seed 42+```++**抽样策略:**++1. **文档级抽样(推荐)**:使用 `--num-docs N` 首先抽样 N 个文档,保留文档中的所有 QA+2. **QA 级抽样**:使用 `--sample-size N` 随机选择文档,直到 QA 计数达到 N+3. **完整数据集**:使用 `--full` 或不指定抽样参数来使用完整数据集++##### 一键准备++使用 `prepare_dataset.py` 一步完成下载和抽样:++```bash+# 准备所有数据集(使用完整数据集,不抽样)+python scripts/prepare_dataset.py++# 准备特定数据集,抽样 5 个文档+python scripts/prepare_dataset.py --dataset Locomo --num-docs 5++# 使用完整数据集(显式,不抽样)+python scripts/prepare_dataset.py --dataset Locomo --full++# 跳过下载,只抽样现有数据+python scripts/prepare_dataset.py --dataset Locomo --num-docs 5 --skip-download++# 跳过抽样,只下载+python scripts/prepare_dataset.py --dataset Locomo --skip-sampling+```++##### 更新配置文件++准备数据集后,需要更新评估配置文件中的 `dataset_path`。++**配置文件位置:**++```+benchmark/RAG/config/+├── config.yaml # 主配置文件+├── locomo_config.yaml+├── syllabusqa_config.yaml+├── qasper_config.yaml+└── financebench_config.yaml+```++**数据集配置示例:**++- **Locomo**:+ ```yaml+ dataset_name: "Locomo"+ paths:+ dataset_path: "datasets/Locomo/locomo10.json"+ ```+- **SyllabusQA**:+ ```yaml+ dataset_name: "SyllabusQA"+ paths:+ dataset_path: "datasets/SyllabusQA"+ ```+- **Qasper**:+ ```yaml+ dataset_name: "Qasper"+ paths:+ dataset_path: "datasets/Qasper"+ ```+- **FinanceBench**:+ ```yaml+ dataset_name: "FinanceBench"+ paths:+ dataset_path: "datasets/FinanceBench/financebench_open_source.jsonl"+ ```++**注意:** 对于像 SyllabusQA 和 Qasper 这样有多个文件的数据集,`dataset_path` 应设置为目录路径,适配器会自动查找并加载所有相关文件。++#### 3. 配置 LLM++在 `config/*.yaml` 中编辑 LLM 配置。此配置用于:++- **答案生成**:从检索的上下文生成答案+- **LLM 作为评判者评估**:使用 LLM 评估生成答案的质量++#### 4. 配置 OpenViking++如果需要使用自定义 OpenViking 配置(用于数据摄取和检索),在 benchmark/RAG 目录中创建 `ov.conf` 文件。这将覆盖默认的 OpenViking 设置。++您可以参考 OpenViking 根目录中的 `examples/ov.conf.example` 了解配置格式。++#### 5. 运行评估++```bash+cd benchmark/RAG++# 运行完整评估(数据摄取、答案生成、评估和数据删除)+python run.py --config config/locomo_config.yaml++# 只运行数据摄取和答案生成阶段+python run.py --config config/locomo_config.yaml --step gen++# 只运行评估阶段(需要前一步生成的答案)+python run.py --config config/locomo_config.yaml --step eval++# 只运行数据删除阶段+python run.py --config config/locomo_config.yaml --step del+```++### 支持的数据集++| 数据集 | 类型 | 文档数 | 问题数 | 特点 |+| ---------------- | ---- | ---- | ---- | ---------------------------------------------- |+| **Locomo** | 多轮对话 | 10 | 1540 | 长对话理解,4 种问题类型(事实性、时间性、推理、理解) |+| **SyllabusQA** | 教学大纲 | 39 | 5078 | 教育领域,6 种问题类型(单一事实、多事实、单一推理、多推理、总结、是/否) |+| **Qasper** | 学术论文 | 1585 | 5049 | 研究领域,1585 篇 NLP 论文,3 种答案类型(抽取式、自由形式、是/否) |+| **FinanceBench** | 金融领域 | 84 | 150 | 金融领域,开源子集包含 150 个 QA 对,3 种问题类型(领域相关、指标生成、新颖生成) |++### 如何使用不同的数据集++每个数据集在 `config/` 目录中都有自己的配置文件。要使用特定数据集:++1. **选择数据集配置文件**:+ - `config/locomo_config.yaml` - 用于 Locomo 数据集+ - `config/syllabusqa_config.yaml` - 用于 SyllabusQA 数据集+ - `config/qasper_config.yaml` - 用于 Qasper 数据集+ - `config/financebench_config.yaml` - 用于 FinanceBench 数据集+2. **使用选定的配置运行评估**:+ ```bash+ # 使用 Locomo 数据集评估+ python run.py --config config/locomo_config.yaml++ # 使用 SyllabusQA 数据集评估+ python run.py --config config/syllabusqa_config.yaml++ # 使用 Qasper 数据集评估+ python run.py --config config/qasper_config.yaml++ # 使用 FinanceBench 数据集评估+ python run.py --config config/financebench_config.yaml+ ```+3. **自定义配置(可选)**:+ 您可以复制数据集配置文件并修改它以满足您的需求:+ ```bash+ cp config/locomo_config.yaml config/my_custom_config.yaml+ # 编辑 config/my_custom_config.yaml 以满足您的偏好+ python run.py --config config/my_custom_config.yaml+ ```++### 配置指南++RAG 使用 YAML 配置文件来控制评估过程。每个数据集在 `config/` 目录中都有自己的配置文件。++**关键配置部分:**++1. **基本配置**:+ - `dataset_name`:正在评估的数据集名称+2. **适配器配置**:+ - `adapter.module`:数据集适配器的 Python 模块路径+ - `adapter.class_name`:数据集适配器的类名+3. **执行配置**:+ - `max_workers`:并发工作线程数+ - `ingest_workers`:文档摄取的工作线程数+ - `retrieval_topk`:要检索的文档数+ - `max_queries`:限制要处理的查询数(null = 全部)+ - `skip_ingestion`:跳过文档摄取(使用现有索引)+ - `ingest_mode`:文档摄取模式("directory" 或 "per\_file")+ - `retrieval_instruction`:检索的自定义指令(默认为空)+4. **路径配置**:+ - `dataset_dir`:数据集文件或目录的路径+ - `doc_output_dir`:处理文档的目录+ - `vector_store`:向量索引存储的目录+ - `output_dir`:评估结果的目录+ - `log_file`:日志文件的路径+5. **LLM 配置**:+ - `llm.model`:LLM 模型名称+ - `llm.temperature`:生成温度+ - `llm.base_url`:API 基础 URL+ - `llm.api_key`:API 密钥(保持安全)++### 评估流程概述++评估过程包括 5 个主要阶段:++1. **数据准备**+ - 将原始数据集转换为 OpenViking 友好格式+ - 处理文档以进行摄取+2. **数据摄取**+ - 将处理后的文档摄取到 OpenViking 向量存储中+ - 为文档创建嵌入+ - 存储向量索引以进行检索+3. **答案生成**+ - 对于每个问题,从向量存储中检索相关文档+ - 使用检索的上下文和问题构建提示+ - 使用 LLM 生成答案+4. **评估**+ - 使用 LLM 作为评判者评估生成的答案与黄金答案的质量+ - 计算指标(召回率、F1、准确率)+5. **数据删除**+ - 清理向量存储并删除摄取的文档++### 评估指标++- **Recall**:检索召回率+- **F1 Score**:答案 F1 分数+- **Accuracy**:LLM 评判分数(0-4)+- **Latency**:检索延迟+- **Token Usage**:令牌使用量++### 输出文件++评估结果保存在 `Output/` 目录中,结构如下:++```+Output/+└── {dataset_name}/+ └── experiment_{experiment_name}/+ ├── generated_answers.json # LLM 生成的答案+ ├── qa_eval_detailed_results.json # 详细评估结果+ ├── benchmark_metrics_report.json # 聚合指标报告+ ├── docs/ # 处理后的文档(如果 skip_ingestion=false)+ └── benchmark.log # 日志文件+```++**向量存储数据库位置:**+向量索引(文档数据库)存储在配置文件中 `vector_store` 指定的路径中。默认情况下,这是:++```+datasets/{dataset_name}/viking_store_index_dir+```++#### 文件描述和示例++**1.** **`benchmark_metrics_report.json`** **- 摘要报告**++- **包含内容**:聚合指标报告,包含整体性能分数++示例:++```json+{+ "Insertion Efficiency (Total Dataset)": {+ "Total Insertion Time (s)": 131.98,+ "Total Input Tokens": 142849,+ "Total Output Tokens": 52077,+ "Total Embedding Tokens": 95626+ },+ "Query Efficiency (Average Per Query)": {+ "Average Retrieval Time (s)": 0.17,+ "Average Input Tokens": 3364.46,+ "Average Output Tokens": 15.5+ },+ "Dataset": "Locomo",+ "Total Queries Evaluated": 100,+ "Performance Metrics": {+ "Average F1 Score": 0.318,+ "Average Recall": 0.724,+ "Average Accuracy (Hit 0-4)": 2.36,+ "Average Accuracy (normalization)": 0.59+ }+}+```++**字段描述:**++- `Insertion Efficiency`:文档摄取性能统计+- `Query Efficiency`:每个查询的性能平均值+- `Performance Metrics`:核心评估分数(Accuracy 为 0-4 分制)++***++**2.** **`generated_answers.json`** **- 生成的答案**++- **包含内容**:所有问题、检索的上下文和 LLM 生成的答案++示例(单个结果):++```json+{+ "_global_index": 0,+ "sample_id": "conv-26",+ "question": "Would Caroline pursue writing as a career option?",+ "gold_answers": ["LIkely no; though she likes reading, she wants to be a counselor"],+ "category": "3",+ "evidence": ["D7:5", "D7:9"],+ "retrieval": {+ "latency_sec": 0.288,+ "uris": ["viking://resources/...", "viking://resources/..."]+ },+ "llm": {+ "final_answer": "Not mentioned"+ },+ "metrics": {+ "Recall": 1.0+ },+ "token_usage": {+ "total_input_tokens": 2643,+ "llm_output_tokens": 2+ }+}+```++**字段描述:**++- `_global_index`:唯一查询标识符+- `question`:正在询问的问题+- `gold_answers`:真实答案+- `retrieval.uris`:检索文档的 URI+- `llm.final_answer`:LLM 生成的答案+- `metrics.Recall`:检索召回分数(0-1)+- `token_usage`:令牌消耗统计++***++**3.** **`qa_eval_detailed_results.json`** **- 详细评估**++- **包含内容**:每个问题的评估,包括 LLM 评判者的推理和分数++示例(单个结果):++```json+{+ "_global_index": 2,+ "question": "When did Caroline go to the LGBTQ support group?",+ "gold_answers": ["7 May 2023"],+ "llm": {+ "final_answer": "7 May 2023 (the day before the chat at 1:56 pm on 8 May, 2023)"+ },+ "metrics": {+ "Recall": 1.0,+ "F1": 0.375,+ "Accuracy": 4+ },+ "llm_evaluation": {+ "prompt_used": "Locomo_0or4",+ "reasoning": "The generated answer explicitly includes the exact date 7 May 2023 that matches the gold answer...",+ "normalized_score": 4+ }+}+```++**字段描述:**++- `metrics.F1`:答案 F1 分数(0-1)+- `metrics.Accuracy`:LLM 评判分数(0-4,4 = 完美)+- `llm_evaluation.reasoning`:LLM 评判者对分数的推理+- `llm_evaluation.normalized_score`:最终标准化分数++***++**4.** **`benchmark.log`** **- 执行日志**++- **包含内容**:详细的执行日志,带有时间戳、警告和错误+- **如何查看**:在任何文本编辑器中直接打开++***++**5.** **`docs/`** **- 处理后的文档**++- **包含内容**:Markdown 格式的处理文档(如果 `skip_ingestion=false`)+- **如何查看**:在任何 Markdown 查看器或文本编辑器中直接打开 `.md` 文件++### 基准测试结果参考++以下是基准测试结果(top-5 检索),仅供参考:++| 数据集 | 评估查询数 | 平均 F1 分数 | 平均召回率 | 平均准确率(0-4)| 标准化准确率 |+| ---------------- | ------ | -------- | ----- | ------------ | ------- |+| **FinanceBench** | 12 | 0.224 | 0.694 | 2.5 | 0.625 |+| **Locomo** | 80 | 0.254 | 0.592 | 2.4 | 0.600 |+| **Qasper** | 60 | 0.293 | 0.614 | 2.12 | 0.529 |+| **SyllabusQA** | 90 | 0.344 | 0.675 | 2.54 | 0.636 |++**测试配置详情:**++- **LLM 模型:** `doubao-seed-2-0-pro-260215`+- **API 基础地址:** `https://ark.cn-beijing.volces.com/api/v3`+- **温度参数:** 0(确定性输出)+- **检索 Top-K:** 5+- **最大工作线程数:** 8+- **摄取工作线程数:** 8+- **摄取模式:** directory+- **检索指令:** (空)+- **评估指标:** Recall、F1 分数、Accuracy(0-4 分制)++所有数据集使用相同的 LLM 和执行配置,特定于数据集的适配器和路径在各自的 YAML 文件中配置。++### 复现实验++要复现基准测试结果,请按照以下步骤操作:++```bash+cd OpenViking/benchmark/RAG++# 1. 安装依赖(如果尚未安装)+uv pip install -e ".[benchmark]"+source .venv/bin/activate++# 2. 下载所有数据集+python scripts/download_dataset.py++# 3. 对所有数据集运行一键抽样,使用与基准测试相同的参数+python scripts/run_sampling.py++# 4. 配置您的 LLM API 密钥+# 编辑 config/ 目录下的配置文件,在 llm.api_key 字段中设置您的 API 密钥++# 5. 为每个数据集运行评估+python run.py --config config/locomo_config.yaml+python run.py --config config/syllabusqa_config.yaml+python run.py --config config/qasper_config.yaml+python run.py --config config/financebench_config.yaml++# 6. 在 Output/{dataset_name}/experiment_test_top_5/ 中查看结果+```++**注意:** `run_sampling.py` 脚本将进行以下抽样:+- Locomo:3 个文档,80 个 QA+- SyllabusQA:7 个文档,90 个 QA+- Qasper:8 个文档,60 个 QA+- FinanceBench:3 个文档,12 个 QA+所有抽样使用 seed=42 以确保可重现性。++### 高级配置++#### 检索指令配置++您可以在 `config.yaml` 文件中配置自定义检索指令,以指导检索过程。此指令在检索期间添加到每个查询的前面。++**配置示例:**++```yaml+# ===========Execution Configuration============+# Instruction for retrieval, empty by default+# Recommended format: "Target_modality: xxx.\nInstruction:xxx.\nQuery:"+retrieval_instruction: "Target_modality: text.\nInstruction:Locate the part of the conversation where the speakers discuss.\nQuery:"+```++**推荐格式:**++- `Target_modality: xxx.` - 指定目标模态(例如,文本、图像、音频)+- `Instruction: xxx.` - 为检索提供具体指令+- `Query:` - 标记实际查询的开始++当 `retrieval_instruction` 为空时,系统将使用原始问题进行检索。++#### 自定义提示++RAG 使用特定于数据集和问题类型的提示来指导 LLM 答案生成。您可以在 `src/adapters/` 下的适配器文件中自定义这些提示,以提高评估结果。++##### Locomo 数据集提示(src/adapters/locomo\_adapter.py)++Locomo 有 4 个问题类别,每个类别都有特定的指令:++- **类别 1(事实提取)**:+ ```+ 从对话中提取准确的事实答案。+ - 尽可能使用上下文中的确切词语+ - 如果有多个项目,用逗号分隔+ ```+- **类别 2(时间相关)**:+ ```+ 回答与时间相关的问题。+ - 密切关注对话中的 DATE 标签+ - 必要时计算相对时间(例如,"10 年前")+ - 使用上下文中的确切日期+ ```+- **类别 3(推理)**:+ ```+ 基于对话进行推理和推断。+ - 仅使用上下文中的事实+ - 清楚地陈述您的结论(例如,"可能是","可能不是")+ - 不要解释您的推理或提供任何依据/理由+ - 只输出您的最终结论,别无其他+ - 不要编造信息+ ```+- **类别 4(理解/意义)**:+ ```+ 理解含义和意义。+ - 关注说话者的意思,而不仅仅是他们说的话+ - 识别象征意义或隐含意义+ - 尽可能使用上下文中的措辞+ ```++##### SyllabusQA 数据集提示(src/adapters/syllabusqa\_adapter.py)++SyllabusQA 有 6 种问题类型:++- **single factual**:提取单个事实答案+- **multi factual**:提取多个事实答案+- **single reasoning**:简单逻辑推理+- **multi reasoning**:复杂推理+- **summarization**:总结相关信息+- **yes/no**:是/否问题++##### Qasper 数据集提示(src/adapters/qasper\_adapter.py)++Qasper 有 3 种答案类型:++- **extractive**:从论文中提取准确答案+- **free\_form**:用自己的话自由回答+- **yes\_no**:是/否问题++##### FinanceBench 数据集提示(src/adapters/financebench\_adapter.py)++FinanceBench 有 3 种问题类型:++- **domain-relevant**:金融领域问题+- **metrics-generated**:计算金融指标+- **novel-generated**:新颖的金融问题++##### 如何自定义提示++1. 打开您的数据集的适配器文件(例如,`src/adapters/locomo_adapter.py`)+2. 找到 `CATEGORY_INSTRUCTIONS` 字典+3. 修改您想要改进的问题类型的提示文本+4. 使用修改后的提示重新运行评估++### 添加新数据集++1. 在 `src/adapters/` 中创建一个新的适配器类,继承自 `BaseAdapter`+2. 在 `config/` 中创建相应的配置文件+3. 实现必要的方法:+ - `data_prepare()`:数据预处理+ - `load_and_transform()`:加载和转换数据+ - `build_prompt()`:构建提示+ - `post_process_answer()`:后处理答案++### 与 OpenViking 集成++本项目通过以下方式与 OpenViking 集成:++- 使用 `openviking` 客户端进行数据摄取和检索+- 通过 `ov.conf` 配置 OpenViking 连接+- 支持动态加载 OpenViking 的最新功能++### 常见问题(FAQ)++**问:如果我已经有向量索引,如何跳过数据摄取阶段?**+答:在配置文件中设置 `skip_ingestion: true`。这将使用现有的向量索引。++**问:我可以只运行评估阶段而不重新摄取文档吗?**+答:可以!首先运行 `--step gen` 生成答案,然后运行 `--step eval` 评估生成的答案。++**问:如果我收到 API 密钥错误,应该怎么办?**+答:确保您在配置文件的 `llm.api_key` 字段中设置了有效的 API 密钥。保持您的 API 密钥安全,不要将其提交到版本控制中。++**问:如何限制测试处理的查询数量?**+答:在配置文件中设置 `max_queries` 为您想要处理的查询数量(例如,`max_queries: 10`)。++**问:"directory" 和 "per\_file" 摄取模式有什么区别?**+答:++- "directory":将整个目录视为一个文档+- "per\_file":将每个文件视为一个单独的文档++**问:如何自定义检索指令?**+答:在配置文件中设置 `retrieval_instruction`。推荐格式为:+`"Target_modality: xxx.\nInstruction:xxx.\nQuery:"`++**问:我在哪里可以找到评估结果?**+答:结果保存在配置文件中 `output_dir` 指定的目录中。默认情况下,这是 `Output/{dataset_name}/experiment_{experiment_name}/`。++### 许可证++与 OpenViking 相同的许可证。diff --git a/benchmark/RAG/config/config.yaml b/benchmark/RAG/config/config.yamlnew file mode 100644index 000000000..dfa51affd--- /dev/null+++ b/benchmark/RAG/config/config.yaml@@ -0,0 +1,60 @@+# RAG Benchmark Configuration File+project_name: "RAG_Benchmark"++# ===========Basic Configuration============+# Dataset name to identify the current dataset being evaluated+# Supported datasets: FinanceBench, Qasper, Locomo, SyllabusQA, etc.+dataset_name: "choose_your_dataset_here"++# ===========Adapter Configuration============+# Dataset adapter configuration for loading and processing dataset-specific formats+adapter:+ # Adapter module path (Python import path)+ module: "src.adapters.your_dataset_adapter"+ # Adapter class name+ class_name: "your_dataset_Adapter"++# ===========Execution Configuration============+execution:+ # Maximum number of concurrent worker threads for parallel QA task processing+ max_workers: 10+ # Number of concurrent worker threads for document ingestion+ ingest_workers: 10+ # Number of documents to retrieve (top-k)+ retrieval_topk: 5+ # Limit the maximum number of queries to process, null means process all queries+ max_queries: null+ # Whether to skip document ingestion phase (true means use existing vector index)+ skip_ingestion: false+ # Document ingestion mode+ # - "directory": Unified directory ingestion mode, treats the entire directory as one document+ # - "per_file": Per-file ingestion mode, each file is treated as a separate document+ ingest_mode: "per_file"+ # Instruction for retrieval, empty by default+ retrieval_instruction: ""++# ===========Path Configuration============+# All paths support {dataset_name} placeholder, which will be automatically replaced with the current dataset name+paths:+ # Dataset directory or file path+ dataset_path: "datasets/{dataset_name}"+ # Output directory for processed documents+ doc_output_dir: "datasets/{dataset_name}/{dataset_name}_processed_docs"+ # Vector index storage directory+ vector_store: "datasets/{dataset_name}/viking_store_index_dir"+ # Results output directory+ output_dir: "Output/{dataset_name}/experiment_dev"+ # Log file path+ log_file: "Output/{dataset_name}/experiment_dev/benchmark.log"++# ===========LLM Configuration============+# Large language model configuration for answer generation and evaluation+llm:+ # Model name+ model: "doubao-seed-2-0-pro-260215"+ # Generation temperature (0 means deterministic output, higher values mean more randomness)+ temperature: 0+ # API base URL+ base_url: "https://ark.cn-beijing.volces.com/api/v3"+ # API key (keep this secure, do not commit to code repositories)+ api_key: ""diff --git a/benchmark/RAG/config/financebench_config.yaml b/benchmark/RAG/config/financebench_config.yamlnew file mode 100644index 000000000..e05900277--- /dev/null+++ b/benchmark/RAG/config/financebench_config.yaml@@ -0,0 +1,32 @@+project_name: "RAG_Benchmark"++# ===========Modify Configuration================+dataset_name: "FinanceBench"++adapter:+ module: "src.adapters.financebench_adapter"+ class_name: "FinanceBenchAdapter"++execution:+ max_workers: 8+ ingest_workers: 8+ retrieval_topk: 10+ max_queries: null+ skip_ingestion: false+ ingest_mode: "directory"+ retrieval_instruction: ""+# ==================================++paths:+ dataset_path: "datasets/{dataset_name}/financebench_open_source.jsonl"+ doc_output_dir: "ov_storage/{dataset_name}/{dataset_name}_processed_docs"+ vector_store: "ov_storage/{dataset_name}/{dataset_name}_viking_store_index"+ output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}"+ log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log"+++llm:+ model: "doubao-seed-2-0-pro-260215"+ temperature: 0+ base_url: "https://ark.cn-beijing.volces.com/api/v3"+ api_key: "your_api_key_here"diff --git a/benchmark/RAG/config/locomo_config.yaml b/benchmark/RAG/config/locomo_config.yamlnew file mode 100644index 000000000..ac5c98620--- /dev/null+++ b/benchmark/RAG/config/locomo_config.yaml@@ -0,0 +1,32 @@+project_name: "RAG_Benchmark"++# ===========Modify Configuration================+dataset_name: "Locomo"++adapter:+ module: "src.adapters.locomo_adapter"+ class_name: "LocomoAdapter"++execution:+ max_workers: 8+ ingest_workers: 8+ retrieval_topk: 5+ max_queries: 20+ skip_ingestion: false+ ingest_mode: "directory"+ retrieval_instruction: ""+# ==================================++paths:+ dataset_path: "datasets/{dataset_name}/locomo10.json"+ doc_output_dir: "ov_storage/{dataset_name}/{dataset_name}_processed_docs"+ vector_store: "ov_storage/{dataset_name}/{dataset_name}_viking_store_index"+ output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}"+ log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log"+++llm:+ model: "doubao-seed-2-0-pro-260215"+ temperature: 0+ base_url: "https://ark.cn-beijing.volces.com/api/v3"+ api_key: "your_api_key_here"diff --git a/benchmark/RAG/config/qasper_config.yaml b/benchmark/RAG/config/qasper_config.yamlnew file mode 100644index 000000000..0ba0c3b77--- /dev/null+++ b/benchmark/RAG/config/qasper_config.yaml@@ -0,0 +1,32 @@+project_name: "RAG_Benchmark"++# ===========Modify Configuration================+dataset_name: "Qasper"++adapter:+ module: "src.adapters.qasper_adapter"+ class_name: "QasperAdapter"++execution:+ max_workers: 8+ ingest_workers: 8+ retrieval_topk: 5+ max_queries: 10+ skip_ingestion: false+ ingest_mode: "directory"+ retrieval_instruction: ""+# ==================================++paths:+ dataset_path: "datasets/{dataset_name}"+ doc_output_dir: "ov_storage/{dataset_name}/{dataset_name}_processed_docs"+ vector_store: "ov_storage/{dataset_name}/{dataset_name}_viking_store_index"+ output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}"+ log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log"+++llm:+ model: "doubao-seed-2-0-pro-260215"+ temperature: 0+ base_url: "https://ark.cn-beijing.volces.com/api/v3"+ api_key: "your_api_key_here"diff --git a/benchmark/RAG/config/syllabusqa_config.yaml b/benchmark/RAG/config/syllabusqa_config.yamlnew file mode 100644index 000000000..07244a19b--- /dev/null+++ b/benchmark/RAG/config/syllabusqa_config.yaml@@ -0,0 +1,32 @@+project_name: "RAG_Benchmark"++# ===========Modify Configuration================+dataset_name: "SyllabusQA"++adapter:+ module: "src.adapters.syllabusqa_adapter"+ class_name: "SyllabusQAAdapter"++execution:+ max_workers: 8+ ingest_workers: 8+ retrieval_topk: 5+ max_queries: 20+ skip_ingestion: false+ ingest_mode: "directory"+ retrieval_instruction: ""+# ==================================++paths:+ dataset_path: "datasets/{dataset_name}"+ doc_output_dir: "ov_storage/{dataset_name}/{dataset_name}_processed_docs"+ vector_store: "ov_storage/{dataset_name}/{dataset_name}_viking_store_index"+ output_dir: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}"+ log_file: "Output/{dataset_name}/experiment_test_top_{retrieval_topk}/benchmark.log"+++llm:+ model: "doubao-seed-2-0-pro-260215"+ temperature: 0+ base_url: "https://ark.cn-beijing.volces.com/api/v3"+ api_key: "your_api_key_here"diff --git a/benchmark/RAG/ov.conf.example b/benchmark/RAG/ov.conf.examplenew file mode 100644index 000000000..e41a79d9a--- /dev/null+++ b/benchmark/RAG/ov.conf.example@@ -0,0 +1,34 @@+{+ "storage": {+ "agfs": {+ "port": 1876+ }+ },+ "log": {+ "level": "INFO",+ "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s",+ "output": "stdout"+ },+ "embedding": {+ "dense": {+ "model": "doubao-embedding-vision-250615",+ "api_key": "your_api_key_here",+ "api_base": "https://ark.cn-beijing.volces.com/api/v3",+ "dimension": 1024,+ "provider": "volcengine",+ "input": "multimodal",+ "batch_size": 8+ },+ "max_concurrent": 10+ },+ "vlm": {+ "model": "doubao-seed-2-0-pro-260215",+ "api_key": "your_api_key_here",+ "api_base": "https://ark.cn-beijing.volces.com/api/v3",+ "temperature": 0.0,+ "max_retries": 240,+ "provider": "volcengine",+ "thinking": false,+ "max_concurrent": 64+ }+}diff --git a/benchmark/RAG/run.py b/benchmark/RAG/run.pynew file mode 100644index 000000000..0d2d0a57b--- /dev/null+++ b/benchmark/RAG/run.py@@ -0,0 +1,169 @@+import os+import sys+import yaml+import importlib+from argparse import ArgumentParser+from pathlib import Path++sys.path.append(str(Path(__file__).parent))++from src.core.logger import setup_logging+# ==========================================+# 1. Environment Initialization+# ==========================================+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))+PROJECT_ROOT = SCRIPT_DIR++ov_config_path = os.path.join(SCRIPT_DIR, "ov.conf")+if os.path.exists(ov_config_path):+ os.environ["OPENVIKING_CONFIG_FILE"] = ov_config_path+ print(f"[Init] Auto-detected OpenViking config: {ov_config_path}")++try:+ from src.pipeline import BenchmarkPipeline+ from src.core.vector_store import VikingStoreWrapper+ from src.core.llm_client import LLMClientWrapper+except SyntaxError as e:+ print(f"\n[Fatal Error] Syntax error while importing modules: {e}")+ sys.exit(1)+except ImportError as e:+ print(f"\n[Fatal Error] Cannot import modules: {e}")+ print(f"Current sys.path: {sys.path}\n")+ sys.exit(1)++# ==========================================+# 2. Helper Functions+# ==========================================++def load_config(config_path):+ if not os.path.exists(config_path):+ raise FileNotFoundError(f"Config file not found: {config_path}")+ with open(config_path, 'r', encoding='utf-8') as f:+ return yaml.safe_load(f)++def resolve_path(path_str, base_path):+ """+ Convert relative path to absolute path based on base_path.+ If path_str is already absolute, keep it unchanged.+ """+ if not path_str:+ return path_str+ if os.path.isabs(path_str):+ return path_str+ return os.path.normpath(os.path.join(base_path, path_str))++# ==========================================+# 3. Main Program+# ==========================================++def main():+ parser = ArgumentParser(description="Run RAG Benchmark (Smart Path Handling)")+ default_config_path = os.path.join(SCRIPT_DIR, "config/config.yaml")++ parser.add_argument("--config", default=default_config_path,+ help=f"Path to config file. Default: {default_config_path}")++ parser.add_argument("--step", choices=["all", "gen", "eval", "del"], default="all",+ help="Execution step: 'gen' (Retrieval+LLM), 'eval' (Judge), or 'all'")++ args = parser.parse_args()++ # --- B. Load and Parse Config ---+ config_path = os.path.abspath(args.config)+ print(f"[Init] Loading configuration from: {config_path}")++ try:+ config = load_config(config_path)+ except FileNotFoundError as e:+ print(f"[Error] {e}")+ return++ # --- C. Path Resolution ---+ print(f"[Init] Resolving paths relative to Project Root: {PROJECT_ROOT}")+ dataset_name = config.get('dataset_name', 'UnknownDataset')+ retrieval_topk = config.get('execution', {}).get('retrieval_topk', 5)++ format_vars = {+ 'dataset_name': dataset_name,+ 'retrieval_topk': retrieval_topk+ }++ path_keys = ['dataset_path', 'output_dir', 'vector_store', 'log_file', 'doc_output_dir']+ for key in path_keys:+ if key in config.get('paths', {}):+ original = config['paths'][key]+ rendered_path = original.format(**format_vars)+ resolved = resolve_path(rendered_path, PROJECT_ROOT)+ config['paths'][key] = resolved+ # print(f" - {key}: {resolved}")++ # --- D. Initialize Components ---+ try:+ logger = setup_logging(config['paths']['log_file'])+ logger.info(">>> Benchmark Session Started")++ # 1. Adapter (Dynamic Loading)+ adapter_cfg = config.get('adapter', {})+ module_path = adapter_cfg.get('module', 'src.adapters.locomo_adapter')+ class_name = adapter_cfg.get('class_name', 'LocomoAdapter')++ logger.info(f"Dynamically loading Adapter: {class_name} from {module_path}")+ logger.info(f"Loading dataset from: {config['paths']['dataset_path']}")++ try:+ mod = importlib.import_module(module_path)+ AdapterClass = getattr(mod, class_name)+ adapter = AdapterClass(raw_file_path=config['paths']['dataset_path'])+ except ImportError as e:+ logger.error(f"Could not import module '{module_path}'. Please check your config 'adapter.module'. Error: {e}")+ raise e+ except AttributeError as e:+ logger.error(f"Class '{class_name}' not found in module '{module_path}'. Please check your config 'adapter.class_name'. Error: {e}")+ raise e++ # 2. Vector Store+ vector_store = VikingStoreWrapper(store_path=config['paths']['vector_store'])++ # 3. LLM Client+ api_key = os.environ.get(+ config['llm'].get('api_key_env_var', ''),+ config['llm'].get('api_key')+ )+ if not api_key:+ logger.warning("No API Key found in config or environment variables!")++ llm_client = LLMClientWrapper(config=config['llm'], api_key=api_key)++ # 4. Pipeline+ pipeline = BenchmarkPipeline(+ config=config,+ adapter=adapter,+ vector_db=vector_store,+ llm=llm_client+ )++ # --- E. Execute Tasks ---+ if args.step in ["all", "gen"]:+ logger.info("Stage: Generation (Ingest -> Retrieve -> Generate)")+ pipeline.run_generation()++ if args.step in ["all", "eval"]:+ logger.info("Stage: Evaluation (Judge -> Metrics)")+ pipeline.run_evaluation()++ if args.step in ["all", "del"]:+ logger.info("Stage: Delete Vector Store")+ pipeline.run_deletion()++ logger.info("Benchmark finished successfully.")++ except KeyboardInterrupt:+ print("\n[Stop] Execution interrupted by user.")+ except Exception as e:+ if 'logger' in locals():+ logger.exception("Fatal error during execution")+ print(f"\n[Fatal Error] Program execution error: {str(e)}")+ sys.exit(1)++if __name__ == "__main__":+ main()diff --git a/benchmark/RAG/scripts/__init__.py b/benchmark/RAG/scripts/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/benchmark/RAG/scripts/download_dataset.py b/benchmark/RAG/scripts/download_dataset.pynew file mode 100644index 000000000..3a32ff639--- /dev/null+++ b/benchmark/RAG/scripts/download_dataset.py@@ -0,0 +1,364 @@+#!/usr/bin/env python3+"""+Download datasets from public sources.+Supports URL downloads from GitHub, S3, etc.+"""++import argparse+import hashlib+import json+import os+import shutil+import sys+from pathlib import Path+from typing import Dict, List, Optional+from urllib.parse import urlparse++import requests+from tqdm import tqdm++sys.path.append(str(Path(__file__).parent.parent))++DATASET_SOURCES = {+ # 示例配置:取消注释并根据需要修改+ #+ "Locomo": {+ "source_type": "url",+ "url": "https://raw.githubusercontent.com/snap-research/locomo/main/data/locomo10.json",+ "checksum": "",+ "files": [+ "locomo10.json"+ ]+ },+ #+ "SyllabusQA": {+ "source_type": "url",+ "url": "https://github.com/umass-ml4ed/SyllabusQA/archive/refs/heads/main.zip",+ "checksum": "",+ "files": [+ "data/dataset_split/train.csv",+ "data/dataset_split/val.csv",+ "data/dataset_split/test.csv",+ "syllabi/syllabi_redacted/word/"+ ],+ "extract_subdir": "SyllabusQA-main"+ },+ #+ "Qasper": {+ "source_type": "url",+ "urls": [+ "https://qasper-dataset.s3.us-west-2.amazonaws.com/qasper-train-dev-v0.3.tgz",+ "https://qasper-dataset.s3.us-west-2.amazonaws.com/qasper-test-and-evaluator-v0.3.tgz"+ ],+ "checksum": "",+ "files": [+ "qasper-train-v0.3.json",+ "qasper-dev-v0.3.json",+ "qasper-test-v0.3.json"+ ]+ },+ #+ "FinanceBench": {+ "source_type": "url",+ "url": "https://github.com/patronus-ai/financebench/archive/refs/heads/main.zip",+ "checksum": "",+ "files": [+ "data/financebench_open_source.jsonl",+ "data/financebench_document_information.jsonl",+ "pdfs/"+ ],+ "extract_subdir": "financebench-main"+ },+}++def calculate_checksum(file_path: Path, algorithm: str = "sha256") -> str:+ """Calculate checksum of a file."""+ hash_obj = hashlib.new(algorithm)+ with open(file_path, "rb") as f:+ for chunk in iter(lambda: f.read(8192), b""):+ hash_obj.update(chunk)+ return hash_obj.hexdigest()+++def download_file(url: str, dest_path: Path, chunk_size: int = 8192) -> bool:+ """Download a file with progress bar."""+ dest_path.parent.mkdir(parents=True, exist_ok=True)++ try:+ response = requests.get(url, stream=True, timeout=30)+ response.raise_for_status()++ total_size = int(response.headers.get("content-length", 0))++ with open(dest_path, "wb") as f, tqdm(+ desc=f"Downloading {dest_path.name}",+ total=total_size,+ unit="B",+ unit_scale=True,+ unit_divisor=1024,+ ) as pbar:+ for chunk in response.iter_content(chunk_size=chunk_size):+ if chunk:+ f.write(chunk)+ pbar.update(len(chunk))+ return True+ except Exception as e:+ print(f"Error downloading {url}: {e}")+ if dest_path.exists():+ dest_path.unlink()+ return False+++def extract_archive(archive_path: Path, extract_to: Path, extract_subdir: Optional[str] = None) -> bool:+ """Extract archive file (zip, tar.gz, etc.)."""+ import zipfile+ import tarfile++ try:+ temp_extract_dir = extract_to / ".temp_extract"+ temp_extract_dir.mkdir(parents=True, exist_ok=True)++ if archive_path.suffix == ".zip":+ with zipfile.ZipFile(archive_path, "r") as zip_ref:+ zip_ref.extractall(temp_extract_dir)+ elif archive_path.suffix in [".tar", ".gz", ".tgz"]:+ with tarfile.open(archive_path, "r:*") as tar_ref:+ tar_ref.extractall(temp_extract_dir)+ else:+ print(f"Unsupported archive format: {archive_path.suffix}")+ shutil.rmtree(temp_extract_dir)+ return False++ if extract_subdir:+ source_dir = temp_extract_dir / extract_subdir+ if source_dir.exists() and source_dir.is_dir():+ for item in source_dir.iterdir():+ dest_item = extract_to / item.name+ if dest_item.exists():+ if dest_item.is_dir():+ shutil.rmtree(dest_item)+ else:+ dest_item.unlink()+ shutil.move(str(item), str(dest_item))+ else:+ print(f"Warning: Subdirectory {extract_subdir} not found in archive")+ else:+ for item in temp_extract_dir.iterdir():+ dest_item = extract_to / item.name+ if dest_item.exists():+ if dest_item.is_dir():+ shutil.rmtree(dest_item)+ else:+ dest_item.unlink()+ shutil.move(str(item), str(dest_item))++ shutil.rmtree(temp_extract_dir)+ return True+ except Exception as e:+ print(f"Error extracting {archive_path}: {e}")+ if temp_extract_dir.exists():+ shutil.rmtree(temp_extract_dir)+ return False+++def verify_dataset(dataset_name: str, dataset_dir: Path) -> bool:+ """Verify that all required files exist for a dataset."""+ if dataset_name not in DATASET_SOURCES:+ print(f"Unknown dataset: {dataset_name}")+ return False++ source = DATASET_SOURCES[dataset_name]+ missing_files = []++ for file_path in source["files"]:+ full_path = dataset_dir / file_path+ # Check if path exists (either file or directory)+ if not full_path.exists():+ missing_files.append(file_path)++ if missing_files:+ print(f"Missing files for {dataset_name}: {missing_files}")+ return False++ print(f"✓ {dataset_name} verified successfully")+ return True+++def is_archive_file(file_path: Path) -> bool:+ """Check if a file is an archive based on extension."""+ archive_extensions = [".zip", ".tar", ".tar.gz", ".tgz", ".gz"]+ return any(str(file_path).lower().endswith(ext) for ext in archive_extensions)+++def download_from_url(+ source: Dict,+ output_dir: Path,+ dataset_name: str,+ force: bool = False,+ verify: bool = True+) -> bool:+ """Download dataset from URL. Supports both archives and single files.+ Supports single url or multiple urls via urls field.+ """+ dataset_dir = output_dir / dataset_name++ if dataset_dir.exists() and not force:+ print(f"{dataset_name} already exists at {dataset_dir}, skipping download")+ if verify:+ return verify_dataset(dataset_name, dataset_dir)+ return True++ print(f"Downloading {dataset_name}...")++ # Support single url or multiple urls+ urls = source.get("urls", [source.get("url")]) if source.get("urls") else [source.get("url")]++ success = True+ for url in urls:+ if not url:+ continue++ parsed_url = urlparse(url)+ file_name = Path(parsed_url.path).name+ downloaded_path = output_dir / file_name++ if not download_file(url, downloaded_path):+ success = False+ continue++ if "checksum" in source and source["checksum"]:+ algo, expected_checksum = source["checksum"].split(":", 1)+ actual_checksum = calculate_checksum(downloaded_path, algo)+ if actual_checksum != expected_checksum:+ print(f"Checksum mismatch for {dataset_name}")+ print(f"Expected: {expected_checksum}")+ print(f"Actual: {actual_checksum}")+ downloaded_path.unlink()+ success = False+ continue+ print(f"✓ Checksum verified for {dataset_name}")++ if is_archive_file(downloaded_path):+ extract_subdir = source.get("extract_subdir")+ if not extract_archive(downloaded_path, dataset_dir, extract_subdir):+ downloaded_path.unlink()+ success = False+ continue+ downloaded_path.unlink()+ else:+ dataset_dir.mkdir(parents=True, exist_ok=True)+ dest_path = dataset_dir / file_name+ shutil.move(str(downloaded_path), str(dest_path))+ print(f"✓ Saved single file to {dest_path}")++ if verify and not verify_dataset(dataset_name, dataset_dir):+ return False++ if success:+ print(f"✓ {dataset_name} downloaded successfully to {dataset_dir}")+ return success+++def download_dataset(+ dataset_name: str,+ output_dir: Path,+ force: bool = False,+ verify: bool = True+) -> bool:+ """Download a single dataset."""+ if dataset_name not in DATASET_SOURCES:+ print(f"Unknown dataset: {dataset_name}")+ return False++ source = DATASET_SOURCES[dataset_name]+ dataset_dir = output_dir / dataset_name++ if dataset_dir.exists() and not force:+ print(f"{dataset_name} already exists at {dataset_dir}, skipping download")+ if verify:+ return verify_dataset(dataset_name, dataset_dir)+ return True++ success = download_from_url(source, output_dir, dataset_name, force, verify)++ if success and verify:+ return verify_dataset(dataset_name, dataset_dir)++ return success+++def main():+ # Check if any datasets are configured+ configured_datasets = [k for k in DATASET_SOURCES.keys() if not k.startswith('#')]++ if not configured_datasets:+ print("=" * 80)+ print("No datasets configured!")+ print("=" * 80)+ print()+ print("Please edit DATASET_SOURCES in download_dataset.py")+ print("and uncomment the example configurations or add your own.")+ print()+ print("See README_DATASET_CONFIG.md for detailed instructions.")+ print("=" * 80)+ return 1++ parser = argparse.ArgumentParser(+ description="Download datasets for RAG benchmark"+ )+ parser.add_argument(+ "--dataset", "-d",+ type=str,+ choices=configured_datasets + ["all"],+ default="all",+ help="Dataset to download (default: all)"+ )+ parser.add_argument(+ "--output-dir", "-o",+ type=Path,+ default=Path(__file__).parent.parent / "raw_data",+ help="Output directory (default: raw_data/)"+ )+ parser.add_argument(+ "--force", "-f",+ action="store_true",+ help="Force re-download even if dataset exists"+ )+ parser.add_argument(+ "--no-verify",+ action="store_true",+ help="Skip dataset verification"+ )++ args = parser.parse_args()++ output_dir = args.output_dir.resolve()+ output_dir.mkdir(parents=True, exist_ok=True)++ datasets = (+ configured_datasets+ if args.dataset == "all"+ else [args.dataset]+ )++ print(f"Downloading datasets to: {output_dir}")+ print(f"Datasets: {', '.join(datasets)}")+ print()++ success_count = 0+ for dataset in datasets:+ if download_dataset(+ dataset,+ output_dir,+ args.force,+ not args.no_verify+ ):+ success_count += 1+ print()++ print(f"Download complete: {success_count}/{len(datasets)} successful")+ return 0 if success_count == len(datasets) else 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/benchmark/RAG/scripts/prepare_dataset.py b/benchmark/RAG/scripts/prepare_dataset.pynew file mode 100644index 000000000..a734cee8d--- /dev/null+++ b/benchmark/RAG/scripts/prepare_dataset.py@@ -0,0 +1,254 @@+#!/usr/bin/env python3+"""+Unified dataset preparation script.+Orchestrates download and sampling for end-to-end data preparation.+"""++import argparse+import sys+from pathlib import Path+from typing import List, Optional++sys.path.append(str(Path(__file__).parent))++from download_dataset import download_dataset, DATASET_SOURCES as DOWNLOAD_SOURCES+from sample_dataset import sample_dataset, DATASET_SAMPLERS+++def prepare_dataset(+ dataset_name: str,+ download_dir: Path,+ output_dir: Path,+ sample_size: Optional[int] = None,+ num_docs: Optional[int] = None,+ seed: int = 42,+ skip_download: bool = False,+ skip_sampling: bool = False,+ force_download: bool = False,+ use_full: bool = False,+ sample_mode: str = "stratified"+) -> bool:+ """Prepare a single dataset end-to-end."""+ print("\n" + "=" * 80)+ print(f"Preparing dataset: {dataset_name}")+ print("=" * 80)++ success = True++ # Step 1: Download+ if not skip_download:+ print("\n[Step 1/2] Downloading dataset...")+ download_success = download_dataset(+ dataset_name,+ download_dir,+ force=force_download,+ verify=True+ )+ if not download_success:+ print(f"❌ Failed to download {dataset_name}")+ success = False+ else:+ print("\n[Step 1/2] Skipping download (--skip-download)")++ # Step 2: Sample+ if not skip_sampling and success:+ print("\n[Step 2/2] Sampling dataset...")+ input_dir = download_dir / dataset_name+ dataset_output_dir = output_dir / dataset_name++ actual_sample_size = None if use_full else sample_size+ actual_num_docs = None if use_full else num_docs++ sample_success = sample_dataset(+ dataset_name,+ input_dir,+ dataset_output_dir,+ actual_sample_size,+ actual_num_docs,+ seed,+ sample_mode+ )+ if not sample_success:+ print(f"❌ Failed to sample {dataset_name}")+ success = False+ elif skip_sampling:+ print("\n[Step 2/2] Skipping sampling (--skip-sampling)")++ return success+++def main():+ parser = argparse.ArgumentParser(+ description="Unified dataset preparation for RAG benchmark",+ formatter_class=argparse.RawDescriptionHelpFormatter,+ epilog="""+Examples:+ # Prepare all datasets with default settings+ python prepare_dataset.py++ # Prepare only Locomo dataset with 50 samples+ python prepare_dataset.py --dataset Locomo --sample-size 50++ # Prepare all datasets using full data (no sampling)+ python prepare_dataset.py --full++ # Skip download, only sample existing datasets+ python prepare_dataset.py --skip-download++ # Skip sampling, only download datasets+ python prepare_dataset.py --skip-sampling+ """+ )++ # Dataset selection+ parser.add_argument(+ "--dataset", "-d",+ type=str,+ choices=list(DOWNLOAD_SOURCES.keys()) + ["all"],+ default="all",+ help="Dataset to prepare (default: all)"+ )++ # Directories+ parser.add_argument(+ "--download-dir",+ type=Path,+ default=Path(__file__).parent.parent / "raw_data",+ help="Directory to download datasets to (default: raw_data/)"+ )+ parser.add_argument(+ "--output-dir", "-o",+ type=Path,+ default=Path(__file__).parent.parent / "datasets",+ help="Directory for final prepared datasets (default: datasets/)"+ )++ # Sampling options+ parser.add_argument(+ "--sample-size", "-n",+ type=int,+ default=None,+ help="Number of samples to use (default: use full dataset)"+ )+ parser.add_argument(+ "--num-docs",+ type=int,+ default=None,+ help="Number of documents to sample (for document-level sampling)"+ )+ parser.add_argument(+ "--seed", "-s",+ type=int,+ default=42,+ help="Random seed for reproducibility (default: 42)"+ )+ parser.add_argument(+ "--full",+ action="store_true",+ help="Use full dataset (no sampling, overrides --sample-size)"+ )+ parser.add_argument(+ "--sample-mode",+ type=str,+ choices=["random", "stratified"],+ default="random",+ help="Sampling mode: 'random' (default) for random sampling, 'stratified' for stratified sampling by category"+ )++ # Skip options+ parser.add_argument(+ "--skip-download",+ action="store_true",+ help="Skip download step (use existing data)"+ )+ parser.add_argument(+ "--skip-sampling",+ action="store_true",+ help="Skip sampling step"+ )++ # Force options+ parser.add_argument(+ "--force-download", "-f",+ action="store_true",+ help="Force re-download even if dataset exists"+ )++ args = parser.parse_args()++ # Validate dataset choices+ available_datasets = set(DOWNLOAD_SOURCES.keys()) & set(DATASET_SAMPLERS.keys())+ if args.dataset != "all" and args.dataset not in available_datasets:+ print(f"Error: Dataset '{args.dataset}' not available")+ print(f"Available datasets: {', '.join(sorted(available_datasets))}")+ return 1++ # Handle --full flag - use full dataset, no sampling+ if args.full:+ args.sample_size = None+ args.num_docs = None++ # Resolve paths+ download_dir = args.download_dir.resolve()+ output_dir = args.output_dir.resolve()++ # Determine datasets to process+ datasets = (+ sorted(available_datasets)+ if args.dataset == "all"+ else [args.dataset]+ )++ # Print configuration+ print("=" * 80)+ print("RAG Benchmark - Unified Dataset Preparation")+ print("=" * 80)+ print(f"Download directory: {download_dir}")+ print(f"Output directory: {output_dir}")+ print(f"Sample size: {args.sample_size if args.sample_size else 'full'}")+ print(f"Number of docs: {args.num_docs if args.num_docs else 'not set'}")+ print(f"Sample mode: {args.sample_mode}")+ print(f"Random seed: {args.seed}")+ print(f"Datasets: {', '.join(datasets)}")+ print(f"Skip download: {args.skip_download}")+ print(f"Skip sampling: {args.skip_sampling}")+ print(f"Force download: {args.force_download}")+ print("=" * 80)++ # Prepare datasets+ success_count = 0+ for dataset in datasets:+ if prepare_dataset(+ dataset,+ download_dir,+ output_dir,+ args.sample_size,+ args.num_docs,+ args.seed,+ args.skip_download,+ args.skip_sampling,+ args.force_download,+ args.full,+ args.sample_mode+ ):+ success_count += 1++ # Final summary+ print("\n" + "=" * 80)+ print("Preparation Complete")+ print("=" * 80)+ print(f"Success: {success_count}/{len(datasets)} datasets")++ if success_count == len(datasets):+ print("\n✅ All datasets prepared successfully!")+ print(f"\nPrepared datasets are in: {output_dir}")+ print("\nYou can now use these datasets with the RAG benchmark.")+ print("\nUpdate your config file's 'dataset_path' path to point to the prepared dataset.")+ return 0+ else:+ print(f"\n❌ {len(datasets) - success_count} dataset(s) failed to prepare")+ return 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/benchmark/RAG/scripts/run_sampling.py b/benchmark/RAG/scripts/run_sampling.pynew file mode 100644index 000000000..b615a5cb3--- /dev/null+++ b/benchmark/RAG/scripts/run_sampling.py@@ -0,0 +1,95 @@+#!/usr/bin/env python3+"""Run sampling for all datasets with specific parameters."""++import sys+from pathlib import Path++sys.path.append(str(Path(__file__).parent))++from sample_dataset import sample_dataset+++def main():+ input_dir = Path(__file__).parent.parent / "raw_data"+ output_dir = Path(__file__).parent.parent / "datasets"++ print("=" * 60)+ print("Running sampling for all datasets with custom parameters")+ print("=" * 60)++ success_count = 0+ total_count = 0++ # Locomo: 3 documents, 80 QAs, stratified+ total_count += 1+ print("\n" + "=" * 60)+ print("Sampling Locomo: 3 documents, 80 QAs")+ print("=" * 60)+ if sample_dataset(+ "Locomo",+ input_dir / "Locomo",+ output_dir / "Locomo",+ sample_size=80,+ num_docs=3,+ seed=42,+ sample_mode="stratified"+ ):+ success_count += 1++ # SyllabusQA: 7 documents, 90 QAs, stratified+ total_count += 1+ print("\n" + "=" * 60)+ print("Sampling SyllabusQA: 7 documents, 90 QAs")+ print("=" * 60)+ if sample_dataset(+ "SyllabusQA",+ input_dir / "SyllabusQA",+ output_dir / "SyllabusQA",+ sample_size=90,+ num_docs=7,+ seed=42,+ sample_mode="stratified"+ ):+ success_count += 1++ # Qasper: 8 documents, 60 QAs, stratified+ total_count += 1+ print("\n" + "=" * 60)+ print("Sampling Qasper: 8 documents, 60 QAs")+ print("=" * 60)+ if sample_dataset(+ "Qasper",+ input_dir / "Qasper",+ output_dir / "Qasper",+ sample_size=60,+ num_docs=8,+ seed=42,+ sample_mode="stratified"+ ):+ success_count += 1++ # FinanceBench: 3 documents, 12 QAs, stratified+ total_count += 1+ print("\n" + "=" * 60)+ print("Sampling FinanceBench: 3 documents, 12 QAs")+ print("=" * 60)+ if sample_dataset(+ "FinanceBench",+ input_dir / "FinanceBench",+ output_dir / "FinanceBench",+ sample_size=12,+ num_docs=3,+ seed=42,+ sample_mode="stratified"+ ):+ success_count += 1++ print("\n" + "=" * 60)+ print(f"Sampling complete: {success_count}/{total_count} successful")+ print("=" * 60)++ return 0 if success_count == total_count else 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/benchmark/RAG/scripts/sample_dataset.py b/benchmark/RAG/scripts/sample_dataset.pynew file mode 100644index 000000000..d7b4b7f7a--- /dev/null+++ b/benchmark/RAG/scripts/sample_dataset.py@@ -0,0 +1,1296 @@+#!/usr/bin/env python3+"""+Sample datasets to create subsets with configurable size.+Supports both full dataset and sampled subsets with seed-based reproducibility.+"""++import argparse+import json+import os+import random+import shutil+import sys+from pathlib import Path+from typing import Any, Dict, List, Optional, Tuple, Callable++sys.path.append(str(Path(__file__).parent.parent))+++def load_json_data(file_path: Path) -> Any:+ """Load JSON data from file."""+ with open(file_path, "r", encoding="utf-8") as f:+ return json.load(f)+++def load_jsonl_data(file_path: Path) -> List[Dict]:+ """Load JSONL data from file."""+ data = []+ with open(file_path, "r", encoding="utf-8") as f:+ for line in f:+ line = line.strip()+ if line:+ data.append(json.loads(line))+ return data+++def save_json_data(data: Any, file_path: Path) -> None:+ """Save JSON data to file."""+ file_path.parent.mkdir(parents=True, exist_ok=True)+ with open(file_path, "w", encoding="utf-8") as f:+ json.dump(data, f, indent=2, ensure_ascii=False)+++def save_jsonl_data(data: List[Dict], file_path: Path) -> None:+ """Save JSONL data to file."""+ file_path.parent.mkdir(parents=True, exist_ok=True)+ with open(file_path, "w", encoding="utf-8") as f:+ for item in data:+ f.write(json.dumps(item, ensure_ascii=False) + "\n")+++def calculate_category_targets(+ sample_size: int,+ categories: List[str],+ print_info: bool = True+) -> Tuple[Dict[str, int], bool]:+ """+ Calculate category targets for stratified sampling.++ Args:+ sample_size: Total number of samples to take+ categories: List of category names+ print_info: Whether to print information++ Returns:+ Tuple of (category_targets dict, should_fallback_to_random bool)+ """+ num_categories = len(categories)+ if num_categories == 0:+ return {}, True++ base_per_category = sample_size // num_categories+ remainder = sample_size % num_categories++ if base_per_category == 0:+ if print_info:+ print(f"Warning: Sample size {sample_size} is too small for {num_categories} categories")+ print("Falling back to random sampling")+ return {}, True++ category_targets = {}+ for i, cat in enumerate(categories):+ category_targets[cat] = base_per_category + (1 if i < remainder else 0)++ if remainder > 0 and print_info:+ print(f"Cannot split {sample_size} QAs evenly into {num_categories} categories")+ print(f"Distributing {remainder} extra QA(s) to first {remainder} category(ies)")++ if print_info:+ print("Category targets:")+ for cat in categories:+ print(f" {cat}: {category_targets[cat]} QAs")++ return category_targets, False+++def stratified_sample_with_reallocation(+ sample_size: int,+ category_qas: Dict[str, List[Any]],+ seed: int,+ print_info: bool = True+) -> List[Any]:+ """+ Perform stratified sampling with remainder reallocation.++ Args:+ sample_size: Total number of samples to take+ category_qas: Dictionary mapping category to list of QAs+ seed: Random seed+ print_info: Whether to print information++ Returns:+ List of sampled QAs+ """+ random.seed(seed)+ categories = sorted(category_qas.keys())++ category_targets, should_fallback = calculate_category_targets(sample_size, categories, print_info)++ if should_fallback:+ all_qas = []+ for qas in category_qas.values():+ all_qas.extend(qas)+ random.seed(seed)+ if len(all_qas) > sample_size:+ return random.sample(all_qas, sample_size)+ return all_qas++ sampled_items = []+ remaining_quota = sample_size++ category_actual = {}+ for cat in categories:+ if cat not in category_targets or category_targets[cat] <= 0:+ category_actual[cat] = 0+ continue++ cat_qas = category_qas[cat].copy()+ random.shuffle(cat_qas)+ sample_count = min(len(cat_qas), category_targets[cat])+ category_actual[cat] = sample_count+ remaining_quota -= sample_count+ sampled_items.extend(cat_qas[:sample_count])++ if remaining_quota > 0 and print_info:+ print(f"Reallocating remaining {remaining_quota} QA(s) to categories with available QAs")++ category_available = {}+ for cat in categories:+ if cat in category_qas:+ total_available = len(category_qas[cat])+ used = category_actual.get(cat, 0)+ category_available[cat] = total_available - used++ while remaining_quota > 0:+ allocated_this_round = 0+ for cat in categories:+ if remaining_quota <= 0:+ break+ if category_available.get(cat, 0) > 0:+ cat_qas = category_qas[cat].copy()+ random.shuffle(cat_qas)+ for qa in cat_qas:+ if qa not in sampled_items:+ sampled_items.append(qa)+ category_actual[cat] += 1+ category_available[cat] -= 1+ remaining_quota -= 1+ allocated_this_round += 1+ break++ if allocated_this_round == 0:+ if print_info:+ print(f"Warning: No more QAs available to sample. Stopping with {remaining_quota} unallocated.")+ break++ if print_info:+ print("Actual category counts after reallocation:")+ for cat in categories:+ print(f" {cat}: {category_actual.get(cat, 0)} QAs")++ return sampled_items+++def random_sample_qas(+ sample_size: int,+ all_qas: List[Any],+ seed: int+) -> List[Any]:+ """+ Perform random sampling of QAs.++ Args:+ sample_size: Number of samples to take+ all_qas: List of all available QAs+ seed: Random seed++ Returns:+ List of sampled QAs+ """+ random.seed(seed)+ if len(all_qas) > sample_size:+ return random.sample(all_qas, sample_size)+ return all_qas+++def sample_docs_stratified(+ sample_size: int,+ category_qas: Dict[str, List[Tuple[Any, Any]]],+ doc_category_qas: Dict[Any, Dict[str, List[Any]]],+ all_doc_ids: List[Any],+ seed: int,+ print_info: bool = True+) -> List[Any]:+ """+ Sample documents using stratified sampling by selecting complete documents.++ Args:+ sample_size: Target number of QAs+ category_qas: Dict mapping category to list of (doc_id, qa) tuples+ doc_category_qas: Dict mapping doc_id to dict of category to QAs+ all_doc_ids: List of all document IDs+ seed: Random seed+ print_info: Whether to print information++ Returns:+ List of selected document IDs+ """+ random.seed(seed)+ categories = sorted(category_qas.keys())++ category_targets, should_fallback = calculate_category_targets(sample_size, categories, print_info)++ if should_fallback:+ return sample_docs_random(sample_size, doc_category_qas, all_doc_ids, seed, print_info)++ selected_docs = []+ selected_qas_by_cat = {cat: 0 for cat in categories}+ doc_used = {doc_id: False for doc_id in all_doc_ids}++ for cat in categories:+ target = category_targets[cat]+ if target == 0:+ continue++ cat_qas = category_qas[cat].copy()+ random.shuffle(cat_qas)++ for doc_id, qa in cat_qas:+ if doc_used[doc_id]:+ continue++ doc_cat_qas = doc_category_qas[doc_id]+ new_count = selected_qas_by_cat[cat] + len(doc_cat_qas.get(cat, []))+ if new_count > target:+ continue++ selected_docs.append(doc_id)+ doc_used[doc_id] = True++ for c, qs in doc_cat_qas.items():+ selected_qas_by_cat[c] += len(qs)++ if selected_qas_by_cat[cat] >= target:+ break++ total_selected = sum(selected_qas_by_cat.values())+ if print_info:+ print(f"Sampled {len(selected_docs)} documents with {total_selected} QAs")+ for cat in categories:+ print(f" {cat}: {selected_qas_by_cat[cat]} QAs (target: {category_targets[cat]})")++ return selected_docs+++def sample_docs_random(+ sample_size: int,+ doc_qas_count: Dict[Any, int],+ all_doc_ids: List[Any],+ seed: int,+ print_info: bool = True+) -> List[Any]:+ """+ Sample documents using random sampling by selecting complete documents.++ Args:+ sample_size: Target number of QAs+ doc_qas_count: Dict mapping doc_id to number of valid QAs+ all_doc_ids: List of all document IDs+ seed: Random seed+ print_info: Whether to print information++ Returns:+ List of selected document IDs+ """+ random.seed(seed)+ shuffled_docs = all_doc_ids.copy()+ random.shuffle(shuffled_docs)++ selected_docs = []+ selected_qas_count = 0++ for doc_id in shuffled_docs:+ doc_qas = doc_qas_count.get(doc_id, 0)++ if doc_qas == 0:+ continue++ if selected_qas_count + doc_qas <= sample_size or not selected_docs:+ selected_docs.append(doc_id)+ selected_qas_count += doc_qas+ else:+ if selected_qas_count >= sample_size:+ break++ if print_info:+ print(f"Sampled {len(selected_docs)} documents with {selected_qas_count} QAs (seed={seed})")++ return selected_docs+++def sample_locomo(+ input_dir: Path,+ output_dir: Path,+ sample_size: Optional[int] = None,+ num_docs: Optional[int] = None,+ seed: int = 42,+ sample_mode: str = "random"+) -> Dict[str, Any]:+ """Sample Locomo dataset with stratified sampling support."""+ input_file = input_dir / "locomo10.json"+ if not input_file.exists():+ raise FileNotFoundError(f"locomo10.json not found at {input_file}")++ data = load_json_data(input_file)+ if not isinstance(data, list):+ data = [data]++ original_num_docs = len(data)+ print(f"Locomo original size: {original_num_docs} documents")++ category_qas = {}+ doc_category_qas = []+ for doc in data:+ doc_cat_qas = {}+ if "qa" in doc:+ for q in doc["qa"]:+ cat = str(q.get("category"))+ if cat != "5":+ if cat not in category_qas:+ category_qas[cat] = []+ category_qas[cat].append((doc, q))+ if cat not in doc_cat_qas:+ doc_cat_qas[cat] = []+ doc_cat_qas[cat].append(q)+ doc_category_qas.append(doc_cat_qas)++ total_qas = sum(len(qas) for qas in category_qas.values())+ categories = sorted(category_qas.keys())+ print(f"Total QAs (excluding category 5): {total_qas}")+ print(f"Categories: {categories}")+ for cat in categories:+ print(f" Category {cat}: {len(category_qas[cat])} QAs")++ is_full = (sample_size is None and num_docs is None)+ if is_full:+ selected_docs = data+ print("Using full Locomo dataset")+ else:+ if num_docs is not None:+ if num_docs >= original_num_docs:+ selected_docs = data+ print("Using all documents")+ else:+ random.seed(seed)+ selected_docs = random.sample(data, num_docs)+ print(f"Sampled {len(selected_docs)} documents (seed={seed})")++ if sample_size is not None:+ print(f"Further sampling {sample_size} QAs from selected documents (mode: {sample_mode})")++ selected_doc_category_qas = {}+ selected_doc_indices = [data.index(doc) for doc in selected_docs]++ for doc_idx in selected_doc_indices:+ doc = data[doc_idx]+ doc_cat_qas = doc_category_qas[doc_idx]+ for cat, qs in doc_cat_qas.items():+ if cat not in selected_doc_category_qas:+ selected_doc_category_qas[cat] = []+ for q in qs:+ selected_doc_category_qas[cat].append((doc_idx, q))++ if sample_mode == "stratified":+ sampled_q_tuples = stratified_sample_with_reallocation(+ sample_size, selected_doc_category_qas, seed+ )++ keep_q_indices = set()+ for doc_idx, q in sampled_q_tuples:+ doc = data[doc_idx]+ for q_idx, qa_item in enumerate(doc.get("qa", [])):+ if qa_item == q:+ keep_q_indices.add((doc_idx, q_idx))+ break++ for doc in selected_docs:+ doc_idx = data.index(doc)+ new_qas = []+ for q_idx, q in enumerate(doc.get("qa", [])):+ if (doc_idx, q_idx) in keep_q_indices or str(q.get("category")) == "5":+ new_qas.append(q)+ doc["qa"] = new_qas++ if sample_mode == "random":+ all_valid_q_indices = []+ for doc_idx_in_selected, doc in enumerate(selected_docs):+ doc_idx = data.index(doc)+ doc_cat_qas = doc_category_qas[doc_idx]+ for cat, qs in doc_cat_qas.items():+ for q in qs:+ for q_idx, qa_item in enumerate(doc.get("qa", [])):+ if qa_item == q:+ all_valid_q_indices.append((doc_idx_in_selected, q_idx))+ break++ sampled_q_indices = random_sample_qas(sample_size, all_valid_q_indices, seed)+ keep_q_indices = set(sampled_q_indices)++ for doc_idx_in_selected, doc in enumerate(selected_docs):+ new_qas = []+ for q_idx, q in enumerate(doc.get("qa", [])):+ if (doc_idx_in_selected, q_idx) in keep_q_indices or str(q.get("category")) == "5":+ new_qas.append(q)+ doc["qa"] = new_qas+ else:+ doc_qas_count = {}+ for doc_idx, doc_cat_qas in enumerate(doc_category_qas):+ count = 0+ for qs in doc_cat_qas.values():+ count += len(qs)+ doc_qas_count[doc_idx] = count++ if sample_mode == "stratified":+ print(f"Using stratified sampling (seed={seed})")+ category_qas_with_indices = {}+ for cat, qas in category_qas.items():+ category_qas_with_indices[cat] = []+ for doc, q in qas:+ doc_idx = data.index(doc)+ category_qas_with_indices[cat].append((doc_idx, q))++ doc_category_qas_dict = {i: d for i, d in enumerate(doc_category_qas)}+ all_doc_indices = list(range(len(data)))++ selected_doc_indices = sample_docs_stratified(+ sample_size, category_qas_with_indices, doc_category_qas_dict, all_doc_indices, seed+ )+ selected_docs = [data[i] for i in selected_doc_indices]++ if sample_mode == "random":+ print(f"Using random sampling (seed={seed})")+ all_doc_indices = list(range(len(data)))+ selected_doc_indices = sample_docs_random(+ sample_size, doc_qas_count, all_doc_indices, seed+ )+ selected_docs = [data[i] for i in selected_doc_indices]++ output_data = selected_docs+ output_file = output_dir / "locomo10.json"+ save_json_data(output_data, output_file)++ sampled_qas = 0+ for doc in selected_docs:+ if "qa" in doc:+ for q in doc["qa"]:+ if str(q.get("category")) != "5":+ sampled_qas += 1++ metadata = {+ "dataset": "Locomo",+ "original_num_docs": original_num_docs,+ "original_total_qas": total_qas,+ "sampled_num_docs": len(selected_docs),+ "sampled_total_qas": sampled_qas,+ "sample_size": sample_size,+ "num_docs": num_docs,+ "seed": seed,+ "sample_mode": sample_mode,+ "is_full": is_full,+ "note": "Category 5 questions are excluded from QA count"+ }++ return metadata+++def sample_syllabusqa(+ input_dir: Path,+ output_dir: Path,+ sample_size: Optional[int] = None,+ num_docs: Optional[int] = None,+ seed: int = 42,+ sample_mode: str = "random"+) -> Dict[str, Any]:+ """Sample SyllabusQA dataset with stratified sampling support."""+ from collections import defaultdict+ import csv++ dataset_split_dir = input_dir / "data" / "dataset_split"+ if not dataset_split_dir.exists():+ raise FileNotFoundError(f"data/dataset_split not found at {dataset_split_dir}")++ all_data = []+ csv_files = ["train.csv", "val.csv", "test.csv"]+ for csv_file in csv_files:+ file_path = dataset_split_dir / csv_file+ if file_path.exists():+ with open(file_path, "r", encoding="utf-8") as f:+ reader = csv.DictReader(f)+ file_data = list(reader)+ for item in file_data:+ item["_source_file"] = csv_file+ all_data.extend(file_data)++ doc_groups = defaultdict(list)+ for item in all_data:+ syllabus_name = item.get("syllabus_name", "unknown")+ doc_groups[syllabus_name].append(item)++ category_qas = {}+ doc_category_qas = {}+ for doc_name, items in doc_groups.items():+ doc_cat_qas = {}+ for item in items:+ q_type = item.get("question_type")+ if q_type != "no answer":+ if q_type not in category_qas:+ category_qas[q_type] = []+ category_qas[q_type].append((doc_name, item))+ if q_type not in doc_cat_qas:+ doc_cat_qas[q_type] = []+ doc_cat_qas[q_type].append(item)+ doc_category_qas[doc_name] = doc_cat_qas++ total_valid_qas = sum(len(qas) for qas in category_qas.values())+ categories = sorted(category_qas.keys())++ doc_valid_qas = {}+ for doc_name, items in doc_groups.items():+ valid_count = 0+ for item in items:+ if item.get("question_type") != "no answer":+ valid_count += 1+ doc_valid_qas[doc_name] = valid_count++ all_doc_names = list(doc_groups.keys())+ original_num_docs = len(all_doc_names)+ original_total_qas = len(all_data)+ print(f"SyllabusQA original size: {original_num_docs} documents, {original_total_qas} QAs (from {len(csv_files)} files)")+ print(f"Total valid QAs (excluding 'no answer'): {total_valid_qas}")+ print(f"Categories: {categories}")+ for cat in categories:+ print(f" {cat}: {len(category_qas[cat])} QAs")++ is_full = (sample_size is None and num_docs is None)+ if is_full:+ selected_docs = all_doc_names+ print("Using full SyllabusQA dataset")+ else:+ if num_docs is not None:+ if num_docs >= original_num_docs:+ selected_docs = all_doc_names+ print("Using all documents")+ else:+ random.seed(seed)+ selected_docs = random.sample(all_doc_names, num_docs)+ print(f"Sampled {len(selected_docs)} documents (seed={seed})")++ if sample_size is not None:+ print(f"Further sampling {sample_size} QAs from selected documents (mode: {sample_mode})")++ selected_doc_category_qas = {}+ for doc_name in selected_docs:+ doc_cat_qas = doc_category_qas[doc_name]+ for cat, items in doc_cat_qas.items():+ if cat not in selected_doc_category_qas:+ selected_doc_category_qas[cat] = []+ for item in items:+ selected_doc_category_qas[cat].append(item)++ if sample_mode == "stratified":+ sampled_items = stratified_sample_with_reallocation(+ sample_size, selected_doc_category_qas, seed+ )++ for doc_name in selected_docs:+ doc_items = doc_groups[doc_name]+ for item in doc_items:+ if item.get("question_type") == "no answer":+ sampled_items.append(item)++ new_doc_groups = defaultdict(list)+ for item in sampled_items:+ doc_name = item.get("syllabus_name", "unknown")+ new_doc_groups[doc_name].append(item)+ doc_groups = new_doc_groups++ if sample_mode == "random":+ all_valid_items = []+ for doc_name in selected_docs:+ items = doc_groups[doc_name]+ for item in items:+ if item.get("question_type") != "no answer":+ all_valid_items.append(item)++ sampled_items = random_sample_qas(sample_size, all_valid_items, seed)++ for doc_name in selected_docs:+ items = doc_groups[doc_name]+ for item in items:+ if item.get("question_type") == "no answer":+ sampled_items.append(item)++ new_doc_groups = defaultdict(list)+ for item in sampled_items:+ doc_name = item.get("syllabus_name", "unknown")+ new_doc_groups[doc_name].append(item)+ doc_groups = new_doc_groups+ else:+ if sample_mode == "stratified":+ print(f"Using stratified sampling (seed={seed})")+ selected_docs = sample_docs_stratified(+ sample_size, category_qas, doc_category_qas, all_doc_names, seed+ )++ if sample_mode == "random":+ print(f"Using random sampling (seed={seed})")+ selected_docs = sample_docs_random(+ sample_size, doc_valid_qas, all_doc_names, seed+ )++ selected_data = []+ for doc_name in selected_docs:+ selected_data.extend(doc_groups[doc_name])++ output_dir.mkdir(parents=True, exist_ok=True)+ for csv_file in csv_files:+ file_data = [item for item in selected_data if item.get("_source_file") == csv_file]+ if file_data:+ for item in file_data:+ item.pop("_source_file", None)+ output_file = output_dir / csv_file+ with open(output_file, "w", encoding="utf-8", newline="") as f:+ if file_data:+ writer = csv.DictWriter(f, fieldnames=file_data[0].keys())+ writer.writeheader()+ writer.writerows(file_data)+ print(f"Saved {len(file_data)} samples to {csv_file}")++ syllabi_src = input_dir / "syllabi"+ syllabi_dst = output_dir / "syllabi"+ if syllabi_src.exists():+ syllabi_dst.mkdir(parents=True, exist_ok=True)++ syllabus_names = set()+ for doc_name in selected_docs:+ items = doc_groups[doc_name]+ for item in items:+ syllabus_name = item.get("syllabus_name")+ if syllabus_name:+ syllabus_names.add(syllabus_name)++ print(f"Copying syllabi for {len(syllabus_names)} unique syllabus files")++ for subdir in ["pdf", "text", "word"]:+ src_subdir = syllabi_src / "syllabi_redacted" / subdir+ dst_subdir = syllabi_dst / "syllabi_redacted" / subdir+ if src_subdir.exists():+ dst_subdir.mkdir(parents=True, exist_ok=True)++ for syllabus_name in syllabus_names:+ for ext in [".pdf", ".txt", ".docx"]:+ src_file = src_subdir / f"{syllabus_name}{ext}"+ if src_file.exists():+ shutil.copy2(src_file, dst_subdir / f"{syllabus_name}{ext}")+ print(f"Copied {subdir}/{syllabus_name}{ext}")+ break++ sampled_valid_qas = 0+ for doc_name in selected_docs:+ items = doc_groups[doc_name]+ for item in items:+ if item.get("question_type") != "no answer":+ sampled_valid_qas += 1++ metadata = {+ "dataset": "SyllabusQA",+ "original_num_docs": original_num_docs,+ "original_total_qas": original_total_qas,+ "original_valid_qas": total_valid_qas,+ "sampled_num_docs": len(selected_docs),+ "sampled_total_qas": len(selected_data),+ "sampled_valid_qas": sampled_valid_qas,+ "sample_size": sample_size,+ "num_docs": num_docs,+ "seed": seed,+ "sample_mode": sample_mode,+ "is_full": is_full,+ "note": "'no answer' type questions are excluded from QA count"+ }++ return metadata+++def sample_qasper(+ input_dir: Path,+ output_dir: Path,+ sample_size: Optional[int] = None,+ num_docs: Optional[int] = None,+ seed: int = 42,+ sample_mode: str = "random"+) -> Dict[str, Any]:+ """Sample Qasper dataset with stratified sampling support."""+ json_files = ["qasper-train-v0.3.json", "qasper-dev-v0.3.json", "qasper-test-v0.3.json"]+ all_paper_ids = []+ paper_data_map = {}++ category_qas = {}+ paper_category_qas = {}++ for json_file in json_files:+ file_path = input_dir / json_file+ if file_path.exists():+ data = load_json_data(file_path)+ for paper_id, paper_data in data.items():+ all_paper_ids.append(paper_id)+ paper_data_map[paper_id] = (paper_data, json_file)++ paper_cat_qas = {}+ if "qas" in paper_data:+ for qa_item in paper_data["qas"]:+ is_unanswerable = all(+ ans.get("answer", {}).get("unanswerable", False)+ for ans in qa_item.get("answers", [])+ )+ if is_unanswerable:+ continue++ answer_types = set()+ for ans in qa_item.get("answers", []):+ ans_obj = ans.get("answer", {})+ if ans_obj.get("unanswerable", False):+ continue+ if ans_obj.get("extractive_spans"):+ answer_types.add("extractive")+ elif ans_obj.get("free_form_answer", "").strip():+ answer_types.add("free_form")+ elif ans_obj.get("yes_no") is not None:+ answer_types.add("yes_no")++ primary_type = next(iter(answer_types), "extractive")+ if primary_type not in category_qas:+ category_qas[primary_type] = []+ category_qas[primary_type].append((paper_id, qa_item))++ if primary_type not in paper_cat_qas:+ paper_cat_qas[primary_type] = []+ paper_cat_qas[primary_type].append(qa_item)++ paper_category_qas[paper_id] = paper_cat_qas++ original_num_docs = len(all_paper_ids)+ print(f"Qasper original size: {original_num_docs} documents (from {len(json_files)} files)")++ total_qas = sum(len(qas) for qas in category_qas.values())+ categories = sorted(category_qas.keys())+ print(f"Total QAs (excluding unanswerable): {total_qas}")+ print(f"Categories: {categories}")+ for cat in categories:+ print(f" {cat}: {len(category_qas[cat])} QAs")++ is_full = (sample_size is None and num_docs is None)+ if is_full:+ selected_ids = all_paper_ids+ print("Using full Qasper dataset")+ else:+ if num_docs is not None:+ if num_docs >= original_num_docs:+ selected_ids = all_paper_ids+ print("Using all documents")+ else:+ if sample_size is not None:+ paper_qas_count = {}+ for paper_id in all_paper_ids:+ count = 0+ for cat_qas in paper_category_qas[paper_id].values():+ count += len(cat_qas)+ paper_qas_count[paper_id] = count++ random.seed(seed)+ shuffled_papers = all_paper_ids.copy()+ random.shuffle(shuffled_papers)+ shuffled_papers.sort(key=lambda pid: paper_qas_count[pid], reverse=True)++ selected_ids = shuffled_papers[:num_docs]+ print(f"Sampled {len(selected_ids)} documents with highest QA counts (seed={seed})")+ else:+ random.seed(seed)+ selected_ids = random.sample(all_paper_ids, num_docs)+ print(f"Sampled {len(selected_ids)} documents (seed={seed})")++ if sample_size is not None:+ print(f"Further sampling {sample_size} QAs from selected documents (mode: {sample_mode})")++ qa_with_indices = []+ for paper_id in selected_ids:+ paper_data, source_file = paper_data_map[paper_id]+ for i, qa_item in enumerate(paper_data.get("qas", [])):+ is_unanswerable = all(+ ans.get("answer", {}).get("unanswerable", False)+ for ans in qa_item.get("answers", [])+ )+ if not is_unanswerable:+ answer_types = set()+ for ans in qa_item.get("answers", []):+ ans_obj = ans.get("answer", {})+ if ans_obj.get("unanswerable", False):+ continue+ if ans_obj.get("extractive_spans"):+ answer_types.add("extractive")+ elif ans_obj.get("free_form_answer", "").strip():+ answer_types.add("free_form")+ elif ans_obj.get("yes_no") is not None:+ answer_types.add("yes_no")+ primary_type = next(iter(answer_types), "extractive")+ qa_with_indices.append((paper_id, primary_type, i, qa_item))++ if sample_mode == "stratified":+ selected_doc_category_qas = {}+ for paper_id, cat, i, qa_item in qa_with_indices:+ if cat not in selected_doc_category_qas:+ selected_doc_category_qas[cat] = []+ selected_doc_category_qas[cat].append((paper_id, i, qa_item))++ category_targets, should_fallback = calculate_category_targets(+ sample_size, sorted(selected_doc_category_qas.keys())+ )++ if not should_fallback:+ random.seed(seed)+ sampled_qas_indices = set()+ remaining_quota = sample_size++ category_actual = {}+ cats = sorted(selected_doc_category_qas.keys())+ for cat in cats:+ if cat not in category_targets or category_targets[cat] <= 0:+ category_actual[cat] = 0+ continue++ cat_qas = selected_doc_category_qas[cat].copy()+ random.shuffle(cat_qas)+ sample_count = min(len(cat_qas), category_targets[cat])+ category_actual[cat] = sample_count+ remaining_quota -= sample_count++ for paper_id, i, qa_item in cat_qas[:sample_count]:+ sampled_qas_indices.add((paper_id, i))++ if remaining_quota > 0:+ print(f"Reallocating remaining {remaining_quota} QA(s) to categories with available QAs")++ category_available = {}+ for cat in cats:+ if cat in selected_doc_category_qas:+ total_available = len(selected_doc_category_qas[cat])+ used = category_actual.get(cat, 0)+ category_available[cat] = total_available - used++ while remaining_quota > 0:+ allocated_this_round = 0+ for cat in cats:+ if remaining_quota <= 0:+ break+ if category_available.get(cat, 0) > 0:+ cat_qas = selected_doc_category_qas[cat].copy()+ random.shuffle(cat_qas)+ for paper_id, i, qa_item in cat_qas:+ if (paper_id, i) not in sampled_qas_indices:+ sampled_qas_indices.add((paper_id, i))+ category_actual[cat] += 1+ category_available[cat] -= 1+ remaining_quota -= 1+ allocated_this_round += 1+ break++ if allocated_this_round == 0:+ print(f"Warning: No more QAs available to sample. Stopping with {remaining_quota} unallocated.")+ break++ print("Actual category counts after reallocation:")+ for cat in cats:+ print(f" {cat}: {category_actual.get(cat, 0)} QAs")++ for paper_id in selected_ids:+ paper_data, source_file = paper_data_map[paper_id]+ new_qas = []++ for i, qa_item in enumerate(paper_data.get("qas", [])):+ is_unanswerable = all(+ ans.get("answer", {}).get("unanswerable", False)+ for ans in qa_item.get("answers", [])+ )+ if is_unanswerable or (paper_id, i) in sampled_qas_indices:+ new_qas.append(qa_item)++ paper_data["qas"] = new_qas++ if sample_mode == "random":+ sampled_qas = random_sample_qas(sample_size, qa_with_indices, seed)++ keep_qas_indices = set()+ for paper_id, cat, i, qa_item in sampled_qas:+ keep_qas_indices.add((paper_id, i))++ for paper_id in selected_ids:+ paper_data, source_file = paper_data_map[paper_id]+ new_qas = []+ for i, qa_item in enumerate(paper_data.get("qas", [])):+ is_unanswerable = all(+ ans.get("answer", {}).get("unanswerable", False)+ for ans in qa_item.get("answers", [])+ )+ if is_unanswerable or (paper_id, i) in keep_qas_indices:+ new_qas.append(qa_item)+ paper_data["qas"] = new_qas+ else:+ paper_qas_count = {}+ for paper_id in all_paper_ids:+ count = 0+ for cat_qas in paper_category_qas[paper_id].values():+ count += len(cat_qas)+ paper_qas_count[paper_id] = count++ if sample_mode == "stratified":+ print(f"Using stratified sampling (seed={seed})")+ selected_ids = sample_docs_stratified(+ sample_size, category_qas, paper_category_qas, all_paper_ids, seed+ )++ if sample_mode == "random":+ print(f"Using random sampling (seed={seed})")+ selected_ids = sample_docs_random(+ sample_size, paper_qas_count, all_paper_ids, seed+ )++ output_dir.mkdir(parents=True, exist_ok=True)+ data_by_file = {}+ for paper_id in selected_ids:+ paper_data, source_file = paper_data_map[paper_id]+ if source_file not in data_by_file:+ data_by_file[source_file] = {}+ data_by_file[source_file][paper_id] = paper_data++ for json_file, output_data in data_by_file.items():+ output_file = output_dir / json_file+ save_json_data(output_data, output_file)+ print(f"Saved {len(output_data)} papers to {json_file}")++ sampled_qas = 0+ for paper_id in selected_ids:+ paper_data, source_file = paper_data_map[paper_id]+ if "qas" in paper_data:+ for qa_item in paper_data["qas"]:+ is_unanswerable = all(+ ans.get("answer", {}).get("unanswerable", False)+ for ans in qa_item.get("answers", [])+ )+ if not is_unanswerable:+ sampled_qas += 1++ metadata = {+ "dataset": "Qasper",+ "original_num_docs": original_num_docs,+ "original_total_qas": total_qas,+ "sampled_num_docs": len(selected_ids),+ "sampled_total_qas": sampled_qas,+ "sample_size": sample_size,+ "num_docs": num_docs,+ "seed": seed,+ "sample_mode": sample_mode,+ "is_full": is_full,+ "note": "Unanswerable questions are excluded from QA count"+ }++ return metadata+++def sample_financebench(+ input_dir: Path,+ output_dir: Path,+ sample_size: Optional[int] = None,+ num_docs: Optional[int] = None,+ seed: int = 42,+ sample_mode: str = "random"+) -> Dict[str, Any]:+ """Sample Financebench dataset with stratified sampling support."""+ from collections import defaultdict++ input_file = input_dir / "data" / "financebench_open_source.jsonl"+ if not input_file.exists():+ raise FileNotFoundError(f"financebench_open_source.jsonl not found at {input_file}")++ data = load_jsonl_data(input_file)++ doc_groups = defaultdict(list)+ for item in data:+ doc_name = item.get("doc_name", "unknown")+ doc_groups[doc_name].append(item)++ category_qas = {}+ doc_category_qas = {}+ for doc_name, items in doc_groups.items():+ doc_cat_qas = {}+ for item in items:+ q_type = item.get("question_type", "domain-relevant")+ if q_type not in category_qas:+ category_qas[q_type] = []+ category_qas[q_type].append((doc_name, item))+ if q_type not in doc_cat_qas:+ doc_cat_qas[q_type] = []+ doc_cat_qas[q_type].append(item)+ doc_category_qas[doc_name] = doc_cat_qas++ all_doc_names = list(doc_groups.keys())+ original_num_docs = len(all_doc_names)+ original_total_qas = len(data)+ total_qas = sum(len(qas) for qas in category_qas.values())+ categories = sorted(category_qas.keys())+ print(f"Financebench original size: {original_num_docs} documents, {original_total_qas} QAs")+ print(f"Categories: {categories}")+ for cat in categories:+ print(f" {cat}: {len(category_qas[cat])} QAs")++ is_full = (sample_size is None and num_docs is None)+ if is_full:+ selected_docs = all_doc_names+ print("Using full Financebench dataset")+ else:+ if num_docs is not None:+ if num_docs >= original_num_docs:+ selected_docs = all_doc_names+ print("Using all documents")+ else:+ random.seed(seed)+ sorted_docs = sorted(all_doc_names, key=lambda x: len(doc_groups[x]), reverse=True)+ random.shuffle(sorted_docs)+ selected_docs = sorted(sorted_docs, key=lambda x: len(doc_groups[x]), reverse=True)[:num_docs]+ print(f"Sampled {len(selected_docs)} documents with highest QA counts (seed={seed})")+ print("Selected documents:")+ for doc in selected_docs:+ print(f" {doc}: {len(doc_groups[doc])} QAs")++ if sample_size is not None:+ print(f"Further sampling {sample_size} QAs from selected documents (mode: {sample_mode})")++ selected_doc_category_qas = {}+ for doc_name in selected_docs:+ doc_cat_qas = doc_category_qas[doc_name]+ for cat, items in doc_cat_qas.items():+ if cat not in selected_doc_category_qas:+ selected_doc_category_qas[cat] = []+ for item in items:+ selected_doc_category_qas[cat].append(item)++ if sample_mode == "stratified":+ sampled_items = stratified_sample_with_reallocation(+ sample_size, selected_doc_category_qas, seed+ )++ new_doc_groups = defaultdict(list)+ for item in sampled_items:+ doc_name = item.get("doc_name", "unknown")+ new_doc_groups[doc_name].append(item)+ doc_groups = new_doc_groups++ if sample_mode == "random":+ all_items = []+ for doc_name in selected_docs:+ items = doc_groups[doc_name]+ all_items.extend(items)++ sampled_items = random_sample_qas(sample_size, all_items, seed)++ new_doc_groups = defaultdict(list)+ for item in sampled_items:+ doc_name = item.get("doc_name", "unknown")+ new_doc_groups[doc_name].append(item)+ doc_groups = new_doc_groups+ else:+ doc_qas_count = {doc_name: len(items) for doc_name, items in doc_groups.items()}++ if sample_mode == "stratified":+ print(f"Using stratified sampling (seed={seed})")+ selected_docs = sample_docs_stratified(+ sample_size, category_qas, doc_category_qas, all_doc_names, seed+ )++ if sample_mode == "random":+ print(f"Using random sampling (seed={seed})")+ selected_docs = sample_docs_random(+ sample_size, doc_qas_count, all_doc_names, seed+ )++ selected_data = []+ for doc_name in selected_docs:+ selected_data.extend(doc_groups[doc_name])++ output_file = output_dir / "financebench_open_source.jsonl"+ save_jsonl_data(selected_data, output_file)++ pdfs_src = input_dir / "pdfs"+ pdfs_dst = output_dir / "pdfs"++ if pdfs_src.exists():+ pdfs_dst.mkdir(parents=True, exist_ok=True)++ for doc_name in selected_docs:+ src_pdf = pdfs_src / f"{doc_name}.pdf"+ if src_pdf.exists():+ shutil.copy2(src_pdf, pdfs_dst / f"{doc_name}.pdf")+ print(f"Copied PDF: {doc_name}.pdf")++ metadata = {+ "dataset": "Financebench",+ "original_num_docs": original_num_docs,+ "original_total_qas": original_total_qas,+ "sampled_num_docs": len(selected_docs),+ "sampled_total_qas": len(selected_data),+ "sample_size": sample_size,+ "num_docs": num_docs,+ "seed": seed,+ "sample_mode": sample_mode,+ "is_full": is_full+ }++ return metadata+++DATASET_SAMPLERS = {+ "Locomo": sample_locomo,+ "SyllabusQA": sample_syllabusqa,+ "Qasper": sample_qasper,+ "FinanceBench": sample_financebench,+}+++def sample_dataset(+ dataset_name: str,+ input_dir: Path,+ output_dir: Path,+ sample_size: Optional[int] = None,+ num_docs: Optional[int] = None,+ seed: int = 42,+ sample_mode: str = "stratified"+) -> bool:+ """Sample a single dataset."""+ if dataset_name not in DATASET_SAMPLERS:+ print(f"Unknown dataset: {dataset_name}")+ return False++ print(f"\nProcessing {dataset_name}...")+ print(f"Input: {input_dir}")+ print(f"Output: {output_dir}")++ try:+ sampler = DATASET_SAMPLERS[dataset_name]+ metadata = sampler(input_dir, output_dir, sample_size, num_docs, seed, sample_mode)++ metadata_file = output_dir / "sampling_metadata.json"+ save_json_data(metadata, metadata_file)+ print(f"✓ Saved metadata to {metadata_file}")++ return True+ except Exception as e:+ print(f"Error sampling {dataset_name}: {e}")+ import traceback+ traceback.print_exc()+ return False+++def main():+ parser = argparse.ArgumentParser(+ description="Sample datasets for RAG benchmark"+ )+ parser.add_argument(+ "--dataset", "-d",+ type=str,+ choices=list(DATASET_SAMPLERS.keys()) + ["all"],+ default="all",+ help="Dataset to sample (default: all)"+ )+ parser.add_argument(+ "--input-dir", "-i",+ type=Path,+ default=Path(__file__).parent.parent / "raw_data",+ help="Input directory with full datasets (default: raw_data/)"+ )+ parser.add_argument(+ "--output-dir", "-o",+ type=Path,+ default=Path(__file__).parent.parent / "datasets",+ help="Output directory for sampled datasets (default: datasets/)"+ )+ parser.add_argument(+ "--sample-size", "-n",+ type=int,+ default=None,+ help="Number of samples to use (default: use full dataset)"+ )+ parser.add_argument(+ "--num-docs",+ type=int,+ default=None,+ help="Number of documents to sample (for document-level sampling)"+ )+ parser.add_argument(+ "--seed", "-s",+ type=int,+ default=42,+ help="Random seed for reproducibility (default: 42)"+ )+ parser.add_argument(+ "--full",+ action="store_true",+ help="Use full dataset (overrides --sample-size)"+ )+ parser.add_argument(+ "--sample-mode",+ type=str,+ choices=["stratified", "random"],+ default="stratified",+ help="Sampling mode (default: stratified)"+ )++ args = parser.parse_args()++ if args.full:+ args.sample_size = None+ args.num_docs = None++ input_dir = args.input_dir.resolve()+ output_dir = args.output_dir.resolve()++ datasets = (+ list(DATASET_SAMPLERS.keys())+ if args.dataset == "all"+ else [args.dataset]+ )++ print("=" * 60)+ print("RAG Benchmark Dataset Sampler")+ print("=" * 60)+ print(f"Input directory: {input_dir}")+ print(f"Output directory: {output_dir}")+ print(f"Sample size: {args.sample_size if args.sample_size else 'full'}")+ print(f"Number of docs: {args.num_docs if args.num_docs else 'not set'}")+ print(f"Sample mode: {args.sample_mode}")+ print(f"Random seed: {args.seed}")+ print(f"Datasets: {', '.join(datasets)}")+ print("=" * 60)++ success_count = 0+ for dataset in datasets:+ dataset_input_dir = input_dir / dataset+ dataset_output_dir = output_dir / dataset++ if sample_dataset(+ dataset,+ dataset_input_dir,+ dataset_output_dir,+ args.sample_size,+ args.num_docs,+ args.seed,+ args.sample_mode+ ):+ success_count += 1++ print("\n" + "=" * 60)+ print(f"Sampling complete: {success_count}/{len(datasets)} successful")+ print("=" * 60)++ return 0 if success_count == len(datasets) else 1+++if __name__ == "__main__":+ sys.exit(main())diff --git a/benchmark/RAG/src/__init__.py b/benchmark/RAG/src/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/benchmark/RAG/src/adapters/__init__.py b/benchmark/RAG/src/adapters/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/benchmark/RAG/src/adapters/base.py b/benchmark/RAG/src/adapters/base.pynew file mode 100644index 000000000..954b22273--- /dev/null+++ b/benchmark/RAG/src/adapters/base.py@@ -0,0 +1,79 @@+from abc import ABC, abstractmethod+from dataclasses import dataclass, field+from typing import List, Dict, Any, Union, Optional+import sys+from pathlib import Path++sys.path.append(str(Path(__file__).parent.parent))++from core.logger import get_logger+++@dataclass+class StandardQA:+ """Standardized single question-answer pair"""+ question: str+ gold_answers: List[str]+ evidence: List[str] = field(default_factory=list)+ category: Optional[Union[int, str]] = None+ metadata: Dict[str, Any] = field(default_factory=dict)+++@dataclass+class StandardSample:+ """Standardized sample containing document content and corresponding QA list"""+ sample_id: str+ qa_pairs: List[StandardQA]+ metadata: Dict[str, Any] = field(default_factory=dict)+++@dataclass+class StandardDoc:+ """Standardized sampleid to doc_path mapping structure"""+ sample_id:str+ doc_path:str+++class BaseAdapter(ABC):+ """Base class for all dataset adapters"""++ def __init__(self, raw_file_path: str):+ self.raw_file_path = raw_file_path+ self.logger = get_logger()++ @abstractmethod+ def data_prepare(self, doc_dir:str) -> List[StandardDoc]:+ """+ Data preparation.+ 1. Convert dataset format to OpenViking-friendly format+ 2. Return converted (or unconverted) file paths++ Returns:+ List[StandardDoc]: Array of file paths expected to be input to OpenViking+ """+ pass++ @abstractmethod+ def load_and_transform(self) -> List[StandardSample]:+ """+ Read raw files and convert to standard format list.+ Must be implemented by subclasses.+ """+ pass++ @abstractmethod+ def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]:+ """+ Build final prompt to send to LLM based on retrieved context and QA pair.++ Returns:+ - full_prompt (str): Complete prompt string+ - meta (Dict): Metadata to pass to post-processing function (e.g., option mapping for multiple choice)+ """+ pass++ def post_process_answer(self, qa: StandardQA, raw_answer: str, meta: Dict[str, Any]) -> str:+ """+ Post-process raw LLM output (default implementation only strips whitespace).+ """+ return raw_answer.strip()diff --git a/benchmark/RAG/src/adapters/financebench_adapter.py b/benchmark/RAG/src/adapters/financebench_adapter.pynew file mode 100644index 000000000..40923e18f--- /dev/null+++ b/benchmark/RAG/src/adapters/financebench_adapter.py@@ -0,0 +1,166 @@+# src/adapters/finance_bench_adapter.py+"""+FinanceBench Dataset Adapter++FinanceBench is a financial domain QA dataset with SEC financial report PDFs as documents.+Data format: JSONL, each line contains question, answer, doc_name, evidence, etc.+evidence_text in evidence is used for recall calculation.+"""++import json+import os+from collections import defaultdict+from typing import List, Dict, Any+from pathlib import Path+import sys++sys.path.append(str(Path(__file__).parent))++from base import BaseAdapter, StandardDoc, StandardSample, StandardQA++CATEGORY_INSTRUCTIONS = {+ "domain-relevant": """Answer the financial question based on the document.+- Use ONLY facts from the context+- If numerical, include units (e.g., USD millions, %)+- Provide concise, direct answer+- Do NOT invent information""",++ "metrics-generated": """Calculate the financial metric based on the document.+- Use ONLY numbers from the context+- Show your calculations clearly+- Round to appropriate decimal places+- Include units (e.g., USD millions, %)+- Do NOT invent numbers""",++ "novel-generated": """Answer the financial question based on the document.+- Use ONLY facts from the context+- If numerical, include units (e.g., USD millions, %)+- Provide clear, complete answer+- Do NOT invent information"""+}++MISSING_RULE = "If the provided context does not contain sufficient information to answer the question, respond with 'Insufficient information'."+++class FinanceBenchAdapter(BaseAdapter):+ """+ FinanceBench Dataset Adapter.+ Processes financial domain QA data with SEC financial report PDFs as documents.+ """++ def __init__(self, raw_file_path: str):+ super().__init__(raw_file_path)+ data_dir = os.path.dirname(self.raw_file_path)+ # Check if pdfs is in data directory or parent directory+ self.pdf_dir = os.path.join(data_dir, "pdfs")+ if not os.path.exists(self.pdf_dir):+ # Try parent directory (new structure: raw_data/dataset_name/data/ and raw_data/dataset_name/pdfs/)+ parent_dir = os.path.dirname(data_dir)+ self.pdf_dir = os.path.join(parent_dir, "pdfs")++ def data_prepare(self, doc_dir: str) -> List[StandardDoc]:+ """+ Prepare document list for ingestion. Only ingest documents referenced in JSONL.+ """+ if not os.path.exists(self.pdf_dir):+ raise FileNotFoundError(f"PDF directory not found: {self.pdf_dir}")++ doc_names = set()+ with open(self.raw_file_path, 'r', encoding='utf-8') as f:+ for line in f:+ line = line.strip()+ if line:+ doc_names.add(json.loads(line)["doc_name"])++ docs: List[StandardDoc] = []+ for doc_name in sorted(doc_names):+ pdf_path = os.path.join(self.pdf_dir, f"{doc_name}.pdf")+ if not os.path.exists(pdf_path):+ self.logger.warning(f"PDF not found: {pdf_path}, skipping")+ continue+ docs.append(StandardDoc(sample_id=doc_name, doc_path=pdf_path))++ self.logger.info(f"[FinanceBench] Prepared {len(docs)} documents for ingestion (referenced only)")+ return docs++ def load_and_transform(self) -> List[StandardSample]:+ """+ Parse JSONL question file, group by doc_name into StandardSample.+ evidence uses evidence_text field from each evidence.+ """+ if not os.path.exists(self.raw_file_path):+ raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}")++ groups: Dict[str, List[Dict]] = defaultdict(list)+ with open(self.raw_file_path, 'r', encoding='utf-8') as f:+ for line in f:+ line = line.strip()+ if not line:+ continue+ item = json.loads(line)+ groups[item["doc_name"]].append(item)++ samples: List[StandardSample] = []+ for doc_name, items in groups.items():+ qa_pairs = []+ for item in items:+ evidence_texts = [+ ev["evidence_text"]+ for ev in item.get("evidence", [])+ if ev.get("evidence_text")+ ]++ qa_pairs.append(StandardQA(+ question=item["question"],+ gold_answers=[item["answer"]],+ evidence=evidence_texts,+ category=item.get("question_type"),+ metadata={+ "financebench_id": item.get("financebench_id"),+ "question_reasoning": item.get("question_reasoning"),+ "justification": item.get("justification", ""),+ "company": item.get("company"),+ }+ ))++ samples.append(StandardSample(+ sample_id=doc_name,+ qa_pairs=qa_pairs,+ ))++ self.logger.info(+ f"[FinanceBench] Loaded {sum(len(s.qa_pairs) for s in samples)} questions "+ f"across {len(samples)} documents"+ )+ return samples++ def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]:+ context_text = "\n\n".join(context_blocks)++ category = qa.category+ category_instruction = CATEGORY_INSTRUCTIONS.get(category, "")++ if category_instruction:+ full_prompt = f"""{context_text}++{category_instruction}++{MISSING_RULE}++Question: {qa.question}++Answer:"""+ else:+ full_prompt = f"""{context_text}++{MISSING_RULE}++Question: {qa.question}++Answer:"""++ meta = {+ "question_type": qa.category,+ "financebench_id": qa.metadata.get("financebench_id"),+ }+ return full_prompt, metadiff --git a/benchmark/RAG/src/adapters/locomo_adapter.py b/benchmark/RAG/src/adapters/locomo_adapter.pynew file mode 100644index 000000000..326e29a53--- /dev/null+++ b/benchmark/RAG/src/adapters/locomo_adapter.py@@ -0,0 +1,192 @@+# src/adapters/locomo_adapter.py+import json+import os+from typing import List, Dict, Any++from .base import BaseAdapter, StandardDoc, StandardSample, StandardQA+++MISSING_RULE = "If no information is available to answer the question, write 'Not mentioned'."+++CATEGORY_INSTRUCTIONS = {+ "1": """Extract the exact factual answer from the conversation.+- Use the exact words from the context when possible+- If multiple items, separate with commas""",++ "2": """Answer the time-related question.+- Pay close attention to DATE labels in the conversation+- Calculate relative time (e.g., "10 years ago") when needed+- Use the exact dates from the context""",++ "3": """Reason and infer based on the conversation.+- Use ONLY the facts in the context+- State your conclusion clearly (e.g., "Likely yes", "Probably no")+- Do NOT explain your reasoning or provide any basis/justification+- Only output your final conclusion, nothing else+- Do NOT invent information""",++ "4": """Understand the meaning and significance.+- Focus on what the speakers mean, not just what they say+- Identify symbolism or implied meaning+- Use wording from the context when possible""",+}+++class LocomoAdapter(BaseAdapter):+ """+ Adapter specifically for processing the LocoMo dataset.+ Converts session-format JSON to Markdown with time information.+ """+ def data_prepare(self,doc_dir:str) -> List[StandardDoc]:+ """+ Load raw data and convert to OpenViking-friendly format+ """+ if not os.path.exists(self.raw_file_path):+ raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}")++ res:List[StandardDoc] = []++ with open(self.raw_file_path, 'r', encoding='utf-8') as f:+ data = json.load(f)+ dataset = [data] if isinstance(data, dict) else data+ os.makedirs(doc_dir, exist_ok=True)+ for item in dataset:+ sample_id = item.get("sample_id", "unknown")+ doc_content = self._convert_conversation_to_markdown(sample_id, item.get("conversation", {}))++ try:+ doc_path = os.path.join(doc_dir, f"{sample_id}_doc.md")+ with open(doc_path, "w", encoding="utf-8") as f:+ f.write(doc_content)+ res.append(StandardDoc(sample_id, doc_path))+ except Exception as e:+ self.logger.error(f"[locomo adapter] doc:{sample_id} prepare error {e}")+ raise e+ return res++ def load_and_transform(self) -> List[StandardSample]:+ """+ Load raw JSON data and convert to standardized StandardSample object list.+ """+ if not os.path.exists(self.raw_file_path):+ raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}")++ with open(self.raw_file_path, 'r', encoding='utf-8') as f:+ data = json.load(f)+ dataset = [data] if isinstance(data, dict) else data++ standard_samples = []++ for item in dataset:+ sample_id = item.get("sample_id", "unknown")++ qa_pairs = []+ for q in item.get("qa", []):+ if str(q.get("category")) == "5":+ continue+ raw_ans = q.get("answer")++ if isinstance(raw_ans, list):+ golds = raw_ans+ elif raw_ans is None or raw_ans == "":+ golds = ["Not mentioned"]+ else:+ golds = [raw_ans]++ qa_pairs.append(StandardQA(+ question=q["question"],+ gold_answers=[str(g) for g in golds],+ evidence=q.get("evidence", []),+ category=q.get("category"),+ metadata={"original_id": q.get("id")}+ ))++ standard_samples.append(StandardSample(+ sample_id=sample_id,+ qa_pairs=qa_pairs+ ))++ return standard_samples++ def _convert_conversation_to_markdown(self, sample_id: str, conv: Dict[str, Any]) -> str:+ """+ Convert LocoMo session structure to flat Markdown string.+ """+ md_lines = [f"# Chat History: {sample_id}"]++ session_idx = 1+ while f"session_{session_idx}" in conv:+ s_key = f"session_{session_idx}"+ dt_key = f"session_{session_idx}_date_time"+ sum_key = f"session_{session_idx}_summary"++ md_lines.append(f"\n## Session {session_idx}")++ session_dt = conv.get(dt_key)+ if session_dt:+ md_lines.append(f"DATE: {session_dt}")++ session_sum = conv.get(sum_key)+ if session_sum:+ md_lines.append(f"SUMMARY: {session_sum}")++ for turn in conv[s_key]:+ spk = turn.get("speaker", "Unknown")+ txt = turn.get("text", "")++ raw_id = turn.get("dia_id") or turn.get("id")++ image_suffix = ""+ img_url = turn.get("img_url", [])+ blip_caption = turn.get("blip_caption", "")++ if img_url and blip_caption:+ if len(img_url) == 1:+ image_suffix = f"[Attached image:{blip_caption}]"+ else:+ for i, caption in enumerate([blip_caption] * len(img_url)):+ image_suffix += f"[Attached image {i+1}:{caption}]"++ dia_suffix = f" [{raw_id}]" if raw_id else ""++ md_lines.append(f"**{spk}**: {txt}{image_suffix}{dia_suffix}")++ session_idx += 1++ return "\n".join(md_lines)++ def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]:+ category = str(qa.category)+ context_text = "\n\n".join(context_blocks)++ category_instruction = CATEGORY_INSTRUCTIONS.get(category, "")++ if category_instruction:+ full_prompt = f"""{context_text}++{MISSING_RULE}++---+{category_instruction}++Question: {qa.question}++Answer:"""+ else:+ full_prompt = f"""{context_text}++{MISSING_RULE}++Based on the conversation above, answer the following question.+Use ONLY the provided context. Do NOT invent any information.++Question: {qa.question}++Answer:"""++ meta = {"category": category}+ return full_prompt, meta++ def post_process_answer(self, qa: StandardQA, raw_answer: str, meta: Dict[str, Any]) -> str:+ return raw_answer.strip()diff --git a/benchmark/RAG/src/adapters/qasper_adapter.py b/benchmark/RAG/src/adapters/qasper_adapter.pynew file mode 100644index 000000000..b17a5a46e--- /dev/null+++ b/benchmark/RAG/src/adapters/qasper_adapter.py@@ -0,0 +1,417 @@+# src/adapters/qasper_adapter.py+"""+Qasper Dataset Adapter++Qasper is an academic paper QA dataset containing 1585 NLP papers and 5049 questions.+Each question is answered by multiple annotators, with answer types including:+- extractive_spans: text spans extracted from the paper+- free_form_answer: free-form answers+- yes_no: yes/no answers+- unanswerable: no answer can be found in the paper++Dataset characteristics:+1. Each paper includes title, abstract, section content, and figure/table information+2. Each question may have answers from multiple annotators+3. Each answer has corresponding evidence (evidence text)++Adapter functions:+- data_prepare: Convert papers to Markdown format, preserving section structure+- load_and_transform: Parse QA data, preserving answer-evidence correspondence+- build_prompt: Build QA prompt+- post_process_answer: Post-process LLM output+"""++import json+import os+from typing import List, Dict, Any++from .base import BaseAdapter, StandardDoc, StandardSample, StandardQA++# Specific instructions for different answer types+CATEGORY_INSTRUCTIONS = {+ "extractive": """Extract the exact answer from the paper.+- Use EXACT wording from the context+- Do NOT rephrase or add explanation+- Provide concise, direct answer""",++ "free_form": """Answer using information from the paper.+- Use ONLY facts from the context+- You may rephrase or summarize in your own words+- Provide clear, complete answer+- Do NOT invent information""",++ "yes_no": """Answer Yes/No question based on the paper.+- First respond "Yes" or "No"+- Do NOT add explanation+- Use ONLY info from context+- Do NOT invent information"""+}++# Rule for when answer cannot be found+MISSING_RULE = "If no information is available to answer the question, write 'Not mentioned'."+++class QasperAdapter(BaseAdapter):+ """+ Adapter specifically for processing Qasper dataset.++ Converts academic papers to Markdown documents with section structure,+ and converts QA data to standardized StandardSample format.++ Attributes:+ raw_file_path: Raw JSON data file path+ logger: Logger+ """++ def data_prepare(self, doc_dir: str) -> List[StandardDoc]:+ """+ Load raw data and convert to OpenViking-friendly format.++ Converts each paper to Markdown document, preserving:+ - Title (# Title)+ - Abstract (## Abstract)+ - Sections (## Section Name)+ - Figures and Tables (## Figures and Tables)++ Args:+ doc_dir: Document output directory path++ Returns:+ List[StandardDoc]: List of standardized document objects, each containing paper_id and document path++ Raises:+ FileNotFoundError: Raw data file not found+ """+ res: List[StandardDoc] = []+ data = {}++ if os.path.isdir(self.raw_file_path):+ json_files = [f for f in os.listdir(self.raw_file_path) if f.endswith('.json') and f != 'sampling_metadata.json']+ for json_file in json_files:+ file_path = os.path.join(self.raw_file_path, json_file)+ with open(file_path, 'r', encoding='utf-8') as f:+ file_data = json.load(f)+ data.update(file_data)+ else:+ if not os.path.exists(self.raw_file_path):+ raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}")+ with open(self.raw_file_path, 'r', encoding='utf-8') as f:+ data = json.load(f)++ os.makedirs(doc_dir, exist_ok=True)++ for paper_id, paper_data in data.items():+ doc_content = self._convert_paper_to_markdown(paper_id, paper_data)++ try:+ doc_path = os.path.join(doc_dir, f"{paper_id}_doc.md")+ with open(doc_path, "w", encoding="utf-8") as f:+ f.write(doc_content)+ res.append(StandardDoc(paper_id, doc_path))+ except Exception as e:+ self.logger.error(f"[qasper adapter] doc:{paper_id} prepare error {e}")+ raise e+ return res++ def load_and_transform(self) -> List[StandardSample]:+ """+ Load raw JSON data and convert to standardized StandardSample object list.++ Processing logic:+ 1. Iterate through QA list of each paper+ 2. For each question, collect answers from all annotators+ 3. Preserve answer-evidence correspondence (stored in metadata)+ 4. Format question as "Based on the paper \"{title}\", {question}"++ Answer type handling:+ - extractive_spans: directly use extracted text spans+ - free_form_answer: use free-form answer+ - yes_no: convert to "Yes" or "No"+ - unanswerable: convert to "Not mentioned"++ Returns:+ List[StandardSample]: List of standardized sample objects++ Raises:+ FileNotFoundError: Raw data file not found+ """+ data = {}++ if os.path.isdir(self.raw_file_path):+ json_files = [f for f in os.listdir(self.raw_file_path) if f.endswith('.json') and f != 'sampling_metadata.json']+ for json_file in json_files:+ file_path = os.path.join(self.raw_file_path, json_file)+ with open(file_path, 'r', encoding='utf-8') as f:+ file_data = json.load(f)+ data.update(file_data)+ else:+ if not os.path.exists(self.raw_file_path):+ raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}")+ with open(self.raw_file_path, 'r', encoding='utf-8') as f:+ data = json.load(f)++ standard_samples = []++ for paper_id, paper_data in data.items():+ qa_pairs = []+ paper_title = paper_data.get("title", "Unknown Title")++ for qa_item in paper_data.get("qas", []):++ # --- Unanswerable filtering logic ---+ # Check if all answers are marked as unanswerable+ is_unanswerable = all(+ ans.get("answer", {}).get("unanswerable", False)+ for ans in qa_item.get("answers", [])+ )+ if is_unanswerable:+ continue+ # ------------------++ raw_question = qa_item.get("question", "")+ question_id = qa_item.get("question_id", "")+ # Append paper title to question for easier retrieval+ question = f'Based on the paper "{paper_title}", {raw_question}'++ gold_answers = []+ evidence_list = []+ answer_types = []+ answer_evidence_pairs = []++ # Iterate through all annotator answers+ for answer_wrapper in qa_item.get("answers", []):+ answer_obj = answer_wrapper.get("answer", {})++ current_answer = None+ answer_type = self._get_answer_type(answer_obj)++ # Process different answer types+ if answer_obj.get("unanswerable", False):+ current_answer = "Not mentioned"+ gold_answers.append(current_answer)+ else:+ extractive_spans = answer_obj.get("extractive_spans", [])+ free_form_answer = answer_obj.get("free_form_answer", "")+ yes_no = answer_obj.get("yes_no")++ if extractive_spans:+ valid_spans = [span.strip() for span in extractive_spans if span and span.strip()]+ if valid_spans:+ combined_answer = "; ".join(valid_spans)+ gold_answers.append(combined_answer)+ current_answer = combined_answer+ else:+ current_answer = None+ elif free_form_answer and free_form_answer.strip():+ current_answer = free_form_answer.strip()+ gold_answers.append(current_answer)+ elif yes_no is not None:+ current_answer = "Yes" if yes_no else "No"+ gold_answers.append(current_answer)++ # Collect evidence text+ current_evidence = []+ evidence = answer_obj.get("evidence", [])+ for ev in evidence:+ if ev and ev.strip():+ current_evidence.append(ev)+ if ev not in evidence_list:+ evidence_list.append(ev)++ # Record answer type (deduplicated)+ if answer_type not in answer_types:+ answer_types.append(answer_type)++ # Save answer-evidence correspondence+ if current_answer:+ answer_evidence_pairs.append({+ "answer": current_answer,+ "evidence": current_evidence,+ "answer_type": answer_type+ })++ # If no answers, default to "Not mentioned"+ if not gold_answers:+ gold_answers = ["Not mentioned"]++ # Deduplicate (preserve order)+ gold_answers = list(dict.fromkeys(gold_answers))++ qa_pairs.append(StandardQA(+ question=question,+ gold_answers=gold_answers,+ evidence=evidence_list,+ category=None,+ metadata={+ "question_id": question_id,+ "answer_types": answer_types,+ "answer_evidence_pairs": answer_evidence_pairs+ }+ ))++ standard_samples.append(StandardSample(+ sample_id=paper_id,+ qa_pairs=qa_pairs+ ))++ return standard_samples++ def _get_answer_type(self, answer_obj: Dict[str, Any]) -> str:+ """+ Determine answer type from answer object.++ Args:+ answer_obj: Answer object, containing extractive_spans, free_form_answer, yes_no, etc.++ Returns:+ str: Answer type, possible values:+ - "unanswerable": cannot answer+ - "extractive": extractive answer+ - "free_form": free-form answer+ - "yes_no": yes/no answer+ - "unknown": unknown type+ """+ if answer_obj.get("unanswerable", False):+ return "unanswerable"+ if answer_obj.get("extractive_spans"):+ return "extractive"+ if answer_obj.get("free_form_answer", "").strip():+ return "free_form"+ if answer_obj.get("yes_no") is not None:+ return "yes_no"+ return "unknown"++ def _convert_paper_to_markdown(self, paper_id: str, paper_data: Dict[str, Any]) -> str:+ """+ Convert Qasper paper structure to Markdown string.++ Conversion format:+ ```markdown+ # {title}+ Paper ID: {paper_id}++ ## Abstract+ {abstract}++ ## Section Name 1+ {paragraph 1}+ {paragraph 2}++ ## Section Name 2+ ...++ ## Figures and Tables+ ### Figure 1+ Caption: {caption}+ File: {filename}++ ### Table 1+ Caption: {caption}+ File: {filename}+ ```++ Args:+ paper_id: Paper ID+ paper_data: Paper data, containing title, abstract, full_text, figures_and_tables++ Returns:+ str: Markdown formatted paper content+ """+ md_lines = []++ # Title+ title = paper_data.get("title", "Unknown Title")+ md_lines.append(f"# {title}")+ md_lines.append(f"Paper ID: {paper_id}\n")++ # Abstract+ abstract = paper_data.get("abstract", "")+ if abstract:+ md_lines.append("## Abstract")+ md_lines.append(abstract)+ md_lines.append("")++ # Main text sections+ full_text = paper_data.get("full_text", [])+ for section in full_text:+ section_name = section.get("section_name", "")+ paragraphs = section.get("paragraphs", [])++ if section_name:+ md_lines.append(f"## {section_name}")++ for para in paragraphs:+ if para and para.strip():+ md_lines.append(para.strip())+ md_lines.append("")++ # Figure and table information+ figures_and_tables = paper_data.get("figures_and_tables", [])+ if figures_and_tables:+ md_lines.append("## Figures and Tables")+ for idx, fig in enumerate(figures_and_tables, 1):+ caption = fig.get("caption", "")+ file_name = fig.get("file", "")++ # Determine if figure or table based on filename or caption+ if "Figure" in file_name or "figure" in caption.lower():+ md_lines.append(f"### Figure {idx}")+ else:+ md_lines.append(f"### Table {idx}")++ if caption:+ md_lines.append(f"Caption: {caption}")+ if file_name:+ md_lines.append(f"File: {file_name}")+ md_lines.append("")++ return "\n".join(md_lines)++ def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]:+ context_text = "\n\n".join(context_blocks) if context_blocks else "No relevant context found."++ answer_types = qa.metadata.get("answer_types", [])+ primary_type = answer_types[0] if answer_types else None++ category_instruction = CATEGORY_INSTRUCTIONS.get(primary_type, "")++ if category_instruction:+ full_prompt = f"""{context_text}++{category_instruction}++{MISSING_RULE}++Question: {qa.question}++Answer:"""+ else:+ full_prompt = f"""{context_text}++{MISSING_RULE}++Question: {qa.question}++Answer:"""++ meta = {+ "question_id": qa.metadata.get("question_id", ""),+ "answer_types": answer_types+ }+ return full_prompt, meta++ def post_process_answer(self, qa: StandardQA, raw_answer: str, meta: Dict[str, Any]) -> str:+ """+ Post-process raw answer generated by LLM.++ Current implementation only strips leading/trailing whitespace.++ Args:+ qa: Standardized QA object+ raw_answer: Raw answer generated by LLM+ meta: Metadata dictionary++ Returns:+ str: Processed answer+ """+ return raw_answer.strip()diff --git a/benchmark/RAG/src/adapters/syllabusqa_adapter.py b/benchmark/RAG/src/adapters/syllabusqa_adapter.pynew file mode 100644index 000000000..69fcf3fb9--- /dev/null+++ b/benchmark/RAG/src/adapters/syllabusqa_adapter.py@@ -0,0 +1,485 @@+# src/adapters/syllabusqa_adapter.py+"""+SyllabusQA Dataset Adapter++SyllabusQA is a syllabus QA dataset containing 39 syllabi and 5078 questions.+Each question is about a specific syllabus, with answer types including:+- single factual: single factual question+- multi factual: multi factual question+- single reasoning: single reasoning question+- multi reasoning: multi reasoning question+- summarization: summarization question+- yes/no: yes/no question+- no answer: no answer can be found in the syllabus++Dataset characteristics:+1. Each question has only one answer+2. Each answer has corresponding answer_span (evidence span)+3. reasoning type questions have reasoning steps+4. Original documents are in docx format++Adapter functions:+- data_prepare: Convert docx to Markdown format+- load_and_transform: Parse QA data, category stores question type+- build_prompt: Build QA prompt+- post_process_answer: Post-process LLM output+"""++import json+import os+import csv+from typing import List, Dict, Any++from .base import BaseAdapter, StandardDoc, StandardSample, StandardQA++# Rule for when answer cannot be found+MISSING_RULE = "If no information is available to answer the question, write 'Not mentioned'."++# Specific instructions for different categories+CATEGORY_INSTRUCTIONS = {+ "single factual": """Extract the single factual answer from the syllabus.+- Use EXACT wording from context when possible+- Provide concise, direct answer+- Do NOT add extra info or explanation""",++ "multi factual": """Extract multiple factual answers from the syllabus.+- Use EXACT wording from context when possible+- List items separated by commas+- Include all relevant facts""",++ "single reasoning": """Answer using simple logical reasoning based on the syllabus.+- Use ONLY facts from context+- Make clear, direct conclusion+- Do NOT explain reasoning+- Do NOT invent information""",++ "multi reasoning": """Answer using reasoning based on the syllabus.+- Use ONLY facts from context+- Do NOT invent information""",++ "summarization": """Summarize relevant information from the syllabus.+- Provide concise summary covering key points+- Use wording from syllabus when possible+- Include all important details""",++ "yes/no": """Answer Yes/No question based on the syllabus.+- First respond "Yes" or "No"+- Do NOT add explanation+- Use ONLY info from context+- Do NOT invent information"""+}+++class SyllabusQAAdapter(BaseAdapter):+ """+ Adapter specifically for processing SyllabusQA dataset.++ Converts syllabi (docx) to Markdown documents,+ and converts QA data to standardized StandardSample format.++ Attributes:+ raw_file_path: Raw CSV data file path+ syllabus_dir: docx file directory path+ logger: Logger+ """++ def __init__(self, raw_file_path: str, **kwargs):+ """+ Initialize SyllabusQAAdapter.++ Args:+ raw_file_path: Raw data file path (CSV or merged JSON)+ **kwargs: Other parameters (ignored, for compatibility)+ """+ super().__init__(raw_file_path)+ # docx file directory, defaults to syllabi subdirectory under data directory+ if os.path.isdir(raw_file_path):+ base_dir = raw_file_path+ else:+ base_dir = os.path.dirname(raw_file_path)++ # Check for official repo structure first+ official_syllabus_dir = os.path.join(base_dir, 'syllabi', 'syllabi_redacted', 'word')+ if os.path.exists(official_syllabus_dir):+ self.syllabus_dir = official_syllabus_dir+ else:+ # Fallback to original structure+ self.syllabus_dir = os.path.join(base_dir, 'syllabi')++ def data_prepare(self, doc_dir: str) -> List[StandardDoc]:+ """+ Load raw docx files and convert to OpenViking-friendly format.++ Only process syllabus documents mentioned in CSV, avoid processing irrelevant documents.+ Requires python-docx library to parse docx files.++ Args:+ doc_dir: Document output directory path++ Returns:+ List[StandardDoc]: List of standardized document objects++ Raises:+ FileNotFoundError: syllabus directory not found+ """+ if not os.path.exists(self.syllabus_dir):+ raise FileNotFoundError(f"Syllabus directory not found: {self.syllabus_dir}")++ res: List[StandardDoc] = []+ os.makedirs(doc_dir, exist_ok=True)++ # Get list of syllabus_name mentioned in CSV+ required_syllabi = self._get_required_syllabi()+ self.logger.info(f"[SyllabusQAAdapter] Required syllabi from CSV: {len(required_syllabi)}")++ # Get all docx files+ docx_files = [f for f in os.listdir(self.syllabus_dir) if f.endswith('.docx')]++ for docx_file in docx_files:+ syllabus_id = docx_file.replace('.docx', '')++ # Only process syllabi mentioned in CSV+ if syllabus_id not in required_syllabi:+ continue++ docx_path = os.path.join(self.syllabus_dir, docx_file)++ try:+ # Convert docx to Markdown+ doc_content = self._convert_docx_to_markdown(docx_path)++ doc_path = os.path.join(doc_dir, f"{syllabus_id}_doc.md")+ with open(doc_path, "w", encoding="utf-8") as f:+ f.write(doc_content)+ res.append(StandardDoc(syllabus_id, doc_path))+ except Exception as e:+ self.logger.error(f"[syllabusqa adapter] doc:{syllabus_id} prepare error {e}")+ # If python-docx not installed, try using plain text+ if "No module named 'docx'" in str(e):+ self.logger.warning("python-docx not installed, skipping docx conversion")+ break+ raise e++ self.logger.info(f"[SyllabusQAAdapter] Processed {len(res)} syllabus documents")+ return res++ def _get_required_syllabi(self) -> set:+ """+ Get list of syllabus_name mentioned in CSV or JSON.++ Returns:+ set: syllabus_name set+ """+ required = set()++ # Determine data source type+ if self.raw_file_path.endswith('.json'):+ # Load from JSON+ if not os.path.exists(self.raw_file_path):+ return required++ with open(self.raw_file_path, 'r', encoding='utf-8') as f:+ data = json.load(f)++ for syllabus_name in data.keys():+ if syllabus_name:+ required.add(syllabus_name)+ elif self.raw_file_path.endswith('.csv'):+ csv_files = [self.raw_file_path]+ elif os.path.isdir(self.raw_file_path):+ csv_files = [os.path.join(self.raw_file_path, f)+ for f in os.listdir(self.raw_file_path)+ if f.endswith('.csv')]+ else:+ return required++ # Process CSV files if any+ if 'csv_files' in locals():+ for csv_file in csv_files:+ if not os.path.exists(csv_file):+ continue+ with open(csv_file, 'r', encoding='utf-8') as f:+ import csv+ reader = csv.DictReader(f)+ for row in reader:+ syllabus_name = row.get('syllabus_name', '')+ if syllabus_name:+ required.add(syllabus_name)++ return required++ def _convert_docx_to_markdown(self, docx_path: str) -> str:+ """+ Convert docx file to Markdown string.++ Args:+ docx_path: docx file path++ Returns:+ str: Markdown formatted content+ """+ try:+ from docx import Document+ except ImportError:+ raise ImportError("python-docx is required. Install with: pip install python-docx")++ doc = Document(docx_path)+ md_lines = []++ # Extract filename as title+ filename = os.path.basename(docx_path).replace('.docx', '')+ md_lines.append(f"# {filename}")+ md_lines.append("")++ # Iterate through all paragraphs+ for para in doc.paragraphs:+ text = para.text.strip()+ if text:+ # Check if heading style+ if para.style.name.startswith('Heading'):+ level = para.style.name.replace('Heading ', '')+ try:+ level_num = int(level)+ md_lines.append(f"{'#' * level_num} {text}")+ except ValueError:+ md_lines.append(f"## {text}")+ else:+ md_lines.append(text)+ md_lines.append("")++ # Extract tables+ for table in doc.tables:+ md_lines.append("## Table")+ for row in table.rows:+ cells = [cell.text.strip() for cell in row.cells]+ md_lines.append("| " + " | ".join(cells) + " |")+ md_lines.append("")++ return "\n".join(md_lines)++ def load_and_transform(self) -> List[StandardSample]:+ """+ Load raw CSV data and convert to standardized StandardSample object list.++ Processing logic:+ 1. Read CSV file (supports single CSV or directory)+ 2. Group by syllabus_name+ 3. Format question as "Based on the syllabus \"{syllabus_name}\", {question}"+ 4. category stores question_type+ 5. answer_span as evidence++ Returns:+ List[StandardSample]: List of standardized sample objects++ Raises:+ FileNotFoundError: Raw data file not found+ """+ # Determine if CSV file or JSON file+ if self.raw_file_path.endswith('.json'):+ return self._load_from_json()+ elif self.raw_file_path.endswith('.csv'):+ return self._load_from_csv([self.raw_file_path])+ elif os.path.isdir(self.raw_file_path):+ # Directory, find all CSV files+ csv_files = [os.path.join(self.raw_file_path, f)+ for f in os.listdir(self.raw_file_path)+ if f.endswith('.csv')]+ return self._load_from_csv(csv_files)+ else:+ raise FileNotFoundError(f"Unsupported file format: {self.raw_file_path}")++ def _load_from_json(self) -> List[StandardSample]:+ """+ Load data from JSON file.+ """+ if not os.path.exists(self.raw_file_path):+ raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}")++ with open(self.raw_file_path, 'r', encoding='utf-8') as f:+ data = json.load(f)++ standard_samples = []++ for syllabus_name, qa_list in data.items():+ qa_pairs = []++ for qa_item in qa_list:+ question = qa_item.get("question", "")+ answer = qa_item.get("answer", "")+ question_type = qa_item.get("question_type", "")+ qa_id = qa_item.get("id", "")++ # Skip "no answer" type questions as RAG results cannot be evaluated+ if question_type == "no answer":+ continue++ # Collect answer_span as evidence+ evidence = []+ for i in range(1, 6):+ span = qa_item.get(f"answer_span_{i}", "")+ if span and span.strip():+ evidence.append(span.strip())++ # Collect reasoning_steps, also as evidence (for reasoning type questions)+ reasoning_steps = []+ for i in range(1, 6):+ step = qa_item.get(f"reasoning_step_{i}", "")+ if step and step.strip():+ reasoning_steps.append(step.strip())+ # reasoning_steps also added to evidence for recall calculation+ if step.strip() not in evidence:+ evidence.append(step.strip())++ # Format question+ formatted_question = f'Based on the syllabus "{syllabus_name}", {question}'++ qa_pairs.append(StandardQA(+ question=formatted_question,+ gold_answers=[answer] if answer else ["Not mentioned"],+ evidence=evidence,+ category=question_type,+ metadata={+ "id": qa_id,+ "reasoning_steps": reasoning_steps+ }+ ))++ # Only add samples with QA pairs to result+ if qa_pairs:+ standard_samples.append(StandardSample(+ sample_id=syllabus_name,+ qa_pairs=qa_pairs+ ))++ return standard_samples++ def _load_from_csv(self, csv_files: List[str]) -> List[StandardSample]:+ """+ Load data from CSV files.++ Args:+ csv_files: List of CSV file paths++ Returns:+ List[StandardSample]: List of standardized sample objects+ """+ # Group by syllabus_name+ syllabus_qa_map: Dict[str, List] = {}++ for csv_file in csv_files:+ if not os.path.exists(csv_file):+ self.logger.warning(f"CSV file not found: {csv_file}")+ continue++ with open(csv_file, 'r', encoding='utf-8') as f:+ reader = csv.DictReader(f)+ for row in reader:+ syllabus_name = row.get('syllabus_name', '')+ if syllabus_name not in syllabus_qa_map:+ syllabus_qa_map[syllabus_name] = []+ syllabus_qa_map[syllabus_name].append(row)++ standard_samples = []++ for syllabus_name, qa_list in syllabus_qa_map.items():+ qa_pairs = []++ for qa_item in qa_list:+ question = qa_item.get("question", "")+ answer = qa_item.get("answer", "")+ question_type = qa_item.get("question_type", "")+ qa_id = qa_item.get("id", "")++ # Skip "no answer" type questions as RAG results cannot be evaluated+ if question_type == "no answer":+ continue++ # Collect answer_span as evidence+ evidence = []+ for i in range(1, 6):+ span = qa_item.get(f"answer_span_{i}", "")+ if span and span.strip():+ evidence.append(span.strip())++ # Collect reasoning_steps, also as evidence (for reasoning type questions)+ reasoning_steps = []+ for i in range(1, 6):+ step = qa_item.get(f"reasoning_step_{i}", "")+ if step and step.strip():+ reasoning_steps.append(step.strip())+ # reasoning_steps also added to evidence for recall calculation+ if step.strip() not in evidence:+ evidence.append(step.strip())++ # Format question+ formatted_question = f'Based on the syllabus "{syllabus_name}", {question}'++ qa_pairs.append(StandardQA(+ question=formatted_question,+ gold_answers=[answer] if answer else ["Not mentioned"],+ evidence=evidence,+ category=question_type,+ metadata={+ "id": qa_id,+ "reasoning_steps": reasoning_steps+ }+ ))++ # Only add samples with QA pairs to result+ if qa_pairs:+ standard_samples.append(StandardSample(+ sample_id=syllabus_name,+ qa_pairs=qa_pairs+ ))++ return standard_samples++ def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]:+ """+ Build complete prompt to send to LLM.++ Prompt structure:+ 1. Context content (retrieved document fragments)+ 2. Category-specific instructions+ 3. Rule for when answer cannot be found+ 4. Question++ Args:+ qa: Standardized QA object+ context_blocks: List of retrieved context text blocks++ Returns:+ tuple[str, Dict[str, Any]]:+ - Complete prompt string+ - Metadata dictionary, containing id+ """+ eff_q = qa.question+ category = qa.category++ category_instruction = CATEGORY_INSTRUCTIONS.get(category, "")++ context_text = "\n\n".join(context_blocks)++ if category_instruction:+ full_prompt = f"{context_text}\n\n{category_instruction}\n\n{MISSING_RULE}\n\nQuestion: {eff_q}\n\nAnswer:"+ else:+ full_prompt = f"{context_text}\n\n{MISSING_RULE}\n\nQuestion: {eff_q}\n\nAnswer:"++ meta = {"id": qa.metadata.get("id", "")}+ return full_prompt, meta++ def post_process_answer(self, qa: StandardQA, raw_answer: str, meta: Dict[str, Any]) -> str:+ """+ Post-process raw answer generated by LLM.++ Current implementation only strips leading/trailing whitespace.++ Args:+ qa: Standardized QA object+ raw_answer: Raw answer generated by LLM+ meta: Metadata dictionary++ Returns:+ str: Processed answer+ """+ return raw_answer.strip()diff --git a/benchmark/RAG/src/core/__init__.py b/benchmark/RAG/src/core/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/benchmark/RAG/src/core/judge_util.py b/benchmark/RAG/src/core/judge_util.pynew file mode 100644index 000000000..b5549a65a--- /dev/null+++ b/benchmark/RAG/src/core/judge_util.py@@ -0,0 +1,164 @@+import json+import re++from langchain_core.messages import HumanMessage, SystemMessage+++def llm_grader(+ llm_client,+ model: str,+ question: str,+ gold_answer: str or list,+ response: str,+ dataset_name: str = "Locomo"+) -> dict:+ """+ Use an LLM as a judge to score a generated answer against a gold answer.++ Return format:+ {+ "score": int, # LoCoMo: 0 or 4; Qasper/Generic: 0~4+ "reasoning": str, # grading explanation or fallback parse info+ "prompt_type": str # which prompt template was used+ }+ """++ dataset_name_lower = (dataset_name or "").lower()+ content = ""+ score = 0+ reasoning = "No reasoning provided."+ prompt_type = "Generic_0-4"++ # Handle case when gold_answer is a list+ if isinstance(gold_answer, list):+ gold_answer_str = " | ".join(gold_answer)+ else:+ gold_answer_str = gold_answer++ # -------------------------+ # 1) Route by dataset+ # -------------------------+ if "locomo" in dataset_name_lower:+ prompt_type = "Locomo_0or4"++ system_prompt = """+You are an expert grader that determines if answers to questions match a gold standard answer+"""++ ACCURACY_PROMPT = f"""+Your task is to label an answer to a question by assigning a score of 4 or 0. You will be given the following data:+(1) a question (posed by one user to another user),+(2) a 'gold' (ground truth) answer,+(3) a generated answer++which you will score as 4 or 0.+The point of the question is to ask about something one user should know about the other user based on their prior conversations.+The gold answer will usually be a concise and short answer that includes the referenced topic, for example:+Question: Do you remember what I got the last time I went to Hawaii?+Gold answer: A shell necklace+The generated answer might be much longer, but you should be generous with your grading - as long as it touches on the same topic as the gold answer, it should be counted as correct.+For time related questions, the gold answer will be a specific date, month, year, etc. The generated answer might be much longer or use relative time references (like "last Tuesday" or "next month"), but you should be generous with your grading - as long as it refers to the same date or time period as the gold answer, it should be counted as correct. Even if the format differs (e.g., "May 7th" vs "7 May"), consider it correct if it's the same date.++Scoring rule:+- Output score 4 if the generated answer should be considered CORRECT.+- Output score 0 if the generated answer should be considered WRONG.++Now it's time for the real question:+Question: {question}+Gold answer: {gold_answer_str}+Generated answer: {response}++First, provide a short (one sentence) explanation of your reasoning.+Respond with JSON only: {{"score": 4 or 0, "reasoning": "your explanation"}}+"""++ else:+ prompt_type = "Generic_0-4"++ system_prompt = """+You are an expert evaluator scoring how well an AI-generated answer matches a gold standard (ground truth).+"""++ ACCURACY_PROMPT = f"""+Please score the Generated Answer against the Gold Answers on a scale of 0 to 4.++[Evaluation Rubric]+- Score 4 (Perfect): Fully and accurately captures the core meaning and key facts of any of the Gold Answers. Additional relevant explanation or context is acceptable and does NOT reduce the score, as long as it is consistent with and does not contradict the Gold Answers. Minor differences in wording, capitalization, punctuation, or phrasing are acceptable if the core meaning is preserved.+- Score 3 (Good): Correctly captures the main answer and most key facts, but has minor issues such as slight imprecision, small omissions of non-critical details, or wording that is somewhat vague or ambiguous. The overall answer is still clearly correct.+- Score 2 (Partial): Partially correct, but missing at least one important fact, condition, or detail needed for a fully correct answer. The answer is related to the correct topic, but is incomplete or insufficient.+- Score 1 (Poor): Mostly incorrect, seriously incomplete, or only weakly related to the Gold Answers.+- Score 0 (Wrong): Incorrect, contradictory to the Gold Answers, or contains fabricated / hallucinated core content.++Important Notes:+- Gold answers are multiple possible correct answers separated by " | ". The generated answer only needs to match any one of them.+- The gold answers may be concise, but the generated answer can be longer and include additional explanations - this is acceptable for Score 4 as long as the core information is correct.+- Do NOT penalize for additional relevant information that doesn't contradict the gold answers. Examples of acceptable extra information: titles ("King Padella" vs "Padella"), locations ("Paflagonia" vs "the capital of Paflagonia"), or additional context that supports the answer.+- Only penalize for actual incorrect information, missing key facts, or contradictions.+- Ignore minor differences in capitalization (e.g., "CRIM TARTARY" vs "Crim Tartary") or punctuation (e.g., with or without a period at the end).++Question: {question}+Gold Answers: {gold_answer_str}+Generated Answer: {response}++First, briefly explain the rating in 1 sentence. Then output the integer score.+Respond ONLY with a JSON object: {{"score": 0 to 4, "reasoning": "string"}}+"""++ messages = [+ SystemMessage(content=system_prompt),+ HumanMessage(content=ACCURACY_PROMPT),+ ]++ # -------------------------+ # 2) Unified invoke + parse+ # -------------------------+ try:+ resp = llm_client.invoke(messages)+ content = resp.content if resp and hasattr(resp, "content") else ""++ result = json.loads(content)+ score = int(result.get("score", 0))+ reasoning = result.get("reasoning", "No reasoning provided.")++ # Clamp score by dataset+ if "locomo" in dataset_name_lower:+ # LoCoMo only allows 0 or 4+ score = 4 if score == 4 else 0+ else:+ # Other datasets allow 0~4+ score = max(0, min(4, score))++ except Exception:+ # -------------------------+ # 3) Unified fallback parse+ # -------------------------+ text = (content or "").strip()+ reasoning = (+ f"Parse fallback from raw output: {text}"+ if text+ else "Parse failed or model invocation failed. Defaulted to 0."+ )++ # First try: JSON-like score field+ match = re.search(r'"score"\s*:\s*([0-4])', text)+ if match:+ score = int(match.group(1))+ else:+ # Second try: any standalone integer 0~4 in text+ match = re.search(r'\b([0-4])\b', text)+ if match:+ score = int(match.group(1))+ else:+ score = 0++ # Dataset-specific clamp+ if "locomo" in dataset_name_lower:+ score = 4 if score == 4 else 0+ else:+ score = max(0, min(4, score))++ return {+ "score": score,+ "reasoning": reasoning,+ "prompt_type": prompt_type,+ }diff --git a/benchmark/RAG/src/core/llm_client.py b/benchmark/RAG/src/core/llm_client.pynew file mode 100644index 000000000..a18505b14--- /dev/null+++ b/benchmark/RAG/src/core/llm_client.py@@ -0,0 +1,30 @@+import time+from langchain_openai import ChatOpenAI+from langchain_core.messages import HumanMessage+++class LLMClientWrapper:+ def __init__(self, config: dict, api_key: str):+ self.llm = ChatOpenAI(+ model=config['model'],+ temperature=config['temperature'],+ api_key=api_key,+ base_url=config['base_url']+ )+ self.retry_count = 3++ def generate(self, prompt: str) -> str:+ """Call LLM to generate answer with simple exponential backoff retry"""+ last_err = None+ for attempt in range(self.retry_count):+ try:+ resp = self.llm.invoke([HumanMessage(content=prompt)])+ return resp.content+ except Exception as e:+ last_err = e+ if attempt < self.retry_count - 1:+ time.sleep(1.5 * (attempt + 1))++ raise RuntimeError(+ f"LLM generate failed after {self.retry_count} retries: {type(last_err).__name__}: {last_err}"+ ) from last_errdiff --git a/benchmark/RAG/src/core/logger.py b/benchmark/RAG/src/core/logger.pynew file mode 100644index 000000000..ab3847305--- /dev/null+++ b/benchmark/RAG/src/core/logger.py@@ -0,0 +1,26 @@+import logging+import os+++def setup_logging(log_file):+ os.makedirs(os.path.dirname(log_file), exist_ok=True)++ logger = logging.getLogger("Benchmark")+ logger.setLevel(logging.INFO)+ logger.handlers = []++ formatter = logging.Formatter("%(asctime)s | %(levelname)-7s | %(message)s")++ fh = logging.FileHandler(log_file, mode="a", encoding="utf-8")+ fh.setFormatter(formatter)+ logger.addHandler(fh)++ sh = logging.StreamHandler()+ sh.setFormatter(formatter)+ logger.addHandler(sh)++ return logger+++def get_logger() -> logging.Logger:+ return logging.getLogger("Benchmark")diff --git a/benchmark/RAG/src/core/metrics.py b/benchmark/RAG/src/core/metrics.pynew file mode 100644index 000000000..d40d8de1e--- /dev/null+++ b/benchmark/RAG/src/core/metrics.py@@ -0,0 +1,86 @@+import re+import string+import collections+from typing import List+++class MetricsCalculator:+ @staticmethod+ def normalize_answer(s):+ """Normalize answer text: remove punctuation, convert to lowercase, remove articles"""+ s = str(s).replace(',', "")+ def remove_articles(text): return re.sub(r'\b(a|an|the|and)\b', ' ', text)+ def white_space_fix(text): return ' '.join(text.split())+ def remove_punc(text):+ exclude = set(string.punctuation)+ return ''.join(ch for ch in text if ch not in exclude)+ return white_space_fix(remove_articles(remove_punc(s.lower())))++ @staticmethod+ def calculate_f1(prediction: str, ground_truth: str) -> float:+ pred_tokens = MetricsCalculator.normalize_answer(prediction).split()+ truth_tokens = MetricsCalculator.normalize_answer(ground_truth).split()+ common = collections.Counter(pred_tokens) & collections.Counter(truth_tokens)+ num_same = sum(common.values())+ if num_same == 0: return 0.0+ precision = 1.0 * num_same / len(pred_tokens)+ recall = 1.0 * num_same / len(truth_tokens)+ return (2 * precision * recall) / (precision + recall)++ @staticmethod+ def check_refusal(text: str) -> bool:+ refusals = ["not mentioned", "no information", "cannot be answered", "none", "unknown", "don't know"]+ return any(r in text.lower() for r in refusals)++ @staticmethod+ def check_recall(retrieved_texts: List[str], evidence_list: List[str], soft_threshold: float = 0.8, min_soft_match_tokens: int = 4) -> float:+ """+ Calculate retrieval recall combining strict substring matching with dynamic token-based soft matching.++ Approach:+ - Combine and preprocess: concatenate multiple retrieved text chunks into a single string.+ - Strict matching first: check if evidence exists as a complete substring in the combined retrieved text.+ - Length blocking mechanism: calculate effective token count of evidence. If below threshold (e.g., short IDs or entities), directly determine no hit after strict match failure, prohibiting soft matching.+ - Soft matching fallback: for long text evidence, calculate token coverage in retrieved text, consider hit if threshold is met.+ - Equal weighting: each evidence has equal weight, final score is hit count / total count.++ Args:+ retrieved_texts: List[str], list of text chunks returned by retrieval module (required)+ evidence_list: List[str], ground truth evidence list containing IDs or long text evidence (required)+ soft_threshold: float, coverage threshold for soft matching to be considered a hit (optional, 0.0~1.0, default 0.8)+ min_soft_match_tokens: int, minimum effective token count threshold allowing fallback to soft matching (optional, default 4. Short texts below this length require strict matching)++ Returns:+ float, retrieval recall score, range 0.0 to 1.0+ """+ if not evidence_list:+ return 0.0++ combined_retrieved = " ".join(retrieved_texts)++ normalized_retrieved = MetricsCalculator.normalize_answer(combined_retrieved)+ ret_tokens = set(normalized_retrieved.split())++ hit_count = 0++ for evidence in evidence_list:+ if evidence in combined_retrieved:+ hit_count += 1+ continue++ normalized_ev = MetricsCalculator.normalize_answer(evidence)+ ev_tokens = set(normalized_ev.split())++ if not ev_tokens:+ continue++ if len(ev_tokens) < min_soft_match_tokens:+ continue++ overlap_count = len(ev_tokens & ret_tokens)+ coverage = overlap_count / len(ev_tokens)++ if coverage >= soft_threshold:+ hit_count += 1++ return hit_count / len(evidence_list)diff --git a/benchmark/RAG/src/core/monitor.py b/benchmark/RAG/src/core/monitor.pynew file mode 100644index 000000000..4c25fb300--- /dev/null+++ b/benchmark/RAG/src/core/monitor.py@@ -0,0 +1,51 @@+import threading+import time+from dataclasses import dataclass+++@dataclass+class MonitorStats:+ active_threads: int = 0+ completed_tasks: int = 0+ failed_tasks: int = 0+ total_tokens: int = 0+ start_time: float = 0.0+++class BenchmarkMonitor:+ def __init__(self):+ self._lock = threading.Lock()+ self.stats = MonitorStats()+ self.stats.start_time = time.time()++ def worker_start(self):+ with self._lock:+ self.stats.active_threads += 1++ def worker_end(self, tokens=0, success=True):+ with self._lock:+ self.stats.active_threads -= 1+ self.stats.completed_tasks += 1+ self.stats.total_tokens += tokens+ if not success:+ self.stats.failed_tasks += 1++ def get_status_dict(self):+ """Return real-time status dictionary for tqdm progress bar display"""+ elapsed = time.time() - self.stats.start_time+ qps = self.stats.completed_tasks / elapsed if elapsed > 0 else 0++ tokens = self.stats.total_tokens+ if tokens > 1_000_000:+ token_str = f"{tokens/1_000_000:.1f}M"+ elif tokens > 1_000:+ token_str = f"{tokens/1_000:.1f}k"+ else:+ token_str = str(tokens)++ return {+ "Active": self.stats.active_threads,+ "QPS": f"{qps:.2f}",+ "Tokens": token_str,+ "Errs": self.stats.failed_tasks+ }diff --git a/benchmark/RAG/src/core/vector_store.py b/benchmark/RAG/src/core/vector_store.pynew file mode 100644index 000000000..feb06fe21--- /dev/null+++ b/benchmark/RAG/src/core/vector_store.py@@ -0,0 +1,105 @@+import os+import time+from typing import List+import sys+from pathlib import Path++sys.path.append(str(Path(__file__).parent.parent))++from adapters.base import StandardDoc, StandardSample+import tiktoken+import openviking as ov+++class VikingStoreWrapper:+ def __init__(self, store_path: str):+ self.store_path = store_path+ if not os.path.exists(store_path):+ os.makedirs(store_path)++ self.client = ov.SyncOpenViking(path=store_path)++ try:+ self.enc = tiktoken.get_encoding("cl100k_base")+ except Exception as e:+ print(f"[Warning] tiktoken init failed: {e}")+ self.enc = None++ def count_tokens(self, text: str) -> int:+ if not text or not self.enc:+ return 0+ return len(self.enc.encode(str(text)))++ def ingest(self, samples: List[StandardDoc], max_workers=10, monitor=None, ingest_mode="per_file") -> dict:+ start_time = time.time()+ total_input_tokens = 0+ total_output_tokens = 0+ total_embedding_tokens = 0++ if not samples:+ return {+ "time": time.time() - start_time,+ "input_tokens": 0,+ "output_tokens": 0+ }++ if ingest_mode == "directory":+ doc_paths = [os.path.abspath(s.doc_path) for s in samples]+ common_ancestor = None+ if doc_paths:+ try:+ common_ancestor = os.path.commonpath(doc_paths)+ except ValueError:+ common_ancestor = None++ if common_ancestor:+ result = self.client.add_resource(common_ancestor, wait=True, telemetry=True)+ telemetry = result.get("telemetry", {})+ summary = telemetry.get("summary", {})+ tokens = summary.get("tokens", {})+ llm_tokens = tokens.get("llm", {})+ embedding_tokens = tokens.get("embedding", {})+ total_input_tokens = llm_tokens.get("input", 0)+ total_output_tokens = llm_tokens.get("output", 0)+ total_embedding_tokens = embedding_tokens.get("total", 0)+ else:+ for sample in samples:+ result = self.client.add_resource(sample.doc_path, wait=True, telemetry=True)+ telemetry = result.get("telemetry", {})+ summary = telemetry.get("summary", {})+ tokens = summary.get("tokens", {})+ llm_tokens = tokens.get("llm", {})+ embedding_tokens = tokens.get("embedding", {})+ total_input_tokens += llm_tokens.get("input", 0)+ total_output_tokens += llm_tokens.get("output", 0)+ total_embedding_tokens += embedding_tokens.get("total", 0)+ else:+ for sample in samples:+ result = self.client.add_resource(sample.doc_path, wait=True, telemetry=True)+ telemetry = result.get("telemetry", {})+ summary = telemetry.get("summary", {})+ tokens = summary.get("tokens", {})+ llm_tokens = tokens.get("llm", {})+ embedding_tokens = tokens.get("embedding", {})+ total_input_tokens += llm_tokens.get("input", 0)+ total_output_tokens += llm_tokens.get("output", 0)+ total_embedding_tokens += embedding_tokens.get("total", 0)++ return {+ "time": time.time() - start_time,+ "input_tokens": total_input_tokens,+ "output_tokens": total_output_tokens,+ "embedding_tokens": total_embedding_tokens+ }++ def retrieve(self, query: str, topk: int, target_uri: str = "viking://resources"):+ """Execute retrieval"""+ return self.client.find(query=query, limit=topk, target_uri=target_uri)++ def read_resource(self, uri: str) -> str:+ """Read resource content"""+ return str(self.client.read(uri))++ def clear(self):+ """Clear the store"""+ self.client.rm("viking://resources", recursive=True)diff --git a/benchmark/RAG/src/pipeline.py b/benchmark/RAG/src/pipeline.pynew file mode 100644index 000000000..54bd67937--- /dev/null+++ b/benchmark/RAG/src/pipeline.py@@ -0,0 +1,352 @@+import os+import json+import time+import random+import re+from concurrent.futures import ThreadPoolExecutor, as_completed+from tqdm import tqdm+from pathlib import Path+import sys++sys.path.append(str(Path(__file__).parent))++from adapters.base import BaseAdapter+from core.logger import get_logger+from core.vector_store import VikingStoreWrapper+from core.monitor import BenchmarkMonitor+from core.metrics import MetricsCalculator+from core.judge_util import llm_grader+++class BenchmarkPipeline:+ def __init__(self, config, adapter: BaseAdapter, vector_db: VikingStoreWrapper, llm):+ self.config = config+ self.adapter = adapter+ self.db = vector_db+ self.llm = llm+ self.logger = get_logger()+ self.monitor = BenchmarkMonitor()++ self.output_dir = self.config['paths']['output_dir']+ if not os.path.exists(self.output_dir):+ os.makedirs(self.output_dir, exist_ok=True)+ self.generated_file = os.path.join(self.output_dir, "generated_answers.json")+ self.eval_file = os.path.join(self.output_dir, "qa_eval_detailed_results.json")+ self.report_file = os.path.join(self.output_dir, "benchmark_metrics_report.json")++ self.metrics_summary = {+ "insertion": {"time": 0, "input_tokens": 0, "output_tokens": 0, "embedding_tokens": 0},+ "deletion": {"time": 0, "input_tokens": 0, "output_tokens": 0, "embedding_tokens": 0}+ }++ def run_generation(self):+ """Step 1: Data Preparation"""+ self.logger.info(">>> Stage: Ingestion & Generation")+ skip_ingestion = self.config['execution'].get('skip_ingestion', False)+ doc_dir = self.config['paths'].get('doc_output_dir')+ if not doc_dir:+ doc_dir = os.path.join(self.output_dir, "docs")++ if skip_ingestion:+ self.logger.info(f"Skipping ingestion. Reusing existing vector index at: {self.db.store_path}")+ self.metrics_summary["insertion"] = {"time": 0, "input_tokens": 0, "output_tokens": 0, "embedding_tokens": 0}+ else:+ try:+ doc_info = self.adapter.data_prepare(doc_dir)+ except Exception as e:+ self.logger.exception(f"Data preparation failed: {e}")+ exit(1)++ ingest_workers = self.config['execution'].get('ingest_workers', 10)+ ingest_mode = self.config['execution'].get('ingest_mode', 'per_file')++ mode_desc = {+ 'directory': 'Unified directory mode',+ 'per_file': 'Per-file mode'+ }+ self.logger.info(f"Ingestion mode: {ingest_mode} ({mode_desc.get(ingest_mode, 'Unknown mode')})")+ self.logger.info(f"Number of documents: {len(doc_info)}")++ ingest_stats = self.db.ingest(+ doc_info,+ max_workers=ingest_workers,+ monitor=self.monitor,+ ingest_mode=ingest_mode+ )+ self.metrics_summary["insertion"] = ingest_stats+ self.logger.info(f"Insertion finished. Time: {ingest_stats['time']:.2f}s")++ self._update_report({+ "Insertion Efficiency (Total Dataset)": {+ "Total Insertion Time (s)": self.metrics_summary["insertion"]["time"],+ "Total Input Tokens": self.metrics_summary["insertion"]["input_tokens"],+ "Total Output Tokens": self.metrics_summary["insertion"]["output_tokens"],+ "Total Embedding Tokens": self.metrics_summary["insertion"].get("embedding_tokens", 0)+ }+ })++ samples = self.adapter.load_and_transform()+ tasks = self._prepare_tasks(samples)+ results_map = {}+ max_workers = self.config['execution']['max_workers']+ task_errors = []++ with ThreadPoolExecutor(max_workers=max_workers) as executor:+ future_to_task = {+ executor.submit(self._process_generation_task, task): task+ for task in tasks+ }++ pbar = tqdm(total=len(tasks), desc="Generating Answers", unit="task")+ for future in as_completed(future_to_task):+ task = future_to_task[future]+ try:+ res = future.result()+ results_map[res['_global_index']] = res+ except Exception as e:+ self.logger.error(f"Generation failed for task {task['id']}: {e}")+ task_errors.append((task['id'], e))+ pbar.set_postfix(self.monitor.get_status_dict())+ pbar.update(1)+ pbar.close()++ if task_errors:+ first_id, first_err = task_errors[0]+ raise RuntimeError(+ f"Generation failed for {len(task_errors)} tasks; first failure task_id={first_id}: {type(first_err).__name__}: {first_err}"+ ) from first_err++ sorted_results = [results_map[i] for i in sorted(results_map.keys())]+ dataset_name = self.config.get('dataset_name', 'Unknown_Dataset')+ save_data = {+ "summary": {"dataset": dataset_name, "total_queries": len(sorted_results)},+ "results": sorted_results+ }+ total = len(sorted_results)+ if total > 0:+ self._update_report({+ "Query Efficiency (Average Per Query)": {+ "Average Retrieval Time (s)": sum(r['retrieval']['latency_sec'] for r in sorted_results) / total,+ "Average Input Tokens": sum(r['token_usage']['total_input_tokens'] for r in sorted_results) / total,+ "Average Output Tokens": sum(r['token_usage']['llm_output_tokens'] for r in sorted_results) / total,+ }+ }+ )+ with open(self.generated_file, "w", encoding="utf-8") as f:+ json.dump(save_data, f, indent=2, ensure_ascii=False)++ def run_evaluation(self):+ """Step 4: Evaluation"""+ self.logger.info(">>> Stage: Evaluation")++ if not os.path.exists(self.generated_file):+ self.logger.error("Generated answers file not found.")+ return++ with open(self.generated_file, "r", encoding="utf-8") as f:+ data = json.load(f)+ items = data.get("results", [])++ eval_items = items+ eval_results_map = {}++ with ThreadPoolExecutor(max_workers=self.config['execution']['max_workers']) as executor:+ future_to_item = {+ executor.submit(self._process_evaluation_task, item): item+ for item in eval_items+ }++ pbar = tqdm(total=len(eval_items), desc="Evaluating", unit="item")+ for future in as_completed(future_to_item):+ try:+ res = future.result()+ eval_results_map[res['_global_index']] = res+ except Exception as e:+ self.logger.error(f"Evaluation failed: {e}")+ pbar.update(1)+ pbar.close()++ eval_records = list(eval_results_map.values())+ total = len(eval_records)++ with open(self.eval_file, "w", encoding="utf-8") as f:+ json.dump({"results": eval_records}, f, indent=2, ensure_ascii=False)++ if total > 0:+ self._update_report({+ "Dataset": self.config.get('dataset_name', 'Unknown_Dataset'),+ "Total Queries Evaluated": total,+ "Performance Metrics": {+ "Average F1 Score": sum(r['metrics']['F1'] for r in eval_records) / total,+ "Average Recall": sum(r['metrics']['Recall'] for r in eval_records) / total,+ "Average Accuracy (Hit 0-4)": sum(r['metrics']['Accuracy'] for r in eval_records) / total,+ "Average Accuracy (normalization)": (sum(r['metrics']['Accuracy'] for r in eval_records) / total)/4,+ }+ })++ def run_deletion(self):+ """Step 5: Cleanup"""+ self.logger.info(">>> Stage: Deletion")+ start_time = time.time()+ self.db.clear()+ duration = time.time() - start_time+ self.metrics_summary["deletion"] = {"time": duration, "input_tokens": 0, "output_tokens": 0}+ self.logger.info(f"Deletion finished. Time: {duration:.2f}s")++ self._update_report({+ "Deletion Efficiency (Total Dataset)": {+ "Total Deletion Time (s)": duration,+ "Total Input Tokens": 0,+ "Total Output Tokens": 0+ }+ })++ def _prepare_tasks(self, samples):+ tasks = []+ global_idx = 0+ max_queries = self.config['execution'].get('max_queries')+ for sample in samples:+ for qa in sample.qa_pairs:+ if max_queries is not None and global_idx >= max_queries:+ break+ tasks.append({"id": global_idx, "sample_id": sample.sample_id, "qa": qa})+ global_idx += 1+ if max_queries is not None and global_idx >= max_queries:+ break+ return tasks++ def _process_generation_task(self, task):+ self.monitor.worker_start()+ try:+ qa = task['qa']++ t0 = time.time()+ # Get retrieval instruction from config, default to empty+ retrieval_instruction = self.config['execution'].get('retrieval_instruction', '')+ # Build enhanced query with instruction if provided+ if retrieval_instruction:+ enhanced_query = f"{retrieval_instruction} {qa.question}"+ self.logger.debug(f"[Query-{task['id']}] Using retrieval instruction: {retrieval_instruction}")+ self.logger.debug(f"[Query-{task['id']}] Enhanced query: {enhanced_query}")+ else:+ enhanced_query = qa.question+ self.logger.debug(f"[Query-{task['id']}] No retrieval instruction, using raw query")+ search_res = self.db.retrieve(query=enhanced_query, topk=self.config['execution']['retrieval_topk'])+ latency = time.time() - t0++ retrieved_texts = []+ retrieved_uris = []+ context_blocks = []++ for r in search_res.resources:+ retrieved_uris.append(r.uri)+ content = self.db.read_resource(r.uri) if getattr(r, 'level', 2) == 2 else f"{getattr(r, 'abstract', '')}\n{getattr(r, 'overview', '')}"+ retrieved_texts.append(content)+ clean = content[:8000]+ context_blocks.append(clean)++ recall = MetricsCalculator.check_recall(retrieved_texts, qa.evidence)++ full_prompt, meta = self.adapter.build_prompt(qa, context_blocks)++ ans_raw = self.llm.generate(full_prompt)++ ans = self.adapter.post_process_answer(qa, ans_raw, meta)++ in_tokens = self.db.count_tokens(full_prompt) + self.db.count_tokens(qa.question)+ out_tokens = self.db.count_tokens(ans)+ self.monitor.worker_end(tokens=in_tokens + out_tokens)++ self.logger.info(f"[Query-{task['id']}] Q: {qa.question[:30]}... | Recall: {recall:.2f} | Latency: {latency:.2f}s")++ return {+ "_global_index": task['id'], "sample_id": task['sample_id'], "question": qa.question,+ "gold_answers": qa.gold_answers, "category": str(qa.category), "evidence": qa.evidence,+ "retrieval": {"latency_sec": latency, "uris": retrieved_uris},+ "llm": {"final_answer": ans},+ "metrics": {"Recall": recall}, "token_usage": {"total_input_tokens": in_tokens, "llm_output_tokens": out_tokens}+ }+ except Exception:+ self.monitor.worker_end(success=False)+ raise++ def _process_evaluation_task(self, item):+ """+ Process a single evaluation task, computing F1 and Accuracy metrics.++ For multi-annotator scenarios (like Qasper dataset), a question may have multiple gold answers.+ Evaluation logic:+ - F1: Compute for each gold answer separately and take the maximum+ - Accuracy: Pass all gold answers to LLM at once for comprehensive judgment++ This correctly handles multi-annotator scenarios while maintaining compatibility with single-answer datasets (like Locomo).+ """+ ans, golds = item['llm']['final_answer'], item['gold_answers']++ f1 = max((MetricsCalculator.calculate_f1(ans, gt) for gt in golds), default=0.0)++ dataset_name = self.config.get('dataset_name', 'Unknown_Dataset')++ eval_record = {+ "score": 0.0,+ "reasoning": "",+ "prompt_type": ""+ }++ try:+ eval_res = llm_grader(+ self.llm.llm,+ self.config['llm']['model'],+ item['question'],+ golds,+ ans,+ dataset_name=dataset_name+ )+ eval_record = eval_res++ except Exception as e:+ self.logger.error(f"Grader error: {e}")++ if MetricsCalculator.check_refusal(ans) and any(MetricsCalculator.check_refusal(gt) for gt in golds):+ f1 = 1.0+ eval_record["score"] = 4.0+ eval_record["reasoning"] = "System successfully identified Unanswerable/Refusal condition."+ eval_record["prompt_type"] = "Heuristic_Refusal_Check"++ acc = eval_record["score"]++ item["metrics"].update({"F1": f1, "Accuracy": acc})++ item["llm_evaluation"] = {+ "prompt_used": eval_record["prompt_type"],+ "reasoning": eval_record["reasoning"],+ "normalized_score": acc+ }++ detailed_info = (+ f"\n" + "="*60 ++ f"\n[Query ID]: {item['_global_index']}"+ f"\n[Question]: {item['question']}"+ f"\n[Retrieved URIs]: {item['retrieval'].get('uris', [])}"+ f"\n[LLM Answer]: {ans}"+ f"\n[Gold Answer]: {golds}"+ f"\n[Metrics]: {item['metrics']}"+ f"\n[LLM Judge Reasoning]: {eval_record['reasoning']}"+ f"\n" + "="*60+ )+ self.logger.info(detailed_info)+ return item++ def _update_report(self, data):+ """Read existing report, merge new data, and write back"""+ report = {}+ if os.path.exists(self.report_file):+ with open(self.report_file, "r", encoding="utf-8") as f:+ try:+ report = json.load(f)+ except json.JSONDecodeError:+ report = {}+ report.update(data)+ with open(self.report_file, "w", encoding="utf-8") as f:+ json.dump(report, f, indent=4, ensure_ascii=False)+ self.logger.info(f"Report updated -> {self.report_file}")diff --git a/pyproject.toml b/pyproject.tomlindex ee4153a32..98fa54ced 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -159,6 +159,14 @@ bot-opencode = ["opencode-ai>=0.1.0a0"]bot-full = ["openviking[bot,bot-langfuse,bot-telegram,bot-feishu,bot-dingtalk,bot-slack,bot-qq,bot-sandbox,bot-fuse,bot-opencode]",]+benchmark = [+ "langchain>=1.0.0",+ "langchain-core>=1.0.0",+ "langchain-openai>=1.0.0",+ "tiktoken>=0.5.0",+ "datasets>=2.0.0",+ "pandas>=2.0.0",+][project.urls]Homepage = "https://github.com/volcengine/openviking"